Full-text / keyword search

.gitignore

@@ -41,3 +41,5 @@
 *.javac
 .gwt/
 target/
+
+unittests.txt

pom.xml

@@ -17,7 +17,6 @@
 		<!-- >version>1.5.9.RELEASE</version -->
 	</parent>
 	<build>
-		<sourceDirectory>src</sourceDirectory>
 		<plugins>
 			<plugin>
 				<groupId>org.springframework.boot</groupId>

src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java

@@ -50,15 +50,17 @@ public ResponseEntity<Product> bundleByLidvid(@ApiParam(value = "lidvid (urn)",r
     	return this.getProductResponseEntity(lidvid);
     }
 
-    public ResponseEntity<Products> getBundles(@ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue="0") Integer start
-,@ApiParam(value = "maximum number of matching results returned, for pagination", defaultValue = "100") @Valid @RequestParam(value = "limit", required = false, defaultValue="100") Integer limit
-,@ApiParam(value = "search query, complex query uses eq,ne,gt,ge,lt,le,(,),not,and,or. Properties are named as in 'properties' attributes, literals are strings between \" or numbers. Detailed query specification is available at https://bit.ly/393i1af") @Valid @RequestParam(value = "q", required = false) String q
-,@ApiParam(value = "returned fields, syntax field0,field1") @Valid @RequestParam(value = "fields", required = false) List<String> fields
-,@ApiParam(value = "sort results, syntax asc(field0),desc(field1)") @Valid @RequestParam(value = "sort", required = false) List<String> sort
-,@ApiParam(value = "only return the summary, useful to get the list of available properties", defaultValue = "false") @Valid @RequestParam(value = "only-summary", required = false, defaultValue="false") Boolean onlySummary
-) {
-    	return this.getProductsResponseEntity(q, start, limit, fields, sort, onlySummary);
-    }
+    public ResponseEntity<Products> getBundles(
+            @ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue = "0") Integer start,
+            @ApiParam(value = "maximum number of matching results returned, for pagination", defaultValue = "100") @Valid @RequestParam(value = "limit", required = false, defaultValue = "100") Integer limit,
+            @ApiParam(value = "search query, complex query uses eq,ne,gt,ge,lt,le,(,),not,and,or. Properties are named as in 'properties' attributes, literals are strings between \" or numbers. Detailed query specification is available at https://bit.ly/393i1af") @Valid @RequestParam(value = "q", required = false) String q,
+            @ApiParam(value = "keyword search query") @Valid @RequestParam(value = "keyword", required = false) String keyword,
+            @ApiParam(value = "returned fields, syntax field0,field1") @Valid @RequestParam(value = "fields", required = false) List<String> fields,
+            @ApiParam(value = "sort results, syntax asc(field0),desc(field1)") @Valid @RequestParam(value = "sort", required = false) List<String> sort,
+            @ApiParam(value = "only return the summary, useful to get the list of available properties", defaultValue = "false") @Valid @RequestParam(value = "only-summary", required = false, defaultValue = "false") Boolean onlySummary)
+    {
+        return this.getProductsResponseEntity(q, keyword, start, limit, fields, sort, onlySummary);
+    }    
 
 
     public ResponseEntity<Products> collectionsOfABundle(@ApiParam(value = "lidvid (urn)",required=true) @PathVariable("lidvid") String lidvid

src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java

@@ -54,18 +54,18 @@ public ResponseEntity<Product> collectionsByLidvid(@ApiParam(value = "lidvid (ur
     }
 
 
+    public ResponseEntity<Products> getCollection(
+            @ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue = "0") Integer start,
+            @ApiParam(value = "maximum number of matching results returned, for pagination", defaultValue = "100") @Valid @RequestParam(value = "limit", required = false, defaultValue = "100") Integer limit,
+            @ApiParam(value = "search query, complex query uses eq,ne,gt,ge,lt,le,(,),not,and,or. Properties are named as in 'properties' attributes, literals are strings between \" or numbers. Detailed query specification is available at https://bit.ly/393i1af") @Valid @RequestParam(value = "q", required = false) String q,
+            @ApiParam(value = "keyword search query") @Valid @RequestParam(value = "keyword", required = false) String keyword,
+            @ApiParam(value = "returned fields, syntax field0,field1") @Valid @RequestParam(value = "fields", required = false) List<String> fields,
+            @ApiParam(value = "sort results, syntax asc(field0),desc(field1)") @Valid @RequestParam(value = "sort", required = false) List<String> sort,
+            @ApiParam(value = "only return the summary, useful to get the list of available properties", defaultValue = "false") @Valid @RequestParam(value = "only-summary", required = false, defaultValue = "false") Boolean onlySummary)
+    {
+        return this.getProductsResponseEntity(q, keyword, start, limit, fields, sort, onlySummary);
+    }    
 
-    public ResponseEntity<Products> getCollection(@ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue="0") Integer start
-    		,@ApiParam(value = "maximum number of matching results returned, for pagination", defaultValue = "100") @Valid @RequestParam(value = "limit", required = false, defaultValue="100") Integer limit
-    		,@ApiParam(value = "search query, complex query uses eq,ne,gt,ge,lt,le,(,),not,and,or. Properties are named as in 'properties' attributes, literals are strings between \" or numbers. Detailed query specification is available at https://bit.ly/393i1af") @Valid @RequestParam(value = "q", required = false) String q
-    		,@ApiParam(value = "returned fields, syntax field0,field1") @Valid @RequestParam(value = "fields", required = false) List<String> fields
-    		,@ApiParam(value = "sort results, syntax asc(field0),desc(field1)") @Valid @RequestParam(value = "sort", required = false) List<String> sort
-    		,@ApiParam(value = "only return the summary, useful to get the list of available properties", defaultValue = "false") @Valid @RequestParam(value = "only-summary", required = false, defaultValue="false") Boolean onlySummary
-    		) {
-
-    	return this.getProductsResponseEntity(q, start, limit, fields, sort, onlySummary);
-    }
-
 
     public ResponseEntity<Products> productsOfACollection(@ApiParam(value = "lidvid (urn)",required=true) @PathVariable("lidvid") String lidvid
     		,@ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue="0") Integer start

src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiBareController.java

@@ -113,9 +113,9 @@ protected void fillProductsFromParents (Products products, HashSet<String> uniqu
     }
 
     @SuppressWarnings("unchecked")
-	protected Products getProducts(String q, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException {
+	protected Products getProducts(String q, String keyword, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException {
 
-    	SearchRequest searchRequest = this.searchRequestBuilder.getSearchProductsRequest(q, fields, start, limit, this.presetCriteria);
+    	SearchRequest searchRequest = this.searchRequestBuilder.getSearchProductsRequest(q, keyword, fields, start, limit, this.presetCriteria);
 
     	SearchResponse searchResponse = this.esRegistryConnection.getRestHighLevelClient().search(searchRequest, 
     			RequestOptions.DEFAULT);
@@ -178,7 +178,7 @@ protected Products getProducts(String q, int start, int limit, List<String> fiel
 
 
 
-    protected ResponseEntity<Products> getProductsResponseEntity(String q, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) {
+    protected ResponseEntity<Products> getProductsResponseEntity(String q, String keyword, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) {
         String accept = this.request.getHeader("Accept");
         log.info("accept value is " + accept);
         if ((accept != null 
@@ -191,7 +191,7 @@ protected ResponseEntity<Products> getProductsResponseEntity(String q, int start
         	try {
 
 
-        		Products products = this.getProducts(q, start, limit, fields, sort, onlySummary);
+        		Products products = this.getProducts(q, keyword, start, limit, fields, sort, onlySummary);
 
 	        	return new ResponseEntity<Products>(products, HttpStatus.OK);
 

src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java

@@ -45,17 +45,17 @@ public MyProductsApiController(ObjectMapper objectMapper, HttpServletRequest req
     }
 
 
-    public ResponseEntity<Products> products(@ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue="0") Integer start
-,@ApiParam(value = "maximum number of matching results returned, for pagination", defaultValue = "100") @Valid @RequestParam(value = "limit", required = false, defaultValue="100") Integer limit
-,@ApiParam(value = "search query") @Valid @RequestParam(value = "q", required = false) String q
-,@ApiParam(value = "returned fields, syntax field0,field1") @Valid @RequestParam(value = "fields", required = false) List<String> fields
-,@ApiParam(value = "sort results, syntax asc(field0),desc(field1)") @Valid @RequestParam(value = "sort", required = false) List<String> sort
-,@ApiParam(value = "only return the summary, useful to get the list of available properties", defaultValue = "false") @Valid @RequestParam(value = "only-summary", required = false, defaultValue="false") Boolean onlySummary
-)  {
-    	return this.getProductsResponseEntity(q, start, limit, fields, sort, onlySummary);
-
+    public ResponseEntity<Products> products(
+            @ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue = "0") Integer start,
+            @ApiParam(value = "maximum number of matching results returned, for pagination", defaultValue = "100") @Valid @RequestParam(value = "limit", required = false, defaultValue = "100") Integer limit,
+            @ApiParam(value = "search query") @Valid @RequestParam(value = "q", required = false) String q,
+            @ApiParam(value = "keyword search query") @Valid @RequestParam(value = "keyword", required = false) String keyword,
+            @ApiParam(value = "returned fields, syntax field0,field1") @Valid @RequestParam(value = "fields", required = false) List<String> fields,
+            @ApiParam(value = "sort results, syntax asc(field0),desc(field1)") @Valid @RequestParam(value = "sort", required = false) List<String> sort,
+            @ApiParam(value = "only return the summary, useful to get the list of available properties", defaultValue = "false") @Valid @RequestParam(value = "only-summary", required = false, defaultValue = "false") Boolean onlySummary)
+    {
+        return this.getProductsResponseEntity(q, keyword, start, limit, fields, sort, onlySummary);
     }
-
 
 
     public ResponseEntity<Product> productsByLidvid(@ApiParam(value = "lidvid (urn)",required=true) @PathVariable("lidvid") String lidvid) {

src/main/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchRegistrySearchRequestBuilder.java

@@ -24,10 +24,12 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.QueryStringQueryBuilder;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.ExistsQueryBuilder;
 import org.elasticsearch.index.query.MatchQueryBuilder;
 import org.elasticsearch.index.query.PrefixQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.common.unit.TimeValue;
 
 
@@ -169,78 +171,42 @@ public GetRequest getGetProductRequest(String lidvid) {
 
 
 	public SearchRequest getSearchProductsRequest(
-			String queryString, 
+			String queryString,
+			String keyword,
 			List<String> fields, 
 			int start, int limit, 
-			Map<String,String> presetCriteria) {
-
-		BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
-
-
-		if (queryString != null) {
-			boolQuery = this.parseQueryString(queryString);
-		} 
-
+			Map<String,String> presetCriteria) 
+	{
+        QueryBuilder query = null;
 
-
-        for (Map.Entry<String, String> e : presetCriteria.entrySet()) {
-        	//example "product_class", "Product_Collection"
-        	boolQuery.must(QueryBuilders.termQuery(e.getKey(), e.getValue() ));
+        // "keyword" parameter provided. Run full-text query.
+        if(keyword != null && !keyword.isBlank())
+        {
+            query = createKeywordQuery(keyword, presetCriteria);
+        }
+        // Run PDS query language ("q" parameter) query
+        else
+        {
+            query = createPqlQuery(queryString, fields, presetCriteria);
         }
 
-
-        String[] includedFields = null;
-		if (fields != null) {
-			boolQuery.must(this.parseFields(fields));
-			HashSet<String> esFields =  new HashSet<String>(Arrays.asList(EntityProduct.JSON_PROPERTIES)); 
-			for (int i=0 ; i<fields.size() ; i++) {
-				String includedField = ElasticSearchUtil.jsonPropertyToElasticProperty((String)fields.get(i));
-				ElasticSearchRegistrySearchRequestBuilder.log.info("add property " + includedField + " to search");
-	    		esFields.add(includedField);
-	    	}
-			includedFields = esFields.toArray(new String[esFields.size()]);
-		}
-
-		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
-    	searchSourceBuilder.query(boolQuery);
-    	searchSourceBuilder.from(start); 
-    	searchSourceBuilder.size(limit); 
-
-    	FetchSourceContext fetchSourceContext = new FetchSourceContext(
-    			true, 
-    			includedFields, 
-    			new String[] {  EntitytProductWithBlob.BLOB_PROPERTY });
-
-    	searchSourceBuilder.fetchSource(fetchSourceContext);
-
-  	    searchSourceBuilder.timeout(new TimeValue(this.timeOutSeconds, 
-    			TimeUnit.SECONDS)); 
-
-    	SearchRequest searchRequest = new SearchRequest();
-    	searchRequest.source(searchSourceBuilder);
-
-
-    	searchRequest.indices(this.registryIndex);
-
-        ElasticSearchRegistrySearchRequestBuilder.log.info("q value: " + queryString);
-    	ElasticSearchRegistrySearchRequestBuilder.log.info("request elasticSearch :" + searchRequest.toString());
-
-    	return searchRequest;
+        String[] includedFields = createIncludedFields(fields);
+
+        SearchRequest searchRequest = createSearchRequest(query, start, limit, includedFields);
 
-		
+        return searchRequest;
 	}
+
 
-	public SearchRequest getSearchProductRequest(String queryString, List<String> fields, int start, int limit) {
+	public SearchRequest getSearchProductRequest(String queryString, String keyword, List<String> fields, int start, int limit) {
 		Map<String, String> presetCriteria = new HashMap<String, String>();
-		return getSearchProductsRequest(queryString, fields, start, limit, presetCriteria);		
+		return getSearchProductsRequest(queryString, keyword, fields, start, limit, presetCriteria);		
 	}
 
-	public SearchRequest getSearchCollectionRequest(String queryString, List<String> fields, int start, int limit) {
-
+	public SearchRequest getSearchCollectionRequest(String queryString, String keyword, List<String> fields, int start, int limit) {
 		Map<String, String> presetCriteria = new HashMap<String, String>();
 		presetCriteria.put("product_class", "Product_Collection");
-		return getSearchProductsRequest(queryString, fields, start, limit, presetCriteria);
-
+		return getSearchProductsRequest(queryString, keyword, fields, start, limit, presetCriteria);
 	}
 
 	static public SearchRequest getQueryFieldFromLidvid (String lidvid, String field, String es_index)
@@ -324,4 +290,108 @@ static public SearchRequest getQueryForKVPs (Map<String,List<String>> kvps, List
     	}
     	return request;
 	}
+
+
+    /**
+     * Create full-text / keyword query (Uses Lucene query language for now)
+     * @param req request parameters
+     * @param presetCriteria preset criteria
+     * @return a query
+     */
+    private QueryBuilder createKeywordQuery(String keyword, Map<String, String> presetCriteria)
+    {
+        // Lucene query
+        QueryStringQueryBuilder luceneQuery = QueryBuilders.queryStringQuery(keyword);
+        // Search in following fields only
+        luceneQuery.field("title");
+        luceneQuery.field("description");
+
+        // Boolean (root) query
+        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
+        boolQuery.must(luceneQuery);
+
+        // Preset criteria filter
+        if(presetCriteria != null)
+        {
+            presetCriteria.forEach((key, value) -> 
+            {
+                boolQuery.filter(QueryBuilders.termQuery(key, value));
+            });
+        }
+
+        return boolQuery;
+    }
+
+
+    /**
+     * Create PDS query language query
+     * @param req request parameters
+     * @param presetCriteria preset criteria
+     * @return a query
+     */
+    private QueryBuilder createPqlQuery(String queryString, List<String> fields, Map<String, String> presetCriteria)
+    {
+        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
+
+        if (queryString != null)
+        {
+            ElasticSearchRegistrySearchRequestBuilder.log.debug("q value: " + queryString);
+            boolQuery = this.parseQueryString(queryString);
+        }
+
+        for (Map.Entry<String, String> e : presetCriteria.entrySet())
+        {
+            // example "product_class", "Product_Collection"
+            boolQuery.must(QueryBuilders.termQuery(e.getKey(), e.getValue()));
+        }
+
+        if (fields != null)
+        {
+            boolQuery.must(this.parseFields(fields));
+        }
+
+        return boolQuery;
+    }
+
+
+    private String[] createIncludedFields(List<String> fields)
+    {
+        if(fields == null || fields.isEmpty()) return null;
+
+        HashSet<String> esFields = new HashSet<String>(Arrays.asList(EntityProduct.JSON_PROPERTIES));
+        for (int i = 0; i < fields.size(); i++)
+        {
+            String includedField = ElasticSearchUtil.jsonPropertyToElasticProperty((String) fields.get(i));
+            ElasticSearchRegistrySearchRequestBuilder.log.debug("add field " + includedField + " to search");
+            esFields.add(includedField);
+        }
+
+        String[] includedFields = esFields.toArray(new String[esFields.size()]);
+
+        return includedFields;
+    }
+
+
+    private SearchRequest createSearchRequest(QueryBuilder query, int from, int size, String[] includedFields)
+    {
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchSourceBuilder.query(query);
+        searchSourceBuilder.from(from);
+        searchSourceBuilder.size(size);
+
+        String[] excludeFields = { EntitytProductWithBlob.BLOB_PROPERTY };
+        FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includedFields, excludeFields);
+        searchSourceBuilder.fetchSource(fetchSourceContext);
+
+        searchSourceBuilder.timeout(new TimeValue(this.timeOutSeconds, TimeUnit.SECONDS));
+
+        SearchRequest searchRequest = new SearchRequest();
+        searchRequest.source(searchSourceBuilder);
+        searchRequest.indices(this.registryIndex);
+
+        ElasticSearchRegistrySearchRequestBuilder.log.debug("request elasticSearch :" + searchRequest.toString());
+
+        return searchRequest;
+    }
+
 }

src/test/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchRegistrySearchRequestBuilderTest.java

@@ -109,7 +109,7 @@ void testGetSearchRequest() {
 				queryString = queryEntry.getKey();
 				List<String> fields = new ArrayList<String>(Arrays.asList("title","ops:Label_File_Info.ops:md5_checksum"));
 				searchRequest = this.requestBuilder.getSearchCollectionRequest(
-						queryString,
+						queryString, null,
 						fields, 0, 10);
 
 				/* Write test input in a file */