Skip to content
This repository has been archived by the owner on Dec 22, 2022. It is now read-only.

Commit

Permalink
Keyword search
Browse files Browse the repository at this point in the history
  • Loading branch information
Eugene committed Aug 11, 2021
1 parent 454e6a4 commit 1ffd10c
Show file tree
Hide file tree
Showing 8 changed files with 168 additions and 95 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,5 @@
*.javac
.gwt/
target/

unittests.txt
1 change: 0 additions & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
<!-- >version>1.5.9.RELEASE</version -->
</parent>
<build>
<sourceDirectory>src</sourceDirectory>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
Expand Down
20 changes: 11 additions & 9 deletions src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,17 @@ public ResponseEntity<Product> bundleByLidvid(@ApiParam(value = "lidvid (urn)",r
return this.getProductResponseEntity(lidvid);
}

public ResponseEntity<Products> getBundles(@ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue="0") Integer start
,@ApiParam(value = "maximum number of matching results returned, for pagination", defaultValue = "100") @Valid @RequestParam(value = "limit", required = false, defaultValue="100") Integer limit
,@ApiParam(value = "search query, complex query uses eq,ne,gt,ge,lt,le,(,),not,and,or. Properties are named as in 'properties' attributes, literals are strings between \" or numbers. Detailed query specification is available at https://bit.ly/393i1af") @Valid @RequestParam(value = "q", required = false) String q
,@ApiParam(value = "returned fields, syntax field0,field1") @Valid @RequestParam(value = "fields", required = false) List<String> fields
,@ApiParam(value = "sort results, syntax asc(field0),desc(field1)") @Valid @RequestParam(value = "sort", required = false) List<String> sort
,@ApiParam(value = "only return the summary, useful to get the list of available properties", defaultValue = "false") @Valid @RequestParam(value = "only-summary", required = false, defaultValue="false") Boolean onlySummary
) {
return this.getProductsResponseEntity(q, start, limit, fields, sort, onlySummary);
}
public ResponseEntity<Products> getBundles(
@ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue = "0") Integer start,
@ApiParam(value = "maximum number of matching results returned, for pagination", defaultValue = "100") @Valid @RequestParam(value = "limit", required = false, defaultValue = "100") Integer limit,
@ApiParam(value = "search query, complex query uses eq,ne,gt,ge,lt,le,(,),not,and,or. Properties are named as in 'properties' attributes, literals are strings between \" or numbers. Detailed query specification is available at https://bit.ly/393i1af") @Valid @RequestParam(value = "q", required = false) String q,
@ApiParam(value = "keyword search query") @Valid @RequestParam(value = "keyword", required = false) String keyword,
@ApiParam(value = "returned fields, syntax field0,field1") @Valid @RequestParam(value = "fields", required = false) List<String> fields,
@ApiParam(value = "sort results, syntax asc(field0),desc(field1)") @Valid @RequestParam(value = "sort", required = false) List<String> sort,
@ApiParam(value = "only return the summary, useful to get the list of available properties", defaultValue = "false") @Valid @RequestParam(value = "only-summary", required = false, defaultValue = "false") Boolean onlySummary)
{
return this.getProductsResponseEntity(q, keyword, start, limit, fields, sort, onlySummary);
}


public ResponseEntity<Products> collectionsOfABundle(@ApiParam(value = "lidvid (urn)",required=true) @PathVariable("lidvid") String lidvid
Expand Down
22 changes: 11 additions & 11 deletions src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,18 +54,18 @@ public ResponseEntity<Product> collectionsByLidvid(@ApiParam(value = "lidvid (ur
}


public ResponseEntity<Products> getCollection(
@ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue = "0") Integer start,
@ApiParam(value = "maximum number of matching results returned, for pagination", defaultValue = "100") @Valid @RequestParam(value = "limit", required = false, defaultValue = "100") Integer limit,
@ApiParam(value = "search query, complex query uses eq,ne,gt,ge,lt,le,(,),not,and,or. Properties are named as in 'properties' attributes, literals are strings between \" or numbers. Detailed query specification is available at https://bit.ly/393i1af") @Valid @RequestParam(value = "q", required = false) String q,
@ApiParam(value = "keyword search query") @Valid @RequestParam(value = "keyword", required = false) String keyword,
@ApiParam(value = "returned fields, syntax field0,field1") @Valid @RequestParam(value = "fields", required = false) List<String> fields,
@ApiParam(value = "sort results, syntax asc(field0),desc(field1)") @Valid @RequestParam(value = "sort", required = false) List<String> sort,
@ApiParam(value = "only return the summary, useful to get the list of available properties", defaultValue = "false") @Valid @RequestParam(value = "only-summary", required = false, defaultValue = "false") Boolean onlySummary)
{
return this.getProductsResponseEntity(q, keyword, start, limit, fields, sort, onlySummary);
}

public ResponseEntity<Products> getCollection(@ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue="0") Integer start
,@ApiParam(value = "maximum number of matching results returned, for pagination", defaultValue = "100") @Valid @RequestParam(value = "limit", required = false, defaultValue="100") Integer limit
,@ApiParam(value = "search query, complex query uses eq,ne,gt,ge,lt,le,(,),not,and,or. Properties are named as in 'properties' attributes, literals are strings between \" or numbers. Detailed query specification is available at https://bit.ly/393i1af") @Valid @RequestParam(value = "q", required = false) String q
,@ApiParam(value = "returned fields, syntax field0,field1") @Valid @RequestParam(value = "fields", required = false) List<String> fields
,@ApiParam(value = "sort results, syntax asc(field0),desc(field1)") @Valid @RequestParam(value = "sort", required = false) List<String> sort
,@ApiParam(value = "only return the summary, useful to get the list of available properties", defaultValue = "false") @Valid @RequestParam(value = "only-summary", required = false, defaultValue="false") Boolean onlySummary
) {

return this.getProductsResponseEntity(q, start, limit, fields, sort, onlySummary);
}


public ResponseEntity<Products> productsOfACollection(@ApiParam(value = "lidvid (urn)",required=true) @PathVariable("lidvid") String lidvid
,@ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue="0") Integer start
Expand Down
8 changes: 4 additions & 4 deletions src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiBareController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,9 @@ protected void fillProductsFromParents (Products products, HashSet<String> uniqu
}

@SuppressWarnings("unchecked")
protected Products getProducts(String q, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException {
protected Products getProducts(String q, String keyword, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException {

SearchRequest searchRequest = this.searchRequestBuilder.getSearchProductsRequest(q, fields, start, limit, this.presetCriteria);
SearchRequest searchRequest = this.searchRequestBuilder.getSearchProductsRequest(q, keyword, fields, start, limit, this.presetCriteria);

SearchResponse searchResponse = this.esRegistryConnection.getRestHighLevelClient().search(searchRequest,
RequestOptions.DEFAULT);
Expand Down Expand Up @@ -178,7 +178,7 @@ protected Products getProducts(String q, int start, int limit, List<String> fiel



protected ResponseEntity<Products> getProductsResponseEntity(String q, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) {
protected ResponseEntity<Products> getProductsResponseEntity(String q, String keyword, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) {
String accept = this.request.getHeader("Accept");
log.info("accept value is " + accept);
if ((accept != null
Expand All @@ -191,7 +191,7 @@ protected ResponseEntity<Products> getProductsResponseEntity(String q, int start
try {


Products products = this.getProducts(q, start, limit, fields, sort, onlySummary);
Products products = this.getProducts(q, keyword, start, limit, fields, sort, onlySummary);

return new ResponseEntity<Products>(products, HttpStatus.OK);

Expand Down
20 changes: 10 additions & 10 deletions src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,17 @@ public MyProductsApiController(ObjectMapper objectMapper, HttpServletRequest req
}


public ResponseEntity<Products> products(@ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue="0") Integer start
,@ApiParam(value = "maximum number of matching results returned, for pagination", defaultValue = "100") @Valid @RequestParam(value = "limit", required = false, defaultValue="100") Integer limit
,@ApiParam(value = "search query") @Valid @RequestParam(value = "q", required = false) String q
,@ApiParam(value = "returned fields, syntax field0,field1") @Valid @RequestParam(value = "fields", required = false) List<String> fields
,@ApiParam(value = "sort results, syntax asc(field0),desc(field1)") @Valid @RequestParam(value = "sort", required = false) List<String> sort
,@ApiParam(value = "only return the summary, useful to get the list of available properties", defaultValue = "false") @Valid @RequestParam(value = "only-summary", required = false, defaultValue="false") Boolean onlySummary
) {
return this.getProductsResponseEntity(q, start, limit, fields, sort, onlySummary);

public ResponseEntity<Products> products(
@ApiParam(value = "offset in matching result list, for pagination", defaultValue = "0") @Valid @RequestParam(value = "start", required = false, defaultValue = "0") Integer start,
@ApiParam(value = "maximum number of matching results returned, for pagination", defaultValue = "100") @Valid @RequestParam(value = "limit", required = false, defaultValue = "100") Integer limit,
@ApiParam(value = "search query") @Valid @RequestParam(value = "q", required = false) String q,
@ApiParam(value = "keyword search query") @Valid @RequestParam(value = "keyword", required = false) String keyword,
@ApiParam(value = "returned fields, syntax field0,field1") @Valid @RequestParam(value = "fields", required = false) List<String> fields,
@ApiParam(value = "sort results, syntax asc(field0),desc(field1)") @Valid @RequestParam(value = "sort", required = false) List<String> sort,
@ApiParam(value = "only return the summary, useful to get the list of available properties", defaultValue = "false") @Valid @RequestParam(value = "only-summary", required = false, defaultValue = "false") Boolean onlySummary)
{
return this.getProductsResponseEntity(q, keyword, start, limit, fields, sort, onlySummary);
}



public ResponseEntity<Product> productsByLidvid(@ApiParam(value = "lidvid (urn)",required=true) @PathVariable("lidvid") String lidvid) {
Expand Down
188 changes: 129 additions & 59 deletions ...gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchRegistrySearchRequestBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,12 @@
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.ExistsQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.PrefixQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.common.unit.TimeValue;


Expand Down Expand Up @@ -169,78 +171,42 @@ public GetRequest getGetProductRequest(String lidvid) {


public SearchRequest getSearchProductsRequest(
String queryString,
String queryString,
String keyword,
List<String> fields,
int start, int limit,
Map<String,String> presetCriteria) {

BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();


if (queryString != null) {
boolQuery = this.parseQueryString(queryString);
}

Map<String,String> presetCriteria)
{
QueryBuilder query = null;


for (Map.Entry<String, String> e : presetCriteria.entrySet()) {
//example "product_class", "Product_Collection"
boolQuery.must(QueryBuilders.termQuery(e.getKey(), e.getValue() ));
// "keyword" parameter provided. Run full-text query.
if(keyword != null && !keyword.isBlank())
{
query = createKeywordQuery(keyword, presetCriteria);
}
// Run PDS query language ("q" parameter) query
else
{
query = createPqlQuery(queryString, fields, presetCriteria);
}


String[] includedFields = null;
if (fields != null) {
boolQuery.must(this.parseFields(fields));
HashSet<String> esFields = new HashSet<String>(Arrays.asList(EntityProduct.JSON_PROPERTIES));
for (int i=0 ; i<fields.size() ; i++) {
String includedField = ElasticSearchUtil.jsonPropertyToElasticProperty((String)fields.get(i));
ElasticSearchRegistrySearchRequestBuilder.log.info("add property " + includedField + " to search");
esFields.add(includedField);
}
includedFields = esFields.toArray(new String[esFields.size()]);
}

SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(boolQuery);
searchSourceBuilder.from(start);
searchSourceBuilder.size(limit);

FetchSourceContext fetchSourceContext = new FetchSourceContext(
true,
includedFields,
new String[] { EntitytProductWithBlob.BLOB_PROPERTY });

searchSourceBuilder.fetchSource(fetchSourceContext);

searchSourceBuilder.timeout(new TimeValue(this.timeOutSeconds,
TimeUnit.SECONDS));

SearchRequest searchRequest = new SearchRequest();
searchRequest.source(searchSourceBuilder);


searchRequest.indices(this.registryIndex);

ElasticSearchRegistrySearchRequestBuilder.log.info("q value: " + queryString);
ElasticSearchRegistrySearchRequestBuilder.log.info("request elasticSearch :" + searchRequest.toString());

return searchRequest;
String[] includedFields = createIncludedFields(fields);

SearchRequest searchRequest = createSearchRequest(query, start, limit, includedFields);

return searchRequest;
}


public SearchRequest getSearchProductRequest(String queryString, List<String> fields, int start, int limit) {
public SearchRequest getSearchProductRequest(String queryString, String keyword, List<String> fields, int start, int limit) {
Map<String, String> presetCriteria = new HashMap<String, String>();
return getSearchProductsRequest(queryString, fields, start, limit, presetCriteria);
return getSearchProductsRequest(queryString, keyword, fields, start, limit, presetCriteria);
}

public SearchRequest getSearchCollectionRequest(String queryString, List<String> fields, int start, int limit) {

public SearchRequest getSearchCollectionRequest(String queryString, String keyword, List<String> fields, int start, int limit) {
Map<String, String> presetCriteria = new HashMap<String, String>();
presetCriteria.put("product_class", "Product_Collection");
return getSearchProductsRequest(queryString, fields, start, limit, presetCriteria);

return getSearchProductsRequest(queryString, keyword, fields, start, limit, presetCriteria);
}

static public SearchRequest getQueryFieldFromLidvid (String lidvid, String field, String es_index)
Expand Down Expand Up @@ -324,4 +290,108 @@ static public SearchRequest getQueryForKVPs (Map<String,List<String>> kvps, List
}
return request;
}


/**
* Create full-text / keyword query (Uses Lucene query language for now)
* @param req request parameters
* @param presetCriteria preset criteria
* @return a query
*/
private QueryBuilder createKeywordQuery(String keyword, Map<String, String> presetCriteria)
{
// Lucene query
QueryStringQueryBuilder luceneQuery = QueryBuilders.queryStringQuery(keyword);
// Search in following fields only
luceneQuery.field("title");
luceneQuery.field("description");

// Boolean (root) query
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(luceneQuery);

// Preset criteria filter
if(presetCriteria != null)
{
presetCriteria.forEach((key, value) ->
{
boolQuery.filter(QueryBuilders.termQuery(key, value));
});
}

return boolQuery;
}


/**
* Create PDS query language query
* @param req request parameters
* @param presetCriteria preset criteria
* @return a query
*/
private QueryBuilder createPqlQuery(String queryString, List<String> fields, Map<String, String> presetCriteria)
{
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();

if (queryString != null)
{
ElasticSearchRegistrySearchRequestBuilder.log.debug("q value: " + queryString);
boolQuery = this.parseQueryString(queryString);
}

for (Map.Entry<String, String> e : presetCriteria.entrySet())
{
// example "product_class", "Product_Collection"
boolQuery.must(QueryBuilders.termQuery(e.getKey(), e.getValue()));
}

if (fields != null)
{
boolQuery.must(this.parseFields(fields));
}

return boolQuery;
}


private String[] createIncludedFields(List<String> fields)
{
if(fields == null || fields.isEmpty()) return null;

HashSet<String> esFields = new HashSet<String>(Arrays.asList(EntityProduct.JSON_PROPERTIES));
for (int i = 0; i < fields.size(); i++)
{
String includedField = ElasticSearchUtil.jsonPropertyToElasticProperty((String) fields.get(i));
ElasticSearchRegistrySearchRequestBuilder.log.debug("add field " + includedField + " to search");
esFields.add(includedField);
}

String[] includedFields = esFields.toArray(new String[esFields.size()]);

return includedFields;
}


private SearchRequest createSearchRequest(QueryBuilder query, int from, int size, String[] includedFields)
{
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(query);
searchSourceBuilder.from(from);
searchSourceBuilder.size(size);

String[] excludeFields = { EntitytProductWithBlob.BLOB_PROPERTY };
FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includedFields, excludeFields);
searchSourceBuilder.fetchSource(fetchSourceContext);

searchSourceBuilder.timeout(new TimeValue(this.timeOutSeconds, TimeUnit.SECONDS));

SearchRequest searchRequest = new SearchRequest();
searchRequest.source(searchSourceBuilder);
searchRequest.indices(this.registryIndex);

ElasticSearchRegistrySearchRequestBuilder.log.debug("request elasticSearch :" + searchRequest.toString());

return searchRequest;
}

}
2 changes: 1 addition & 1 deletion ...nasa/pds/api/engineering/elasticsearch/ElasticSearchRegistrySearchRequestBuilderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ void testGetSearchRequest() {
queryString = queryEntry.getKey();
List<String> fields = new ArrayList<String>(Arrays.asList("title","ops:Label_File_Info.ops:md5_checksum"));
searchRequest = this.requestBuilder.getSearchCollectionRequest(
queryString,
queryString, null,
fields, 0, 10);

/* Write test input in a file */
Expand Down

0 comments on commit 1ffd10c

Please sign in to comment.