Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add filter query parameter supporting query string syntax #645

Merged
merged 1 commit into from
Dec 13, 2017
Merged

Add filter query parameter supporting query string syntax #645

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 34 additions & 33 deletions web/app/controllers/resources/Application.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,13 @@
import play.mvc.Result;
import play.test.Helpers;
import play.twirl.api.Html;

import views.html.api;
import views.html.dataset;
import views.html.details;
import views.html.details_item;
import views.html.index;
import views.html.query;
import views.html.stars;


/**
* The main application controller.
Expand Down Expand Up @@ -169,13 +167,15 @@ public static String currentUri() {
* @param aggs The comma separated aggregation fields
* @param location A single "lat,lon" point or space delimited points polygon
* @param nested The nested object path. If non-empty, use q as nested query
* @param filter A filter to apply to the query, supports query string syntax
* @return The search results
*/
public static Promise<Result> query(final String q, final String agent,
final String name, final String subject, final String id,
final String publisher, final String issued, final String medium,
final int from, final int size, final String owner, String t, String sort,
String set, String format, String aggs, String location, String nested) {
String set, String format, String aggs, String location, String nested,
String filter) {
final String aggregations = aggs == null ? "" : aggs;
if (!aggregations.isEmpty() && !Index.SUPPORTED_AGGREGATIONS
.containsAll(Arrays.asList(aggregations.split(",")))) {
Expand Down Expand Up @@ -214,7 +214,7 @@ public static Promise<Result> query(final String q, final String agent,
} else {
result = Promise.promise(() -> {
Index queryResources = index.queryResources(queryString, from, size,
sort, owner, aggregations, location, nested);
sort, owner, aggregations, location, nested, filter);
boolean returnSuggestions = responseFormat.startsWith("json:");
JsonNode json = returnSuggestions
? toSuggestions(queryResources.getResult(), format.split(":")[1])
Expand Down Expand Up @@ -253,29 +253,30 @@ private static Promise<Result> bulkResult(final String q, final String owner,
Chunks<String> chunks = StringChunks.whenReady(out -> {
SearchResponse lastResponse =
index.<SearchResponse> withClient((Client client) -> {
QueryBuilder query = owner.isEmpty()
? QueryBuilders.queryStringQuery(q) : Index.ownerQuery(q, owner);
Index.validate(client, query);
Logger.trace("bulkResources: q={}, owner={}, query={}", q, owner,
query);
TimeValue keepAlive = new TimeValue(60000);
SearchResponse scrollResp = client.prepareSearch(Index.INDEX_NAME)
.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
.setScroll(keepAlive).setQuery(query)
.setSize(100 /* hits per shard for each scroll */).get();
String scrollId = scrollResp.getScrollId();
while (scrollResp.getHits().iterator().hasNext()) {
scrollResp.getHits().forEach((hit) -> {
out.write(hit.getSourceAsString());
out.write("\n");
QueryBuilder query =
owner.isEmpty() ? QueryBuilders.queryStringQuery(q)
: Index.ownerQuery(q, owner);
Index.validate(client, query);
Logger.trace("bulkResources: q={}, owner={}, query={}", q, owner,
query);
TimeValue keepAlive = new TimeValue(60000);
SearchResponse scrollResp = client.prepareSearch(Index.INDEX_NAME)
.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
.setScroll(keepAlive).setQuery(query)
.setSize(100 /* hits per shard for each scroll */).get();
String scrollId = scrollResp.getScrollId();
while (scrollResp.getHits().iterator().hasNext()) {
scrollResp.getHits().forEach((hit) -> {
out.write(hit.getSourceAsString());
out.write("\n");
});
scrollResp = client.prepareSearchScroll(scrollId)
.setScroll(keepAlive).execute().actionGet();
scrollId = scrollResp.getScrollId();
}
out.close();
return scrollResp;
});
scrollResp = client.prepareSearchScroll(scrollId)
.setScroll(keepAlive).execute().actionGet();
scrollId = scrollResp.getScrollId();
}
out.close();
return scrollResp;
});
Logger.trace("Last search response for bulk request: " + lastResponse);
});
return ok(chunks).as(Accept.Format.BULK.types[0]);
Expand Down Expand Up @@ -548,12 +549,13 @@ private static void uncache(List<String> ids) {
* @param set The set
* @param location A single "lat,lon" point or space delimited points polygon
* @param nested The nested object path. If non-empty, use q as nested query
* @param filter A filter to apply to the query, supports query string syntax
* @return The search results
*/
public static Promise<Result> facets(String q, String agent, String name,
String subject, String id, String publisher, String issued, String medium,
int from, int size, String owner, String t, String field, String sort,
String set, String location, String nested) {
String set, String location, String nested, String filter) {

String key = String.format("facets.%s.%s.%s.%s.%s.%s.%s.%s.%s.%s.%s.%s.%s",
field, q, agent, name, id, publisher, set, subject, issued, medium,
Expand Down Expand Up @@ -625,11 +627,10 @@ public static Promise<Result> facets(String q, String agent, String name,
boolean current =
current(subject, agent, medium, owner, t, field, term, location);

String routeUrl = routes.Application
.query(q, agentQuery, name, subjectQuery, id, publisher, issuedQuery,
mediumQuery, from, size, ownerQuery, typeQuery,
sort(sort, subjectQuery), set, null, field, locationQuery, nested)
.url();
String routeUrl = routes.Application.query(q, agentQuery, name,
subjectQuery, id, publisher, issuedQuery, mediumQuery, from, size,
ownerQuery, typeQuery, sort(sort, subjectQuery), set, null, field,
locationQuery, nested, filter).url();

String result = String.format(
"<li " + (current ? "class=\"active\"" : "")
Expand All @@ -644,7 +645,7 @@ public static Promise<Result> facets(String q, String agent, String name,

Promise<Result> promise =
query(q, agent, name, subject, id, publisher, issued, medium, from,
size, owner, t, sort, set, "json", field, location, nested)
size, owner, t, sort, set, "json", field, location, nested, filter)
.map(result -> {
JsonNode json = Json.parse(Helpers.contentAsString(result))
.get("aggregation");
Expand Down
13 changes: 8 additions & 5 deletions web/app/controllers/resources/Index.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,9 +118,8 @@ public String buildQueryString(String q, String... values) {
String fullQuery = q.isEmpty() ? "*" : "(" + q + ")";
for (int i = 0; i < values.length; i++) {
String fieldValue = values[i];
String fieldName =
fieldValue.contains("http") ? QUERY_FIELDS[i].replace(".label", ".id")
: QUERY_FIELDS[i];
String fieldName = fieldValue.contains("http")
? QUERY_FIELDS[i].replace(".label", ".id") : QUERY_FIELDS[i];
if (fieldName.toLowerCase().endsWith("date")
&& fieldValue.matches("(\\d{1,4}|\\*)-(\\d{1,4}|\\*)")) {
String[] fromTo = fieldValue.split("-");
Expand Down Expand Up @@ -162,7 +161,7 @@ private static String buildFieldQuery(String fieldValue, String fieldName) {
public long totalHits(String q) {
try {
return Cache.getOrElse("total-" + q,
() -> queryResources(q, 0, 0, "", "", "", "", "").getTotal(),
() -> queryResources(q, 0, 0, "", "", "", "", "", "").getTotal(),
Application.ONE_DAY);
} catch (Exception e) {
e.printStackTrace();
Expand All @@ -179,12 +178,13 @@ public long totalHits(String q) {
* @param aggregations The comma separated aggregation fields
* @param location A single "lat,lon" point or space delimited points polygon
* @param nested The nested object path. If non-empty, use q as nested query
* @param filter A filter to apply to the query, supports query string syntax
* @return This index, get results via {@link #getResult()} and
* {@link #getTotal()}
*/
public Index queryResources(String q, int from, int size, String sort,
String owner, @SuppressWarnings("hiding") String aggregations,
String location, String nested) {
String location, String nested, String filter) {
Index resultIndex = withClient((Client client) -> {
QueryBuilder query = owner.isEmpty() ? QueryBuilders.queryStringQuery(q)
: ownerQuery(q, owner);
Expand All @@ -209,6 +209,9 @@ public Index queryResources(String q, int from, int size, String sort,
requestBuilder =
withAggregations(client, requestBuilder, aggregations.split(","));
}
if (!filter.isEmpty()) {
requestBuilder.setPostFilter(QueryBuilders.queryStringQuery(filter));
}
SearchResponse response = requestBuilder.execute().actionGet();
SearchHits hits = response.getHits();
List<JsonNode> results = new ArrayList<>();
Expand Down
4 changes: 2 additions & 2 deletions web/conf/resources.routes
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ GET /*path/ controllers.resources.Application.redirectTo(path:
GET /resources controllers.resources.Application.index()
GET /resources/api controllers.resources.Application.api()
GET /resources/advanced controllers.resources.Application.advanced()
GET /resources/search controllers.resources.Application.query(q?="", agent?="", name?="", subject?="", id?="", publisher?="", issued?="", medium ?= "", from:Int?=0, size:Int?=15, owner?="", t?="", sort ?= "", set?="", format ?= null, aggregations ?= "", location ?= "", nested ?= "")
GET /resources/facets controllers.resources.Application.facets(q,agent?="", name?="", subject?="", id?="", publisher?="", issued?="", medium ?= "", from:Int,size:Int,owner,t,field,sort,set?="", location ?= "", nested ?= "")
GET /resources/search controllers.resources.Application.query(q?="", agent?="", name?="", subject?="", id?="", publisher?="", issued?="", medium ?= "", from:Int?=0, size:Int?=15, owner?="", t?="", sort ?= "", set?="", format ?= null, aggregations ?= "", location ?= "", nested ?= "", filter ?= "")
GET /resources/facets controllers.resources.Application.facets(q,agent?="", name?="", subject?="", id?="", publisher?="", issued?="", medium ?= "", from:Int,size:Int,owner,t,field,sort,set?="", location ?= "", nested ?= "", filter ?= "")

GET /resources/stars controllers.resources.Application.showStars(format?="", ids?="")
GET /resources/stars/clear controllers.resources.Application.clearStars(ids ?= "")
Expand Down
2 changes: 1 addition & 1 deletion web/test/tests/InputStringsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ public void setUp() throws Exception {
public void test() {
running(testServer(3333), () -> {
String uri = controllers.resources.Application.query(input, "", "", "",
"", "", "", "", 0, 10, "", "", "", "", "", "", "", "").toString();
"", "", "", "", 0, 10, "", "", "", "", "", "", "", "", "").toString();
try {
URLDecoder.decode(input, StandardCharsets.UTF_8.name());
} catch (IllegalArgumentException | UnsupportedEncodingException x) {
Expand Down
15 changes: 13 additions & 2 deletions web/test/tests/IntegrationTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public void testFacets() {
String queryString =
index.buildQueryString("köln", "", "", "", "", "", "", "", "");
Index queryResources = index.queryResources(queryString, 0, 0, "", "",
Joiner.on(",").join(Index.SUPPORTED_AGGREGATIONS), "", "");
Joiner.on(",").join(Index.SUPPORTED_AGGREGATIONS), "", "", "");
Aggregations facets = queryResources.getAggregations();
Terms terms = facets.get("type");
Stream<String> values =
Expand Down Expand Up @@ -120,7 +120,7 @@ public void responseJsonFilterGet() {
public void responseJsonFilterSearch() {
running(testServer(3333), () -> {
Index index = new Index();
index = index.queryResources("*", 0, 100, "", "", "", "", "");
index = index.queryResources("*", 0, 100, "", "", "", "", "", "");
assertThat(index.getTotal()).isGreaterThanOrEqualTo(100);
JsonNode hits = index.getResult();
assertThat(hits.isArray()).as("hits is an array").isTrue();
Expand All @@ -137,6 +137,17 @@ public void responseJsonFilterSearch() {
});
}

@Test
public void queryFilter() {
running(testServer(3333), () -> {
Index all =
new Index().queryResources("*", 0, 100, "", "", "", "", "", "");
Index nwbib = new Index().queryResources("*", 0, 100, "", "", "", "", "",
"inCollection.id:HT014176012");
assertThat(all.getTotal()).isGreaterThan(nwbib.getTotal());
});
}

@Test
public void contextContentTypeAndCorsHeaderContext() {
testJsonld("/resources/context.jsonld");
Expand Down
3 changes: 2 additions & 1 deletion web/test/tests/NestedQueryTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ public void testResultCount() {

private long hitsForQuery() {
return nestedString.isEmpty() ? index.totalHits(queryString)
: index.queryResources(queryString, 0, 1, "", "", "", "", nestedString)
: index
.queryResources(queryString, 0, 1, "", "", "", "", nestedString, "")
.getTotal();
}

Expand Down