Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Return additional field->value mapping for terms queries #4278

Merged
merged 1 commit into from
Oct 25, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import org.joda.time.DateTime;

import javax.annotation.Nullable;
import java.util.Collections;
import java.util.List;
import java.util.Map;

public class TermsHistogramResult extends IndexQueryResult {
Expand All @@ -34,7 +36,7 @@ public class TermsHistogramResult extends IndexQueryResult {
private final Map<Long, TermsResult> result;
private AbsoluteRange boundaries;

public TermsHistogramResult(@Nullable DateHistogramAggregation result, String originalQuery, String builtQuery, long size, long tookMs, Searches.DateHistogramInterval interval) {
public TermsHistogramResult(@Nullable DateHistogramAggregation result, String originalQuery, String builtQuery, long size, long tookMs, Searches.DateHistogramInterval interval, List<String> fields) {
super(originalQuery, builtQuery, tookMs);
this.size = size;
this.interval = interval;
Expand All @@ -45,7 +47,7 @@ public TermsHistogramResult(@Nullable DateHistogramAggregation result, String or
final DateTime keyAsDate = new DateTime(histogram.getKey());
final TermsAggregation termsAggregation = histogram.getFilterAggregation(Searches.AGG_FILTER).getTermsAggregation(Searches.AGG_TERMS);
final MissingAggregation missingAgregation = histogram.getMissingAggregation("missing");
final TermsResult termsResult = new TermsResult(termsAggregation, missingAgregation.getMissing(), histogram.getCount(), "", "", tookMs);
final TermsResult termsResult = new TermsResult(termsAggregation, missingAgregation.getMissing(), histogram.getCount(), "", "", tookMs, fields);

this.result.put(keyAsDate.getMillis() / 1000L, termsResult);
}
Expand Down Expand Up @@ -78,6 +80,6 @@ public AbsoluteRange getHistogramBoundaries() {
}

public static TermsHistogramResult empty(String originalQuery, String builtQuery, long size, Searches.DateHistogramInterval interval) {
return new TermsHistogramResult(null, originalQuery, builtQuery, size, 0L, interval);
return new TermsHistogramResult(null, originalQuery, builtQuery, size, 0L, interval, Collections.emptyList());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,28 +16,37 @@
*/
package org.graylog2.indexer.results;

import io.searchbox.core.search.aggregation.Bucket;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import io.searchbox.core.search.aggregation.TermsAggregation;
import org.graylog2.indexer.searches.Searches;

import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

public class TermsResult extends IndexQueryResult {

private final long total;
private final long missing;
private final long other;
private final Map<String, Long> terms;
private final Map<String, Long> terms = new HashMap<>();
private final Map<String, List<Map<String, String>>> termsMapping = new HashMap<>();

public TermsResult(TermsAggregation terms, long missingCount, long totalCount, String originalQuery, String builtQuery, long tookMs) {
this(terms, missingCount, totalCount, originalQuery, builtQuery, tookMs, Collections.emptyList());
}

public TermsResult(TermsAggregation terms, long missingCount, long totalCount, String originalQuery, String builtQuery, long tookMs, List<String> fields) {
super(originalQuery, builtQuery, tookMs);

this.total = totalCount;
this.missing = missingCount;
this.other = terms.getSumOtherDocCount();
this.terms = terms.getBuckets().stream()
.collect(Collectors.toMap(TermsAggregation.Entry::getKey, Bucket::getCount));

processTermsBuckets(terms, fields, this.terms, this.termsMapping);
}

private TermsResult(String originalQuery, String builtQuery) {
Expand All @@ -46,7 +55,29 @@ private TermsResult(String originalQuery, String builtQuery) {
this.total = 0;
this.missing = 0;
this.other = 0;
this.terms = Collections.emptyMap();
}

private static void processTermsBuckets(TermsAggregation buckets,
List<String> fields,
Map<String, Long> terms,
Map<String, List<Map<String, String>>> termsMapping) {
for (final TermsAggregation.Entry entry : buckets.getBuckets()) {
// Use the "special" character to split up the terms value so we can create a field->value mapping.
final List<String> valueList = Splitter.on(Searches.STACKED_TERMS_AGG_SEPARATOR).splitToList(entry.getKey());

// We need a human readable value here because the word separator we are using might not be readable
final String value = entry.getKey().replace(Searches.STACKED_TERMS_AGG_SEPARATOR, " - ");

// For every field in the field list, get the value from the split up terms value list. After this, we
// have a mapping of field->value for each bucket.
final ImmutableList.Builder<Map<String, String>> mapping = ImmutableList.builder();
for (int i = 0; i < fields.size(); i++) {
mapping.add(ImmutableMap.of("field", fields.get(i), "value", valueList.get(i)));
}

terms.put(value, entry.getCount());
termsMapping.put(value, mapping.build());
}
}

public static TermsResult empty(String originalQuery, String builtQuery) {
Expand All @@ -69,4 +100,7 @@ public Map<String, Long> getTerms() {
return terms;
}

public Map<String, List<Map<String, String>>> termsMapping() {
return termsMapping;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.codahale.metrics.Timer;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedSet;
import io.searchbox.client.JestClient;
import io.searchbox.client.JestResult;
Expand Down Expand Up @@ -113,6 +114,10 @@ public class Searches {
public static final String AGG_VALUE_COUNT = "gl2_value_count";
private static final Pattern filterStreamIdPattern = Pattern.compile("^(.+[^\\p{Alnum}])?streams:([\\p{XDigit}]+)");

// This is the "WORD SEPARATOR MIDDLE DOT" unicode character. It's used to join and split the term values in a
// stacked terms query.
public static final String STACKED_TERMS_AGG_SEPARATOR = "\u2E31";

public enum TermsStatsOrder {
TERM,
REVERSE_TERM,
Expand Down Expand Up @@ -313,7 +318,10 @@ public AbstractAggregationBuilder createTermsBuilder(String field, List<String>

// Add all other fields
stackedFields.forEach(f -> {
scriptStringBuilder.append(" + ' - ' + ");
// There is no way to use some kind of structured value for the stacked fields in the painless script
// so we have to use a "special" character (that is hopefully not showing up in any value) to join the
// stacked field values. That allows us to split the result again later to create a field->value mapping.
scriptStringBuilder.append(" + \"").append(STACKED_TERMS_AGG_SEPARATOR).append("\" + ");
scriptStringBuilder.append("doc['").append(f).append("'].value");
filterQuery.must(QueryBuilders.existsQuery(f));
});
Expand Down Expand Up @@ -357,7 +365,9 @@ public TermsResult terms(String field, List<String> stackedFields, int size, Str
searchResult.getTotal(),
query,
searchSourceBuilder.toString(),
tookMsFromSearchResult(searchResult)
tookMsFromSearchResult(searchResult),
// Concat field and stacked fields into one fields list
ImmutableList.<String>builder().add(field).addAll(stackedFields).build()
);
}

Expand Down Expand Up @@ -413,7 +423,9 @@ public TermsHistogramResult termsHistogram(String field,
searchSourceBuilder.toString(),
size,
tookMsFromSearchResult(searchResult),
interval);
interval,
// Concat field and stacked fields into one fields list
ImmutableList.<String>builder().add(field).addAll(stackedFields).build());
}

public TermsStatsResult termsStats(String keyField, String valueField, TermsStatsOrder order, int size, String query, String filter, TimeRange range) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
import com.google.auto.value.AutoValue;
import org.graylog.autovalue.WithBeanGetter;

import java.util.Collections;
import java.util.List;
import java.util.Map;

@JsonAutoDetect
Expand All @@ -33,6 +35,9 @@ public abstract class TermsResult {
@JsonProperty
public abstract Map<String, Long> terms();

@JsonProperty
public abstract Map<String, List<Map<String, String>>> termsMapping();

@JsonProperty
public abstract long missing();

Expand All @@ -51,6 +56,16 @@ public static TermsResult create(long time,
long other,
long total,
String builtQuery) {
return new AutoValue_TermsResult(time, terms, missing, other, total, builtQuery);
return create(time, terms, Collections.emptyMap(), missing, other, total, builtQuery);
}

public static TermsResult create(long time,
Map<String, Long> terms,
Map<String, List<Map<String, String>>> termsMapping,
long missing,
long other,
long total,
String builtQuery) {
return new AutoValue_TermsResult(time, terms, termsMapping, missing, other, total, builtQuery);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ protected org.graylog2.indexer.results.HistogramResult fieldHistogram(String fie
}

protected TermsResult buildTermsResult(org.graylog2.indexer.results.TermsResult tr) {
return TermsResult.create(tr.tookMs(), tr.getTerms(), tr.getMissing(), tr.getOther(), tr.getTotal(), tr.getBuiltQuery());
return TermsResult.create(tr.tookMs(), tr.getTerms(), tr.termsMapping(), tr.getMissing(), tr.getOther(), tr.getTotal(), tr.getBuiltQuery());
}

protected TermsStatsResult buildTermsStatsResult(org.graylog2.indexer.results.TermsStatsResult tr) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public static TermsHistogramResult buildTermsHistogramResult(org.graylog2.indexe
final Map<Long, TermsResult> result = termsHistogram.getResults().entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, entry -> {
final org.graylog2.indexer.results.TermsResult tr = entry.getValue();
return TermsResult.create(tr.tookMs(), tr.getTerms(), tr.getMissing(), tr.getOther(), tr.getTotal(), tr.getBuiltQuery());
return TermsResult.create(tr.tookMs(), tr.getTerms(), tr.termsMapping(), tr.getMissing(), tr.getOther(), tr.getTotal(), tr.getBuiltQuery());
}));

return TermsHistogramResult.create(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ const QuickValuesVisualization = React.createClass({
others: undefined,
missing: undefined,
terms: Immutable.List(),
termsMapping: {},
};
},
componentDidMount() {
Expand Down Expand Up @@ -136,6 +137,7 @@ const QuickValuesVisualization = React.createClass({
others: quickValues.other,
missing: quickValues.missing,
terms: formattedTerms,
termsMapping: quickValues.terms_mapping,
}, this.drawData);
}
},
Expand Down Expand Up @@ -210,7 +212,7 @@ const QuickValuesVisualization = React.createClass({
table.selectAll('td.dc-table-column button').on('click', () => {
// noinspection Eslint
const term = $(d3.event.target).closest('button').data('term');
SearchStore.addSearchTerm(props.id, term);
SearchStore.addSearchTermWithMapping(this.state.termsMapping, props.id, term);
});
});

Expand Down
10 changes: 10 additions & 0 deletions graylog2-web-interface/src/stores/search/SearchStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,16 @@ class SearchStore {
this.addQueryTerm(term, effectiveOperator);
}

addSearchTermWithMapping(mapping, field, value, operator) {
if (!mapping[value]) {
return this.addSearchTerm(field, value, operator);
}

mapping[value].forEach((m) => {
this.addSearchTerm(m.field, m.value, operator);
});
}

changeTimeRange(newRangeType: string, newRangeParams: Object) {
this.rangeType = newRangeType;
this.rangeParams = Immutable.fromJS(newRangeParams);
Expand Down