From 5556e63bca7ba423d09fa86045fc1b667e62c807 Mon Sep 17 00:00:00 2001 From: markharwood Date: Mon, 15 Jul 2019 14:08:54 +0100 Subject: [PATCH] =?UTF-8?q?Search=20enhancement:=20-=20new=20query=20type?= =?UTF-8?q?=20allows=20selected=20documents=20to=20be=20promoted=20above?= =?UTF-8?q?=20=E2=80=9Cany=20organic=E2=80=9D=20search=20results.=20This?= =?UTF-8?q?=20is=20the=20first=20feature=20in=20a=20new=20module=20`search?= =?UTF-8?q?-business-rules`=20which=20will=20house=20licensed=20(non=20OSS?= =?UTF-8?q?)=20logic=20for=20rewriting=20queries=20according=20to=20busine?= =?UTF-8?q?ss=20rules.=20The=20PinnedQueryBuilder=20class=20offers=20a=20n?= =?UTF-8?q?ew=20`pinned`=20query=20in=20the=20DSL=20that=20takes=20an=20ar?= =?UTF-8?q?ray=20of=20promoted=20IDs=20and=20an=20=E2=80=9Corganic?= =?UTF-8?q?=E2=80=9D=20query=20and=20ensures=20the=20documents=20with=20th?= =?UTF-8?q?e=20promoted=20IDs=20rank=20higher=20than=20the=20organic=20mat?= =?UTF-8?q?ches.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #44074 --- .../reference/query-dsl/pinned-query.asciidoc | 37 +++ .../query-dsl/special-queries.asciidoc | 6 +- .../plugin/search-business-rules/.gitignore | 1 + .../plugin/search-business-rules/build.gradle | 45 ++++ .../lucene/search/CappedScoreQuery.java | 175 ++++++++++++++ .../lucene/search/CappedScoreWeight.java | 72 ++++++ .../apache/lucene/search/CappedScorer.java | 34 +++ .../PinnedQueryBuilder.java | 220 ++++++++++++++++++ .../SearchBusinessRules.java | 23 ++ .../PinnedQueryBuilderIT.java | 173 ++++++++++++++ .../PinnedQueryBuilderTests.java | 173 ++++++++++++++ 11 files changed, 958 insertions(+), 1 deletion(-) create mode 100644 docs/reference/query-dsl/pinned-query.asciidoc create mode 100644 x-pack/plugin/search-business-rules/.gitignore create mode 100644 x-pack/plugin/search-business-rules/build.gradle create mode 100644 x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreQuery.java create mode 100644 x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreWeight.java create mode 100644 x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScorer.java create mode 100644 x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java create mode 100644 x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java create mode 100644 x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java create mode 100644 x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java diff --git a/docs/reference/query-dsl/pinned-query.asciidoc b/docs/reference/query-dsl/pinned-query.asciidoc new file mode 100644 index 0000000000000..9768a1aa81793 --- /dev/null +++ b/docs/reference/query-dsl/pinned-query.asciidoc @@ -0,0 +1,37 @@ +[role="xpack"] +[testenv="basic"] +[[query-dsl-pinned-query]] +=== Pinned Query +Promotes selected documents to rank higher than those matching a given query. +This feature is typically used to guide searchers to curated documents that are +promoted over and above any "organic" matches for a search. +The promoted or "pinned" documents are identified using the document IDs stored in +the <> field. + +==== Example request + +[source,js] +-------------------------------------------------- +GET /_search +{ + "query": { + "pinned" : { + "ids" : ["1", "4", "100"], + "organic" : { + "match":{ + "description": "iphone" + } + } + } + } +} +-------------------------------------------------- +// CONSOLE + +[[pinned-query-top-level-parameters]] +==== Top-level parameters for `pinned` + +`ids`:: +An array of <> listed in the order they are to appear in results. +`organic`:: +Any choice of query used to rank documents which will be ranked below the "pinned" document ids. \ No newline at end of file diff --git a/docs/reference/query-dsl/special-queries.asciidoc b/docs/reference/query-dsl/special-queries.asciidoc index c2e95a4d818cf..06f7cc98a734e 100644 --- a/docs/reference/query-dsl/special-queries.asciidoc +++ b/docs/reference/query-dsl/special-queries.asciidoc @@ -31,6 +31,8 @@ A query that allows to modify the score of a sub-query with a script. <>:: A query that accepts other queries as json or yaml string. +<>:: +A query that promotes selected documents over others matching a given query. include::distance-feature-query.asciidoc[] @@ -44,4 +46,6 @@ include::script-query.asciidoc[] include::script-score-query.asciidoc[] -include::wrapper-query.asciidoc[] \ No newline at end of file +include::wrapper-query.asciidoc[] + +include::pinned-query.asciidoc[] \ No newline at end of file diff --git a/x-pack/plugin/search-business-rules/.gitignore b/x-pack/plugin/search-business-rules/.gitignore new file mode 100644 index 0000000000000..ae3c1726048cd --- /dev/null +++ b/x-pack/plugin/search-business-rules/.gitignore @@ -0,0 +1 @@ +/bin/ diff --git a/x-pack/plugin/search-business-rules/build.gradle b/x-pack/plugin/search-business-rules/build.gradle new file mode 100644 index 0000000000000..5789ec5f08114 --- /dev/null +++ b/x-pack/plugin/search-business-rules/build.gradle @@ -0,0 +1,45 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'search-business-rules' + description 'A plugin for applying business rules to search result rankings' + classname 'org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRules' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-searchbusinessrules' + + +integTest.enabled = false + +// Instead we create a separate task to run the +// tests based on ESIntegTestCase +task internalClusterTest(type: Test) { + description = 'Java fantasy integration tests' + mustRunAfter test + + include '**/*IT.class' +} + +check.dependsOn internalClusterTest + +dependencies { + compileOnly project(path: xpackModule('core'), configuration: 'default') + testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') + testCompile project(":test:framework") + if (isEclipse) { + testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts') + } +} + +// copied from CCR +dependencyLicenses { + ignoreSha 'x-pack-core' +} + +//testingConventions.naming { +// IT { +// baseClass "org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilderIT" +// } +//} diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreQuery.java b/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreQuery.java new file mode 100644 index 0000000000000..140e26c5e974c --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreQuery.java @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.apache.lucene.search; + +import java.io.IOException; +import java.util.Objects; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.Bits; + +/** + * A query that wraps another query and ensures scores do not exceed a maximum value + */ +public final class CappedScoreQuery extends Query { + private final Query query; + private final float maxScore; + + /** Caps scores from the passed in Query to the supplied maxScore parameter */ + public CappedScoreQuery(Query query, float maxScore) { + this.query = Objects.requireNonNull(query, "Query must not be null"); + if (maxScore > 0 == false) { + throw new IllegalArgumentException(this.getClass().getName() + " maxScore must be >0, " + maxScore + " supplied."); + } + this.maxScore = maxScore; + } + + /** Returns the encapsulated query. */ + public Query getQuery() { + return query; + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + Query rewritten = query.rewrite(reader); + + if (rewritten != query) { + return new CappedScoreQuery(rewritten, maxScore); + } + + if (rewritten.getClass() == CappedScoreQuery.class) { + return rewritten; + } + + if (rewritten.getClass() == BoostQuery.class) { + return new CappedScoreQuery(((BoostQuery) rewritten).getQuery(), maxScore); + } + + return super.rewrite(reader); + } + + /** + * We return this as our {@link BulkScorer} so that if the CSQ wraps a query with its own optimized top-level scorer (e.g. + * BooleanScorer) we can use that top-level scorer. + */ + protected static class CappedBulkScorer extends BulkScorer { + final BulkScorer bulkScorer; + final Weight weight; + final float maxScore; + + public CappedBulkScorer(BulkScorer bulkScorer, Weight weight, float maxScore) { + this.bulkScorer = bulkScorer; + this.weight = weight; + this.maxScore = maxScore; + } + + @Override + public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException { + return bulkScorer.score(wrapCollector(collector), acceptDocs, min, max); + } + + private LeafCollector wrapCollector(LeafCollector collector) { + return new FilterLeafCollector(collector) { + @Override + public void setScorer(Scorable scorer) throws IOException { + // we must wrap again here, but using the scorer passed in as parameter: + in.setScorer(new FilterScorable(scorer) { + @Override + public float score() throws IOException { + return Math.min(maxScore, in.score()); + } + + @Override + public void setMinCompetitiveScore(float minScore) throws IOException { + scorer.setMinCompetitiveScore(minScore); + } + + }); + } + }; + } + + @Override + public long cost() { + return bulkScorer.cost(); + } + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + final Weight innerWeight = searcher.createWeight(query, scoreMode, boost); + if (scoreMode.needsScores()) { + return new CappedScoreWeight(this, innerWeight, maxScore) { + @Override + public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { + final BulkScorer innerScorer = innerWeight.bulkScorer(context); + if (innerScorer == null) { + return null; + } + return new CappedBulkScorer(innerScorer, this, maxScore); + } + + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + ScorerSupplier innerScorerSupplier = innerWeight.scorerSupplier(context); + if (innerScorerSupplier == null) { + return null; + } + return new ScorerSupplier() { + @Override + public Scorer get(long leadCost) throws IOException { + final Scorer innerScorer = innerScorerSupplier.get(leadCost); + // short-circuit if scores will not need capping + innerScorer.advanceShallow(0); + if (innerScorer.getMaxScore(DocIdSetIterator.NO_MORE_DOCS) <= maxScore) { + return innerScorer; + } + return new CappedScorer(innerWeight, innerScorer, maxScore); + } + + @Override + public long cost() { + return innerScorerSupplier.cost(); + } + }; + } + + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return innerWeight.matches(context, doc); + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + ScorerSupplier scorerSupplier = scorerSupplier(context); + if (scorerSupplier == null) { + return null; + } + return scorerSupplier.get(Long.MAX_VALUE); + } + }; + } else { + return innerWeight; + } + } + + @Override + public String toString(String field) { + return new StringBuilder("CappedScore(").append(query.toString(field)).append(')').toString(); + } + + @Override + public boolean equals(Object other) { + return sameClassAs(other) && maxScore == ((CappedScoreQuery) other).maxScore && + query.equals(((CappedScoreQuery) other).query); + } + + @Override + public int hashCode() { + return 31 * classHash() + query.hashCode() + Float.hashCode(maxScore); + } +} diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreWeight.java b/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreWeight.java new file mode 100644 index 0000000000000..4d8badbaaecc6 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreWeight.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.apache.lucene.search; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; + +import java.io.IOException; +import java.util.Set; + +/** + * A Weight that caps scores of the wrapped query to a maximum value + * + * @lucene.internal + */ +public abstract class CappedScoreWeight extends Weight { + + private final float maxScore; + private final Weight innerWeight; + + protected CappedScoreWeight(Query query, Weight innerWeight, float maxScore) { + super(query); + this.maxScore = maxScore; + this.innerWeight = innerWeight; + } + + @Override + public void extractTerms(Set terms) { + innerWeight.extractTerms(terms); + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return innerWeight.isCacheable(ctx); + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + return new CappedScorer(this, innerWeight.scorer(context), maxScore); + } + + @Override + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + + final Scorer s = scorer(context); + final boolean exists; + if (s == null) { + exists = false; + } else { + final TwoPhaseIterator twoPhase = s.twoPhaseIterator(); + if (twoPhase == null) { + exists = s.iterator().advance(doc) == doc; + } else { + exists = twoPhase.approximation().advance(doc) == doc && twoPhase.matches(); + } + } + + Explanation sub = innerWeight.explain(context, doc); + if (sub.isMatch() && sub.getValue().floatValue() > maxScore) { + return Explanation.match(maxScore, "Capped score of " + innerWeight.getQuery() + ", max of", + sub, + Explanation.match(maxScore, "maximum score")); + } else { + return sub; + } + } + +} diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScorer.java b/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScorer.java new file mode 100644 index 0000000000000..a97fe51629bdd --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScorer.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.apache.lucene.search; + +import java.io.IOException; + +public class CappedScorer extends FilterScorer { + private final float maxScore; + + public CappedScorer(Weight weight, Scorer delegate, float maxScore) { + super(delegate, weight); + this.maxScore = maxScore; + } + + @Override + public float getMaxScore(int upTo) throws IOException { + return Math.min(maxScore, in.getMaxScore(upTo)); + } + + @Override + public int advanceShallow(int target) throws IOException { + return in.advanceShallow(target); + } + + @Override + public float score() throws IOException { + return Math.min(maxScore, in.score()); + } + +} \ No newline at end of file diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java new file mode 100644 index 0000000000000..5e2adc5c67440 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilder.java @@ -0,0 +1,220 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchbusinessrules; + +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.CappedScoreQuery; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.DisjunctionMaxQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; +import org.elasticsearch.index.query.QueryShardContext; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +/** + * A query that will promote selected documents (identified by ID) above matches produced by an "organic" query. In practice, some upstream + * system will identify the promotions associated with a user's query string and use this object to ensure these are "pinned" to the top of + * the other search results. + */ +public class PinnedQueryBuilder extends AbstractQueryBuilder { + public static final String NAME = "pinned"; + public static final int MAX_NUM_PINNED_HITS = 100; + + private static final ParseField IDS_FIELD = new ParseField("ids"); + public static final ParseField ORGANIC_QUERY_FIELD = new ParseField("organic"); + + private final List ids; + private QueryBuilder organicQuery; + + // Organic queries will have their scores capped to this number range, + // We reserve the highest float exponent for scores of pinned queries + private static final float MAX_ORGANIC_SCORE = Float.intBitsToFloat((0xfe << 23)) - 1; + + /** + * Creates a new PinnedQueryBuilder + */ + public PinnedQueryBuilder(QueryBuilder organicQuery, String... ids) { + if (organicQuery == null) { + throw new IllegalArgumentException("[" + NAME + "] organicQuery cannot be null"); + } + this.organicQuery = organicQuery; + if (ids == null) { + throw new IllegalArgumentException("[" + NAME + "] ids cannot be null"); + } + if (ids.length > MAX_NUM_PINNED_HITS) { + throw new IllegalArgumentException("[" + NAME + "] Max of "+MAX_NUM_PINNED_HITS+" ids exceeded: "+ + ids.length+" provided."); + } + LinkedHashSet deduped = new LinkedHashSet<>(); + for (String id : ids) { + if (id == null) { + throw new IllegalArgumentException("[" + NAME + "] id cannot be null"); + } + if(deduped.add(id) == false) { + throw new IllegalArgumentException("[" + NAME + "] duplicate id found in list: "+id); + } + } + this.ids = new ArrayList<>(); + Collections.addAll(this.ids, ids); + + } + + /** + * Read from a stream. + */ + public PinnedQueryBuilder(StreamInput in) throws IOException { + super(in); + ids = in.readStringList(); + organicQuery = in.readNamedWriteable(QueryBuilder.class); + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + out.writeStringCollection(this.ids); + out.writeNamedWriteable(organicQuery); + } + + /** + * @return the organic query set in the constructor + */ + public QueryBuilder organicQuery() { + return this.organicQuery; + } + + /** + * Returns the pinned ids for the query. + */ + public List ids() { + return Collections.unmodifiableList(this.ids); + } + + @Override + protected void doXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + if (organicQuery != null) { + builder.field(ORGANIC_QUERY_FIELD.getPreferredName()); + organicQuery.toXContent(builder, params); + } + builder.startArray(IDS_FIELD.getPreferredName()); + for (String value : ids) { + builder.value(value); + } + builder.endArray(); + printBoostAndQueryName(builder); + builder.endObject(); + } + + + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, + a -> + { + QueryBuilder organicQuery = (QueryBuilder) a[0]; + @SuppressWarnings("unchecked") + List ids = (List) a[1]; + return new PinnedQueryBuilder(organicQuery, ids.toArray(String[]::new)); + } + ); + static { + PARSER.declareObject(constructorArg(), (p, c) -> parseInnerQueryBuilder(p), ORGANIC_QUERY_FIELD); + PARSER.declareStringArray(constructorArg(), IDS_FIELD); + declareStandardFields(PARSER); + } + + public static PinnedQueryBuilder fromXContent(XContentParser parser) { + try { + return PARSER.apply(parser, null); + } catch (IllegalArgumentException e) { + throw new ParsingException(parser.getTokenLocation(), e.getMessage(), e); + } + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + protected QueryBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException { + QueryBuilder newOrganicQuery = organicQuery.rewrite(queryShardContext); + if (newOrganicQuery != organicQuery) { + PinnedQueryBuilder result = new PinnedQueryBuilder(newOrganicQuery, ids.toArray(String[]::new)); + result.boost(this.boost); + return result; + } + return this; + } + + @Override + protected Query doToQuery(QueryShardContext context) throws IOException { + MappedFieldType idField = context.fieldMapper(IdFieldMapper.NAME); + if (idField == null) { + return new MatchNoDocsQuery("No mappings"); + } + if (this.ids.isEmpty()) { + return new CappedScoreQuery(organicQuery.toQuery(context), MAX_ORGANIC_SCORE); + } else { + BooleanQuery.Builder pinnedQueries = new BooleanQuery.Builder(); + + // Ensure each pin order using a Boost query with the relevant boost factor + int minPin = NumericUtils.floatToSortableInt(MAX_ORGANIC_SCORE) + 1; + int boostNum = minPin + ids.size(); + float lastScore = Float.MAX_VALUE; + for (String id : ids) { + float pinScore = NumericUtils.sortableIntToFloat(boostNum); + assert pinScore < lastScore; + lastScore = pinScore; + boostNum--; + // Ensure the pin order using a Boost query with the relevant boost factor + Query idQuery = new BoostQuery(new ConstantScoreQuery(idField.termQuery(id, context)), pinScore); + pinnedQueries.add(idQuery, BooleanClause.Occur.SHOULD); + } + + // Score for any pinned query clause should be used, regardless of any organic clause score, to preserve pin order. + // Use dismax to always take the larger (ie pinned) of the organic vs pinned scores + List organicAndPinned = new ArrayList<>(); + organicAndPinned.add(pinnedQueries.build()); + // Cap the scores of the organic query + organicAndPinned.add(new CappedScoreQuery(organicQuery.toQuery(context), MAX_ORGANIC_SCORE)); + return new DisjunctionMaxQuery(organicAndPinned, 0); + } + + } + + @Override + protected int doHashCode() { + return Objects.hash(ids, organicQuery); + } + + @Override + protected boolean doEquals(PinnedQueryBuilder other) { + return Objects.equals(ids, other.ids) && Objects.equals(organicQuery, other.organicQuery) && boost == other.boost; + } +} diff --git a/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java new file mode 100644 index 0000000000000..d479a471e193f --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/main/java/org/elasticsearch/xpack/searchbusinessrules/SearchBusinessRules.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchbusinessrules; + +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SearchPlugin; + +import java.util.List; + +import static java.util.Collections.singletonList; + +public class SearchBusinessRules extends Plugin implements SearchPlugin { + + @Override + public List> getQueries() { + return singletonList(new QuerySpec<>(PinnedQueryBuilder.NAME, PinnedQueryBuilder::new, PinnedQueryBuilder::fromXContent)); + } + +} diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java new file mode 100644 index 0000000000000..6ef9436c81579 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchbusinessrules; + +import org.apache.lucene.search.Explanation; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.Operator; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.test.ESIntegTestCase; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSecondHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThirdHit; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; + + +public class PinnedQueryBuilderIT extends ESIntegTestCase { + + public void testIdInsertionOrderRetained() { + String[] ids = generateRandomStringArray(10, 50, false); + PinnedQueryBuilder pqb = new PinnedQueryBuilder(new MatchAllQueryBuilder(), ids); + List addedIds = pqb.ids(); + int pos = 0; + for (String key : addedIds) { + assertEquals(ids[pos++], key); + } + } + + @Override + protected Collection> nodePlugins() { + List> plugins = new ArrayList<>(); + plugins.add(SearchBusinessRules.class); + return plugins; + } + + public void testPinnedPromotions() throws Exception { + assertAcked(prepareCreate("test") + .addMapping("type1", + jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("field1") + .field("analyzer", "whitespace").field("type", "text").endObject().endObject().endObject().endObject()) + .setSettings(Settings.builder().put(indexSettings()).put("index.number_of_shards", 2))); + + int numRelevantDocs = randomIntBetween(1, 100); + for (int i = 0; i < numRelevantDocs; i++) { + if (i % 2 == 0) { + // add lower-scoring text + client().prepareIndex("test", "type1", Integer.toString(i)).setSource("field1", "the quick brown fox").get(); + } else { + // add higher-scoring text + client().prepareIndex("test", "type1", Integer.toString(i)).setSource("field1", "red fox").get(); + } + } + // Add docs with no relevance + int numIrrelevantDocs = randomIntBetween(1, 10); + for (int i = numRelevantDocs; i <= numRelevantDocs + numIrrelevantDocs; i++) { + client().prepareIndex("test", "type1", Integer.toString(i)).setSource("field1", "irrelevant").get(); + } + refresh(); + + // Test doc pinning + int totalDocs = numRelevantDocs + numIrrelevantDocs; + for (int i = 0; i < 100; i++) { + int numPromotions = randomIntBetween(0, totalDocs); + + LinkedHashSet pins = new LinkedHashSet<>(); + for (int j = 0; j < numPromotions; j++) { + pins.add(Integer.toString(randomIntBetween(0, totalDocs))); + } + QueryBuilder organicQuery = null; + if (i % 5 == 0) { + // Occasionally try a query with no matches to check all pins still show + organicQuery = QueryBuilders.matchQuery("field1", "matchNoDocs"); + } else { + organicQuery = QueryBuilders.matchQuery("field1", "red fox"); + } + PinnedQueryBuilder pqb = new PinnedQueryBuilder(organicQuery, pins.toArray(new String[0])); + + int from = randomIntBetween(0, numRelevantDocs); + int size = randomIntBetween(10, 100); + SearchResponse searchResponse = client().prepareSearch().setQuery(pqb).setTrackTotalHits(true).setSize(size).setFrom(from) + .get(); + + long numHits = searchResponse.getHits().getTotalHits().value; + assertThat(numHits, lessThanOrEqualTo((long) numRelevantDocs + pins.size())); + + // Check pins are sorted by increasing score, (unlike organic, there are no duplicate scores) + float lastScore = Float.MAX_VALUE; + SearchHit[] hits = searchResponse.getHits().getHits(); + for (int hitNumber = 0; hitNumber < Math.min(hits.length, pins.size() - from); hitNumber++) { + assertThat("Hit " + hitNumber + " in iter " + i + " wrong" + pins, hits[hitNumber].getScore(), lessThan(lastScore)); + lastScore = hits[hitNumber].getScore(); + } + // Check that the pins appear in the requested order (globalHitNumber is cursor independent of from and size window used) + int globalHitNumber = 0; + for (String id : pins) { + if (globalHitNumber < size && globalHitNumber >= from) { + assertThat("Hit " + globalHitNumber + " in iter " + i + " wrong" + pins, hits[globalHitNumber - from].getId(), + equalTo(id)); + } + globalHitNumber++; + } + // Test the organic hits are sorted by text relevance + boolean highScoresExhausted = false; + for (; globalHitNumber < hits.length + from; globalHitNumber++) { + if (globalHitNumber >= from) { + int id = Integer.parseInt(hits[globalHitNumber - from].getId()); + if (id % 2 == 0) { + highScoresExhausted = true; + } else { + assertFalse("All odd IDs should have scored higher than even IDs in organic results", highScoresExhausted); + } + } + + } + + } + + } + + public void testExplain() throws Exception { + assertAcked(prepareCreate("test").addMapping("type1", + jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("field1") + .field("analyzer", "whitespace").field("type", "text").endObject().endObject().endObject().endObject())); + ensureGreen(); + client().prepareIndex("test", "type1", "1").setSource("field1", "the quick brown fox").get(); + client().prepareIndex("test", "type1", "2").setSource("field1", "pinned").get(); + client().prepareIndex("test", "type1", "3").setSource("field1", "irrelevant").get(); + client().prepareIndex("test", "type1", "4").setSource("field1", "slow brown cat").get(); + refresh(); + + PinnedQueryBuilder pqb = new PinnedQueryBuilder(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR), "2"); + + SearchResponse searchResponse = client().prepareSearch().setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(pqb) + .setExplain(true).get(); + assertHitCount(searchResponse, 3); + assertFirstHit(searchResponse, hasId("2")); + assertSecondHit(searchResponse, hasId("1")); + assertThirdHit(searchResponse, hasId("4")); + + Explanation pinnedExplanation = searchResponse.getHits().getAt(0).getExplanation(); + assertThat(pinnedExplanation, notNullValue()); + assertThat(pinnedExplanation.isMatch(), equalTo(true)); + assertThat(pinnedExplanation.getDetails().length, equalTo(1)); + assertThat(pinnedExplanation.getDetails()[0].isMatch(), equalTo(true)); + assertThat(pinnedExplanation.getDetails()[0].getDescription(), containsString("ConstantScore")); + + + } + +} diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java new file mode 100644 index 0000000000000..57db7972655e1 --- /dev/null +++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderTests.java @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchbusinessrules; + +import com.fasterxml.jackson.core.io.JsonStringEncoder; + +import org.apache.lucene.search.DisjunctionMaxQuery; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.test.AbstractQueryTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static org.hamcrest.CoreMatchers.instanceOf; + +public class PinnedQueryBuilderTests extends AbstractQueryTestCase { + @Override + protected PinnedQueryBuilder doCreateTestQueryBuilder() { + return new PinnedQueryBuilder(createRandomQuery(), generateRandomStringArray(100, 256, false, true)); + } + + private QueryBuilder createRandomQuery() { + if (randomBoolean()) { + return new MatchAllQueryBuilder(); + } else { + return createTestTermQueryBuilder(); + } + } + + private QueryBuilder createTestTermQueryBuilder() { + String fieldName = null; + Object value; + switch (randomIntBetween(0, 3)) { + case 0: + if (randomBoolean()) { + fieldName = BOOLEAN_FIELD_NAME; + } + value = randomBoolean(); + break; + case 1: + if (randomBoolean()) { + fieldName = randomFrom(STRING_FIELD_NAME, STRING_ALIAS_FIELD_NAME); + } + if (frequently()) { + value = randomAlphaOfLengthBetween(1, 10); + } else { + // generate unicode string in 10% of cases + JsonStringEncoder encoder = JsonStringEncoder.getInstance(); + value = new String(encoder.quoteAsString(randomUnicodeOfLength(10))); + } + break; + case 2: + if (randomBoolean()) { + fieldName = INT_FIELD_NAME; + } + value = randomInt(10000); + break; + case 3: + if (randomBoolean()) { + fieldName = DOUBLE_FIELD_NAME; + } + value = randomDouble(); + break; + default: + throw new UnsupportedOperationException(); + } + + if (fieldName == null) { + fieldName = randomAlphaOfLengthBetween(1, 10); + } + return new TermQueryBuilder(fieldName, value); + } + + @Override + protected void doAssertLuceneQuery(PinnedQueryBuilder queryBuilder, Query query, SearchContext searchContext) throws IOException { + if (queryBuilder.ids().size() == 0 && queryBuilder.organicQuery() == null) { + assertThat(query, instanceOf(MatchNoDocsQuery.class)); + } else { + if (queryBuilder.ids().size() > 0) { + // Have IDs and an organic query - uses DisMax + assertThat(query, instanceOf(DisjunctionMaxQuery.class)); + } + } + } + + @Override + protected Collection> getPlugins() { + List> classpathPlugins = new ArrayList<>(); + classpathPlugins.add(SearchBusinessRules.class); + return classpathPlugins; + } + + public void testIllegalArguments() { + expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), (String)null)); + expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(null, "1")); + expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), "1", null, "2")); + String[] bigList = new String[PinnedQueryBuilder.MAX_NUM_PINNED_HITS + 1]; + for (int i = 0; i < bigList.length; i++) { + bigList[i] = String.valueOf(i); + } + expectThrows(IllegalArgumentException.class, () -> new PinnedQueryBuilder(new MatchAllQueryBuilder(), bigList)); + + } + + public void testEmptyPinnedQuery() throws Exception { + XContentBuilder contentBuilder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + contentBuilder.startObject().startObject("pinned").endObject().endObject(); + try (XContentParser xParser = createParser(contentBuilder)) { + expectThrows(ParsingException.class, () -> parseQuery(xParser).toQuery(createShardContext())); + } + } + + public void testFromJson() throws IOException { + String query = + "{" + + "\"pinned\" : {" + + " \"organic\" : {" + + " \"term\" : {" + + " \"tag\" : {" + + " \"value\" : \"tech\"," + + " \"boost\" : 1.0" + + " }" + + " }" + + " }, "+ + " \"ids\" : [ \"1\",\"2\" ]," + + " \"boost\":1.0 "+ + "}" + + "}"; + + PinnedQueryBuilder queryBuilder = (PinnedQueryBuilder) parseQuery(query); + checkGeneratedJson(query, queryBuilder); + + assertEquals(query, 2, queryBuilder.ids().size()); + assertThat(queryBuilder.organicQuery(), instanceOf(TermQueryBuilder.class)); + } + + /** + * test that unknown query names in the clauses throw an error + */ + public void testUnknownQueryName() throws IOException { + String query = "{\"pinned\" : {\"organic\" : { \"unknown_query\" : { } } } }"; + + ParsingException ex = expectThrows(ParsingException.class, () -> parseQuery(query)); + // BoolQueryBuilder test has this test for a more detailed error message: + // assertEquals("no [query] registered for [unknown_query]", ex.getMessage()); + // But ObjectParser used in PinnedQueryBuilder tends to hide the above message and give this below: + assertEquals("[1:46] [pinned] failed to parse field [organic]", ex.getMessage()); + } + + public void testRewrite() throws IOException { + PinnedQueryBuilder pinnedQueryBuilder = new PinnedQueryBuilder(new TermQueryBuilder("foo", 1), "1"); + QueryBuilder rewritten = pinnedQueryBuilder.rewrite(createShardContext()); + assertThat(rewritten, instanceOf(PinnedQueryBuilder.class)); + } + +}