Search enhancement: - new query type allows selected documents to be …

…promoted above “any organic” search results.

This is the first feature in a new module `search-business-rules` which will house licensed (non OSS) logic for rewriting queries according to business rules.
The PinnedQueryBuilder class offers a new `pinned` query in the DSL that takes an array of promoted IDs and an “organic” query and ensures the documents with the promoted IDs rank higher than the organic matches.

Closes #44074

docs/reference/query-dsl/pinned-query.asciidoc

@@ -0,0 +1,37 @@
+[role="xpack"]
+[testenv="basic"]
+[[query-dsl-pinned-query]]
+=== Pinned Query
+Promotes selected documents to rank higher than those matching a given query.
+This feature is typically used to guide searchers to curated documents that are
+promoted over and above any "organic" matches for a search.  
+The promoted or "pinned" documents are identified using the document IDs stored in
+the <<mapping-id-field,`_id`>> field.
+
+==== Example request
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "query": {
+        "pinned" : {
+            "ids" : ["1", "4", "100"],
+            "organic" : {
+            	"match":{
+            		"description": "iphone"
+            	}
+            }
+        }
+    }
+}    
+--------------------------------------------------
+// CONSOLE
+
+[[pinned-query-top-level-parameters]]
+==== Top-level parameters for `pinned`
+
+`ids`::
+An array of <<mapping-id-field, document IDs>> listed in the order they are to appear in results.
+`organic`::
+Any choice of query used to rank documents which will be ranked below the "pinned" document ids.

docs/reference/query-dsl/special-queries.asciidoc

@@ -31,6 +31,8 @@ A query that allows to modify the score of a sub-query with a script.
 <<query-dsl-wrapper-query,`wrapper` query>>::
 A query that accepts other queries as json or yaml string.
 
+<<query-dsl-pinned-query,`pinned` query>>::
+A query that promotes selected documents over others matching a given query.
 
 include::distance-feature-query.asciidoc[]
 
@@ -44,4 +46,6 @@ include::script-query.asciidoc[]
 
 include::script-score-query.asciidoc[]
 
-include::wrapper-query.asciidoc[]
+include::wrapper-query.asciidoc[]
+
+include::pinned-query.asciidoc[]

x-pack/plugin/search-business-rules/.gitignore

@@ -0,0 +1 @@
+/bin/

x-pack/plugin/search-business-rules/build.gradle

@@ -0,0 +1,45 @@
+evaluationDependsOn(xpackModule('core'))
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+    name 'search-business-rules'
+    description 'A plugin for applying business rules to search result rankings'
+    classname 'org.elasticsearch.xpack.searchbusinessrules.SearchBusinessRules'
+    extendedPlugins = ['x-pack-core']
+}
+archivesBaseName = 'x-pack-searchbusinessrules'
+
+
+integTest.enabled = false
+
+// Instead we create a separate task to run the
+// tests based on ESIntegTestCase
+task internalClusterTest(type: Test) {
+    description = 'Java fantasy integration tests'
+    mustRunAfter test
+
+    include '**/*IT.class'
+}
+
+check.dependsOn internalClusterTest
+
+dependencies {
+    compileOnly project(path: xpackModule('core'), configuration: 'default')
+    testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
+    testCompile project(":test:framework")    
+    if (isEclipse) {
+        testCompile project(path: xpackModule('core-tests'), configuration: 'testArtifacts')
+    }
+}
+
+// copied from CCR
+dependencyLicenses {
+    ignoreSha 'x-pack-core'
+}
+
+//testingConventions.naming {
+//    IT {
+//        baseClass "org.elasticsearch.xpack.searchbusinessrules.PinnedQueryBuilderIT"
+//    }
+//}

x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreQuery.java

@@ -0,0 +1,175 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.util.Bits;
+
+/**
+ * A query that wraps another query and ensures scores do not exceed a maximum value
+ */
+public final class CappedScoreQuery extends Query {
+    private final Query query;
+    private final float maxScore;
+
+    /** Caps scores from the passed in Query to the supplied maxScore parameter */
+    public CappedScoreQuery(Query query, float maxScore) {
+        this.query = Objects.requireNonNull(query, "Query must not be null");
+        if (maxScore > 0 == false) {
+            throw new IllegalArgumentException(this.getClass().getName() + " maxScore must be >0, " + maxScore + " supplied.");
+        }
+        this.maxScore = maxScore;
+    }
+
+    /** Returns the encapsulated query. */
+    public Query getQuery() {
+        return query;
+    }
+
+    @Override
+    public Query rewrite(IndexReader reader) throws IOException {
+        Query rewritten = query.rewrite(reader);
+
+        if (rewritten != query) {
+            return new CappedScoreQuery(rewritten, maxScore);
+        }
+
+        if (rewritten.getClass() == CappedScoreQuery.class) {
+            return rewritten;
+        }
+
+        if (rewritten.getClass() == BoostQuery.class) {
+            return new CappedScoreQuery(((BoostQuery) rewritten).getQuery(), maxScore);
+        }
+
+        return super.rewrite(reader);
+    }
+
+    /**
+     * We return this as our {@link BulkScorer} so that if the CSQ wraps a query with its own optimized top-level scorer (e.g.
+     * BooleanScorer) we can use that top-level scorer.
+     */
+    protected static class CappedBulkScorer extends BulkScorer {
+        final BulkScorer bulkScorer;
+        final Weight weight;
+        final float maxScore;
+
+        public CappedBulkScorer(BulkScorer bulkScorer, Weight weight, float maxScore) {
+            this.bulkScorer = bulkScorer;
+            this.weight = weight;
+            this.maxScore = maxScore;
+        }
+
+        @Override
+        public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
+            return bulkScorer.score(wrapCollector(collector), acceptDocs, min, max);
+        }
+
+        private LeafCollector wrapCollector(LeafCollector collector) {
+            return new FilterLeafCollector(collector) {
+                @Override
+                public void setScorer(Scorable scorer) throws IOException {
+                    // we must wrap again here, but using the scorer passed in as parameter:
+                    in.setScorer(new FilterScorable(scorer) {
+                        @Override
+                        public float score() throws IOException {
+                            return Math.min(maxScore, in.score());
+                        }
+
+                        @Override
+                        public void setMinCompetitiveScore(float minScore) throws IOException {
+                            scorer.setMinCompetitiveScore(minScore);
+                        }
+
+                    });
+                }
+            };
+        }
+
+        @Override
+        public long cost() {
+            return bulkScorer.cost();
+        }
+    }
+
+    @Override
+    public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+        final Weight innerWeight = searcher.createWeight(query, scoreMode, boost);
+        if (scoreMode.needsScores()) {
+            return new CappedScoreWeight(this, innerWeight, maxScore) {
+                @Override
+                public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
+                    final BulkScorer innerScorer = innerWeight.bulkScorer(context);
+                    if (innerScorer == null) {
+                        return null;
+                    }
+                    return new CappedBulkScorer(innerScorer, this, maxScore);
+                }
+
+                @Override
+                public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+                    ScorerSupplier innerScorerSupplier = innerWeight.scorerSupplier(context);
+                    if (innerScorerSupplier == null) {
+                        return null;
+                    }
+                    return new ScorerSupplier() {
+                        @Override
+                        public Scorer get(long leadCost) throws IOException {
+                            final Scorer innerScorer = innerScorerSupplier.get(leadCost);
+                            // short-circuit if scores will not need capping
+                            innerScorer.advanceShallow(0);
+                            if (innerScorer.getMaxScore(DocIdSetIterator.NO_MORE_DOCS) <= maxScore) {
+                              return innerScorer;
+                            }                            
+                            return new CappedScorer(innerWeight, innerScorer, maxScore);
+                        }
+
+                        @Override
+                        public long cost() {
+                            return innerScorerSupplier.cost();
+                        }
+                    };
+                }
+
+                @Override
+                public Matches matches(LeafReaderContext context, int doc) throws IOException {
+                    return innerWeight.matches(context, doc);
+                }
+
+                @Override
+                public Scorer scorer(LeafReaderContext context) throws IOException {
+                    ScorerSupplier scorerSupplier = scorerSupplier(context);
+                    if (scorerSupplier == null) {
+                        return null;
+                    }
+                    return scorerSupplier.get(Long.MAX_VALUE);
+                }
+            };
+        } else {
+            return innerWeight;
+        }
+    }
+
+    @Override
+    public String toString(String field) {
+        return new StringBuilder("CappedScore(").append(query.toString(field)).append(')').toString();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        return sameClassAs(other) && maxScore == ((CappedScoreQuery) other).maxScore && 
+                query.equals(((CappedScoreQuery) other).query);
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * classHash() + query.hashCode() + Float.hashCode(maxScore);
+    }
+}

x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScoreWeight.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.apache.lucene.search;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * A Weight that caps scores of the wrapped query to a maximum value
+ *
+ * @lucene.internal
+ */
+public abstract class CappedScoreWeight extends Weight {
+
+    private final float maxScore;
+    private final Weight innerWeight;
+
+    protected CappedScoreWeight(Query query, Weight innerWeight, float maxScore) {
+        super(query);
+        this.maxScore = maxScore;
+        this.innerWeight = innerWeight;
+    }
+
+    @Override
+    public void extractTerms(Set<Term> terms) {
+        innerWeight.extractTerms(terms);
+    }
+
+    @Override
+    public boolean isCacheable(LeafReaderContext ctx) {
+        return innerWeight.isCacheable(ctx);
+    }    
+
+    @Override
+    public Scorer scorer(LeafReaderContext context) throws IOException {
+        return new CappedScorer(this, innerWeight.scorer(context), maxScore);
+    }
+
+    @Override
+    public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+
+        final Scorer s = scorer(context);
+        final boolean exists;
+        if (s == null) {
+            exists = false;
+        } else {
+            final TwoPhaseIterator twoPhase = s.twoPhaseIterator();
+            if (twoPhase == null) {
+                exists = s.iterator().advance(doc) == doc;
+            } else {
+                exists = twoPhase.approximation().advance(doc) == doc && twoPhase.matches();
+            }
+        }
+
+        Explanation sub = innerWeight.explain(context, doc);
+        if (sub.isMatch() && sub.getValue().floatValue() > maxScore) {
+          return Explanation.match(maxScore, "Capped score of " + innerWeight.getQuery() + ", max of",
+              sub,
+              Explanation.match(maxScore, "maximum score"));
+        } else {
+          return sub;
+        }
+    }
+
+}

x-pack/plugin/search-business-rules/src/main/java/org/apache/lucene/search/CappedScorer.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.apache.lucene.search;
+
+import java.io.IOException;
+
+public class CappedScorer extends FilterScorer {
+    private final float maxScore;
+
+    public CappedScorer(Weight weight, Scorer delegate, float maxScore) {
+        super(delegate, weight);
+        this.maxScore = maxScore;
+    }
+
+    @Override
+    public float getMaxScore(int upTo) throws IOException {
+        return Math.min(maxScore, in.getMaxScore(upTo));
+    }
+
+    @Override
+    public int advanceShallow(int target) throws IOException {
+        return in.advanceShallow(target);
+    }
+
+    @Override
+    public float score() throws IOException {
+        return Math.min(maxScore, in.score());
+    }
+
+}