diff --git a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java
index 6ef9436c81579..f17ff7e258006 100644
--- a/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java
+++ b/x-pack/plugin/search-business-rules/src/test/java/org/elasticsearch/xpack/searchbusinessrules/PinnedQueryBuilderIT.java
@@ -23,6 +23,7 @@
 import java.util.LinkedHashSet;
 import java.util.List;
 
+import static org.elasticsearch.action.search.SearchType.DFS_QUERY_THEN_FETCH;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFirstHit;
@@ -61,7 +62,7 @@ public void testPinnedPromotions() throws Exception {
                 .addMapping("type1",
                         jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("field1")
                                 .field("analyzer", "whitespace").field("type", "text").endObject().endObject().endObject().endObject())
-                .setSettings(Settings.builder().put(indexSettings()).put("index.number_of_shards", 2)));
+                .setSettings(Settings.builder().put(indexSettings()).put("index.number_of_shards", randomIntBetween(2, 5))));
 
         int numRelevantDocs = randomIntBetween(1, 100);
         for (int i = 0; i < numRelevantDocs; i++) {
@@ -101,6 +102,7 @@ public void testPinnedPromotions() throws Exception {
             int from = randomIntBetween(0, numRelevantDocs);
             int size = randomIntBetween(10, 100);
             SearchResponse searchResponse = client().prepareSearch().setQuery(pqb).setTrackTotalHits(true).setSize(size).setFrom(from)
+                    .setSearchType(DFS_QUERY_THEN_FETCH)
                     .get();
 
             long numHits = searchResponse.getHits().getTotalHits().value;