[CI] BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests testReindex failing

[ywangd]

It failed on my PR. But it is reproducible locally on the latest master branch. Also note the original failure is in a FIPS job. But it has nothing to do with FIPS, i.e. it fails without the fips bit.

Build scan:
https://gradle-enterprise.elastic.co/s/vhzjwuaexssgs/tests/:modules:reindex:internalClusterTest/org.elasticsearch.index.reindex.BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests/testReindex

Reproduction line:
./gradlew ':modules:reindex:internalClusterTest' --tests "org.elasticsearch.index.reindex.BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.testReindex" -Dtests.seed=AFA3788DC76760FD -Dtests.locale=ar-QA -Dtests.timezone=America/St_Kitts -Druntime.java=11 -Dtests.fips.enabled=true

Applicable branches:
master

Reproduces locally?:
Yes

Failure history:
https://gradle-enterprise.elastic.co/scans/tests?tests.container=org.elasticsearch.index.reindex.BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests&tests.test=testReindex

Failure excerpt:

java.lang.AssertionError: {"took":53,"errors":true,"items":[{"index":{"_index":"gjifkbqhxr","_id":"13","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":10,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"18","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":11,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"17","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":12,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"90","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":13,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"27","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":14,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"87","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":15,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"21","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":16,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"60","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":17,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"81","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":18,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"73","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":19,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"78","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":20,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"51","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":21,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"62","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":22,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"38","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":23,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"8","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":24,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"22","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":25,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"100","status":409,"error":{"type":"version_conflict_engine_exception","reason":"[100]: version conflict, document already exists (current version [1])","index_uuid":"Vbn3LBPWS_KuUBpAy0UKHw","shard":"0","index":"gjifkbqhxr"}}},{"index":{"_index":"gjifkbqhxr","_id":"66","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":26,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"19","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":27,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"14","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":28,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"2","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":29,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"67","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":30,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"92","status":409,"error":{"type":"version_conflict_engine_exception","reason":"[92]: version conflict, document already exists (current version [1])","index_uuid":"Vbn3LBPWS_KuUBpAy0UKHw","shard":"0","index":"gjifkbqhxr"}}},{"index":{"_index":"gjifkbqhxr","_id":"68","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":31,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"76","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":32,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"41","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":33,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"37","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":34,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"72","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":35,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"45","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":36,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"49","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":37,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"3","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":38,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"4","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":39,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"48","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":40,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"84","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":41,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"89","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":42,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"6","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":43,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"7","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":44,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"95","status":409,"error":{"type":"version_conflict_engine_exception","reason":"[95]: version conflict, document already exists (current version [1])","index_uuid":"Vbn3LBPWS_KuUBpAy0UKHw","shard":"0","index":"gjifkbqhxr"}}},{"index":{"_index":"gjifkbqhxr","_id":"44","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":45,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"53","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":46,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"43","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":47,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"35","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":48,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"39","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":49,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"46","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":50,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"50","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":51,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"10","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":52,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"74","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":53,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"25","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":54,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"99","status":409,"error":{"type":"version_conflict_engine_exception","reason":"[99]: version conflict, document already exists (current version [1])","index_uuid":"Vbn3LBPWS_KuUBpAy0UKHw","shard":"0","index":"gjifkbqhxr"}}},{"index":{"_index":"gjifkbqhxr","_id":"83","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":55,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"47","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":56,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"28","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":57,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"56","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":58,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"80","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":59,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"91","status":409,"error":{"type":"version_conflict_engine_exception","reason":"[91]: version conflict, document already exists (current version [1])","index_uuid":"Vbn3LBPWS_KuUBpAy0UKHw","shard":"0","index":"gjifkbqhxr"}}},{"index":{"_index":"gjifkbqhxr","_id":"94","status":409,"error":{"type":"version_conflict_engine_exception","reason":"[94]: version conflict, document already exists (current version [1])","index_uuid":"Vbn3LBPWS_KuUBpAy0UKHw","shard":"0","index":"gjifkbqhxr"}}},{"index":{"_index":"gjifkbqhxr","_id":"54","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":60,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"26","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":61,"_primary_term":1,"status":201}},{"index":{"_index":"gjifkbqhxr","_id":"96","status":409,"error":{"type":"version_conflict_engine_exception","reason":"[96]: version conflict, document already exists (current version [1])","index_uuid":"Vbn3LBPWS_KuUBpAy0UKHw","shard":"0","index":"gjifkbqhxr"}}},{"index":{"_index":"gjifkbqhxr","_id":"12","_version":1,"result":"created","_shards":{"total":1,"successful":1,"failed":0},"_seq_no":62,"_primary_term":1,"status":201}}]}
Expected: is <false>
     but: was <true>

  at __randomizedtesting.SeedInfo.seed([AFA3788DC76760FD:3CD926614532DAAF]:0)
  at org.hamcrest.MatcherAssert.assertThat(MatcherAssert.java:18)
  at org.junit.Assert.assertThat(Assert.java:956)
  at org.elasticsearch.index.reindex.BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.executeConcurrentUpdatesOnSubsetOfDocs(BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java:244)
  at org.elasticsearch.index.reindex.BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.testReindex(BulkByScrollUsesAllScrollDocumentsAfterConflictsIntegTests.java:126)
  at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(NativeMethodAccessorImpl.java:-2)
  at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  at java.lang.reflect.Method.invoke(Method.java:566)
  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
  at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
  at java.lang.Thread.run(Thread.java:834)

[elasticmachine]

Pinging @elastic/es-distributed (Team:Distributed)

[gwbrown]

Looks like a duplicate of #79300. Also, got another failure scan.

[fcofdez]

I'm closing this in favor of #79300