Make GeoIpDownloaderIT.testInvalidTimestamp more robust (#75330)

This change adds additional assertion in GeoIpDownloaderIT.testInvalidTimestamp which makes sure that validity checks work both ways (so going out of validity and back) and it should fix race in cleanUp method leading to occasional failures.

Closes #75221
Closes #74358

modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java

@@ -61,6 +61,7 @@
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasItems;
 import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.nullValue;
@@ -88,9 +89,9 @@ public void cleanUp() throws Exception {
         ClusterUpdateSettingsResponse settingsResponse = client().admin().cluster()
             .prepareUpdateSettings()
             .setPersistentSettings(Settings.builder()
-                .put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), (String) null)
-                .put(GeoIpDownloader.POLL_INTERVAL_SETTING.getKey(), (String) null)
-                .put("ingest.geoip.database_validity", (String) null))
+                .putNull(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey())
+                .putNull(GeoIpDownloader.POLL_INTERVAL_SETTING.getKey())
+                .putNull("ingest.geoip.database_validity"))
             .get();
         assertTrue(settingsResponse.isAcknowledged());
 
@@ -167,6 +168,21 @@ public void testInvalidTimestamp() throws Exception {
             assertFalse(result.getIngestDocument().hasField("ip-asn"));
             assertFalse(result.getIngestDocument().hasField("ip-country"));
         });
+        settingsResponse =
+            client().admin().cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(Settings.builder()
+                    .putNull("ingest.geoip.database_validity"))
+                .get();
+        assertTrue(settingsResponse.isAcknowledged());
+        assertBusy(() -> {
+            for (Path geoIpTmpDir : geoIpTmpDirs) {
+                try (Stream<Path> files = Files.list(geoIpTmpDir)) {
+                    Set<String> names = files.map(f -> f.getFileName().toString()).collect(Collectors.toSet());
+                    assertThat(names, hasItems("GeoLite2-ASN.mmdb","GeoLite2-City.mmdb","GeoLite2-Country.mmdb"));
+                }
+            }
+        });
     }
 
     public void testUpdatedTimestamp() throws Exception {