Add await busy loop for SimpleKdcLdapServer initialization (#39221) (#…

…39344)

There have been intermittent failures where either
LDAP server could not be started or KDC server could
not be started causing failures during test runs.

`KdcNetwork` class from Apache kerby project does not set reuse
address to `true` on the socket so if the port that we found to be free
is in `TIME_WAIT` state it may fail to bind. As this is an internal
class for kerby, I could not find a way to extend.

This commit adds a retry loop for initialization. It will keep
trying in an await busy loop and fail after 10 seconds if not
initialized.

Closes #35982

x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java

@@ -90,22 +90,43 @@ public Boolean run() throws Exception {
         AccessController.doPrivileged(new PrivilegedExceptionAction<Void>() {
             @Override
             public Void run() throws Exception {
-                init();
+                if (ESTestCase.awaitBusy(() -> init()) == false) {
+                    throw new IllegalStateException("could not initialize SimpleKdcLdapServer");
+                }
                 return null;
             }
         });
         logger.info("SimpleKdcLdapServer started.");
     }
 
     @SuppressForbidden(reason = "Uses Apache Kdc which requires usage of java.io.File in order to create a SimpleKdcServer")
-    private void init() throws Exception {
-        // start ldap server
-        createLdapServiceAndStart();
-        // create ldap backend conf
-        createLdapBackendConf();
-        // Kdc Server
-        simpleKdc = new SimpleKdcServer(this.workDir.toFile(), new KrbConfig());
-        prepareKdcServerAndStart();
+    private boolean init() {
+        boolean initialized = false;
+        try {
+            // start ldap server
+            createLdapServiceAndStart();
+            // create ldap backend conf
+            createLdapBackendConf();
+            // Kdc Server
+            simpleKdc = new SimpleKdcServer(this.workDir.toFile(), new KrbConfig());
+            prepareKdcServerAndStart();
+            initialized = true;
+        } catch (Exception e) {
+            if (simpleKdc != null) {
+                try {
+                    simpleKdc.stop();
+                } catch (KrbException krbException) {
+                    logger.debug("error occurred while cleaning up after init failure for SimpleKdcLdapServer");
+                }
+            }
+            if (ldapServer != null) {
+                ldapServer.shutDown(true);
+            }
+            ldapPort = 0;
+            kdcPort = 0;
+            initialized = false;
+        }
+        return initialized;
     }
 
     private void createLdapServiceAndStart() throws Exception {
@@ -229,12 +250,14 @@ private static int getServerPort(String transport) {
         if (transport != null && transport.trim().equalsIgnoreCase("TCP")) {
             try (ServerSocket serverSocket = ServerSocketFactory.getDefault().createServerSocket(0, 1,
                     InetAddress.getByName("127.0.0.1"))) {
+                serverSocket.setReuseAddress(true);
                 return serverSocket.getLocalPort();
             } catch (Exception ex) {
                 throw new RuntimeException("Failed to get a TCP server socket point");
             }
         } else if (transport != null && transport.trim().equalsIgnoreCase("UDP")) {
             try (DatagramSocket socket = new DatagramSocket(0, InetAddress.getByName("127.0.0.1"))) {
+                socket.setReuseAddress(true);
                 return socket.getLocalPort();
             } catch (Exception ex) {
                 throw new RuntimeException("Failed to get a UDP server socket point");