diff --git a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
index 8888ce33be57f..ec94af9b75fc4 100644
--- a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
+++ b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/SimpleKdcLdapServer.java
@@ -90,7 +90,9 @@ public Boolean run() throws Exception {
         AccessController.doPrivileged(new PrivilegedExceptionAction<Void>() {
             @Override
             public Void run() throws Exception {
-                init();
+                if (ESTestCase.awaitBusy(() -> init()) == false) {
+                    throw new IllegalStateException("could not initialize SimpleKdcLdapServer");
+                }
                 return null;
             }
         });
@@ -98,14 +100,33 @@ public Void run() throws Exception {
     }
 
     @SuppressForbidden(reason = "Uses Apache Kdc which requires usage of java.io.File in order to create a SimpleKdcServer")
-    private void init() throws Exception {
-        // start ldap server
-        createLdapServiceAndStart();
-        // create ldap backend conf
-        createLdapBackendConf();
-        // Kdc Server
-        simpleKdc = new SimpleKdcServer(this.workDir.toFile(), new KrbConfig());
-        prepareKdcServerAndStart();
+    private boolean init() {
+        boolean initialized = false;
+        try {
+            // start ldap server
+            createLdapServiceAndStart();
+            // create ldap backend conf
+            createLdapBackendConf();
+            // Kdc Server
+            simpleKdc = new SimpleKdcServer(this.workDir.toFile(), new KrbConfig());
+            prepareKdcServerAndStart();
+            initialized = true;
+        } catch (Exception e) {
+            if (simpleKdc != null) {
+                try {
+                    simpleKdc.stop();
+                } catch (KrbException krbException) {
+                    logger.debug("error occurred while cleaning up after init failure for SimpleKdcLdapServer");
+                }
+            }
+            if (ldapServer != null) {
+                ldapServer.shutDown(true);
+            }
+            ldapPort = 0;
+            kdcPort = 0;
+            initialized = false;
+        }
+        return initialized;
     }
 
     private void createLdapServiceAndStart() throws Exception {
@@ -229,12 +250,14 @@ private static int getServerPort(String transport) {
         if (transport != null && transport.trim().equalsIgnoreCase("TCP")) {
             try (ServerSocket serverSocket = ServerSocketFactory.getDefault().createServerSocket(0, 1,
                     InetAddress.getByName("127.0.0.1"))) {
+                serverSocket.setReuseAddress(true);
                 return serverSocket.getLocalPort();
             } catch (Exception ex) {
                 throw new RuntimeException("Failed to get a TCP server socket point");
             }
         } else if (transport != null && transport.trim().equalsIgnoreCase("UDP")) {
             try (DatagramSocket socket = new DatagramSocket(0, InetAddress.getByName("127.0.0.1"))) {
+                socket.setReuseAddress(true);
                 return socket.getLocalPort();
             } catch (Exception ex) {
                 throw new RuntimeException("Failed to get a UDP server socket point");