Fix potential race during TcpTransport close

Fixed two potential causes for leaked threads during tests:
1. When adding a channel to serverChannels, we add it under a monitor
that we do not use when reading from it. This is potentially unsafe if
there is no other happens-before relationship ensuring the safety of
this.
2. Long-shot but if the thread pool was shutdown before entering this
code, we would silently forget about closing server channels so added
assert.

Relates to CI failure issue: elastic#37543

server/src/main/java/org/elasticsearch/transport/TcpTransport.java

@@ -553,13 +553,21 @@ protected final void doClose() {
     protected final void doStop() {
         final CountDownLatch latch = new CountDownLatch(1);
         // make sure we run it on another thread than a possible IO handler thread
+        assert threadPool.generic().isShutdown() == false : "Must stop transport before terminating underlying threadpool";
         threadPool.generic().execute(() -> {
             closeLock.writeLock().lock();
             try {
                 keepAlive.close();
 
+                // Copy map of server channels to avoid holding serverChannels monitor while closing channels.
+                Map<String, List<TcpServerChannel>> serverChannelsCopy;
+                synchronized (this.serverChannels) {
+                    serverChannelsCopy = new HashMap<>(this.serverChannels);
+                    this.serverChannels.clear();
+                }
+
                 // first stop to accept any incoming connections so nobody can connect to this transport
-                for (Map.Entry<String, List<TcpServerChannel>> entry : serverChannels.entrySet()) {
+                for (Map.Entry<String, List<TcpServerChannel>> entry : serverChannelsCopy.entrySet()) {
                     String profile = entry.getKey();
                     List<TcpServerChannel> channels = entry.getValue();
                     ActionListener<Void> closeFailLogger = ActionListener.wrap(c -> {
@@ -568,7 +576,6 @@ protected final void doStop() {
                     channels.forEach(c -> c.addCloseListener(closeFailLogger));
                     CloseableChannel.closeChannels(channels, true);
                 }
-                serverChannels.clear();
 
                 // close all of the incoming channels. The closeChannels method takes a list so we must convert the set.
                 CloseableChannel.closeChannels(new ArrayList<>(acceptedChannels), true);