More reliable shutdown of the EnsoContext to save resources (#6468)

There is 572 threads during `RuntimeServerTest` execution create. They stay around because the context isn't closed after each test finishes. Adding `afterEach` cleanup sequence to `runtime-with-instruments` tests. Improving robustness of overall `EnsoContext` [shutdown](#6468 (comment)).

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/BuiltinTypesTest.scala

@@ -97,6 +97,10 @@ class BuiltinTypesTest
     context = new TestContext("Test")
     val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
   }
+  override protected def afterEach(): Unit = {
+    context.executionContext.context.close()
+    context.runtimeServerEmulator.terminate()
+  }
 
   def runCode(contextId: UUID, requestId: UUID, contents: String): Unit = {
     val moduleName = "Enso_Test.Test.Main"

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeAsyncCommandsTest.scala

@@ -100,6 +100,10 @@ class RuntimeAsyncCommandsTest
     context = new TestContext("Test")
     val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
   }
+  override protected def afterEach(): Unit = {
+    context.executionContext.context.close()
+    context.runtimeServerEmulator.terminate()
+  }
 
   it should "interrupt stopped execution context" in {
     val moduleName = "Enso_Test.Test.Main"

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeComponentsTest.scala

@@ -200,6 +200,10 @@ class RuntimeComponentsTest
     context = new TestContext("Test")
     val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
   }
+  override protected def afterEach(): Unit = {
+    context.executionContext.context.close()
+    context.runtimeServerEmulator.terminate()
+  }
 
   it should "load library extended by the component group" in {
     val contextId  = UUID.randomUUID()

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeErrorsTest.scala

@@ -106,6 +106,11 @@ class RuntimeErrorsTest
     val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
   }
 
+  override protected def afterEach(): Unit = {
+    context.executionContext.context.close()
+    context.runtimeServerEmulator.terminate()
+  }
+
   it should "return panic sentinels in method body" in {
     val contextId  = UUID.randomUUID()
     val requestId  = UUID.randomUUID()

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeExecutionEnvironmentTest.scala

@@ -129,6 +129,11 @@ class RuntimeExecutionEnvironmentTest
     val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
   }
 
+  override protected def afterEach(): Unit = {
+    context.executionContext.context.close()
+    context.runtimeServerEmulator.terminate()
+  }
+
   it should "panic when output context is not enabled" in {
 
     val contextId  = UUID.randomUUID()

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeInstrumentTest.scala

@@ -99,6 +99,11 @@ class RuntimeInstrumentTest
     val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
   }
 
+  override protected def afterEach(): Unit = {
+    context.executionContext.context.close()
+    context.runtimeServerEmulator.terminate()
+  }
+
   it should "instrument simple expression" in {
     val contextId  = UUID.randomUUID()
     val requestId  = UUID.randomUUID()

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala

@@ -331,6 +331,11 @@ class RuntimeServerTest
     val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
   }
 
+  override protected def afterEach(): Unit = {
+    context.executionContext.context.close()
+    context.runtimeServerEmulator.terminate()
+  }
+
   "RuntimeServer" should "push and pop functions on the stack" in {
     val contents  = context.Main.code
     val mainFile  = context.writeMain(contents)

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeStdlibTest.scala

@@ -185,6 +185,11 @@ class RuntimeStdlibTest
     val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
   }
 
+  override protected def afterEach(): Unit = {
+    context.executionContext.context.close()
+    context.runtimeServerEmulator.terminate()
+  }
+
   it should "import Base modules" in {
     val contextId  = UUID.randomUUID()
     val requestId  = UUID.randomUUID()

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeSuggestionUpdatesTest.scala

@@ -94,6 +94,11 @@ class RuntimeSuggestionUpdatesTest
     val Some(Api.Response(_, Api.InitializedNotification())) = context.receive
   }
 
+  override protected def afterEach(): Unit = {
+    context.executionContext.context.close()
+    context.runtimeServerEmulator.terminate()
+  }
+
   it should "send suggestion updates after file modification" in {
     val contextId  = UUID.randomUUID()
     val requestId  = UUID.randomUUID()

engine/runtime/src/main/java/org/enso/interpreter/runtime/ThreadExecutors.java

@@ -5,7 +5,9 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
 
 final class ThreadExecutors {
   private final EnsoContext context;
@@ -32,6 +34,17 @@ public void shutdown() {
     while (it.hasNext()) {
       var p = it.next().getKey();
       p.shutdown();
+      boolean success;
+      try {
+        success = p.awaitTermination(10, TimeUnit.SECONDS);
+      } catch (InterruptedException ex) {
+        success = false;
+      }
+      if (!success) {
+        context
+            .getLogger()
+            .log(Level.WARNING, "Cannot shutdown {0} thread pool", it.next().getValue());
+      }
     }
   }
 

engine/runtime/src/main/scala/org/enso/interpreter/instrument/execution/JobExecutionEngine.scala

@@ -175,6 +175,7 @@ final class JobExecutionEngine(
     runtimeContext.executionService.getContext.getThreadManager
       .interruptThreads()
     jobExecutor.shutdownNow()
+    backgroundJobExecutor.shutdownNow()
   }
 
   /** Submit background jobs preserving the stable order. */

engine/runtime/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerEmulator.scala

@@ -80,4 +80,6 @@ class RuntimeServerEmulator(
       }
     } else null
   }
+
+  def terminate() = system.terminate()
 }