Improve job pool configurations

deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/ConfigHelper.java

@@ -64,16 +64,20 @@
 import com.pinterest.teletraan.worker.MetricsEmitter;
 import com.pinterest.teletraan.worker.SimpleAgentJanitor;
 import com.pinterest.teletraan.worker.StateTransitioner;
+import io.dropwizard.setup.Environment;
+import io.dropwizard.util.Duration;
+import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.binder.jvm.ExecutorServiceMetrics;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.TreeMap;
+import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.ThreadPoolExecutor.AbortPolicy;
 import java.util.concurrent.TimeUnit;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.dbcp.BasicDataSource;
@@ -98,8 +102,8 @@ public class ConfigHelper {
     private static final int DEFAULT_MAX_DAYS_TO_KEEP = 180;
     private static final int DEFAULT_MAX_BUILDS_TO_KEEP = 1000;
 
-    public static TeletraanServiceContext setupContext(TeletraanServiceConfiguration configuration)
-            throws Exception {
+    public static TeletraanServiceContext setupContext(
+            TeletraanServiceConfiguration configuration, Environment environment) throws Exception {
         TeletraanServiceContext context = new TeletraanServiceContext();
 
         BasicDataSource dataSource = configuration.getDataSourceFactory().build();
@@ -218,20 +222,20 @@ public static TeletraanServiceContext setupContext(TeletraanServiceConfiguration
             context.setAccountAllowList(configuration.getAccountAllowList());
         }
 
-        /*
-        Lastly, let us create the in-process background job executor, all transient, long
-        running background jobs can be handled by this executor
-        Currently we hard coded the parameters as:
-
-        corePoolSize - the number of threads to keep in the pool, even if they are idle, unless allowCoreThreadTimeOut is set
-        maximumPoolSize - the maximum number of threads to allow in the pool
-        keepAliveTime - when the number of threads is greater than the core, this is the maximum time that excess idle threads will wait for new tasks before terminating.
-        unit - the time unit for the keepAliveTime argument
-        workQueue - the queue to use for holding tasks before they are executed. This queue will hold only the Runnable tasks submitted by the execute method.
-        */
-        // TODO make the thread configurable
+        int poolSize = Runtime.getRuntime().availableProcessors();
+        String jobPoolName = "jobPool";
         ExecutorService jobPool =
-                new ThreadPoolExecutor(1, 10, 30, TimeUnit.SECONDS, new LinkedBlockingQueue<>());
+                environment
+                        .lifecycle()
+                        .executorService(jobPoolName)
+                        .minThreads(poolSize * 4)
+                        .maxThreads(poolSize * 8)
+                        .keepAliveTime(Duration.seconds(30))
+                        .workQueue(new ArrayBlockingQueue<>(poolSize * 5000, false))
+                        .shutdownTime(Duration.seconds(30))
+                        .rejectedExecutionHandler(new AbortPolicy())
+                        .build();
+        new ExecutorServiceMetrics(jobPool, jobPoolName, null).bindTo(Metrics.globalRegistry);
         context.setJobPool(jobPool);
 
         context.setDeployBoardUrlPrefix(configuration.getSystemFactory().getDashboardUrl());

deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/TeletraanAgentService.java

@@ -44,7 +44,7 @@ public void initialize(Bootstrap<TeletraanServiceConfiguration> bootstrap) {
     @Override
     public void run(TeletraanServiceConfiguration configuration, Environment environment)
             throws Exception {
-        TeletraanServiceContext context = ConfigHelper.setupContext(configuration);
+        TeletraanServiceContext context = ConfigHelper.setupContext(configuration, environment);
         environment.jersey().register(context);
         environment
                 .jersey()

deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/TeletraanService.java

@@ -51,7 +51,7 @@ public void initialize(Bootstrap<TeletraanServiceConfiguration> bootstrap) {
     @Override
     public void run(TeletraanServiceConfiguration configuration, Environment environment)
             throws Exception {
-        TeletraanServiceContext context = ConfigHelper.setupContext(configuration);
+        TeletraanServiceContext context = ConfigHelper.setupContext(configuration, environment);
 
         environment.jersey().register(context);
         environment

deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/TeletraanWorker.java

@@ -40,7 +40,7 @@ public void initialize(Bootstrap<TeletraanServiceConfiguration> bootstrap) {
     @Override
     public void run(TeletraanServiceConfiguration configuration, Environment environment)
             throws Exception {
-        TeletraanServiceContext context = ConfigHelper.setupContext(configuration);
+        TeletraanServiceContext context = ConfigHelper.setupContext(configuration, environment);
         ConfigHelper.scheduleWorkers(configuration, context);
         environment.healthChecks().register("generic", new WorkerHealthCheck(context));
     }