downscale gracefully (#647)

* Failing requests in down-scaling scenario

closes #547

Following
https://dzone.com/articles/graceful-shutdown-spring-boot-applications

We let the tomcat finish it's work (send open async responses -
DeferredResult), before it's terminated (SIGTERM coming from
kubernetes).

* #589 Log4j improper shutdown

same thing as in distribution service

* checkstyle complains about missing javadoc

why is it doing that for this method but not for the others without
javadoc?!?! bug in checkstyle?

* ignore null executors

#547

* add missing file comment 'license'

#547

* fix issues found by sonar

#547

* let's use the spring framework solution

#547

https://dzone.com/articles/configuring-graceful-shutdown-readiness-and-livene

* prefer `DisposableBean` over `@PreDestroy`

#589

* use k8s setting `terminationGracePeriodSeconds`

#547

services/submission/src/main/java/app/coronawarn/server/services/submission/ServerApplication.java

@@ -24,8 +24,10 @@
 import io.micrometer.core.instrument.MeterRegistry;
 import java.util.Arrays;
 import java.util.List;
+import org.apache.logging.log4j.LogManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.DisposableBean;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.autoconfigure.domain.EntityScan;
@@ -46,7 +48,7 @@
     "app.coronawarn.server.services.submission"})
 @EnableConfigurationProperties
 @EnableFeignClients
-public class ServerApplication implements EnvironmentAware {
+public class ServerApplication implements EnvironmentAware, DisposableBean {
 
   private static final Logger logger = LoggerFactory.getLogger(ServerApplication.class);
 
@@ -59,6 +61,14 @@ TimedAspect timedAspect(MeterRegistry registry) {
     return new TimedAspect(registry);
   }
 
+  /**
+   * Manual shutdown hook needed to avoid Log4j shutdown issues (see cwa-server/#589).
+   */
+  @Override
+  public void destroy() {
+    LogManager.shutdown();
+  }
+
   @Bean
   ProtobufHttpMessageConverter protobufHttpMessageConverter() {
     return new ProtobufHttpMessageConverter();

services/submission/src/main/resources/application.yaml

@@ -41,6 +41,10 @@ services:
         trust-store-password: ${SSL_VERIFICATION_TRUSTSTORE_PASSWORD}
 
 spring:
+  lifecycle:
+    # keep in sync or lower than the kubernetes setting 'terminationGracePeriodSeconds'
+    # 5s +5s Feign client + 20s DB timeout
+    timeout-per-shutdown-phase: 30s
   transaction:
     default-timeout: 20
   flyway:
@@ -82,6 +86,7 @@ management:
       enabled: true
 
 server:
+  shutdown: graceful
   ssl:
     enabled: true
     enabled-protocols: TLSv1.2,TLSv1.3