Release 1.0.5 (#567)

* Introduce new version 1.0.5

* Enabling roles also for local development & docker setup (#552)

* Restict allowed actuator routes (#540)

* Fix batch counter concurrency issue (#551)

* Cherry-pick: Fix connection pool size configuration (#561)

* Updated timestamp format (#566)

* Add missing NoopHostnameVerifierProvider

Co-authored-by: MKusber <[email protected]>
Co-authored-by: Michael Burwig <[email protected]>
Co-authored-by: Pit Humke <[email protected]>
Co-authored-by: Steve BE <[email protected]>

.env

@@ -6,6 +6,15 @@ POSTGRES_DB=cwa
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 
+POSTGRES_DISTRIBUTION_USER=local_setup_distribution
+POSTGRES_DISTRIBUTION_PASSWORD=local_setup_distribution
+
+POSTGRES_SUBMISSION_USER=local_setup_submission
+POSTGRES_SUBMISSION_PASSWORD=local_setup_submission
+
+POSTGRES_FLYWAY_USER=local_setup_flyway
+POSTGRES_FLYWAY_PASSWORD=local_setup_flyway
+
 # Docker Compose PgAdmin settings
 PGADMIN_DEFAULT_EMAIL=[email protected]
 PGADMIN_DEFAULT_PASSWORD=password

.mvn/maven.config

@@ -1,4 +1,4 @@
--Drevision=1.0.4
+-Drevision=1.0.5
 -Dlicense.projectName=Corona-Warn-App
 -Dlicense.inceptionYear=2020
 -Dlicense.licenseName=apache_v2

README.md

@@ -97,6 +97,30 @@ To prepare your machine to run the CWA project locally, we recommend that you fi
 * [Postgres]
 * [Zenko CloudServer]
 
+If you are already running a local Postgres, you need to create a database `cwa` and run the following setup scripts:
+
+* Create the different CWA roles first by executing [create-roles.sql](setup/create-roles.sql).
+* Create local database users for the specific roles by running [create-users.sql](./local-setup/create-users.sql).
+* It is recommended to also run [enable-test-data-docker-compose.sql](./local-setup/enable-test-data-docker-compose.sql)
+, which enables the test data generation profile. If you already had CWA running before and an existing `diagnosis-key`
+table on your database, you need to run [enable-test-data.sql](./local-setup/enable-test-data.sql) instead.
+
+You can also use `docker-compose` to start Postgres and Zenko. If you do that, you have to
+set the following environment-variables when running the Spring project:
+
+For the distribution module:
+
+```bash
+POSTGRESQL_SERVICE_PORT=8001
+VAULT_FILESIGNING_SECRET=</path/to/your/private_key>
+```
+
+For the submission module:
+
+```bash
+POSTGRESQL_SERVICE_PORT=8001
+```
+
 #### Configure
 
 After you made sure that the specified dependencies are running, configure them in the respective configuration files.

common/persistence/src/main/resources/db/specific/h2/V2__createPermissions.sql

@@ -0,0 +1 @@
+-- no permissions on H2 necessary

common/persistence/src/main/resources/db/specific/postgresql/V2__createPermissions.sql

@@ -0,0 +1,2 @@
+GRANT SELECT, DELETE ON TABLE diagnosis_key TO "cwa_distribution";
+GRANT INSERT ON TABLE diagnosis_key TO "cwa_submission";

docker-compose.yml

@@ -15,10 +15,10 @@ services:
       POSTGRESQL_SERVICE_PORT: '5432'
       POSTGRESQL_SERVICE_HOST: postgres
       POSTGRESQL_DATABASE: ${POSTGRES_DB}
-      POSTGRESQL_PASSWORD_SUBMISION: ${POSTGRES_PASSWORD}
-      POSTGRESQL_USER_SUBMISION: ${POSTGRES_USER}
-      POSTGRESQL_PASSWORD_FLYWAY: ${POSTGRES_PASSWORD}
-      POSTGRESQL_USER_FLYWAY: ${POSTGRES_USER}
+      POSTGRESQL_PASSWORD_SUBMISION: ${POSTGRES_SUBMISSION_PASSWORD}
+      POSTGRESQL_USER_SUBMISION: ${POSTGRES_SUBMISSION_USER}
+      POSTGRESQL_PASSWORD_FLYWAY: ${POSTGRES_FLYWAY_PASSWORD}
+      POSTGRESQL_USER_FLYWAY: ${POSTGRES_FLYWAY_USER}
       VERIFICATION_BASE_URL: http://verification-fake:8004
   distribution:
     build:
@@ -33,10 +33,10 @@ services:
       POSTGRESQL_SERVICE_PORT: '5432'
       POSTGRESQL_SERVICE_HOST: postgres
       POSTGRESQL_DATABASE: ${POSTGRES_DB}
-      POSTGRESQL_PASSWORD_DISTRIBUTION: ${POSTGRES_PASSWORD}
-      POSTGRESQL_USER_DISTRIBUTION: ${POSTGRES_USER}
-      POSTGRESQL_PASSWORD_FLYWAY: ${POSTGRES_PASSWORD}
-      POSTGRESQL_USER_FLYWAY: ${POSTGRES_USER}
+      POSTGRESQL_PASSWORD_DISTRIBUTION: ${POSTGRES_DISTRIBUTION_PASSWORD}
+      POSTGRESQL_USER_DISTRIBUTION: ${POSTGRES_DISTRIBUTION_USER}
+      POSTGRESQL_PASSWORD_FLYWAY: ${POSTGRES_FLYWAY_PASSWORD}
+      POSTGRESQL_USER_FLYWAY: ${POSTGRES_FLYWAY_USER}
       # Settings for the S3 compatible objectstore
       CWA_OBJECTSTORE_ACCESSKEY: ${OBJECTSTORE_ACCESSKEY}
       CWA_OBJECTSTORE_SECRETKEY: ${OBJECTSTORE_SECRETKEY}
@@ -60,6 +60,9 @@ services:
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
     volumes:
        - postgres_volume:/data/postgres
+       - ./setup/setup-roles.sql:/docker-entrypoint-initdb.d/1-roles.sql
+       - ./local-setup/create-users.sql:/docker-entrypoint-initdb.d/2-users.sql
+       - ./local-setup/enable-test-data-docker-compose.sql:/docker-entrypoint-initdb.d/3-enable-testdata.sql
   pgadmin:
     container_name: pgadmin_container
     image: dpage/pgadmin4

local-setup/create-users.sql

@@ -0,0 +1,5 @@
+/* Creates default users for local development */
+
+CREATE USER "local_setup_flyway" WITH INHERIT IN ROLE cwa_flyway ENCRYPTED PASSWORD 'local_setup_flyway';
+CREATE USER "local_setup_submission" WITH INHERIT IN ROLE cwa_submission ENCRYPTED PASSWORD 'local_setup_submission';
+CREATE USER "local_setup_distribution" WITH INHERIT IN ROLE cwa_distribution ENCRYPTED PASSWORD 'local_setup_distribution';

local-setup/enable-test-data-docker-compose.sql

@@ -0,0 +1,3 @@
+/* Testdata creation requires INSERT permission - this is only active on local deployments */
+/* Since the Diagnosis Key Table does not yet exist, this is a workaround to still allow inserts */
+ALTER DEFAULT PRIVILEGES FOR USER local_setup_flyway IN SCHEMA public GRANT INSERT ON TABLES TO cwa_distribution;

local-setup/enable-test-data.sql

@@ -0,0 +1,2 @@
+/* Test Data profile requires insert permissions on diagnois key table */
+GRANT INSERT ON diagnosis_key TO cwa_distribution;

services/distribution/src/main/resources/application-cloud.yaml

@@ -1,8 +1,6 @@
 ---
 spring:
   flyway:
-    enabled: true
-    locations: classpath:db/migration/postgres
     password: ${POSTGRESQL_PASSWORD_FLYWAY}
     user: ${POSTGRESQL_USER_FLYWAY}
   datasource:

services/distribution/src/main/resources/application.yaml

@@ -61,12 +61,12 @@ spring:
       ddl-auto: validate
   flyway:
     enabled: true
-    locations: classpath:db/migration/postgres
-    password: ${POSTGRESQL_PASSWORD_FLYWAY:postgres}
-    user: ${POSTGRESQL_USER_FLYWAY:postgres}
+    locations: classpath:/db/migration, classpath:/db/specific/{vendor}
+    password: ${POSTGRESQL_PASSWORD_FLYWAY:local_setup_flyway}
+    user: ${POSTGRESQL_USER_FLYWAY:local_setup_flyway}
 
   datasource:
     driver-class-name: org.postgresql.Driver
     url: jdbc:postgresql://${POSTGRESQL_SERVICE_HOST:localhost}:${POSTGRESQL_SERVICE_PORT:5432}/${POSTGRESQL_DATABASE:cwa}
-    username: ${POSTGRESQL_USER_DISTRIBUTION:postgres}
-    password: ${POSTGRESQL_PASSWORD_DISTRIBUTION:postgres}
+    username: ${POSTGRESQL_USER_DISTRIBUTION:local_setup_distribution}
+    password: ${POSTGRESQL_PASSWORD_DISTRIBUTION:local_setup_distribution}

services/distribution/src/main/resources/log4j2-dev.xml

@@ -2,7 +2,7 @@
 <Configuration status="WARN">
   <Properties>
     <Property name="LOG_EXCEPTION_CONVERSION_WORD">%xwEx</Property>
-    <Property name="LOG_DATEFORMAT_PATTERN">yyyy-MM-dd'T'HH:mm:ssXXX</Property>
+    <Property name="LOG_DATEFORMAT_PATTERN">yyyy-MM-dd'T'HH:mm:ssZ</Property>
     <Property name="CONSOLE_LOG_PATTERN">%d{${LOG_DATEFORMAT_PATTERN}} %-5level %t %c{1.}[%pid]: %m%n${sys:LOG_EXCEPTION_CONVERSION_WORD}</Property>
   </Properties>
   <Appenders>

services/distribution/src/main/resources/log4j2.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Configuration status="WARN">
   <Properties>
-    <Property name="LOG_DATEFORMAT_PATTERN">yyyy-MM-dd'T'HH:mm:ssXXX</Property>
+    <Property name="LOG_DATEFORMAT_PATTERN">yyyy-MM-dd'T'HH:mm:ssZ</Property>
     <Property name="CONSOLE_LOG_PATTERN">%d{${LOG_DATEFORMAT_PATTERN}} %-5level %t %c{1.}[%pid]: %m %replace{%rException}{\n}{\u2028}%n</Property>
   </Properties>
   <Appenders>

services/distribution/src/test/resources/application.yaml

@@ -55,8 +55,6 @@ spring:
     banner-mode: off
   flyway:
     enabled: true
-# default case is H2 - value will be overwritten by profile cloud or postgres
-    locations: classpath:db/migration/h2
   jpa:
     hibernate:
       ddl-auto: validate

services/submission/src/main/java/app/coronawarn/server/services/submission/config/SecurityConfig.java

@@ -35,7 +35,11 @@
 @EnableWebSecurity
 public class SecurityConfig extends WebSecurityConfigurerAdapter {
 
-  private static final String ACTUATOR_ROUTE = "/actuator/**";
+  private static final String ACTUATOR_ROUTE = "/actuator/";
+  private static final String HEALTH_ROUTE = ACTUATOR_ROUTE + "health";
+  private static final String PROMETHEUS_ROUTE = ACTUATOR_ROUTE + "prometheus";
+  private static final String READINESS_ROUTE = ACTUATOR_ROUTE + "readiness";
+  private static final String LIVENESS_ROUTE = ACTUATOR_ROUTE + "liveness";
   private static final String SUBMISSION_ROUTE =
       "/version/v1" + SubmissionController.SUBMISSION_ROUTE;
 
@@ -51,7 +55,7 @@ protected HttpFirewall strictFirewall() {
   @Override
   protected void configure(HttpSecurity http) throws Exception {
     http.authorizeRequests()
-        .mvcMatchers(HttpMethod.GET, ACTUATOR_ROUTE).permitAll()
+        .mvcMatchers(HttpMethod.GET, HEALTH_ROUTE, PROMETHEUS_ROUTE, READINESS_ROUTE, LIVENESS_ROUTE).permitAll()
         .mvcMatchers(HttpMethod.POST, SUBMISSION_ROUTE).permitAll()
         .anyRequest().denyAll()
         .and().csrf().disable();

services/submission/src/main/java/app/coronawarn/server/services/submission/config/SubmissionServiceConfig.java

@@ -32,6 +32,7 @@ public class SubmissionServiceConfig {
   private Double initialFakeDelayMilliseconds;
   private Double fakeDelayMovingAverageSamples;
   private Integer retentionDays;
+  private Integer connectionPoolSize;
   private Payload payload;
   private Verification verification;
   private Monitoring monitoring;
@@ -60,6 +61,14 @@ public void setRetentionDays(Integer retentionDays) {
     this.retentionDays = retentionDays;
   }
 
+  public Integer getConnectionPoolSize() {
+    return connectionPoolSize;
+  }
+
+  public void setConnectionPoolSize(Integer connectionPoolSize) {
+    this.connectionPoolSize = connectionPoolSize;
+  }
+
   public Integer getMaxNumberOfKeys() {
     return payload.getMaxNumberOfKeys();
   }
@@ -116,13 +125,13 @@ public void setPath(String path) {
   }
 
   private static class Monitoring {
-    private Integer batchSize;
+    private Long batchSize;
 
-    public Integer getBatchSize() {
+    public Long getBatchSize() {
       return batchSize;
     }
 
-    public void setBatchSize(Integer batchSize) {
+    public void setBatchSize(Long batchSize) {
       this.batchSize = batchSize;
     }
   }
@@ -135,11 +144,11 @@ public void setMonitoring(Monitoring monitoring) {
     this.monitoring = monitoring;
   }
 
-  public Integer getMonitoringBatchSize() {
+  public Long getMonitoringBatchSize() {
     return this.monitoring.getBatchSize();
   }
 
-  public void setMonitoringBatchSize(Integer batchSize) {
+  public void setMonitoringBatchSize(Long batchSize) {
     this.monitoring.setBatchSize(batchSize);
   }
 }

services/submission/src/main/java/app/coronawarn/server/services/submission/monitoring/BatchCounter.java

@@ -22,6 +22,7 @@
 
 import io.micrometer.core.instrument.Counter;
 import io.micrometer.core.instrument.MeterRegistry;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * Batch counter for counting requests for monitoring. Counts up in batches, given batch size. This way, single requests
@@ -33,11 +34,11 @@ public class BatchCounter {
   private static final String SUBMISSION_CONTROLLER_REQUESTS_COUNTER_DESCRIPTION
       = "Counts requests to the Submission Controller.";
 
-  private final int batchSize;
+  private final long batchSize;
   private final Counter counter;
-  private Double batch = 0.;
+  private final AtomicLong count = new AtomicLong(0L);
 
-  BatchCounter(MeterRegistry meterRegistry, int batchSize, String type) {
+  BatchCounter(MeterRegistry meterRegistry, long batchSize, String type) {
     this.batchSize = batchSize;
     counter = Counter.builder(SUBMISSION_CONTROLLER_REQUESTS_COUNTER_NAME)
         .tag("type", type)
@@ -50,11 +51,8 @@ public class BatchCounter {
    * counter is incremented.
    */
   public void increment() {
-    if (batch < batchSize) {
-      batch++;
-    } else {
-      counter.increment(batch);
-      batch = 1.;
+    if (0 == count.incrementAndGet() % batchSize) {
+      counter.increment(batchSize);
     }
   }
 }

services/submission/src/main/java/app/coronawarn/server/services/submission/monitoring/SubmissionControllerMonitor.java

@@ -38,7 +38,7 @@ public class SubmissionControllerMonitor {
 
   private final MeterRegistry meterRegistry;
 
-  private final Integer batchSize;
+  private final long batchSize;
   private BatchCounter requests;
   private BatchCounter realRequests;
   private BatchCounter fakeRequests;

services/submission/src/main/java/app/coronawarn/server/services/submission/verification/CloudFeignClientProvider.java

@@ -20,12 +20,12 @@
 
 package app.coronawarn.server.services.submission.verification;
 
+import app.coronawarn.server.services.submission.config.SubmissionServiceConfig;
 import feign.Client;
 import feign.httpclient.ApacheHttpClient;
 import java.io.IOException;
 import java.security.GeneralSecurityException;
 import javax.net.ssl.SSLContext;
-import org.apache.http.conn.ssl.DefaultHostnameVerifier;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.ssl.SSLContextBuilder;
 import org.springframework.cloud.commons.httpclient.ApacheHttpClientConnectionManagerFactory;
@@ -44,8 +44,14 @@ public class CloudFeignClientProvider implements FeignClientProvider {
 
   private final Environment environment;
   private final HostnameVerifierProvider hostnameVerifierProvider;
+  private final Integer connectionPoolSize;
 
-  public CloudFeignClientProvider(Environment environment, HostnameVerifierProvider hostnameVerifierProvider) {
+  /**
+   * Creates a {@link CloudFeignClientProvider} with a fixed connection pool size and SSL key+trust material.
+   */
+  public CloudFeignClientProvider(SubmissionServiceConfig config, Environment environment,
+      HostnameVerifierProvider hostnameVerifierProvider) {
+    this.connectionPoolSize = config.getConnectionPoolSize();
     this.environment = environment;
     this.hostnameVerifierProvider = hostnameVerifierProvider;
   }
@@ -81,6 +87,8 @@ private SSLContext getSslContext() {
   @Bean
   public ApacheHttpClientFactory createHttpClientFactory() {
     return new DefaultApacheHttpClientFactory(HttpClientBuilder.create()
+        .setMaxConnPerRoute(this.connectionPoolSize)
+        .setMaxConnTotal(this.connectionPoolSize)
         .setSSLContext(getSslContext())
         .setSSLHostnameVerifier(this.hostnameVerifierProvider.createHostnameVerifier()));
   }

services/submission/src/main/java/app/coronawarn/server/services/submission/verification/DevelopmentFeignClientProvider.java

@@ -20,6 +20,7 @@
 
 package app.coronawarn.server.services.submission.verification;
 
+import app.coronawarn.server.services.submission.config.SubmissionServiceConfig;
 import feign.Client;
 import feign.httpclient.ApacheHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
@@ -35,6 +36,15 @@
 @Profile("!ssl-client-verification")
 public class DevelopmentFeignClientProvider implements FeignClientProvider {
 
+  private final HostnameVerifierProvider hostnameVerifierProvider;
+  private final Integer connectionPoolSize;
+
+  public DevelopmentFeignClientProvider(SubmissionServiceConfig config,
+      HostnameVerifierProvider hostnameVerifierProvider) {
+    this.connectionPoolSize = config.getConnectionPoolSize();
+    this.hostnameVerifierProvider = hostnameVerifierProvider;
+  }
+
   @Override
   public Client createFeignClient() {
     return new ApacheHttpClient(createHttpClientFactory().createBuilder().build());
@@ -45,7 +55,10 @@ public Client createFeignClient() {
    */
   @Bean
   public ApacheHttpClientFactory createHttpClientFactory() {
-    return new DefaultApacheHttpClientFactory(HttpClientBuilder.create());
+    return new DefaultApacheHttpClientFactory(HttpClientBuilder.create()
+        .setMaxConnPerRoute(this.connectionPoolSize)
+        .setMaxConnTotal(this.connectionPoolSize)
+        .setSSLHostnameVerifier(this.hostnameVerifierProvider.createHostnameVerifier()));
   }
 
   @Bean

services/submission/src/main/resources/application-cloud.yaml

@@ -1,8 +1,6 @@
 ---
 spring:
   flyway:
-    enabled: true
-    locations: classpath:db/migration/postgres
     password: ${POSTGRESQL_PASSWORD_FLYWAY}
     user: ${POSTGRESQL_USER_FLYWAY}
   datasource: