[KOGITO-8024] - Build Performance Tuning (#2517)

* [KOGITO-8024] - Parallel Builds

Signed-off-by: Ricardo Zanini <[email protected]>

* Update .ci/jenkins/Jenkinsfile.deploy

* Add test for KogitoPostgreSqlContainer-s

* Remove deprecated argument

Signed-off-by: Ricardo Zanini <[email protected]>
Co-authored-by: Tristan Radisson <[email protected]>

.ci/jenkins/Jenkinsfile.specific_nightly

@@ -202,6 +202,9 @@ MavenCommand getFullMavenCommand(String directory) {
         if (builderImage) {
             mvnCmd.withProperty('quarkus.native.builder-image', builderImage)
         }
+    } else {
+        // Do not use parallel for native checks
+        mvnCmd.withOptions(['-T 1C'])
     }
     return mvnCmd
 }

.ci/jenkins/dsl/jobs.groovy

@@ -27,13 +27,15 @@ Map getMultijobPRConfig(Folder jobFolder) {
                     // Sonarcloud analysis only on main branch
                     // As we have only Community edition
                     DISABLE_SONARCLOUD: !Utils.isMainBranch(this),
+                    // No parallel build for native
+                    BUILD_MVN_OPTS_CURRENT: !(jobFolder.isNative() || jobFolder.isMandrel() || jobFolder.isMandrelLTS()) ? '-T 1C' : '',
                 ]
             ], [
                 id: 'kogito-apps',
                 dependsOn: 'kogito-runtimes',
                 repository: 'kogito-apps',
                 env : [
-                    ADDITIONAL_TIMEOUT: jobFolder.isNative() || jobFolder.isMandrel() ? '360' : '210',
+                    ADDITIONAL_TIMEOUT: jobFolder.isNative() || jobFolder.isMandrel() || jobFolder.isMandrelLTS() ? '360' : '210',
                 ]
             ], [
                 id: 'kogito-examples',

.github/workflows/kogito-runtimes-pr.yml

@@ -44,6 +44,8 @@ jobs:
         with:
           annotations-prefix: ${{ runner.os }}-${{ matrix.java-version }}/${{ matrix.maven-version }}
           github-token: "${{ secrets.GITHUB_TOKEN }}"
+        env:
+          BUILD_MVN_OPTS_CURRENT: '-T 1.5C'
       - name: Surefire Report
         uses: kiegroup/kogito-pipelines/.ci/actions/surefire-report@main
         if: ${{ always() }}

.gitignore

@@ -19,4 +19,6 @@ target/
 .factorypath
 .vscode
 .run/
-.checkstyle
+.checkstyle
+
+*.log

jbpm/jbpm-bpmn2/pom.xml

@@ -150,6 +150,8 @@
             <!-- enforce strict variable checking for the sake of tests -->
             <org.jbpm.variable.strict>true</org.jbpm.variable.strict>
           </systemPropertyVariables>
+          <forkCount>2</forkCount>
+          <parallel>all</parallel>
         </configuration>
       </plugin>
     </plugins>

kogito-build/kogito-build-no-bom-parent/pom.xml

@@ -53,6 +53,7 @@
     <container.image.wurstmeister.kafka>wurstmeister/kafka:${version.wurstmeister.kafka}</container.image.wurstmeister.kafka>
     <container.image.mongodb>library/mongo:${version.org.mongo-image}</container.image.mongodb>
     <container.image.redis>redislabs/redisearch:${version.org.redis}</container.image.redis>
+    <container.image.postgres>postgres:${version.org.postgres}</container.image.postgres>
 
     <maven.compiler.release>11</maven.compiler.release>
     <maven.compiler.source>11</maven.compiler.source>
@@ -327,11 +328,12 @@
               <exclude>${tests.surefire.include}</exclude>
               <exclude>**/Native*</exclude>
             </excludes>
-            <argLine>-Xmx2048m -Xmx4g</argLine>
+            <argLine>-Xms2048m -Xmx4g  -XX:-TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none</argLine>
             <systemPropertyVariables>
               <maven.repo.local>${session.request.localRepositoryPath.path}</maven.repo.local>
               <maven.settings>${session.request.userSettingsFile.path}</maven.settings>
             </systemPropertyVariables>
+            <parallel>all</parallel>
           </configuration>
         </plugin>
         <plugin>
@@ -345,7 +347,7 @@
             <excludes>
               <exclude>${tests.failsafe.include}</exclude>
             </excludes>
-            <argLine>-Xmx2048m -Dfile.encoding=UTF-8</argLine>
+            <argLine>-Xms2048m -Xmx4g -XX:-TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none -Dfile.encoding=UTF-8</argLine>
             <systemPropertyVariables>
               <apple.awt.UIElement>true</apple.awt.UIElement>
               <org.uberfire.nio.git.daemon.enabled>false</org.uberfire.nio.git.daemon.enabled>
@@ -354,6 +356,7 @@
               <maven.repo.local>${session.request.localRepositoryPath.path}</maven.repo.local>
               <maven.settings>${session.request.userSettingsFile.path}</maven.settings>
             </systemPropertyVariables>
+            <parallel>all</parallel>
           </configuration>
         </plugin>
         <plugin>
@@ -742,7 +745,7 @@ limitations under the License.
             <artifactId>maven-surefire-plugin</artifactId>
             <configuration>
               <!-- Append jacoco.agent.argLine property populated by JaCoCo's prepare-agent goal. -->
-              <argLine>@{jacoco.agent.argLine} -Xms1024m -Xmx4g -Dfile.encoding=UTF-8</argLine>
+              <argLine>@{jacoco.agent.argLine} -Xms1024m -Xmx4g  -XX:-TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none -Dfile.encoding=UTF-8</argLine>
             </configuration>
           </plugin>
           <plugin>

kogito-build/kogito-dependencies-bom/pom.xml

@@ -80,6 +80,7 @@
     <version.org.mongo-image>4.4.14</version.org.mongo-image>
     <version.org.mozilla.rhino>1.7.13</version.org.mozilla.rhino>
     <version.org.redis>2.0.4</version.org.redis>
+    <version.org.postgres>13.4-alpine3.14</version.org.postgres>
 
     <version.org.assertj>3.22.0</version.org.assertj>
     <version.org.json-unit-assertj>2.9.0</version.org.json-unit-assertj>
@@ -106,6 +107,9 @@
 
     <version.plexus>1.6</version.plexus>
     <version.plexus.classworld>2.6.0</version.plexus.classworld>
+
+    <!-- see: https://maven.apache.org/surefire/maven-surefire-plugin/examples/fork-options-and-parallel-execution.html#parallel-test-execution-and-single-thread-execution -->
+    <version.com.github.stephenc.jcip>1.0-1</version.com.github.stephenc.jcip>
   </properties>
 
   <dependencyManagement>

kogito-test-utils/src/main/java/org/kie/kogito/test/utils/SocketUtils.java

@@ -23,14 +23,19 @@
 
 public final class SocketUtils {
 
-    protected static final int PORT_RANGE_MIN = 1024;
-    protected static final int PORT_RANGE_MAX = 65535;
+    /**
+     * Minimal port to use.
+     *
+     * @see <a href="https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers#Dynamic.2C_private_or_ephemeral_ports">Dynamic, private or ephemeral ports</a>
+     */
+    static final int PORT_RANGE_MIN = 49152;
+    static final int PORT_RANGE_MAX = 65535;
     private static final SecureRandom RND = new SecureRandom();
 
     private SocketUtils() {
     }
 
-    public static final int findAvailablePort() {
+    public static int findAvailablePort() {
         int portRange = PORT_RANGE_MAX - PORT_RANGE_MIN;
         int candidatePort;
         int searchCounter = 0;
@@ -47,12 +52,12 @@ public static final int findAvailablePort() {
         return candidatePort;
     }
 
-    private static final int findRandomPort(int minPort, int maxPort) {
+    private static int findRandomPort(int minPort, int maxPort) {
         int portRange = maxPort - minPort;
         return minPort + RND.nextInt(portRange + 1);
     }
 
-    private static final boolean isPortAvailable(int port) {
+    private static boolean isPortAvailable(int port) {
         try {
             ServerSocket serverSocket = ServerSocketFactory.getDefault().createServerSocket(port, 1, InetAddress.getByName("localhost"));
             serverSocket.close();

kogito-test-utils/src/main/java/org/kie/kogito/testcontainers/Constants.java

@@ -19,7 +19,7 @@
 
 public class Constants {
 
-    public static final Duration CONTAINER_START_TIMEOUT = Duration.ofMinutes(5);
+    public static final Duration CONTAINER_START_TIMEOUT = Duration.ofMinutes(15);
     public static final String CONTAINER_NAME_PREFIX = "container.image.";
 
     private Constants() {

kogito-test-utils/src/main/java/org/kie/kogito/testcontainers/KogitoPostgreSqlContainer.java

@@ -30,11 +30,12 @@
  */
 public class KogitoPostgreSqlContainer extends PostgreSQLContainer<KogitoPostgreSqlContainer> implements TestResource {
 
+    public static final String NAME = "postgres";
     public static final String POSTGRESQL_CONNECTION_URI = "kogito.persistence.postgresql.connection.uri";
     private static final Logger LOGGER = LoggerFactory.getLogger(KogitoPostgreSqlContainer.class);
 
     public KogitoPostgreSqlContainer() {
-        super("postgres:13.4-alpine3.14");
+        super(KogitoGenericContainer.getImageName(NAME));
         withLogConsumer(getLogger());
         withLogConsumer(new Slf4jLogConsumer(LOGGER));
         withStartupTimeout(Constants.CONTAINER_START_TIMEOUT);

kogito-test-utils/src/main/resources/testcontainers.properties

@@ -3,4 +3,5 @@ container.image.infinispan=${container.image.infinispan}
 container.image.kafka=${container.image.kafka}
 container.image.keycloak=${container.image.keycloak}
 container.image.redis=${container.image.redis}
-container.image.mongodb=${container.image.mongodb}
+container.image.mongodb=${container.image.mongodb}
+container.image.postgres=${container.image.postgres}

kogito-test-utils/src/test/java/org/kie/kogito/testcontainers/KogitoPostgreSqlContainerTest.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2021 Red Hat, Inc. and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.kie.kogito.testcontainers;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.containers.PostgreSQLContainer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.kie.kogito.testcontainers.Constants.CONTAINER_NAME_PREFIX;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+
+public class KogitoPostgreSqlContainerTest {
+
+    private static final int MAPPED_PORT = 5432;
+    private static final String IMAGE = "postgres";
+
+    private KogitoPostgreSqlContainer container;
+
+    @BeforeEach
+    public void setup() {
+        System.setProperty(CONTAINER_NAME_PREFIX + PostgreSQLContainer.NAME, IMAGE);
+        container = spy(new KogitoPostgreSqlContainer());
+        doNothing().when(container).start();
+        doNothing().when(container).setDockerImageName(IMAGE);
+    }
+
+    @Test
+    public void shouldAddDefaultSettings() {
+        assertTrue(container.getExposedPorts().contains(PostgreSQLContainer.POSTGRESQL_PORT));
+    }
+
+    @Test
+    public void shouldGetResourceName() {
+        assertEquals(PostgreSQLContainer.NAME, container.getResourceName());
+    }
+
+    @Test
+    public void shouldGetMapperPort() {
+        doReturn(MAPPED_PORT).when(container).getMappedPort(PostgreSQLContainer.POSTGRESQL_PORT);
+        assertEquals(MAPPED_PORT, container.getMappedPort());
+    }
+
+}

quarkus/addons/common/deployment/pom.xml

@@ -23,6 +23,20 @@
       <artifactId>junit-jupiter-engine</artifactId>
       <scope>test</scope>
     </dependency>
+    <!-- this is used implicitly by Maven parallel builds so let's make Maven aware of it -->
+    <dependency>
+      <groupId>org.kie.kogito</groupId>
+      <artifactId>kogito-quarkus-deployment</artifactId>
+      <version>${project.version}</version>
+      <type>pom</type>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
   </dependencies>
 
 </project>

quarkus/addons/explainability/integration-tests/pom.xml

@@ -27,6 +27,33 @@
       <groupId>org.kie.kogito</groupId>
       <artifactId>kogito-addons-quarkus-explainability</artifactId>
     </dependency>
+    <!-- this is used implicitly by parallel builds so let's make Maven aware of it -->
+    <dependency>
+      <groupId>org.kie.kogito</groupId>
+      <artifactId>kogito-quarkus-deployment</artifactId>
+      <version>${project.version}</version>
+      <type>pom</type>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.kie.kogito</groupId>
+      <artifactId>kogito-addons-quarkus-explainability-deployment</artifactId>
+      <version>${project.version}</version>
+      <type>pom</type>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
 
     <dependency>
       <groupId>io.quarkus</groupId>
@@ -48,6 +75,11 @@
       <artifactId>rest-assured</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.kie.kogito</groupId>
+      <artifactId>kogito-test-utils</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

quarkus/addons/explainability/integration-tests/src/test/java/org/kie/kogito/explainability/QuarkusExplainableResourceIT.java

@@ -27,6 +27,7 @@
 import org.kie.kogito.conf.ConfigBean;
 import org.kie.kogito.conf.StaticConfigBean;
 import org.kie.kogito.explainability.model.PredictOutput;
+import org.kie.kogito.test.utils.SocketUtils;
 
 import io.quarkus.test.junit.QuarkusIntegrationTest;
 import io.restassured.common.mapper.TypeRef;
@@ -45,7 +46,7 @@ public class QuarkusExplainableResourceIT {
 
     @Singleton
     public static ConfigBean configBeanProducer() {
-        return new StaticConfigBean("http://localhost:8081", true, null);
+        return new StaticConfigBean("http://localhost:" + SocketUtils.findAvailablePort(), true, null);
     }
 
     @Test

quarkus/addons/explainability/integration-tests/src/test/resources/application.properties

@@ -0,0 +1 @@
+quarkus.http.test-port=0

quarkus/addons/kubernetes/runtime/src/test/java/org/kie/kogito/addons/quarkus/k8s/KnativeRouteEndpointDiscoveryTest.java

@@ -22,6 +22,8 @@
 import java.util.Map;
 import java.util.Optional;
 
+import javax.inject.Inject;
+
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.kie.kogito.addons.k8s.Endpoint;
@@ -32,16 +34,19 @@
 import io.fabric8.knative.serving.v1.RouteBuilder;
 import io.fabric8.knative.serving.v1.RouteStatus;
 import io.fabric8.kubernetes.client.KubernetesClient;
-import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.kubernetes.client.WithKubernetesTestServer;
 
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.junit.jupiter.api.Assertions.fail;
 
-@EnableKubernetesMockClient(crud = true)
+@QuarkusTest
+@WithKubernetesTestServer
 public class KnativeRouteEndpointDiscoveryTest {
 
-    static KubernetesClient kubernetesClient;
+    @Inject
+    KubernetesClient kubernetesClient;
     KnativeClient knativeClient;
 
     @BeforeEach

quarkus/addons/kubernetes/runtime/src/test/resources/application.properties

@@ -1 +1 @@
-quarkus.tls.trust-all=true
+quarkus.tls.trust-all=true

quarkus/addons/messaging/common/pom.xml

@@ -83,6 +83,12 @@
       <artifactId>quarkus-junit5-mockito</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.github.stephenc.jcip</groupId>
+      <artifactId>jcip-annotations</artifactId>
+      <version>${version.com.github.stephenc.jcip}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>