fix: tests

.github/workflows/release.yaml

@@ -2,8 +2,6 @@ name: release
 
 on:
   push:
-    branches:
-      - main
     tags:
       - 'v*'
 
@@ -28,7 +26,7 @@ jobs:
         uses: gradle/gradle-build-action@4137be6a8bf7d7133955359dbd952c0ca73b1021
         with:
           arguments: |
-            -Prelease.useLastTag=true -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} djobiAssemble -x test
+            -Prelease.useLastTag=true -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} djobiAssemble
       - name: upload artifacts
         uses: actions/upload-artifact@v2
         with:
@@ -127,7 +125,8 @@ jobs:
       - name: Publish package
         uses: gradle/gradle-build-action@4137be6a8bf7d7133955359dbd952c0ca73b1021
         with:
-          arguments: publish
+          arguments: |
+            -Prelease.useLastTag=true publish
         env:
           MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
           MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }}

build.gradle

@@ -183,7 +183,9 @@ subprojects {
     }
 
     test {
-        useJUnitPlatform()
+        useJUnitPlatform {
+            excludeTags("IntegrationTest", "NeedDocker")
+        }
 
         environment "projectRoot", System.getProperty("user.dir")
         environment "SPARK_LOCAL_IP", "127.0.0.1"

dev/default.conf

@@ -85,6 +85,8 @@ djobi {
       spark {
         master = "local"
 
+        webHistoryUrlForJob = "http://localhost:4040"
+
         data {
           utils_country {
             type: "table"

djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/pojo/StageDefinition.java

@@ -38,16 +38,24 @@ public class StageDefinition {
     public String condition;
 
     public Stage buildStage(final String name) {
-        return new Stage()
+        final Stage s = new Stage()
                 .setAllowFailure(allowFailure)
                 .setCondition(condition)
-                .setName(name)
                 .setEnabled(enabled)
                 .setPreCheckEnabled(check)
                 .setPostCheckEnabled(check)
                 .setSpec(spec)
                 .setKind(kind)
                 .setLabels(labels)
         ;
+
+        // Deprecated: name must come always from "name: " field
+        if (this.name == null || this.name.isEmpty()) {
+            s.setName(name);
+        } else {
+            s.setName(this.name);
+        }
+
+        return s;
     }
 }

djobi-core/src/test/java/io/datatok/djobi/engine/actions/net/scp/output/SCPOutputPostCheckerTest.java

@@ -11,13 +11,11 @@
 import io.datatok.djobi.utils.Bag;
 import io.datatok.djobi.utils.io.IOFileUtils;
 import io.datatok.djobi.utils.templating.TemplateUtils;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.*;
 
 import java.io.IOException;
 
+@Tag("UnitTest")
 class SCPOutputPostCheckerTest {
 
     @Inject
@@ -70,7 +68,7 @@ void testMissingFile() throws Exception {
     void testTooSmall() throws Exception {
         Stage stage = StageTestUtils.getNewStage();
 
-        stage.setParameters(ActionConfiguration.get(SCPOutputConfig.class, new Bag("path", "steam/toto.csv"), stage, templateUtils));
+        stage.setParameters(ActionConfiguration.get(SCPOutputConfig.class, new Bag("path", "/opt/toto.csv"), stage, templateUtils));
 
         CheckResult res = checker.postCheck(stage);
 
@@ -82,7 +80,7 @@ void testTooSmall() throws Exception {
     void testTooOld() throws Exception {
         Stage stage = StageTestUtils.getNewStage();
 
-        stage.setParameters(ActionConfiguration.get(SCPOutputConfig.class, new Bag("path", "steam/junior.csv", "check_service", "http_files_listing_local"), stage, templateUtils));
+        stage.setParameters(ActionConfiguration.get(SCPOutputConfig.class, new Bag("path", "/opt/report_2017_10_10.csv", "check_service", "http_files_listing_local"), stage, templateUtils));
 
         CheckResult res = checker.postCheck(stage);
 

djobi-core/src/test/java/io/datatok/djobi/engine/stages/S3Test.java

@@ -22,15 +22,13 @@
 import io.datatok.djobi.utils.MyMapUtils;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Row;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.extension.ExtendWith;
 
 import javax.inject.Inject;
 
 @ExtendWith(MyTestRunner.class)
+@Tag("IntegrationTest")
 public class S3Test extends ActionTest {
 
     static public final String ACCESS_KEY = "root";

djobi-core/src/test/java/io/datatok/djobi/loaders/yaml/YAMLPipelineLoaderTest.java

@@ -79,6 +79,18 @@ void testWithFilter() throws Exception {
         Assertions.assertEquals(0, load("good.yml", "not_existing").getJobs().size());
     }
 
+    @Test()
+    void testName() throws Exception {
+        final Pipeline p = load("good.yml");
+
+        Assertions.assertNotNull(p);
+
+        final Job job1  = p.getJobs().get(0);
+
+        Assertions.assertEquals("setup-file-overriden-name", job1.getStages().get(0).getName());
+        Assertions.assertEquals("as_table", job1.getStages().get(1).getName());
+    }
+
     @Test()
     void testLabels() throws Exception {
         final Pipeline p = load("good.yml");
@@ -87,11 +99,11 @@ void testLabels() throws Exception {
 
         final Job job1  = p.getJobs().get(0);
 
-        Assertions.assertEquals("t.decaux", p.getLabel("io.datatok/org-author"));
+        Assertions.assertEquals("t.decaux", p.getLabel("io.datatok.djobi/org-author"));
 
         Assertions.assertEquals("low", job1.getLabel("level"));
 
-        Assertions.assertEquals("input", job1.getStages().get(0).getLabel("io.datatok/stage-type"));
+        Assertions.assertEquals("input", job1.getStages().get(0).getLabel("io.datatok.djobi/stage-type"));
     }
 
     private Pipeline load(final String pipeline) throws Exception {

djobi-core/src/test/java/io/datatok/djobi/plugins/logging/WorkLoggerTest.java

@@ -22,6 +22,7 @@
 import io.datatok.djobi.utils.MyMapUtils;
 import io.datatok.djobi.utils.elasticsearch.ElasticsearchUtils;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 
@@ -61,6 +62,7 @@ void testParameterJSONSerialization() throws Exception {
      * @todo implements log structure tests here.
      */
     @Test
+    @Tag("IntegrationTest")
     void testElasticsearchSink() throws Exception {
 
         final String configPrefix = "djobi.plugins.logger.sinks.";

djobi-core/src/test/resources/pipelines/good.yml

@@ -16,8 +16,8 @@ parameters:
   pp: pp
   pd: pipeline
 labels:
-  io.datatok/org-team: datahub
-  io.datatok/org-author: t.decaux
+  io.datatok.djobi/org-team: datahub
+  io.datatok.djobi/org-author: t.decaux
 jobs:
   job1:
     labels:
@@ -38,9 +38,10 @@ jobs:
         file: not_found
     stages:
       setup_file:
+        name: setup-file-overriden-name
         kind: fs-input
         labels:
-          io.djobi/stage-type: input
+          io.datatok.djobi/stage-type: input
         spec:
           path: "../data/json_1"
           format: json
@@ -61,7 +62,7 @@ jobs:
       output:
         kind: "fs-output"
         labels:
-          io.datatok/stage-type: output
+          io.datatok.djobi/stage-type: output
         spec:
           path: "/tmp/djobi_test_engine"
           options:

djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ESOutputPostCheckerTest.java

@@ -12,16 +12,14 @@
 import io.datatok.djobi.utils.http.Http;
 import io.datatok.djobi.utils.templating.TemplateUtils;
 import org.apache.log4j.Logger;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.extension.ExtendWith;
 
 import javax.inject.Inject;
 import java.io.IOException;
 
 @ExtendWith(MyTestRunner.class)
+@Tag("IntegrationTest")
 public class ESOutputPostCheckerTest {
 
     private static Logger logger = Logger.getLogger(ESOutputPostCheckerTest.class);

djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ESOutputRunnerTest.java

@@ -12,12 +12,14 @@
 import org.apache.spark.sql.Row;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
 import javax.inject.Inject;
 import java.io.IOException;
 
+@Tag("IntegrationTest")
 class ESOutputRunnerTest extends ActionTest {
 
     @Inject

djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ElasticsearchEngineTest.java

@@ -9,13 +9,15 @@
 import io.datatok.djobi.test.MyTestRunner;
 import io.datatok.djobi.utils.elasticsearch.ElasticsearchUtils;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 
 import javax.inject.Inject;
 import java.lang.reflect.Method;
 
 @ExtendWith(MyTestRunner.class)
+@Tag("IntegrationTest")
 public class ElasticsearchEngineTest {
 
     @Inject

docker-compose.yml

@@ -19,9 +19,9 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     ports:
-      - 9200:9200
-      - 9000:9000
-      - 9001:9001
+      - "9200:9200"
+      - "9000:9000"
+      - "9001:9001"
 
   ##
   # compile java