From a940ed545faf1ea6bf00ac8c7876376cf3c7835b Mon Sep 17 00:00:00 2001
From: Thomas Decaux <ebuildy@gmail.com>
Date: Thu, 20 Jan 2022 09:39:12 +0100
Subject: [PATCH] fix: tests

---
 .github/workflows/release.yaml                   |  7 +++----
 build.gradle                                     |  4 +++-
 dev/default.conf                                 |  2 ++
 .../djobi/cli/CLIOutUtilsTest.java               |  0
 .../djobi/cli/CLIUtilsTest.java                  |  0
 .../djobi/cli/DumpActionCommandTest.java         |  0
 .../djobi/cli/DumpEnvCommandTest.java            |  0
 .../djobi/cli/DumpPipelineCommandTest.java       |  0
 .../djobi/cli/RunPipelineCommandTest.java        |  0
 .../djobi/loaders/yaml/pojo/StageDefinition.java | 12 ++++++++++--
 .../net/scp/output/SCPOutputPostCheckerTest.java | 10 ++++------
 .../io/datatok/djobi/engine/stages/S3Test.java   |  6 ++----
 .../loaders/yaml/YAMLPipelineLoaderTest.java     | 16 ++++++++++++++--
 .../djobi/plugins/logging/WorkLoggerTest.java    |  2 ++
 djobi-core/src/test/resources/pipelines/good.yml |  9 +++++----
 .../output/ESOutputPostCheckerTest.java          |  6 ++----
 .../elasticsearch/output/ESOutputRunnerTest.java |  2 ++
 .../output/ElasticsearchEngineTest.java          |  2 ++
 docker-compose.yml                               |  6 +++---
 19 files changed, 54 insertions(+), 30 deletions(-)
 rename djobi-cli/src/test/java/io/{qwant => datatok}/djobi/cli/CLIOutUtilsTest.java (100%)
 rename djobi-cli/src/test/java/io/{qwant => datatok}/djobi/cli/CLIUtilsTest.java (100%)
 rename djobi-cli/src/test/java/io/{qwant => datatok}/djobi/cli/DumpActionCommandTest.java (100%)
 rename djobi-cli/src/test/java/io/{qwant => datatok}/djobi/cli/DumpEnvCommandTest.java (100%)
 rename djobi-cli/src/test/java/io/{qwant => datatok}/djobi/cli/DumpPipelineCommandTest.java (100%)
 rename djobi-cli/src/test/java/io/{qwant => datatok}/djobi/cli/RunPipelineCommandTest.java (100%)

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 2865c00..73625d0 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -2,8 +2,6 @@ name: release
 
 on:
   push:
-    branches:
-      - main
     tags:
       - 'v*'
 
@@ -28,7 +26,7 @@ jobs:
         uses: gradle/gradle-build-action@4137be6a8bf7d7133955359dbd952c0ca73b1021
         with:
           arguments: |
-            -Prelease.useLastTag=true -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} djobiAssemble -x test
+            -Prelease.useLastTag=true -Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} djobiAssemble
       - name: upload artifacts
         uses: actions/upload-artifact@v2
         with:
@@ -127,7 +125,8 @@ jobs:
       - name: Publish package
         uses: gradle/gradle-build-action@4137be6a8bf7d7133955359dbd952c0ca73b1021
         with:
-          arguments: publish
+          arguments: |
+            -Prelease.useLastTag=true publish
         env:
           MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
           MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }}
diff --git a/build.gradle b/build.gradle
index c6a5039..ea1608f 100644
--- a/build.gradle
+++ b/build.gradle
@@ -183,7 +183,9 @@ subprojects {
     }
 
     test {
-        useJUnitPlatform()
+        useJUnitPlatform {
+            excludeTags("IntegrationTest", "NeedDocker")
+        }
 
         environment "projectRoot", System.getProperty("user.dir")
         environment "SPARK_LOCAL_IP", "127.0.0.1"
diff --git a/dev/default.conf b/dev/default.conf
index 0ad94e5..2f4da48 100644
--- a/dev/default.conf
+++ b/dev/default.conf
@@ -85,6 +85,8 @@ djobi {
       spark {
         master = "local"
 
+        webHistoryUrlForJob = "http://localhost:4040"
+
         data {
           utils_country {
             type: "table"
diff --git a/djobi-cli/src/test/java/io/qwant/djobi/cli/CLIOutUtilsTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/CLIOutUtilsTest.java
similarity index 100%
rename from djobi-cli/src/test/java/io/qwant/djobi/cli/CLIOutUtilsTest.java
rename to djobi-cli/src/test/java/io/datatok/djobi/cli/CLIOutUtilsTest.java
diff --git a/djobi-cli/src/test/java/io/qwant/djobi/cli/CLIUtilsTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/CLIUtilsTest.java
similarity index 100%
rename from djobi-cli/src/test/java/io/qwant/djobi/cli/CLIUtilsTest.java
rename to djobi-cli/src/test/java/io/datatok/djobi/cli/CLIUtilsTest.java
diff --git a/djobi-cli/src/test/java/io/qwant/djobi/cli/DumpActionCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpActionCommandTest.java
similarity index 100%
rename from djobi-cli/src/test/java/io/qwant/djobi/cli/DumpActionCommandTest.java
rename to djobi-cli/src/test/java/io/datatok/djobi/cli/DumpActionCommandTest.java
diff --git a/djobi-cli/src/test/java/io/qwant/djobi/cli/DumpEnvCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpEnvCommandTest.java
similarity index 100%
rename from djobi-cli/src/test/java/io/qwant/djobi/cli/DumpEnvCommandTest.java
rename to djobi-cli/src/test/java/io/datatok/djobi/cli/DumpEnvCommandTest.java
diff --git a/djobi-cli/src/test/java/io/qwant/djobi/cli/DumpPipelineCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/DumpPipelineCommandTest.java
similarity index 100%
rename from djobi-cli/src/test/java/io/qwant/djobi/cli/DumpPipelineCommandTest.java
rename to djobi-cli/src/test/java/io/datatok/djobi/cli/DumpPipelineCommandTest.java
diff --git a/djobi-cli/src/test/java/io/qwant/djobi/cli/RunPipelineCommandTest.java b/djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java
similarity index 100%
rename from djobi-cli/src/test/java/io/qwant/djobi/cli/RunPipelineCommandTest.java
rename to djobi-cli/src/test/java/io/datatok/djobi/cli/RunPipelineCommandTest.java
diff --git a/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/pojo/StageDefinition.java b/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/pojo/StageDefinition.java
index 2550adb..77bff73 100644
--- a/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/pojo/StageDefinition.java
+++ b/djobi-core/src/main/java/io/datatok/djobi/loaders/yaml/pojo/StageDefinition.java
@@ -38,10 +38,9 @@ public class StageDefinition {
     public String condition;
 
     public Stage buildStage(final String name) {
-        return new Stage()
+        final Stage s = new Stage()
                 .setAllowFailure(allowFailure)
                 .setCondition(condition)
-                .setName(name)
                 .setEnabled(enabled)
                 .setPreCheckEnabled(check)
                 .setPostCheckEnabled(check)
@@ -49,5 +48,14 @@ public Stage buildStage(final String name) {
                 .setKind(kind)
                 .setLabels(labels)
         ;
+
+        // Deprecated: name must come always from "name: " field
+        if (this.name == null || this.name.isEmpty()) {
+            s.setName(name);
+        } else {
+            s.setName(this.name);
+        }
+
+        return s;
     }
 }
diff --git a/djobi-core/src/test/java/io/datatok/djobi/engine/actions/net/scp/output/SCPOutputPostCheckerTest.java b/djobi-core/src/test/java/io/datatok/djobi/engine/actions/net/scp/output/SCPOutputPostCheckerTest.java
index 4678723..7b5aa2d 100644
--- a/djobi-core/src/test/java/io/datatok/djobi/engine/actions/net/scp/output/SCPOutputPostCheckerTest.java
+++ b/djobi-core/src/test/java/io/datatok/djobi/engine/actions/net/scp/output/SCPOutputPostCheckerTest.java
@@ -11,13 +11,11 @@
 import io.datatok.djobi.utils.Bag;
 import io.datatok.djobi.utils.io.IOFileUtils;
 import io.datatok.djobi.utils.templating.TemplateUtils;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.*;
 
 import java.io.IOException;
 
+@Tag("UnitTest")
 class SCPOutputPostCheckerTest {
 
     @Inject
@@ -70,7 +68,7 @@ void testMissingFile() throws Exception {
     void testTooSmall() throws Exception {
         Stage stage = StageTestUtils.getNewStage();
 
-        stage.setParameters(ActionConfiguration.get(SCPOutputConfig.class, new Bag("path", "steam/toto.csv"), stage, templateUtils));
+        stage.setParameters(ActionConfiguration.get(SCPOutputConfig.class, new Bag("path", "/opt/toto.csv"), stage, templateUtils));
 
         CheckResult res = checker.postCheck(stage);
 
@@ -82,7 +80,7 @@ void testTooSmall() throws Exception {
     void testTooOld() throws Exception {
         Stage stage = StageTestUtils.getNewStage();
 
-        stage.setParameters(ActionConfiguration.get(SCPOutputConfig.class, new Bag("path", "steam/junior.csv", "check_service", "http_files_listing_local"), stage, templateUtils));
+        stage.setParameters(ActionConfiguration.get(SCPOutputConfig.class, new Bag("path", "/opt/report_2017_10_10.csv", "check_service", "http_files_listing_local"), stage, templateUtils));
 
         CheckResult res = checker.postCheck(stage);
 
diff --git a/djobi-core/src/test/java/io/datatok/djobi/engine/stages/S3Test.java b/djobi-core/src/test/java/io/datatok/djobi/engine/stages/S3Test.java
index 4f0e2c0..cfd0925 100644
--- a/djobi-core/src/test/java/io/datatok/djobi/engine/stages/S3Test.java
+++ b/djobi-core/src/test/java/io/datatok/djobi/engine/stages/S3Test.java
@@ -22,15 +22,13 @@
 import io.datatok.djobi.utils.MyMapUtils;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Row;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.extension.ExtendWith;
 
 import javax.inject.Inject;
 
 @ExtendWith(MyTestRunner.class)
+@Tag("IntegrationTest")
 public class S3Test extends ActionTest {
 
     static public final String ACCESS_KEY = "root";
diff --git a/djobi-core/src/test/java/io/datatok/djobi/loaders/yaml/YAMLPipelineLoaderTest.java b/djobi-core/src/test/java/io/datatok/djobi/loaders/yaml/YAMLPipelineLoaderTest.java
index e5c57d4..179744d 100644
--- a/djobi-core/src/test/java/io/datatok/djobi/loaders/yaml/YAMLPipelineLoaderTest.java
+++ b/djobi-core/src/test/java/io/datatok/djobi/loaders/yaml/YAMLPipelineLoaderTest.java
@@ -79,6 +79,18 @@ void testWithFilter() throws Exception {
         Assertions.assertEquals(0, load("good.yml", "not_existing").getJobs().size());
     }
 
+    @Test()
+    void testName() throws Exception {
+        final Pipeline p = load("good.yml");
+
+        Assertions.assertNotNull(p);
+
+        final Job job1  = p.getJobs().get(0);
+
+        Assertions.assertEquals("setup-file-overriden-name", job1.getStages().get(0).getName());
+        Assertions.assertEquals("as_table", job1.getStages().get(1).getName());
+    }
+
     @Test()
     void testLabels() throws Exception {
         final Pipeline p = load("good.yml");
@@ -87,11 +99,11 @@ void testLabels() throws Exception {
 
         final Job job1  = p.getJobs().get(0);
 
-        Assertions.assertEquals("t.decaux", p.getLabel("io.datatok/org-author"));
+        Assertions.assertEquals("t.decaux", p.getLabel("io.datatok.djobi/org-author"));
 
         Assertions.assertEquals("low", job1.getLabel("level"));
 
-        Assertions.assertEquals("input", job1.getStages().get(0).getLabel("io.datatok/stage-type"));
+        Assertions.assertEquals("input", job1.getStages().get(0).getLabel("io.datatok.djobi/stage-type"));
     }
 
     private Pipeline load(final String pipeline) throws Exception {
diff --git a/djobi-core/src/test/java/io/datatok/djobi/plugins/logging/WorkLoggerTest.java b/djobi-core/src/test/java/io/datatok/djobi/plugins/logging/WorkLoggerTest.java
index 4212d30..0625dcb 100644
--- a/djobi-core/src/test/java/io/datatok/djobi/plugins/logging/WorkLoggerTest.java
+++ b/djobi-core/src/test/java/io/datatok/djobi/plugins/logging/WorkLoggerTest.java
@@ -22,6 +22,7 @@
 import io.datatok.djobi.utils.MyMapUtils;
 import io.datatok.djobi.utils.elasticsearch.ElasticsearchUtils;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 
@@ -61,6 +62,7 @@ void testParameterJSONSerialization() throws Exception {
      * @todo implements log structure tests here.
      */
     @Test
+    @Tag("IntegrationTest")
     void testElasticsearchSink() throws Exception {
 
         final String configPrefix = "djobi.plugins.logger.sinks.";
diff --git a/djobi-core/src/test/resources/pipelines/good.yml b/djobi-core/src/test/resources/pipelines/good.yml
index c7cca7c..6d9c2c9 100644
--- a/djobi-core/src/test/resources/pipelines/good.yml
+++ b/djobi-core/src/test/resources/pipelines/good.yml
@@ -16,8 +16,8 @@ parameters:
   pp: pp
   pd: pipeline
 labels:
-  io.datatok/org-team: datahub
-  io.datatok/org-author: t.decaux
+  io.datatok.djobi/org-team: datahub
+  io.datatok.djobi/org-author: t.decaux
 jobs:
   job1:
     labels:
@@ -38,9 +38,10 @@ jobs:
         file: not_found
     stages:
       setup_file:
+        name: setup-file-overriden-name
         kind: fs-input
         labels:
-          io.djobi/stage-type: input
+          io.datatok.djobi/stage-type: input
         spec:
           path: "../data/json_1"
           format: json
@@ -61,7 +62,7 @@ jobs:
       output:
         kind: "fs-output"
         labels:
-          io.datatok/stage-type: output
+          io.datatok.djobi/stage-type: output
         spec:
           path: "/tmp/djobi_test_engine"
           options:
diff --git a/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ESOutputPostCheckerTest.java b/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ESOutputPostCheckerTest.java
index 024ae89..3b94824 100644
--- a/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ESOutputPostCheckerTest.java
+++ b/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ESOutputPostCheckerTest.java
@@ -12,16 +12,14 @@
 import io.datatok.djobi.utils.http.Http;
 import io.datatok.djobi.utils.templating.TemplateUtils;
 import org.apache.log4j.Logger;
-import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.*;
 import org.junit.jupiter.api.extension.ExtendWith;
 
 import javax.inject.Inject;
 import java.io.IOException;
 
 @ExtendWith(MyTestRunner.class)
+@Tag("IntegrationTest")
 public class ESOutputPostCheckerTest {
 
     private static Logger logger = Logger.getLogger(ESOutputPostCheckerTest.class);
diff --git a/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ESOutputRunnerTest.java b/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ESOutputRunnerTest.java
index 95724a4..3a5e801 100644
--- a/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ESOutputRunnerTest.java
+++ b/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ESOutputRunnerTest.java
@@ -12,12 +12,14 @@
 import org.apache.spark.sql.Row;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
 import javax.inject.Inject;
 import java.io.IOException;
 
+@Tag("IntegrationTest")
 class ESOutputRunnerTest extends ActionTest {
 
     @Inject
diff --git a/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ElasticsearchEngineTest.java b/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ElasticsearchEngineTest.java
index 18cc9f9..e08e36b 100644
--- a/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ElasticsearchEngineTest.java
+++ b/djobi-elasticsearch/src/test/java/io/datatok/djobi/engine/stages/elasticsearch/output/ElasticsearchEngineTest.java
@@ -9,6 +9,7 @@
 import io.datatok.djobi.test.MyTestRunner;
 import io.datatok.djobi.utils.elasticsearch.ElasticsearchUtils;
 import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 
@@ -16,6 +17,7 @@
 import java.lang.reflect.Method;
 
 @ExtendWith(MyTestRunner.class)
+@Tag("IntegrationTest")
 public class ElasticsearchEngineTest {
 
     @Inject
diff --git a/docker-compose.yml b/docker-compose.yml
index a9bedfb..d2d1b8e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -19,9 +19,9 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     ports:
-      - 9200:9200
-      - 9000:9000
-      - 9001:9001
+      - "9200:9200"
+      - "9000:9000"
+      - "9001:9001"
 
   ##
   # compile java