#356 Create migration to update spark configs with version update

foundation/foundation-upgrade/src/main/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/SparkVersionUpgradeMigration.java

@@ -0,0 +1,48 @@
+package com.boozallen.aissemble.upgrade.migration.v1_10_0;
+
+/*-
+ * #%L
+ * aiSSEMBLE::Foundation::Upgrade
+ * %%
+ * Copyright (C) 2021 Booz Allen
+ * %%
+ * This software package is licensed under the Booz Allen Public License. All Rights Reserved.
+ * #L%
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+
+import com.boozallen.aissemble.upgrade.migration.AbstractAissembleMigration;
+
+public class SparkVersionUpgradeMigration extends AbstractAissembleMigration {
+    private static final String OLD_FAILURE_VALIDITY_INTERVAL = "spark.yarn.executor.failuresValidityInterval";
+    private static final String NEW_FAILURE_VALIDITY_INTERVAL = "spark.executor.failuresValidityInterval";
+    private static final String OLD_MAX_FAILURES = "spark.yarn.max.executor.failures";
+    private static final String NEW_MAX_FAILURES = "spark.executor.maxNumFailures";
+
+    @Override
+    protected boolean shouldExecuteOnFile(File file) {
+        try {
+            System.out.println("Clay: " + file.getName());
+            return (Files.readString(file.toPath()).contains(OLD_FAILURE_VALIDITY_INTERVAL) ||
+                    Files.readString(file.toPath()).contains(OLD_MAX_FAILURES));
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    protected boolean performMigration(File file) {
+        try {
+            Files.writeString(file.toPath(), Files.readString(file.toPath())
+                    .replace(OLD_FAILURE_VALIDITY_INTERVAL, NEW_FAILURE_VALIDITY_INTERVAL));
+            Files.writeString(file.toPath(), Files.readString(file.toPath())
+                    .replace(OLD_MAX_FAILURES, NEW_MAX_FAILURES));
+            return true;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

foundation/foundation-upgrade/src/main/resources/migrations.json

@@ -3,6 +3,18 @@
     "group": "v1-10-0-migrations",
     "type": "ordered",
     "migrations": [
+      {
+        "name": "spark-version-upgrade-migration",
+        "implementation": "com.boozallen.aissemble.upgrade.migration.v1_10_0.SparkVersionUpgradeMigration",
+        "fileSets": [
+          {
+            "includes": [
+              "*-pipelines/*/src/*/resources/apps/*-values.yaml",
+              "*-pipelines/*/src/main/resources/apps/*-values.yaml"
+            ]
+          }
+        ]
+      }
     ]
   },
   {

foundation/foundation-upgrade/src/test/java/com/boozallen/aissemble/upgrade/migration/v1_10_0/SparkVersionUpgradeMigrationSteps.java

@@ -0,0 +1,47 @@
+package com.boozallen.aissemble.upgrade.migration.v1_10_0;
+
+/*-
+ * #%L
+ * aiSSEMBLE::Foundation::Upgrade
+ * %%
+ * Copyright (C) 2021 Booz Allen
+ * %%
+ * This software package is licensed under the Booz Allen Public License. All Rights Reserved.
+ * #L%
+ */
+
+import com.boozallen.aissemble.upgrade.migration.AbstractMigrationTest;
+
+import io.cucumber.java.en.Given;
+import io.cucumber.java.en.Then;
+import io.cucumber.java.en.When;
+
+public class SparkVersionUpgradeMigrationSteps extends AbstractMigrationTest {
+
+    @Given("a project with outdated spark application configuration")
+    public void aProjectWithOutdatedSparkApplicationConfiguration() {
+        testFile = getTestFile("v1_10_0/SparkVersionUpgrade/migration/outdatedApps/base-values.yaml");
+    }
+
+    @Given("a project with up to date spark application configuration")
+    public void aProjectWithUpToDateSparkApplicationConfiguration() {
+        testFile = getTestFile("v1_10_0/SparkVersionUpgrade/migration/updatedApps/base-values.yaml");
+    }
+
+    @When("the spark version upgrade migration executes")
+    public void theSparkVersionUpgradeMigrationExecutes() {
+        performMigration(new SparkVersionUpgradeMigration());
+    }
+
+    @Then("the spark application configs are updated")
+    public void theSparkApplicationConfigsAreUpdated() {
+        assertMigrationSuccess();
+        assertTestFileMatchesExpectedFile("Spark configs were not updated correctly following version migration");
+    }
+
+    @Then("the spark application configs are not updated")
+    public void theSparkApplicationConfigsAreNotUpdated() {
+        assertMigrationSkipped();
+        assertTestFileMatchesExpectedFile("Spark configs were not updated correctly following version migration");
+    }
+}

foundation/foundation-upgrade/src/test/resources/specifications/spark-version-upgrade-migration.feature

@@ -0,0 +1,11 @@
+Feature: Spark is updated to use appropriate v3.5.0 configs
+
+  Scenario: Update a project with outdated spark configuration
+    Given a project with outdated spark application configuration
+    When the spark version upgrade migration executes
+    Then the spark application configs are updated
+
+  Scenario: A project with up to date spark configuration is not migrated
+    Given a project with up to date spark application configuration
+    When the spark version upgrade migration executes
+    Then the spark application configs are not updated

foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/migration/outdatedApps/base-values.yaml

@@ -0,0 +1,27 @@
+metadata:
+    name: spark-persist
+sparkApp:
+  spec:
+    sparkConf:
+      spark.yarn.executor.failuresValidityInterval: "2h"
+      spark.yarn.max.executor.failures: 10
+    volumes: []
+    type: Java
+    deps:
+      packages:
+        - mysql:mysql-connector-java:8.0.30
+        - org.apache.hadoop:hadoop-aws:3.3.4
+        - com.amazonaws:aws-java-sdk-bundle:1.12.262
+      excludePackages: []
+    hadoopConf:
+    driver:
+      volumeMounts: []
+      cores: 1
+      coreLimit: "1200m"
+      memory: "2048m"
+      javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base"
+    executor:
+      volumeMounts: []
+      cores: 1
+      memory: "4096m"
+      javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base"

foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/migration/updatedApps/base-values.yaml

@@ -0,0 +1,27 @@
+metadata:
+    name: spark-persist
+sparkApp:
+  spec:
+    sparkConf:
+      spark.executor.failuresValidityInterval: "2h"
+      spark.executor.maxNumFailures: 10
+    volumes: []
+    type: Java
+    deps:
+      packages:
+        - mysql:mysql-connector-java:8.0.30
+        - org.apache.hadoop:hadoop-aws:3.3.4
+        - com.amazonaws:aws-java-sdk-bundle:1.12.262
+      excludePackages: []
+    hadoopConf:
+    driver:
+      volumeMounts: []
+      cores: 1
+      coreLimit: "1200m"
+      memory: "2048m"
+      javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base"
+    executor:
+      volumeMounts: []
+      cores: 1
+      memory: "4096m"
+      javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base"

foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/validation/outdatedApps/base-values.yaml

@@ -0,0 +1,27 @@
+metadata:
+    name: spark-persist
+sparkApp:
+  spec:
+    sparkConf:
+      spark.executor.failuresValidityInterval: "2h"
+      spark.executor.maxNumFailures: 10
+    volumes: []
+    type: Java
+    deps:
+      packages:
+        - mysql:mysql-connector-java:8.0.30
+        - org.apache.hadoop:hadoop-aws:3.3.4
+        - com.amazonaws:aws-java-sdk-bundle:1.12.262
+      excludePackages: []
+    hadoopConf:
+    driver:
+      volumeMounts: []
+      cores: 1
+      coreLimit: "1200m"
+      memory: "2048m"
+      javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base"
+    executor:
+      volumeMounts: []
+      cores: 1
+      memory: "4096m"
+      javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base"

foundation/foundation-upgrade/src/test/resources/test-files/v1_10_0/SparkVersionUpgrade/validation/updatedApps/base-values.yaml

@@ -0,0 +1,27 @@
+metadata:
+    name: spark-persist
+sparkApp:
+  spec:
+    sparkConf:
+      spark.executor.failuresValidityInterval: "2h"
+      spark.executor.maxNumFailures: 10
+    volumes: []
+    type: Java
+    deps:
+      packages:
+        - mysql:mysql-connector-java:8.0.30
+        - org.apache.hadoop:hadoop-aws:3.3.4
+        - com.amazonaws:aws-java-sdk-bundle:1.12.262
+      excludePackages: []
+    hadoopConf:
+    driver:
+      volumeMounts: []
+      cores: 1
+      coreLimit: "1200m"
+      memory: "2048m"
+      javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base"
+    executor:
+      volumeMounts: []
+      cores: 1
+      memory: "4096m"
+      javaOptions: "-DKRAUSENING_BASE=/opt/spark/krausening/base"