Merge pull request #36 from mishmash-io/publish-internal-code

Publish internal code

.github/workflows/maven.yml

@@ -15,66 +15,154 @@ on:
     branches: [ "main" ]
 
 jobs:
-  build:
-
+  compile:
+    name: Compile everything
     runs-on: ubuntu-latest
 
-    timeout-minutes: 520
-
     # allow 'Update dependency graph' step below to submit the generated graph
     permissions:
       contents: write
       pull-requests: write
 
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up JDK 17
+      uses: actions/setup-java@v3
+      with:
+        java-version: '17'
+        distribution: 'temurin'
+        cache: 'maven'
+    - name: Build with Maven
+      run: mvn -DskipTests install --file pom.xml
+    - name: Cache build artifacts
+      uses: actions/cache/save@v4
+      id: buildcache
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-${{ github.run_id }}-${{ github.run_attempt }}
+    - name: Update dependency graph
+      uses: advanced-security/maven-dependency-submission-action@571e99aab1055c2e71a1e2309b9691de18d6b7d6
+      continue-on-error: true
+
+  test:
+    needs: compile
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 3
+      matrix:
+        testset:
+          - 'coordination/base'
+          - 'storage/block/base'
+          - 'storage/block/balancer'
+          - 'storage/block/blockmanagement'
+          - 'storage/block/datanode'
+          - 'storage/block/dfs-base'
+          - 'storage/block/fs-base'
+          - 'storage/block/misc'
+        include:
+          - telemetryport: 4317
+            testset: 'coordination/base'
+          - telemetryport: 4319
+            testset: 'storage/block/base'
+          - telemetryport: 4321
+            testset: 'storage/block/balancer'
+          - telemetryport: 4323
+            testset: 'storage/block/blockmanagement'
+          - telemetryport: 4325
+            testset: 'storage/block/datanode'
+          - telemetryport: 4327
+            testset: 'storage/block/dfs-base'
+          - telemetryport: 4329
+            testset: 'storage/block/fs-base'
+          - telemetryport: 4331
+            testset: 'storage/block/misc'
+          - telemetryoutput: 'coordination-base'
+            testset: 'coordination/base'
+          - telemetryoutput: 'storage-block-base'
+            testset: 'storage/block/base'
+          - telemetryoutput: 'storage-block-balancer'
+            testset: 'storage/block/balancer'
+          - telemetryoutput: 'storage-block-blockmanagement'
+            testset: 'storage/block/blockmanagement'
+          - telemetryoutput: 'storage-block-datanode'
+            testset: 'storage/block/datanode'
+          - telemetryoutput: 'storage-block-dfs-base'
+            testset: 'storage/block/dfs-base'
+          - telemetryoutput: 'storage-block-fs-base'
+            testset: 'storage/block/fs-base'
+          - telemetryoutput: 'storage-block-misc'
+            testset: 'storage/block/misc'
+          - testname: 'Base distributed coordination tests'
+            testset: 'coordination/base'
+          - testname: 'Base distributed block storage tests'
+            testset: 'storage/block/base'
+          - testname: 'Distributed block storage balancer tests'
+            testset: 'storage/block/balancer'
+          - testname: 'Distributed block storage management tests'
+            testset: 'storage/block/blockmanagement'
+          - testname: 'Distributed block storage datanode tests'
+            testset: 'storage/block/datanode'
+          - testname: 'Base distributed block dfs storage tests'
+            testset: 'storage/block/dfs-base'
+          - testname: 'Base distributed block fs storage tests'
+            testset: 'storage/block/fs-base'
+          - testname: 'Misc distributed block storage tests'
+            testset: 'storage/block/misc'
+    name: Testing, ${{ join(matrix.*, ' - ') }}
+
     services:
       otel-to-parquet:
         image: mishmashio/opentelemetry-parquet-server
         ports:
-          - 4317:4317
-          - 4318:4318
+          - ${{ matrix.telemetryport }}:4317
         volumes:
-          - /tmp/tests-parquet:/parquet
+          - /tmp/tests-parquet/${{ matrix.telemetryoutput }}:/parquet
+
     steps:
     - uses: actions/checkout@v4
     - name: Set up JDK 17
       uses: actions/setup-java@v3
       with:
         java-version: '17'
         distribution: 'temurin'
-        cache: maven
     - name: Download latest OpenTelemetry java agent
       run: curl -L -o /tmp/opentelemetry-javaagent.jar https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar
-    - name: Build with Maven
+    - name: Restore cached build artifacts
+      uses: actions/cache/restore@v4
+      id: buildcache
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-${{ github.run_id }}-${{ github.run_attempt }}
+        fail-on-cache-miss: true
+    - name: Run test set ${{ matrix.testset }}
       env:
         OTEL_SERVICE_NAME: maven-test-service
         OTEL_EXPORTER_OTLP_PROTOCOL: grpc
-        OTEL_EXPORTER_OTLP_ENDPOINT: http://localhost:4317
-      run: mvn -Dsurefire.rerunFailingTestsCount=2 -Dtest.instrumentation.opts=-javaagent:/tmp/opentelemetry-javaagent.jar --fail-at-end -B package --file pom.xml
+        OTEL_EXPORTER_OTLP_ENDPOINT: http://localhost:${{ matrix.telemetryport }}
+      run: cd tests/${{ matrix.testset }} && mvn -Dsurefire.rerunFailingTestsCount=2 -Dtest.instrumentation.opts=-javaagent:/tmp/opentelemetry-javaagent.jar test --file pom.xml
     - name: Upload telemetry logs
       uses: actions/upload-artifact@v4
       if: always()
       continue-on-error: true
       with:
-        name: build-telemetry-logs
-        path: /tmp/tests-parquet/logs-*.parquet
+        name: build-telemetry-logs-${{ matrix.telemetryoutput }}
+        path: /tmp/tests-parquet/${{ matrix.telemetryoutput }}/logs-*.parquet
     - name: Upload telemetry metrics
       uses: actions/upload-artifact@v4
       if: always()
       continue-on-error: true
       with:
-        name: build-telemetry-metrics
-        path: /tmp/tests-parquet/metrics-*.parquet
+        name: build-telemetry-metrics-${{ matrix.telemetryoutput }}
+        path: /tmp/tests-parquet/${{ matrix.telemetryoutput }}/metrics-*.parquet
     - name: Upload telemetry traces
       uses: actions/upload-artifact@v4
       if: always()
       continue-on-error: true
       with:
-        name: build-telemetry-traces
-        path: /tmp/tests-parquet/traces-*.parquet
+        name: build-telemetry-traces-${{ matrix.telemetryoutput }}
+        path: /tmp/tests-parquet/${{ matrix.telemetryoutput }}/traces-*.parquet
     # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive
-    - name: Update dependency graph
-      uses: advanced-security/maven-dependency-submission-action@571e99aab1055c2e71a1e2309b9691de18d6b7d6
-      continue-on-error: true
 
 #    - name: Publish Test Report
 #      if: success() || failure()

hadoop/stable-base/pom.xml

@@ -217,18 +217,7 @@
       </extension>
     </extensions>
 
-    <resources>
-      <resource>
-        <directory>src/main/resources</directory>
-        <filtering>true</filtering>
-      </resource>
-    </resources>
-
     <plugins>
-      <plugin>
-        <groupId>io.github.git-commit-id</groupId>
-        <artifactId>git-commit-id-maven-plugin</artifactId>
-      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>
@@ -367,7 +356,12 @@
                     **/hadoop/util/hash/*.java,
                   </includes>
                   <excludes>
-
+                    **/hadoop/fs/CacheFlag.java,
+                    **/hadoop/fs/Hdfs.java,
+                    **/hadoop/fs/HdfsBlockLocation.java,
+                    **/hadoop/fs/SWebHdfs.java,
+                    **/hadoop/fs/WebHdfs.java,
+                    **/hadoop/fs/package.html
                   </excludes>
                 </artifactItem>
               </artifactItems>