From d1941c428b706eab3bd2e7d5c3194ca6e19f6d41 Mon Sep 17 00:00:00 2001 From: Ian Streeter Date: Fri, 26 Jul 2024 15:40:37 +0100 Subject: [PATCH] Trigger snyk for kafka image --- .github/workflows/cookieless.yml | 37 ----- .github/workflows/deploy.yml | 170 +---------------------- .github/workflows/telemetryIntegTest.yml | 44 ------ .github/workflows/test.yml | 24 ---- 4 files changed, 7 insertions(+), 268 deletions(-) delete mode 100644 .github/workflows/cookieless.yml delete mode 100644 .github/workflows/telemetryIntegTest.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/cookieless.yml b/.github/workflows/cookieless.yml deleted file mode 100644 index 42c7007f3..000000000 --- a/.github/workflows/cookieless.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Test cookieless tracking - -on: push - -jobs: - run_test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: coursier/cache-action@v3 - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Publish Docker image - run: sbt 'project stdout; set Docker / version := "0.0.0"' docker:publishLocal - - name: Run Docker image - run: docker run -d -v "$PWD"/.github/workflows/ssc-collector-config:/snowplow/config -p 12345:12345 snowplow/scala-stream-collector-stdout:0.0.0 --config /snowplow/config/config.hocon - - name: Allow time for collector to start - run: sleep 30 - - name: Test non-anonymous tracking - id: non-anonymous - run: | - output=$(curl -X POST -i http://0.0.0.0:12345/com.snowplowanalytics.snowplow/tp2 -d '{}' 2>&1 | grep -q 'Set-Cookie') - echo "exit_code=$?" >> $GITHUB_OUTPUT - - name: Test anonymous tracking - id: anonymous - if: ${{ steps.non-anonymous.outputs.exit_code == 0 }} - run: | - set +e - output=$(curl -X POST -i http://0.0.0.0:12345/com.snowplowanalytics.snowplow/tp2 -H 'SP-Anonymous: *' -d '{}' 2>&1 | grep -q 'Set-Cookie') - echo "exit_code=$?" >> $GITHUB_OUTPUT - - name: Report outcome - if: ${{ steps.non-anonymous.outputs.exit_code == 0 && steps.anonymous.outputs.exit_code == 1 }} - run: echo "All tests successful!" - - name: Stop Docker container - run: docker stop $(docker ps -aq) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index dc95026a5..d4fc8898b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,177 +1,21 @@ -name: build +name: snyk on: push: - tags: - - '*' + branches: + - "run-snyk" jobs: - deploy_to_github: + snyk: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: coursier/cache-action@v3 - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Build artifacts - run: | - sbt 'project kafka' assembly - sbt 'project kinesis' assembly - sbt 'project nsq' assembly - sbt 'project pubsub' assembly - sbt 'project sqs' assembly - sbt 'project stdout' assembly - - name: Get current version - id: ver - run: | - export PROJECT_VERSION=$(sbt "project core" version -Dsbt.log.noformat=true | perl -ne 'print "$1\n" if /info.*(\d+\.\d+\.\d+[^\r\n]*)/' | tail -n 1 | tr -d '\n') - echo "project_version=$PROJECT_VERSION" >> $GITHUB_OUTPUT - - name: Create GitHub release and attach artifacts - uses: softprops/action-gh-release@v1 - with: - draft: true - prerelease: true - name: ${{ steps.ver.outputs.project_version }} - tag_name: ${{ steps.ver.outputs.project_version }} - files: | - kafka/target/scala-2.13/snowplow-stream-collector-kafka-${{ steps.ver.outputs.project_version }}.jar - kinesis/target/scala-2.13/snowplow-stream-collector-kinesis-${{ steps.ver.outputs.project_version }}.jar - nsq/target/scala-2.13/snowplow-stream-collector-nsq-${{ steps.ver.outputs.project_version }}.jar - pubsub/target/scala-2.13/snowplow-stream-collector-google-pubsub-${{ steps.ver.outputs.project_version }}.jar - sqs/target/scala-2.13/snowplow-stream-collector-sqs-${{ steps.ver.outputs.project_version }}.jar - stdout/target/scala-2.13/snowplow-stream-collector-stdout-${{ steps.ver.outputs.project_version }}.jar - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - deploy_to_docker: - runs-on: ubuntu-latest - strategy: - matrix: - platform: - - kinesis - - sqs - - pubsub - - kafka - - nsq - - stdout - include: - - suffix: "" - - platform: kinesis - run_snyk: ${{ !contains(github.ref, 'rc') }} - - platform: pubsub - run_snyk: ${{ !contains(github.ref, 'rc') }} - - steps: - - uses: actions/checkout@v2 - - uses: coursier/cache-action@v3 - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 11 - - - name: Get current version - id: ver - run: | - export PROJECT_VERSION=$(sbt "project core" version -Dsbt.log.noformat=true | perl -ne 'print "$1\n" if /info.*(\d+\.\d+\.\d+[^\r\n]*)/' | tail -n 1 | tr -d '\n') - echo "project_version=$PROJECT_VERSION" >> $GITHUB_OUTPUT - - - name: Stage the Docker build - run: sbt "project ${{ matrix.platform }}" docker:stage - - - name: Stage the Docker distroless build - run: sbt "project ${{ matrix.platform }}Distroless" docker:stage - - - name: Docker metadata - id: meta - uses: docker/metadata-action@v3 - with: - images: snowplow/scala-stream-collector-${{ matrix.platform }}${{ matrix.suffix }} - tags: | - type=raw,value=latest,enable=${{ !contains(steps.ver.outputs.project_version, 'rc') }} - type=raw,value=latest-focal,enable=${{ !contains(steps.ver.outputs.project_version, 'rc') }} - type=raw,value=${{ steps.ver.outputs.project_version }} - type=raw,value=${{ steps.ver.outputs.project_version }}-focal - flavor: | - latest=false - - - name: Docker metadata distroless - id: distroless-meta - uses: docker/metadata-action@v3 - with: - images: snowplow/scala-stream-collector-${{ matrix.platform }}${{ matrix.suffix }} - tags: | - type=raw,value=latest-distroless,enable=${{ !contains(steps.ver.outputs.project_version, 'rc') }} - type=raw,value=${{ steps.ver.outputs.project_version }}-distroless - flavor: | - latest=false - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Push image - uses: docker/build-push-action@v2 - with: - context: ${{ matrix.platform }}/target/docker/stage - file: ${{ matrix.platform }}/target/docker/stage/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - tags: ${{ steps.meta.outputs.tags }} - push: true - - name: Push distroless image - uses: docker/build-push-action@v2 - with: - context: distroless/${{ matrix.platform }}/target/docker/stage - file: distroless/${{ matrix.platform }}/target/docker/stage/Dockerfile - platforms: linux/amd64,linux/arm64/v8 - tags: ${{ steps.distroless-meta.outputs.tags }} - push: true - - - name: Build local distroless image, which is needed to run Snyk - if: matrix.run_snyk - run: sbt "project ${{ matrix.platform }}Distroless" docker:publishLocal - - name: Show local docker images - run: docker images + - name: Pull image + run: "docker pull snowplow/scala-stream-collector-kafka:3.2.0-distroless" - name: Run Snyk to check for vulnerabilities uses: snyk/actions/docker@master - if: matrix.run_snyk with: - image: "snowplow/scala-stream-collector-${{ matrix.platform }}:${{ github.ref_name }}-distroless" + image: "snowplow/scala-stream-collector-kafka:3.2.0-distroless" args: "--app-vulns --org=data-processing-new" command: monitor env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - qa_framework: - runs-on: ubuntu-latest - needs: deploy_to_docker - steps: - - uses: actions/checkout@v2 - - uses: coursier/cache-action@v3 - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Get current version - id: ver - run: | - export PROJECT_VERSION=$(sbt "project core" version -Dsbt.log.noformat=true | perl -ne 'print "$1\n" if /info.*(\d+\.\d+\.\d+[^\r\n]*)/' | tail -n 1 | tr -d '\n') - echo "project_version=$PROJECT_VERSION" >> $GITHUB_OUTPUT - - name: Execute QA Framework tests - uses: aurelien-baudet/workflow-dispatch@v2 - with: - workflow: test-component - repo: snowplow-devops/qa-framework - token: ${{ secrets.GLOBAL_QA_FRAMEWORK_PAT }} - ref: "refs/heads/main" - inputs: '{ "test_directory": "collector", "qa_collector_version": "${{ steps.ver.outputs.project_version }}" }' - wait-for-completion-timeout: 60m # workflow is much slower in Azure; can take up to 20-something minutes - display-workflow-run-url: true diff --git a/.github/workflows/telemetryIntegTest.yml b/.github/workflows/telemetryIntegTest.yml deleted file mode 100644 index 247f52eb1..000000000 --- a/.github/workflows/telemetryIntegTest.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: telemetryIntegTest - -on: push - -jobs: - integ_test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: coursier/cache-action@v3 - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Set up python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - name: build - run: | - sbt "project stdout" assembly - find $(pwd) -name "*.jar" - - name: run micro - run: > - docker run -d - --name micro - --mount type=bind,source=$(pwd)/.github/workflows/integration_tests/telemetry/micro_config,destination=/config - -p 9191:9191 snowplow/snowplow-micro:1.2.1 - --collector-config /config/config.hocon - --iglu /config/iglu.json - - name: run collectors - run: | - java -jar $(pwd)/stdout/target/scala-*/*.jar --config $(pwd)/.github/workflows/integration_tests/telemetry/sender_config/config.hocon & - PID_C1=$! - java -jar $(pwd)/stdout/target/scala-*/*.jar --config $(pwd)/.github/workflows/integration_tests/telemetry/sender_config/config_disabled.hocon & - PID_C2=$! - sleep 30 - kill $PID_C1 $PID_C2 - - name: assess result - run: | - pip install requests - python3 .github/workflows/integration_tests/telemetry/verify_micro_content.py - - name: clean up - run: docker stop micro \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 4edbc7ed8..000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: test - -on: push - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: coursier/cache-action@v3 - - name: Set up JDK - uses: actions/setup-java@v1 - with: - java-version: 11 - - name: Check formatting - run: sbt scalafmtCheckAll - - name: Run unit tests - run: sbt +test - - name: Run integration tests Kinesis - run: sbt "project kinesisDistroless" IntegrationTest/test - - name: Run integration tests PubSub - run: sbt "project pubsubDistroless" IntegrationTest/test - - name: Run integration tests Kafka - run: sbt "project kafkaDistroless" IntegrationTest/test