Adjustments to Logging and a Python Error

.github/workflows/check_images_x_region.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+tag=$1
+
+function checkImages() {
+  repo=$1
+  primary_sha=$(aws ecr describe-images --region us-east-1 --repository-name $repo --image-ids imageTag=$tag | jq -r '.imageDetails[0].imageDigest')
+
+  for region in us-east-2 us-west-1 us-west-2 eu-west-1 eu-west-2; do
+    sha=$(aws ecr describe-images --region $region --repository-name $repo --image-ids imageTag=$tag | jq -r '.imageDetails[0].imageDigest')
+
+    if [[ "$sha" != "$primary_sha" ]]; then
+      echo "Incomplete ECR propagation for repo $repo (tag $tag) in $region. Expected $primary_sha but found $sha."
+      exit 1
+    fi
+  done
+}
+
+checkImages strelka-frontend
+checkImages strelka-backend
+checkImages strelka-manager
+checkImages strelka-mmrpc

.github/workflows/release-images.yml

@@ -0,0 +1,164 @@
+
+name: Update Strelka Docker Images
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  push_to_registry:
+    name: Build & Push to Registries
+    runs-on: ubuntu-latest
+    environment: production
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@036a4a1ddf2c0e7a782dca6e083c6c53e5d90321 # No released version with OIDC
+        with:
+          # Use a role instead of including credentials in secrets, but still hide that role name to be gated by GH environments.
+          role-to-assume: ${{ secrets.ECR_REPO_ROLE }}
+          role-duration-seconds: 7200 # 2 hours
+          aws-region: us-east-1
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Determine Tags FrontEnd
+        id: meta_frontend
+        uses: crazy-max/ghaction-docker-meta@v2
+        with:
+          images: |
+            ${{ steps.login-ecr.outputs.registry }}/strelka-frontend
+            sublimesec/strelka-frontend
+          flavor: |
+            latest=false
+          tags: |
+            type=semver,pattern={{version}}
+
+      - name: Determine Tags BackEnd
+        id: meta_backend
+        uses: crazy-max/ghaction-docker-meta@v2
+        with:
+          images: |
+            ${{ steps.login-ecr.outputs.registry }}/strelka-backend
+            sublimesec/strelka-backend
+          flavor: |
+            latest=false
+          tags: |
+            type=semver,pattern={{version}}
+
+      - name: Determine Tags Manager
+        id: meta_manager
+        uses: crazy-max/ghaction-docker-meta@v2
+        with:
+          images: |
+            ${{ steps.login-ecr.outputs.registry }}/strelka-manager
+            sublimesec/strelka-manager
+          flavor: |
+            latest=false
+          tags: |
+            type=semver,pattern={{version}}
+
+      - name: Determine Tags MMRPC
+        id: meta_mmrpc
+        uses: crazy-max/ghaction-docker-meta@v2
+        with:
+          images: |
+            ${{ steps.login-ecr.outputs.registry }}/strelka-mmrpc
+            sublimesec/strelka-mmrpc
+          flavor: |
+            latest=false
+          tags: |
+            type=semver,pattern={{version}}
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build FrontEnd
+        uses: docker/build-push-action@v2
+        with:
+          file: build/go/frontend/Dockerfile
+          context: .
+          load: true
+          tags: ${{ steps.meta_frontend.outputs.tags }}
+          labels: ${{ steps.meta_frontend.outputs.labels }}
+
+      - name: Build BackEnd
+        uses: docker/build-push-action@v2
+        with:
+          file: build/python/backend/Dockerfile
+          context: .
+          load: true
+          tags: ${{ steps.meta_backend.outputs.tags }}
+          labels: ${{ steps.meta_backend.outputs.labels }}
+
+      - name: Build Manager
+        uses: docker/build-push-action@v2
+        with:
+          file: build/go/manager/Dockerfile
+          context: .
+          load: true
+          tags: ${{ steps.meta_manager.outputs.tags }}
+          labels: ${{ steps.meta_manager.outputs.labels }}
+
+      - name: Build Manager
+        uses: docker/build-push-action@v2
+        with:
+          file: build/python/mmrpc/Dockerfile
+          context: .
+          load: true
+          tags: ${{ steps.meta_mmrpc.outputs.tags }}
+          labels: ${{ steps.meta_mmrpc.outputs.labels }}
+
+      - name: Push FrontEnd
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        run: |
+          docker push --all-tags $ECR_REGISTRY/strelka-frontend
+          docker push --all-tags sublimesec/strelka-frontend
+
+      - name: Push BackEnd
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        run: |
+          docker push --all-tags $ECR_REGISTRY/strelka-backend
+          docker push --all-tags sublimesec/strelka-backend
+
+      - name: Push Manager
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        run: |
+          docker push --all-tags $ECR_REGISTRY/strelka-manager
+          docker push --all-tags sublimesec/strelka-manager
+
+      - name: Push MMRPC
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        run: |
+          docker push --all-tags $ECR_REGISTRY/strelka-mmrpc
+          docker push --all-tags sublimesec/strelka-mmrpc
+
+      - name: Validate All X-Region Replication
+        run: |
+          SEM_VER=$(echo "${{ github.ref }}" | grep -E -o "[0-9]+\.[0-9]+.[0-9]*")
+          .github/workflows/check_images_x_region.sh $SEM_VER
+          if [ $? != 0 ]; then
+            exit 1
+          fi
+
+      - name: Slack Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_Z_LOG_DOCKER_BUILDS }}
+          SLACK_TITLE: Strelka Images Updated

.github/workflows/tag-latest.yml

@@ -0,0 +1,159 @@
+
+name: Finalize Tags
+# Workflow this will update various mutable tags to match the full semver given in the input tag.
+# E.g. if an image is tagged 0.1.2 and this workflow is triggered with a tag `latest-v0.1.2` then it will update tags:
+# latest, 0, and 0.1 to point to the 0.1.2 image.
+
+on:
+  create:
+
+jobs:
+  update_latest_tag:
+    name: Build & Push to Registries
+    if: ${{ startsWith(github.ref, 'refs/tags/latest-v') }}
+    runs-on: ubuntu-latest
+    environment: production
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@036a4a1ddf2c0e7a782dca6e083c6c53e5d90321 # No released version with OIDC
+        with:
+          # Use a role instead of including credentials in secrets, but still hide that role name to be gated by GH environments.
+          role-to-assume: ${{ secrets.ECR_REPO_ROLE }}
+          role-duration-seconds: 7200 # 2 hours
+          aws-region: us-east-1
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v1
+
+      # If this workflow runs for a semver that hasn't been released, then we'll fail below. This workflow is only
+      # meant to add the `latest` tag onto an existing release.
+
+      - name: Determine the version from the tag
+        id: get_ver
+        run: |
+          SEM_VER=$(echo "${{ github.ref }}" | grep -E -o "[0-9]+\.[0-9]+.[0-9]*")
+          if [ -z $SEM_VER ]; then
+            exit 1
+          fi
+          echo "::set-output name=SEM_VER::$SEM_VER"
+
+          MAJOR_VERSION=$(echo "$SEM_VER" | grep -E -o "^[0-9]+")
+          echo "::set-output name=MAJOR_VERSION::$MAJOR_VERSION"
+
+          MINOR_VERSION=$(echo "$SEM_VER" | grep -E -o "^[0-9]+\.[0-9]+")
+          echo "::set-output name=MINOR_VERSION::$MINOR_VERSION"
+
+      # Right now just pull the image in order to tag it. There might be alternatives:
+      # https://stackoverflow.com/questions/37134929/how-to-tag-image-in-docker-registry-v2/38362476#38362476 (auth unclear)
+      # Use a shared context with original workflow?
+
+      - name: Pull, Tag, Push FrontEnd
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          VERSION: ${{ steps.get_ver.outputs.SEM_VER }}
+          MAJOR_VERSION: ${{ steps.get_ver.outputs.MAJOR_VERSION }}
+          MINOR_VERSION: ${{ steps.get_ver.outputs.MINOR_VERSION }}
+        run: |
+          docker pull $ECR_REGISTRY/strelka-frontend:$VERSION
+
+          docker tag $ECR_REGISTRY/strelka-frontend:$VERSION $ECR_REGISTRY/strelka-frontend:latest
+          docker tag $ECR_REGISTRY/strelka-frontend:$VERSION sublimesec/strelka-frontend:latest
+
+          docker tag $ECR_REGISTRY/strelka-frontend:$VERSION $ECR_REGISTRY/strelka-frontend:$MAJOR_VERSION
+          docker tag $ECR_REGISTRY/strelka-frontend:$VERSION sublimesec/strelka-frontend:$MAJOR_VERSION
+
+          docker tag $ECR_REGISTRY/strelka-frontend:$VERSION $ECR_REGISTRY/strelka-frontend:$MINOR_VERSION
+          docker tag $ECR_REGISTRY/strelka-frontend:$VERSION sublimesec/strelka-frontend:$MINOR_VERSION
+
+          docker push --all-tags $ECR_REGISTRY/strelka-frontend:latest
+          docker push --all-tags sublimesec/strelka-frontend:latest
+
+      - name: Pull, Tag, Push BackEnd
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          VERSION: ${{ steps.get_ver.outputs.SEM_VER }}
+          MAJOR_VERSION: ${{ steps.get_ver.outputs.MAJOR_VERSION }}
+          MINOR_VERSION: ${{ steps.get_ver.outputs.MINOR_VERSION }}
+        run: |
+          docker pull $ECR_REGISTRY/strelka-backend:$VERSION
+
+          docker tag $ECR_REGISTRY/strelka-backend:$VERSION $ECR_REGISTRY/strelka-backend:latest
+          docker tag $ECR_REGISTRY/strelka-backend:$VERSION sublimesec/strelka-backend:latest
+
+          docker tag $ECR_REGISTRY/strelka-backend:$VERSION $ECR_REGISTRY/strelka-backend:$MAJOR_VERSION
+          docker tag $ECR_REGISTRY/strelka-backend:$VERSION sublimesec/strelka-backend:$MAJOR_VERSION
+
+          docker tag $ECR_REGISTRY/strelka-backend:$VERSION $ECR_REGISTRY/strelka-backend:$MINOR_VERSION
+          docker tag $ECR_REGISTRY/strelka-backend:$VERSION sublimesec/strelka-backend:$MINOR_VERSION
+
+          docker push --all-tags $ECR_REGISTRY/strelka-backend:latest
+          docker push --all-tags sublimesec/strelka-backend:latest
+
+      - name: Pull, Tag, Push Manager
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          VERSION: ${{ steps.get_ver.outputs.SEM_VER }}
+          MAJOR_VERSION: ${{ steps.get_ver.outputs.MAJOR_VERSION }}
+          MINOR_VERSION: ${{ steps.get_ver.outputs.MINOR_VERSION }}
+        run: |
+          docker pull $ECR_REGISTRY/strelka-manager:$VERSION
+
+          docker tag $ECR_REGISTRY/strelka-manager:$VERSION $ECR_REGISTRY/strelka-manager:latest
+          docker tag $ECR_REGISTRY/strelka-manager:$VERSION sublimesec/strelka-manager:latest
+
+          docker tag $ECR_REGISTRY/strelka-manager:$VERSION $ECR_REGISTRY/strelka-manager:$MAJOR_VERSION
+          docker tag $ECR_REGISTRY/strelka-manager:$VERSION sublimesec/strelka-manager:$MAJOR_VERSION
+
+          docker tag $ECR_REGISTRY/strelka-manager:$VERSION $ECR_REGISTRY/strelka-manager:$MINOR_VERSION
+          docker tag $ECR_REGISTRY/strelka-manager:$VERSION sublimesec/strelka-manager:$MINOR_VERSION
+
+          docker push --all-tags $ECR_REGISTRY/strelka-manager:latest
+          docker push --all-tags sublimesec/strelka-manager:latest
+
+      - name: Pull, Tag, Push MMRPC
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          VERSION: ${{ steps.get_ver.outputs.SEM_VER }}
+          MAJOR_VERSION: ${{ steps.get_ver.outputs.MAJOR_VERSION }}
+          MINOR_VERSION: ${{ steps.get_ver.outputs.MINOR_VERSION }}
+        run: |
+          docker pull $ECR_REGISTRY/strelka-mmrpc:$VERSION
+
+          docker tag $ECR_REGISTRY/strelka-mmrpc:$VERSION $ECR_REGISTRY/strelka-mmrpc:latest
+          docker tag $ECR_REGISTRY/strelka-mmrpc:$VERSION sublimesec/strelka-mmrpc:latest
+
+          docker tag $ECR_REGISTRY/strelka-mmrpc:$VERSION $ECR_REGISTRY/strelka-mmrpc:$MAJOR_VERSION
+          docker tag $ECR_REGISTRY/strelka-mmrpc:$VERSION sublimesec/strelka-mmrpc:$MAJOR_VERSION
+
+          docker tag $ECR_REGISTRY/strelka-mmrpc:$VERSION $ECR_REGISTRY/strelka-mmrpc:$MINOR_VERSION
+          docker tag $ECR_REGISTRY/strelka-mmrpc:$VERSION sublimesec/strelka-mmrpc:$MINOR_VERSION
+
+          docker push --all-tags $ECR_REGISTRY/strelka-mmrpc:latest
+          docker push --all-tags sublimesec/strelka-mmrpc:latest
+
+      - name: Validate All X-Region Replication
+        run: |
+          .github/workflows/check_images_x_region.sh latest
+          if [ $? != 0 ]; then
+            exit 1
+          fi
+
+      - name: Slack Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_Z_LOG_DOCKER_BUILDS }}
+          SLACK_TITLE: Strelka Images latest tag updated to ${{ steps.get_ver.outputs.SEM_VER }}

.gitignore

@@ -8,7 +8,6 @@ __pycache__/
 
 # Distribution / packaging
 .Python
-build/
 develop-eggs/
 dist/
 downloads/
@@ -143,4 +142,7 @@ cython_debug/
 .idea/
 
 # MacOS Attributes File
-*.DS_Store
+*.DS_Store
+
+# Vendored/downloaded go dependencies
+vendor

build/configs/README.md

@@ -0,0 +1,14 @@
+This configs folder contains configs which are built into images. The contents may appear similar to `strelka/configs`
+but that folder is left as is to cleanly receive/make updates from/to the upstream. These configs use a slightly 
+different structure than the upstream in order to disconnect rules/scan configurations from runtime configurations 
+(those will still be injected by the runner).
+
+Some of this content also appears similar to content in `strelka/misc/kubernets` -- that folder is example K8S manifests
+and includes inlined configuration (which currently differs slightly from other example configurations).
+
+To use these:
+* Change scanners in the main backend config to `scanners: '/strelka/config/scanners.yaml'` (instead of a list of objs)
+* Any scanner which supports the passwords file:
+  * `password_file: '/strelka/config/passwords.dat' `
+* Change `tasting` in main backend config to `yara_rules: '/etc/strelka/taste/'`
+