Implement Github Actions caching

.github/workflows/grapl-build.yml

@@ -33,7 +33,10 @@ jobs:
       - name: Build Grapl Rust services
         run: |
           GRAPL_RELEASE_TARGET=debug TAG=latest ./dobi-linux --no-bind-mount rust
-          docker system prune -f
+
+      - name: Grapl Rust unit tests
+        run: |
+          GRAPL_RELEASE_TARGET=debug TAG=latest ./dobi-linux --no-bind-mount rust-unit-tests
 
   grapl-python-build:
     runs-on: ubuntu-latest
@@ -49,7 +52,10 @@ jobs:
       - name: Build Grapl Python services
         run: |
           GRAPL_RELEASE_TARGET=debug TAG=latest ./dobi-linux --no-bind-mount python
-          docker system prune -f
+
+      - name: Grapl Python unit tests
+        run: |
+          GRAPL_RELEASE_TARGET=debug TAG=latest ./dobi-linux --no-bind-mount python-unit-tests
 
   grapl-js-build:
     runs-on: ubuntu-latest
@@ -65,7 +71,10 @@ jobs:
       - name: Build Grapl JS services
         run: |
           GRAPL_RELEASE_TARGET=debug TAG=latest ./dobi-linux --no-bind-mount js
-          docker system prune -f
+
+      - name: Grapl JS unit tests
+        run: |
+          GRAPL_RELEASE_TARGET=debug TAG=latest ./dobi-linux --no-bind-mount js-unit-tests
 
   grapl-rust-integration-tests:
     runs-on: ubuntu-latest

.github/workflows/grapl-lint.yml

@@ -17,6 +17,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
+          components: rustfmt
 
       - name: Check rust formatting
         run: |

.github/workflows/grapl-release.yml

@@ -5,7 +5,47 @@ on:
     types: [released, prereleased]
 
 jobs:
-  release:
+  rust-unit-tests:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: [3.7]
+
+    steps:
+
+      - uses: actions/checkout@v2
+
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dobi
+        run: |
+          wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
+          chmod +x dobi-linux
+
+      - name: Run unit tests
+        run: |
+          GRAPL_RELEASE_TARGET=release TAG=grapl-test ./dobi-linux --no-bind-mount rust-unit-test
+
+      - name: Run integration tests
+        run: |
+          GRAPL_RELEASE_TARGET=release TAG=grapl-test ./dobi-linux --no-bind-mount rust-integration-test
+
+      - name: Clean Rust build
+        run: |
+          GRAPL_RELEASE_TARGET=release TAG=grapl-test ./dobi-linux --no-bind-mount clean-build
+          docker system prune -f
+
+  release-rust-services:
     runs-on: ubuntu-latest
 
     strategy:
@@ -72,7 +112,7 @@ jobs:
           wget https://github.com/dnephin/dobi/releases/download/v0.13.0/dobi-linux
           chmod +x dobi-linux
 
-      - name: Build Grapl
+      - name: Build Grapl Rust services
         env:
           VERSION: ${{ github.event.release.tag_name }}
         run: |
@@ -85,36 +125,9 @@ jobs:
           GRAPL_RELEASE_TARGET=release TAG="$VERSION-$CHANNEL" ./dobi-linux --no-bind-mount js
           GRAPL_RELEASE_TARGET=release TAG="$VERSION-$CHANNEL" ./dobi-linux --no-bind-mount clean-js-build
           docker system prune -f
-          GRAPL_RELEASE_TARGET=release TAG="$VERSION-CHANNEL" ./dobi-linux --no-bind-mount integration
-          GRAPL_RELEASE_TARGET=release TAG="$VERSION-CHANNEL" ./dobi-linux --no-bind-mount clean-build
-          docker system prune -f
 
-      - name: Log in to Docker registry
-        run: echo "${{ secrets.DOCKERHUB_TOKEN }}" | docker login --username grapl --password-stdin
-
-      - name: Publish Grapl images to DockerHub
-        env:
-          VERSION: ${{ github.event.release.tag_name }}
-        run: |
-          docker push grapl/grapl-sysmon-subgraph-generator:$VERSION-$CHANNEL
-          docker push grapl/grapl-generic-subgraph-generator:$VERSION-$CHANNEL
-          docker push grapl/grapl-node-identifier:$VERSION-$CHANNEL
-          docker push grapl/grapl-node-identifier-retry-handler:$VERSION-$CHANNEL
-          docker push grapl/grapl-graph-merger:$VERSION-$CHANNEL
-          docker push grapl/grapl-analyzer-dispatcher:$VERSION-$CHANNEL
-          docker push grapl/grapl-analyzer-executor:$VERSION-$CHANNEL
-          docker push grapl/grapl-engagement-creator:$VERSION-$CHANNEL
-          docker push grapl/grapl-notebook:$VERSION-$CHANNEL
-          docker push grapl/grapl-engagement-edge:$VERSION-$CHANNEL
-          docker push grapl/grapl-model-plugin-deployer:$VERSION-$CHANNEL
-          docker push grapl/grapl-engagement-view:$VERSION-$CHANNEL
-          docker push grapl/grapl-graphql-endpoint:$VERSION-$CHANNEL
-          docker push grapl/grapl-graph-provision:$VERSION-$CHANNEL
-          docker push grapl/grapl-dynamodb-provision:$VERSION-$CHANNEL
-          docker push grapl/grapl-dgraph-ttl:$VERSION-$CHANNEL
-
-      - name: Extract Grapl artifacts from build
-        id: extract-artifacts
+      - name: Extract Grapl artifacts from Rust build
+        id: extract-rust-artifacts
         env:
           VERSION: ${{ github.event.release.tag_name }}
         run: |
@@ -132,7 +145,7 @@ jobs:
           zip -9 node-identifier-$VERSION-$CHANNEL.zip ./bootstrap
           rm ./bootstrap
           echo "::set-output name=node-identifier::node-identifier-$VERSION-$CHANNEL.zip"
-
+n
           cp dist/node-identifier-retry-handler ./bootstrap
           zip -9 node-identifier-retry-handler-$VERSION-$CHANNEL.zip ./bootstrap
           rm ./bootstrap
@@ -148,6 +161,35 @@ jobs:
           rm ./bootstrap
           echo "::set-output name=analyzer-dispatcher::analyzer-dispatcher-$VERSION-$CHANNEL.zip"
 
+      - name: Log in to Docker registry
+        run: echo "${{ secrets.DOCKERHUB_TOKEN }}" | docker login --username grapl --password-stdin
+
+      - name: Publish Grapl images to DockerHub
+        env:
+          VERSION: ${{ github.event.release.tag_name }}
+        run: |
+          docker push grapl/grapl-sysmon-subgraph-generator:$VERSION-$CHANNEL
+          docker push grapl/grapl-generic-subgraph-generator:$VERSION-$CHANNEL
+          docker push grapl/grapl-node-identifier:$VERSION-$CHANNEL
+          docker push grapl/grapl-node-identifier-retry-handler:$VERSION-$CHANNEL
+          docker push grapl/grapl-graph-merger:$VERSION-$CHANNEL
+          docker push grapl/grapl-analyzer-dispatcher:$VERSION-$CHANNEL
+          docker push grapl/grapl-analyzer-executor:$VERSION-$CHANNEL
+          docker push grapl/grapl-engagement-creator:$VERSION-$CHANNEL
+          docker push grapl/grapl-notebook:$VERSION-$CHANNEL
+          docker push grapl/grapl-engagement-edge:$VERSION-$CHANNEL
+          docker push grapl/grapl-model-plugin-deployer:$VERSION-$CHANNEL
+          docker push grapl/grapl-engagement-view:$VERSION-$CHANNEL
+          docker push grapl/grapl-graphql-endpoint:$VERSION-$CHANNEL
+          docker push grapl/grapl-graph-provision:$VERSION-$CHANNEL
+          docker push grapl/grapl-dynamodb-provision:$VERSION-$CHANNEL
+          docker push grapl/grapl-dgraph-ttl:$VERSION-$CHANNEL
+
+      - name: Extract Grapl artifacts from build
+        id: extract-artifacts
+        env:
+          VERSION: ${{ github.event.release.tag_name }}
+        run: |
           cp dist/analyzer-executor/lambda.zip analyzer-executor-$VERSION-$CHANNEL.zip
           echo "::set-output name=analyzer-executor::analyzer-executor-$VERSION-$CHANNEL.zip"
 

.gitignore

@@ -30,6 +30,7 @@ __pycache__/
 *$py.class
 
 target/
+.cargo
 
 # C extensions
 *.so

dobi.yaml

@@ -273,6 +273,14 @@ image=dynamodb-provision:
 
 # js images
 
+image=engagement-view-build:
+  image: grapl/engagement-view-build
+  context: src/js/engagement_view
+  dockerfile: Dockerfile
+  target: engagement-view-build
+  tags:
+    - latest
+
 image=engagement-view:
   image: grapl/grapl-engagement-view
   context: src/js/engagement_view
@@ -343,7 +351,13 @@ job=build-rust:
   mounts:
     - dist
   artifact:
-    - ./dist/*
+    - ./dist/
+
+job=run-rust-unit-tests:
+  use: rust-build
+  command: /bin/bash -c "cargo test --target=x86_64-unknown-linux-musl"
+  depends:
+    - rust-build
 
 job=run-node-identifier-integration-tests:
   use: rust-build
@@ -373,6 +387,12 @@ job=run-node-identifier-integration-tests:
 
 # python jobs
 
+job=run-grapl-analyzerlib-unit-tests:
+  use: grapl-analyzerlib-build
+  command: /bin/bash -c "source venv/bin/activate && cd grapl_analyzerlib && py.test -n auto -m 'not integration_test'"
+  depends:
+    - grapl-analyzerlib-build
+
 job=run-grapl-analyzerlib-integration-tests:
   use: grapl-analyzerlib-build
   net-mode: grapl-integration-tests_default
@@ -393,43 +413,83 @@ job=build-analyzer-executor:
   artifact:
     - ./dist/analyzer-executor/lambda.zip
 
+job=run-analyzer-executor-unit-tests:
+  use: analyzer-executor-build
+  command: /bin/bash -c "source venv/bin/activate && cd analyzer_executor && py.test -n auto -m 'not integration_test'"
+  depends:
+    - build-analyzer-executor
+
 job=build-engagement-creator:
   use: engagement-creator-build
   mounts:
     - dist
   artifact:
     - ./dist/engagement-creator/lambda.zip
 
+job=run-engagement-creator-unit-tests:
+  use: engagement-creator-build
+  command: /bin/bash -c "source venv/bin/activate && cd engagement-creator && py.test -n auto -m 'not integration_test'"
+  depends:
+    - engagement-creator-build
+
 job=build-engagement-edge:
   use: engagement-edge-build
   mounts:
     - dist
   artifact:
     - ./dist/engagement-edge/lambda.zip
 
+job=run-engagement-edge-unit-tests:
+  use: engagement-edge-build
+  command: /bin/bash -c "source venv/bin/activate && cd engagement_edge && py.test -n auto -m 'not integration_test'"
+  depends:
+    - engagement-edge-build
+
 job=build-dgraph-ttl:
   use: dgraph-ttl-build
   mounts:
     - dist
   artifact:
     - ./dist/dgraph-ttl/lambda.zip
 
+job=run-dgraph-ttl-unit-tests:
+  use: dgraph-ttl-build
+  command: /bin/bash -c "source venv/bin/activate && cd dgraph-ttl && py.test -n auto -m 'not integration_test'"
+  depends:
+    - dgraph-ttl-build
+
 job=build-model-plugin-deployer:
   use: model-plugin-deployer-build
   mounts:
     - dist
   artifact:
     - ./dist/model-plugin-deployer/lambda.zip
 
+job=run-model-plugin-deployer-unit-tests:
+  use: model-plugin-deployer-build
+  command: /bin/bash -c "source venv/bin/activate && cd model-plugin-deployer && py.test -n auto -m 'not integration_test'"
+  depends:
+    - model-plugin-deployer-build
+
 # js jobs
 
+# TODO: more normal build job for engagement-view w/ deploy artifact
+
+job=run-engagement-view-unit-tests:
+  use: engagement-view-build
+  command: /bin/bash -c "CI=true yarn test"
+  depends:
+    - engagement-view-build
+
 job=build-graphql-endpoint:
   use: graphql-endpoint-build
   mounts:
     - dist
   artifact:
     - ./dist/graphql-endpoint/lambda.zip
 
+# TODO: graphql endpoint unit tests
+
 # TODO: graphql endpoint integration tests
 
 #
@@ -454,7 +514,13 @@ alias=clean-rust-build:
   annotations:
     description: "Delete the rust build image"
 
-alias=rust-integration:
+alias=rust-unit-tests:
+  tasks:
+    - run-rust-unit-tests
+  annotations:
+    description: "Run the rust unit tests"
+
+alias=rust-integration-tests:
   tasks:
     - run-node-identifier-integration-tests
   annotations:
@@ -491,7 +557,18 @@ alias=clean-python-build:
   annotations:
     description: "Delete the python build images"
 
-alias=python-integration:
+alias=python-unit-tests:
+  tasks:
+    - run-grapl-analyzerlib-unit-tests
+    - run-analyzer-executor-unit-tests
+    - run-engagement-creator-unit-tests
+    - run-engagement-edge-unit-tests
+    - run-dgraph-ttl-unit-tests
+    - run-model-plugin-deployer-unit-tests
+  annotations:
+    description: "Run the python unit tests"
+
+alias=python-integration-tests:
   tasks:
     - run-grapl-analyzerlib-integration-tests
   annotations:
@@ -511,6 +588,12 @@ alias=clean-js-build:
   annotations:
     description: "Delete the js build images"
 
+alias=js-unit-tests:
+  tasks:
+    - run-engagement-view-unit-tests
+  annotations:
+    description: "Run the js unit tests"
+
 # TODO: js integration tests
 
 alias=build:
@@ -529,10 +612,18 @@ alias=clean-build:
   annotations:
     description: "Delete all the build images"
 
-alias=integration:
+alias=unit-tests:
+  tasks:
+    - rust-unit-tests
+    - python-unit-tests
+    - js-unit-tests
+  annotations:
+    description: "Run all the unit tests"
+
+alias=integration-tests:
   tasks:
-    - rust-integration
-    - python-integration
+    - rust-integration-tests
+    - python-integration-tests
     # TODO: js integration tests
   annotations:
     description: "Run all the integration tests"