Run functional tests against sandbox-bundled (#3581)

Signed-off-by: Jeev B <[email protected]>

.github/workflows/single-binary.yml

@@ -36,14 +36,12 @@ jobs:
         run: |
           make test
 
-  trigger-single-binary-build:
+  build-and-push-single-binary-image:
     runs-on: ubuntu-latest
     needs: [test-bootstrap]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
+        uses: actions/checkout@v3
       - name: Setup Golang caches
         uses: actions/cache@v3
         with:
@@ -72,21 +70,12 @@ jobs:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
-        id: buildx
         uses: docker/setup-buildx-action@v2
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
-          password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Setup destination directories for image tarballs
         run: |
           mkdir -p docker/sandbox-bundled/images/tar/{arm64,amd64}
       - name: Export ARM64 Image
         uses: docker/build-push-action@v3
-        env:
-          DOCKER_CLI_EXPERIMENTAL: enabled
         with:
           context: .
           platforms: linux/arm64
@@ -98,8 +87,6 @@ jobs:
           outputs: type=docker,dest=docker/sandbox-bundled/images/tar/arm64/flyte-binary.tar
       - name: Export AMD64 Image
         uses: docker/build-push-action@v3
-        env:
-          DOCKER_CLI_EXPERIMENTAL: enabled
         with:
           context: .
           platforms: linux/amd64
@@ -109,10 +96,20 @@ jobs:
             FLYTE_VERSION=${{ env.FLYTE_VERSION }}
           file: Dockerfile
           outputs: type=docker,dest=docker/sandbox-bundled/images/tar/amd64/flyte-binary.tar
+      - name: Upload single binary image
+        uses: actions/upload-artifact@v2
+        with:
+          name: single-binary-image
+          path: docker/sandbox-bundled/images/tar
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
+          password: "${{ secrets.FLYTE_BOT_PAT }}"
       - name: Build and push Image
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event_name == 'workflow_dispatch' }}
         uses: docker/build-push-action@v3
-        env:
-          DOCKER_CLI_EXPERIMENTAL: enabled
         with:
           context: .
           platforms: linux/arm64, linux/amd64
@@ -121,25 +118,87 @@ jobs:
             FLYTECONSOLE_VERSION=${{ env.FLYTECONSOLE_VERSION }}
             FLYTE_VERSION=${{ env.FLYTE_VERSION }}
           file: Dockerfile
-          push: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event_name == 'workflow_dispatch' }}
-      - name: Upload Artifact GitHub Action
-        uses: actions/upload-artifact@v2
+          push: true
+
+  sandbox-bundled-functional-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    needs: [build-and-push-single-binary-image]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: image=moby/buildkit:master
+          buildkitd-flags: "--allow-insecure-entitlement security.insecure"
+      - name: Build sandbox image for functional tests
+        uses: docker/build-push-action@v3
+        with:
+          context: docker/sandbox-bundled
+          load: true
+          allow: "security.insecure"
+          tags: flyte-sandbox-bundled:local
+      # Without this, the GHA runner is under disk pressure and evicts all the pods.
+      # Buildx cache uses roughly 50% (7gb) of the GHA runner's disk (14gb).
+      - name: Prune Docker Buildx cache to reclaim storage
+        run: docker buildx prune --all --force
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - uses: unionai/[email protected]
+      - name: Setup sandbox
+        run: |
+          mkdir -p ~/.flyte/sandbox
+          cat << EOF > ~/.flyte/sandbox/config.yaml
+          task_resources:
+            defaults:
+              cpu: "0"
+              memory: "0"
+          EOF
+          flytectl demo start --image flyte-sandbox-bundled:local --imagePullPolicy Never
+      - name: Register examples
+        uses: unionai/[email protected]
+        with:
+          flytesnacks: true
+          config: boilerplate/flyte/end2end/functional-test-config.yaml
+          project: flytesnacks
+          domain: development
+          version: "latest"
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flytekit
+          pip freeze
+      - name: End2End
+        env:
+          PRIORITIES: "P0"
+        run: |
+          make end2end_execute
 
-  trigger-sandbox-bundled-build:
+  build-and-push-sandbox-bundled-image:
+    if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event_name == 'workflow_dispatch' }}
     runs-on: ubuntu-latest
-    needs: [trigger-single-binary-build]
+    needs: [build-and-push-single-binary-image]
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
+        uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
         with:
           name: single-binary-image
           path: docker/sandbox-bundled/images/tar
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: image=moby/buildkit:master
+          buildkitd-flags: "--allow-insecure-entitlement security.insecure"
       - name: Set version
         id: set_version
         run: |
@@ -154,28 +213,18 @@ jobs:
           tags: |
             type=raw,value=latest,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
             type=sha,format=long,
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-        with:
-          driver-opts: image=moby/buildkit:master
-          buildkitd-flags: "--allow-insecure-entitlement security.insecure"
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: "${{ secrets.FLYTE_BOT_USERNAME }}"
           password: "${{ secrets.FLYTE_BOT_PAT }}"
-      - name: Build and push Image
+      - name: Build and push multi-arch image
         uses: docker/build-push-action@v3
-        env:
-          DOCKER_CLI_EXPERIMENTAL: enabled
         with:
           context: docker/sandbox-bundled
           allow: "security.insecure"
           platforms: linux/arm64, linux/amd64
           tags: ${{ steps.image-names.outputs.tags }}
           build-args: "FLYTE_SANDBOX_VERSION=${{ env.FLYTE_SANDBOX_VERSION }}"
-          push: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event_name == 'workflow_dispatch' }}
+          push: true

boilerplate/flyte/end2end/functional-test-config.yaml

@@ -1,5 +1,18 @@
 admin:
-  # For GRPC endpoints you might want to use dns:///flyte.myexample.com
-  endpoint: localhost:30081
-  authType: Pkce
+  endpoint: dns:///localhost:30080
   insecure: true
+  authType: Pkce
+logger:
+  show-source: true
+  level: 0
+storage:
+  connection:
+    access-key: minio
+    auth-type: accesskey
+    disable-ssl: true
+    endpoint: http://localhost:30002
+    region: us-east-1
+    secret-key: miniostorage
+  type: minio
+  container: "my-s3-bucket"
+  enable-multicontainer: true