Fix feature server docker image tag generation in pr integration tests (

#2077) * Fix feature server docker image tag generation in pr integration tests Signed-off-by: Tsotne Tabidze <[email protected]> * Also fix integration_tests.yml Signed-off-by: Tsotne Tabidze <[email protected]>
feast-dev · Nov 20, 2021 · 27bf63c · 27bf63c
1 parent 3917d5a
commit 27bf63c
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 22 deletions.
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -23,17 +23,21 @@ jobs:
       - name: Login to Amazon ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v1
+      - name: Set ECR image tag
+        id: image-tag
+        run: echo '::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`'
       - name: Build and push
         env:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           ECR_REPOSITORY: feast-python-server
-        # Note: the image tags should be in sync with sdk/python/feast/infra/aws.py:_get_docker_image_version
         run: |
           docker build \
             --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
-            --tag $ECR_REGISTRY/$ECR_REPOSITORY:`git rev-parse HEAD` \
+            --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
             .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:`git rev-parse HEAD`
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
+    outputs:
+      DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
   integration-test-python:
     needs: build-docker-image
     runs-on: ${{ matrix.os }}
@@ -99,7 +103,11 @@ jobs:
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test python
-        run: FEAST_USAGE=False IS_TEST=True pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5
+        env:
+          FEAST_SERVER_DOCKER_IMAGE_TAG: ${{needs.build-docker-image.outputs.DOCKER_IMAGE_TAG}}
+          FEAST_USAGE: False
+          IS_TEST: True
+        run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
         with:

diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml
@@ -34,17 +34,21 @@ jobs:
       - name: Login to Amazon ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v1
+      - name: Set ECR image tag
+        id: image-tag
+        run: echo '::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`'
       - name: Build and push
         env:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           ECR_REPOSITORY: feast-python-server
-        # Note: the image tags should be in sync with sdk/python/feast/infra/aws.py:_get_docker_image_version
         run: |
           docker build \
             --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \
-            --tag $ECR_REGISTRY/$ECR_REPOSITORY:`git rev-parse HEAD` \
+            --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \
             .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:`git rev-parse HEAD`
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
+    outputs:
+      DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
   integration-test-python:
     # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes.
     if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test'))
@@ -119,7 +123,11 @@ jobs:
       - name: Install dependencies
         run: make install-python-ci-dependencies
       - name: Test python
-        run: FEAST_USAGE=False IS_TEST=True pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5
+        env:
+          FEAST_SERVER_DOCKER_IMAGE_TAG: ${{needs.build-docker-image.outputs.DOCKER_IMAGE_TAG}}
+          FEAST_USAGE: False
+          IS_TEST: True
+        run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
         with:

diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py
@@ -37,3 +37,6 @@
 
 # Default FTS port
 DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT = 6569
+
+# Environment variable for feature server docker image tag
+DOCKER_IMAGE_TAG_ENV_NAME: str = "FEAST_SERVER_DOCKER_IMAGE_TAG"
diff --git a/sdk/python/feast/infra/aws.py b/sdk/python/feast/infra/aws.py
@@ -2,7 +2,6 @@
 import hashlib
 import logging
 import os
-import subprocess
 import uuid
 from datetime import datetime
 from pathlib import Path
@@ -12,10 +11,10 @@
 
 from colorama import Fore, Style
 
-from feast import flags_helper
 from feast.constants import (
     AWS_LAMBDA_FEATURE_SERVER_IMAGE,
     AWS_LAMBDA_FEATURE_SERVER_REPOSITORY,
+    DOCKER_IMAGE_TAG_ENV_NAME,
     FEAST_USAGE,
     FEATURE_STORE_YAML_ENV_NAME,
 )
@@ -322,27 +321,21 @@ def _get_lambda_name(project: str):
 def _get_docker_image_version() -> str:
     """Returns a version for the feature server Docker image.
 
+    If the feast.constants.DOCKER_IMAGE_TAG_ENV_NAME environment variable is set,
+    we return that (mostly used for integration tests, but can be used for local testing too).
+
     For public Feast releases this equals to the Feast SDK version modified by replacing "." with "_".
     For example, Feast SDK version "0.14.1" would correspond to Docker image version "0_14_1".
 
-    For integration tests this equals to the git commit hash of HEAD. This is necessary,
-    because integration tests need to use images built from the same commit hash.
-
     During development (when Feast is installed in editable mode) this equals to the Feast SDK version
     modified by removing the "dev..." suffix and replacing "." with "_". For example, Feast SDK version
     "0.14.1.dev41+g1cbfa225.d20211103" would correspond to Docker image version "0_14_1". This way,
     Feast SDK will use an already existing Docker image built during the previous public release.
 
     """
-    if flags_helper.is_test():
-        # Note: this should be in sync with https://github.com/feast-dev/feast/blob/6fbe01b6e9a444dc77ec3328a54376f4d9387664/.github/workflows/pr_integration_tests.yml#L41
-        return (
-            subprocess.check_output(
-                ["git", "rev-parse", "HEAD"], cwd=Path(__file__).resolve().parent
-            )
-            .decode()
-            .strip()
-        )
+    tag = os.environ.get(DOCKER_IMAGE_TAG_ENV_NAME)
+    if tag is not None:
+        return tag
     else:
         version = get_version()
         if "dev" in version: