diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 666bb08378..80cf794b09 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -23,17 +23,21 @@ jobs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 + - name: Set ECR image tag + id: image-tag + run: echo '::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`' - name: Build and push env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: feast-python-server - # Note: the image tags should be in sync with sdk/python/feast/infra/aws.py:_get_docker_image_version run: | docker build \ --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - --tag $ECR_REGISTRY/$ECR_REPOSITORY:`git rev-parse HEAD` \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:`git rev-parse HEAD` + docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} + outputs: + DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: needs: build-docker-image runs-on: ${{ matrix.os }} @@ -99,7 +103,11 @@ jobs: - name: Install dependencies run: make install-python-ci-dependencies - name: Test python - run: FEAST_USAGE=False IS_TEST=True pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 + env: + FEAST_SERVER_DOCKER_IMAGE_TAG: ${{needs.build-docker-image.outputs.DOCKER_IMAGE_TAG}} + FEAST_USAGE: False + IS_TEST: True + run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 7699d70381..e3c5fd384c 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -34,17 +34,21 @@ jobs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 + - name: Set ECR image tag + id: image-tag + run: echo '::set-output name=DOCKER_IMAGE_TAG::`git rev-parse HEAD`' - name: Build and push env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: feast-python-server - # Note: the image tags should be in sync with sdk/python/feast/infra/aws.py:_get_docker_image_version run: | docker build \ --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ - --tag $ECR_REGISTRY/$ECR_REPOSITORY:`git rev-parse HEAD` \ + --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:`git rev-parse HEAD` + docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} + outputs: + DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} integration-test-python: # all jobs MUST have this if check for 'ok-to-test' or 'approved' for security purposes. if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) @@ -119,7 +123,11 @@ jobs: - name: Install dependencies run: make install-python-ci-dependencies - name: Test python - run: FEAST_USAGE=False IS_TEST=True pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 + env: + FEAST_SERVER_DOCKER_IMAGE_TAG: ${{needs.build-docker-image.outputs.DOCKER_IMAGE_TAG}} + FEAST_USAGE: False + IS_TEST: True + run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 1e88e60435..e2e987dd77 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -37,3 +37,6 @@ # Default FTS port DEFAULT_FEATURE_TRANSFORMATION_SERVER_PORT = 6569 + +# Environment variable for feature server docker image tag +DOCKER_IMAGE_TAG_ENV_NAME: str = "FEAST_SERVER_DOCKER_IMAGE_TAG" diff --git a/sdk/python/feast/infra/aws.py b/sdk/python/feast/infra/aws.py index 2ff38de9f9..04b4abb48b 100644 --- a/sdk/python/feast/infra/aws.py +++ b/sdk/python/feast/infra/aws.py @@ -2,7 +2,6 @@ import hashlib import logging import os -import subprocess import uuid from datetime import datetime from pathlib import Path @@ -12,10 +11,10 @@ from colorama import Fore, Style -from feast import flags_helper from feast.constants import ( AWS_LAMBDA_FEATURE_SERVER_IMAGE, AWS_LAMBDA_FEATURE_SERVER_REPOSITORY, + DOCKER_IMAGE_TAG_ENV_NAME, FEAST_USAGE, FEATURE_STORE_YAML_ENV_NAME, ) @@ -322,27 +321,21 @@ def _get_lambda_name(project: str): def _get_docker_image_version() -> str: """Returns a version for the feature server Docker image. + If the feast.constants.DOCKER_IMAGE_TAG_ENV_NAME environment variable is set, + we return that (mostly used for integration tests, but can be used for local testing too). + For public Feast releases this equals to the Feast SDK version modified by replacing "." with "_". For example, Feast SDK version "0.14.1" would correspond to Docker image version "0_14_1". - For integration tests this equals to the git commit hash of HEAD. This is necessary, - because integration tests need to use images built from the same commit hash. - During development (when Feast is installed in editable mode) this equals to the Feast SDK version modified by removing the "dev..." suffix and replacing "." with "_". For example, Feast SDK version "0.14.1.dev41+g1cbfa225.d20211103" would correspond to Docker image version "0_14_1". This way, Feast SDK will use an already existing Docker image built during the previous public release. """ - if flags_helper.is_test(): - # Note: this should be in sync with https://github.com/feast-dev/feast/blob/6fbe01b6e9a444dc77ec3328a54376f4d9387664/.github/workflows/pr_integration_tests.yml#L41 - return ( - subprocess.check_output( - ["git", "rev-parse", "HEAD"], cwd=Path(__file__).resolve().parent - ) - .decode() - .strip() - ) + tag = os.environ.get(DOCKER_IMAGE_TAG_ENV_NAME) + if tag is not None: + return tag else: version = get_version() if "dev" in version: