diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 089d9f4733..c441705613 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -20,7 +20,7 @@ jobs: (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm'))) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve @@ -30,7 +30,9 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 + with: + install: true - name: Set up AWS SDK uses: aws-actions/configure-aws-credentials@v1 with: diff --git a/Makefile b/Makefile index e208ab3c45..96558f97de 100644 --- a/Makefile +++ b/Makefile @@ -212,7 +212,7 @@ push-feature-server-python-aws-docker: docker push $(REGISTRY)/feature-server-python-aws:$$VERSION build-feature-server-python-aws-docker: - docker build --build-arg VERSION=$$VERSION \ + docker buildx --build-arg VERSION=$$VERSION \ -t $(REGISTRY)/feature-server-python-aws:$$VERSION \ -f sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile . diff --git a/infra/scripts/release/bump_file_versions.py b/infra/scripts/release/bump_file_versions.py index b706c2c141..e17463c2c7 100644 --- a/infra/scripts/release/bump_file_versions.py +++ b/infra/scripts/release/bump_file_versions.py @@ -33,46 +33,22 @@ def main() -> None: with open(path_to_file_list, "r") as f: files_to_bump = f.read().splitlines() - # The current version should be 0.18.0 or 0.19.0 or 0.20.0 etc, but we should also make sure to support the - # occasional patch release on the master branch like 0.18.1 or 0.18.2 - versions_in_files = 0 - if current_version[-2:] != ".0": - print(current_version[-2:]) - versions_in_files = count_version(current_version, files_to_bump, repo_root) - if versions_in_files != VERSIONS_TO_BUMP: - raise SystemExit(f"Found {versions_in_files} occurrences of {current_version} in files to bump, but " - f"expected {VERSIONS_TO_BUMP}") - else: - found = False - - # Lets make sure the files don't contain a patch version (e.g, 0.x.0 -> 0.x.20) - for patch_version in range(0, 20): - current_version_patch = current_version[:-1] + str(patch_version) - versions_in_files = count_version(current_version_patch, files_to_bump, repo_root) - - # We are using a patch version, let's change our version number - if versions_in_files == VERSIONS_TO_BUMP: - print(f"Found {versions_in_files} occurrences of {current_version_patch}, changing current version to " - f"{current_version_patch}") - current_version = current_version_patch - found = True - break - else: - print(f"Found {versions_in_files} occurrences of {current_version_patch}, instead of {VERSIONS_TO_BUMP}") - if not found: - raise SystemExit(f"Could not find {VERSIONS_TO_BUMP} versions of {current_version} in {path_to_file_list}") - - print(f"Found {versions_in_files} occurrences of {current_version} in files to bump {path_to_file_list}") + # The current version should be 0.18.0 or 0.19.0 or 0.20.0 etc + validate_files_to_bump(current_version, files_to_bump, repo_root) # Bump the version in the files updated_count = 0 for file in files_to_bump: - with open(repo_root.joinpath(file), "r") as f: - file_contents = f.read() - file_contents = file_contents.replace(current_version, new_version) - - with open(repo_root.joinpath(file), "w") as f: - f.write(file_contents) + components = file.split(" ") + file_path = components[0] + lines = components[1:] + with open(repo_root.joinpath(file_path), "r") as f: + file_contents = f.readlines() + for line in lines: + file_contents[int(line) - 1] = file_contents[int(line) - 1].replace(current_version, new_version) + + with open(repo_root.joinpath(file_path), "w") as f: + f.write(''.join(file_contents)) updated_count += 1 print(f"Updated {updated_count} files with new version {new_version}") @@ -88,14 +64,19 @@ def is_semantic_version(version: str) -> bool: return True -def count_version(current_version, files_to_bump, repo_root): - # Count how many of the existing versions we find - total = 0 +def validate_files_to_bump(current_version, files_to_bump, repo_root): for file in files_to_bump: - with open(repo_root.joinpath(file), "r") as f: - file_contents = f.read() - total += file_contents.count(current_version) - return total + components = file.split(" ") + assert len(components) > 1, f"Entry {file} should have a file name, and a list of line numbers with versions" + file_path = components[0] + lines = components[1:] + with open(repo_root.joinpath(file_path), "r") as f: + file_contents = f.readlines() + for line in lines: + assert current_version in file_contents[int(line) - 1], ( + f"File `{file_path}` line `{line}` didn't contain version {current_version}. " + f"Contents: {file_contents[int(line) - 1]}" + ) if __name__ == "__main__": diff --git a/infra/scripts/release/files_to_bump.txt b/infra/scripts/release/files_to_bump.txt index 2c3eece6be..a1e2d29623 100644 --- a/infra/scripts/release/files_to_bump.txt +++ b/infra/scripts/release/files_to_bump.txt @@ -1,13 +1,12 @@ -infra/charts/feast/requirements.yaml -infra/charts/feast/Chart.yaml -infra/charts/feast/charts/transformation-service/Chart.yaml -infra/charts/feast/charts/transformation-service/README.md -infra/charts/feast/charts/transformation-service/values.yaml -infra/charts/feast/charts/feature-server/Chart.yaml -infra/charts/feast/charts/feature-server/README.md -infra/charts/feast/charts/feature-server/values.yaml -infra/charts/feast/README.md -infra/charts/feast-python-server/Chart.yaml -infra/charts/feast-python-server/README.md -java/pom.xml -ui/package.json \ No newline at end of file +infra/charts/feast/requirements.yaml 4 9 +infra/charts/feast/Chart.yaml 4 +infra/charts/feast/charts/transformation-service/Chart.yaml 4 5 +infra/charts/feast/charts/transformation-service/README.md 3 16 +infra/charts/feast/charts/transformation-service/values.yaml 8 +infra/charts/feast/charts/feature-server/Chart.yaml 4 5 +infra/charts/feast/charts/feature-server/README.md 3 20 +infra/charts/feast/charts/feature-server/values.yaml 8 +infra/charts/feast/README.md 11 58 59 +infra/charts/feast-python-server/Chart.yaml 5 +infra/charts/feast-python-server/README.md 3 +java/pom.xml 41 diff --git a/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile b/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile index fe997afcae..929227a810 100644 --- a/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile @@ -1,5 +1,8 @@ FROM public.ecr.aws/lambda/python:3.9 +RUN yum install -y git + + # Copy app handler code COPY sdk/python/feast/infra/feature_servers/aws_lambda/app.py ${LAMBDA_TASK_ROOT} @@ -16,7 +19,7 @@ COPY README.md README.md # git dir to infer the version of feast we're installing. # https://github.com/pypa/setuptools_scm#usage-from-docker # I think it also assumes that this dockerfile is being built from the root of the directory. -RUN SETUPTOOLS_SCM_PRETEND_VERSION=1 pip3 install --no-cache-dir -e '.[aws,redis]' +RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir -e '.[aws,redis]' RUN pip3 install -r sdk/python/feast/infra/feature_servers/aws_lambda/requirements.txt --target "${LAMBDA_TASK_ROOT}" # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile index e0e16f6a14..6e3ff424ea 100644 --- a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile @@ -1,5 +1,7 @@ FROM python:3.9-slim +RUN apt-get update && apt-get install -y git + # Allow statements and log messages to immediately appear in the Knative logs ENV PYTHONUNBUFFERED True @@ -18,7 +20,7 @@ COPY pyproject.toml pyproject.toml COPY README.md ./README.md # Install production dependencies. -RUN pip install --no-cache-dir \ +RUN --mount=source=.git,target=.git,type=bind pip install --no-cache-dir \ -e '.[gcp,redis]' \ -r ./sdk/python/feast/infra/feature_servers/gcp_cloudrun/requirements.txt diff --git a/sdk/python/feast/infra/transformation_servers/Dockerfile b/sdk/python/feast/infra/transformation_servers/Dockerfile index ff276c7a70..5e77144757 100644 --- a/sdk/python/feast/infra/transformation_servers/Dockerfile +++ b/sdk/python/feast/infra/transformation_servers/Dockerfile @@ -1,4 +1,6 @@ -FROM python:3.7-slim +FROM python:3.9-slim + +RUN apt-get update && apt-get install -y git # Copy app handler code COPY sdk/python/feast/infra/transformation_servers/app.py app.py @@ -13,7 +15,7 @@ COPY README.md README.md # Install dependencies -RUN pip3 install -e '.' +RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir -e '.' # Start feature transformation server CMD [ "python", "app.py" ]