ci: Make version bumping script more robust (#2689)

* ci: Make version bumping script more robust Signed-off-by: Achal Shah <[email protected]> * bump dockerfiles too Signed-off-by: Achal Shah <[email protected]> * fix build-docker-image Signed-off-by: Achal Shah <[email protected]> * use buildx instead of build in makefile Signed-off-by: Achal Shah <[email protected]>
feast-dev · May 16, 2022 · 1958df0 · 1958df0
1 parent b3ba8aa
commit 1958df0
Show file tree

Hide file tree

Showing 7 changed files with 52 additions and 63 deletions.
diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml
@@ -20,7 +20,7 @@ jobs:
       (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm')))
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           # pull_request_target runs the workflow in the context of the base repo
           # as such actions/checkout needs to be explicit configured to retrieve
@@ -30,7 +30,9 @@ jobs:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
+        with:
+          install: true
       - name: Set up AWS SDK
         uses: aws-actions/configure-aws-credentials@v1
         with:

diff --git a/Makefile b/Makefile
@@ -212,7 +212,7 @@ push-feature-server-python-aws-docker:
 		docker push $(REGISTRY)/feature-server-python-aws:$$VERSION
 
 build-feature-server-python-aws-docker:
-		docker build --build-arg VERSION=$$VERSION \
+		docker buildx --build-arg VERSION=$$VERSION \
 			-t $(REGISTRY)/feature-server-python-aws:$$VERSION \
 			-f sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile .
 

diff --git a/infra/scripts/release/bump_file_versions.py b/infra/scripts/release/bump_file_versions.py
@@ -33,46 +33,22 @@ def main() -> None:
     with open(path_to_file_list, "r") as f:
         files_to_bump = f.read().splitlines()
 
-    # The current version should be 0.18.0 or 0.19.0 or 0.20.0 etc, but we should also make sure to support the
-    # occasional patch release on the master branch like 0.18.1 or 0.18.2
-    versions_in_files = 0
-    if current_version[-2:] != ".0":
-        print(current_version[-2:])
-        versions_in_files = count_version(current_version, files_to_bump, repo_root)
-        if versions_in_files != VERSIONS_TO_BUMP:
-            raise SystemExit(f"Found {versions_in_files} occurrences of {current_version} in files to bump, but "
-                             f"expected {VERSIONS_TO_BUMP}")
-    else:
-        found = False
-
-        # Lets make sure the files don't contain a patch version (e.g, 0.x.0 -> 0.x.20)
-        for patch_version in range(0, 20):
-            current_version_patch = current_version[:-1] + str(patch_version)
-            versions_in_files = count_version(current_version_patch, files_to_bump, repo_root)
-
-            # We are using a patch version, let's change our version number
-            if versions_in_files == VERSIONS_TO_BUMP:
-                print(f"Found {versions_in_files} occurrences of {current_version_patch}, changing current version to "
-                      f"{current_version_patch}")
-                current_version = current_version_patch
-                found = True
-                break
-            else:
-                print(f"Found {versions_in_files} occurrences of {current_version_patch}, instead of {VERSIONS_TO_BUMP}")
-        if not found:
-            raise SystemExit(f"Could not find {VERSIONS_TO_BUMP} versions of {current_version} in {path_to_file_list}")
-
-    print(f"Found {versions_in_files} occurrences of {current_version} in files to bump {path_to_file_list}")
+    # The current version should be 0.18.0 or 0.19.0 or 0.20.0 etc
+    validate_files_to_bump(current_version, files_to_bump, repo_root)
 
     # Bump the version in the files
     updated_count = 0
     for file in files_to_bump:
-        with open(repo_root.joinpath(file), "r") as f:
-            file_contents = f.read()
-            file_contents = file_contents.replace(current_version, new_version)
-
-        with open(repo_root.joinpath(file), "w") as f:
-            f.write(file_contents)
+        components = file.split(" ")
+        file_path = components[0]
+        lines = components[1:]
+        with open(repo_root.joinpath(file_path), "r") as f:
+            file_contents = f.readlines()
+            for line in lines:
+                file_contents[int(line) - 1] = file_contents[int(line) - 1].replace(current_version, new_version)
+
+        with open(repo_root.joinpath(file_path), "w") as f:
+            f.write(''.join(file_contents))
             updated_count += 1
 
     print(f"Updated {updated_count} files with new version {new_version}")
@@ -88,14 +64,19 @@ def is_semantic_version(version: str) -> bool:
     return True
 
 
-def count_version(current_version, files_to_bump, repo_root):
-    # Count how many of the existing versions we find
-    total = 0
+def validate_files_to_bump(current_version, files_to_bump, repo_root):
     for file in files_to_bump:
-        with open(repo_root.joinpath(file), "r") as f:
-            file_contents = f.read()
-            total += file_contents.count(current_version)
-    return total
+        components = file.split(" ")
+        assert len(components) > 1, f"Entry {file} should have a file name, and a list of line numbers with versions"
+        file_path = components[0]
+        lines = components[1:]
+        with open(repo_root.joinpath(file_path), "r") as f:
+            file_contents = f.readlines()
+            for line in lines:
+                assert current_version in file_contents[int(line) - 1], (
+                    f"File `{file_path}` line `{line}` didn't contain version {current_version}. "
+                    f"Contents: {file_contents[int(line) - 1]}"
+                )
 
 
 if __name__ == "__main__":

diff --git a/infra/scripts/release/files_to_bump.txt b/infra/scripts/release/files_to_bump.txt
@@ -1,13 +1,12 @@
-infra/charts/feast/requirements.yaml
-infra/charts/feast/Chart.yaml
-infra/charts/feast/charts/transformation-service/Chart.yaml
-infra/charts/feast/charts/transformation-service/README.md
-infra/charts/feast/charts/transformation-service/values.yaml
-infra/charts/feast/charts/feature-server/Chart.yaml
-infra/charts/feast/charts/feature-server/README.md
-infra/charts/feast/charts/feature-server/values.yaml
-infra/charts/feast/README.md
-infra/charts/feast-python-server/Chart.yaml
-infra/charts/feast-python-server/README.md
-java/pom.xml
-ui/package.json
+infra/charts/feast/requirements.yaml 4 9
+infra/charts/feast/Chart.yaml 4
+infra/charts/feast/charts/transformation-service/Chart.yaml 4 5
+infra/charts/feast/charts/transformation-service/README.md 3 16
+infra/charts/feast/charts/transformation-service/values.yaml 8
+infra/charts/feast/charts/feature-server/Chart.yaml 4 5
+infra/charts/feast/charts/feature-server/README.md 3 20
+infra/charts/feast/charts/feature-server/values.yaml 8
+infra/charts/feast/README.md 11 58 59
+infra/charts/feast-python-server/Chart.yaml 5
+infra/charts/feast-python-server/README.md 3
+java/pom.xml 41
diff --git a/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile b/sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile
@@ -1,5 +1,8 @@
 FROM public.ecr.aws/lambda/python:3.9
 
+RUN yum install -y git
+
+
 # Copy app handler code
 COPY sdk/python/feast/infra/feature_servers/aws_lambda/app.py ${LAMBDA_TASK_ROOT}
 
@@ -16,7 +19,7 @@ COPY README.md README.md
 # git dir to infer the version of feast we're installing.
 # https://github.com/pypa/setuptools_scm#usage-from-docker
 # I think it also assumes that this dockerfile is being built from the root of the directory.
-RUN SETUPTOOLS_SCM_PRETEND_VERSION=1 pip3 install --no-cache-dir -e '.[aws,redis]'
+RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir -e '.[aws,redis]'
 RUN pip3 install -r sdk/python/feast/infra/feature_servers/aws_lambda/requirements.txt --target "${LAMBDA_TASK_ROOT}"
 
 # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)

diff --git a/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile b/sdk/python/feast/infra/feature_servers/gcp_cloudrun/Dockerfile
@@ -1,5 +1,7 @@
 FROM python:3.9-slim
 
+RUN apt-get update && apt-get install -y git
+
 # Allow statements and log messages to immediately appear in the Knative logs
 ENV PYTHONUNBUFFERED True
 
@@ -18,7 +20,7 @@ COPY pyproject.toml pyproject.toml
 COPY README.md ./README.md
 
 # Install production dependencies.
-RUN pip install --no-cache-dir \
+RUN --mount=source=.git,target=.git,type=bind pip install --no-cache-dir \
     -e '.[gcp,redis]' \
     -r ./sdk/python/feast/infra/feature_servers/gcp_cloudrun/requirements.txt
 

diff --git a/sdk/python/feast/infra/transformation_servers/Dockerfile b/sdk/python/feast/infra/transformation_servers/Dockerfile
@@ -1,4 +1,6 @@
-FROM python:3.7-slim
+FROM python:3.9-slim
+
+RUN apt-get update && apt-get install -y git
 
 # Copy app handler code
 COPY sdk/python/feast/infra/transformation_servers/app.py app.py
@@ -13,7 +15,7 @@ COPY README.md README.md
 
 
 # Install dependencies
-RUN pip3 install -e '.'
+RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir -e '.'
 
 # Start feature transformation server
 CMD [ "python", "app.py" ]