From ad3edb85fde5053619564e92b7a47651215a399f Mon Sep 17 00:00:00 2001 From: edithturn Date: Wed, 16 Mar 2022 16:50:48 -0500 Subject: [PATCH] restructuring files in selective check --- .github/workflows/ci.yml | 15 ++++ dev/breeze/setup.cfg | 1 + .../ci/selective_checks/__init__.py | 17 ++++ .../selective_checks/selective_ci_checks.py | 68 ++++++++++++++++ .../ci/selective_checks/utilities.py | 63 +++++++++++++++ .../validate_selective_checks.py | 81 +++++++++++++++++++ 6 files changed, 245 insertions(+) create mode 100644 dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/__init__.py create mode 100755 dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/selective_ci_checks.py create mode 100755 dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/utilities.py create mode 100644 dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/validate_selective_checks.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd51fbcd84b66..6bc8c99a24979 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,7 @@ concurrency: jobs: build-info: + timeout-minutes: 30 name: "Build info" # The runs-on cannot refer to env. or secrets. context, so we have no # option but to specify a hard-coded list here. This is "safe", as the list @@ -121,6 +122,8 @@ jobs: ) && 'self-hosted' || 'ubuntu-20.04' }} env: GITHUB_CONTEXT: ${{ toJson(github) }} + MOUNT_SELECTED_LOCAL_SOURCES: "true" + PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} outputs: defaultBranch: ${{ steps.selective-checks.outputs.default-branch }} cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }} @@ -198,6 +201,18 @@ jobs: # Run all checks ./scripts/ci/selective_ci_checks.sh fi + - name: "Setup python" + uses: actions/setup-python@v2 + with: + python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} + cache: 'pip' + cache-dependency-path: ./dev/breeze/setup* + - run: python -m pip install --user ./dev/breeze/ + - name: Selective Checks Python + id: selective-checks-python + env: + PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}" + run: airflow-selective-checks build-image # Avoid having to specify the runs-on logic every time. We use the custom # env var AIRFLOW_SELF_HOSTED_RUNNER set only on our runners, but never # on the public runners diff --git a/dev/breeze/setup.cfg b/dev/breeze/setup.cfg index abdc8b9c41be5..9e21ad1216971 100644 --- a/dev/breeze/setup.cfg +++ b/dev/breeze/setup.cfg @@ -73,6 +73,7 @@ console_scripts= breeze=airflow_breeze.breeze:main airflow-freespace=airflow_ci.freespace:main airflow-find-newer-dependencies=airflow_ci.find_newer_dependencies:main + airflow-selective-checks=airflow_breeze.build_image.ci.selective_checks.selective_ci_checks:main [bdist_wheel] python-tag=py3 diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/__init__.py b/dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/__init__.py new file mode 100644 index 0000000000000..e540b25cb395a --- /dev/null +++ b/dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Selective Check.""" diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/selective_ci_checks.py b/dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/selective_ci_checks.py new file mode 100755 index 0000000000000..470655b08dc8b --- /dev/null +++ b/dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/selective_ci_checks.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import os.path + +import click + +from airflow_breeze.build_image.ci.selective_checks.utilities import get_changed_files +from airflow_breeze.build_image.ci.selective_checks.validate_selective_checks import ( + check_if_any_py_files_changed, + validate_github_default_branch, + validate_github_sha, +) + +GITHUB_SHA = os.environ.get('GITHUB_SHA') +DEFAULT_BRANCH = os.environ.get('DEFAULT_BRANCH') +PR_LABELS = os.environ.get('PR_LABELS') +INCOMING_COMMIT_SHA = os.environ.get('GITHUB_SHA') + + +ANY_PY_FILES_CHANGED = r"\.py$" +AIRFLOW_SOURCES_TRIGGERING_TESTS = """ '^.pre-commit-config.yaml$' + '^airflow' + '^chart' + '^tests' + '^kubernetes_tests'""" +upgrade_to_newer_dependencies = "false" +print(AIRFLOW_SOURCES_TRIGGERING_TESTS) +print(ANY_PY_FILES_CHANGED) + + +@click.group() +def main(): + validate_github_sha(GITHUB_SHA) + validate_github_default_branch(DEFAULT_BRANCH) + get_changed_files(INCOMING_COMMIT_SHA) + check_if_any_py_files_changed() + + +@main.command(name='build-image') +def build_image(): + print("BUILD IMAGE") + + +@main.command(name='matrix-strategy') +def matrix_strategy(): + print("PRINT python-versions") + + +if __name__ == '__main__': + main() diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/utilities.py b/dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/utilities.py new file mode 100755 index 0000000000000..4e7f2f822b2f3 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/utilities.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import os +import os.path +import subprocess + +INCOMING_COMMIT_SHA = os.environ.get('GITHUB_SHA') + + +def get_changed_files(INCOMING_COMMIT_SHA): + print("") + print(f"Incoming commit SHA: ${INCOMING_COMMIT_SHA}") + print("") + print(f"Changed files from ${INCOMING_COMMIT_SHA} vs it's first parent") + print("") + CHANGED_FILES = ( + subprocess.check_output( + [ + "git", + "diff-tree", + "--no-commit-id", + "--name-only", + "-r", + f"{INCOMING_COMMIT_SHA}^", + f"{INCOMING_COMMIT_SHA}", + ] + ) + .decode('utf-8') + .strip() + ) + + if not CHANGED_FILES: + print("") + # print(f"${COLOR_YELLOW}WARNING: Could not find any changed files ${COLOR_RESET}" + print("WARNING: Could not find any changed files") + print("Assuming that we should run all tests in this case") + print("") + print("set_outputs_run_everything_and_exit") + else: + print("") + print("Changed files:") + print("") + print(CHANGED_FILES) + + return CHANGED_FILES diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/validate_selective_checks.py b/dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/validate_selective_checks.py new file mode 100644 index 0000000000000..de9da3ce7d6b8 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/build_image/ci/selective_checks/validate_selective_checks.py @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import os +import os.path + + +from airflow_breeze.build_image.ci.selective_checks.utilities import ( + get_changed_files +) + +INCOMING_COMMIT_SHA = os.environ.get('GITHUB_SHA') + +def validate_pull_request_label(PR_LABELS): + if "full tests needed" in PR_LABELS: + print(f"Found the right PR labels in ${PR_LABELS} : 'full tests needed'") + FULL_TESTS_NEEDED_LABEL = True + else: + print(f"Did not find the right PR labels in ${PR_LABELS}: 'full tests needed'") + FULL_TESTS_NEEDED_LABEL = False + return FULL_TESTS_NEEDED_LABEL + + +# Call it from Python Main +def validate_github_sha(GITHUB_SHA): + PR_LABELS = os.environ.get('PR_LABELS') + FULL_TESTS_NEEDED_LABEL = validate_pull_request_label(PR_LABELS) + global INCOMING_COMMIT_SHA + if not GITHUB_SHA: + print("") + print("No Commit SHA - running all tests (likely direct merge, or scheduled run)!") + print("") + INCOMING_COMMIT_SHA = "" + FULL_TESTS_NEEDED_LABEL = "true" + # output_all_basic_variables + # output_all_basic_variables() + # check_upgrade_to_newer_dependencies_needed + # set_outputs_run_everything_and_exit + print(FULL_TESTS_NEEDED_LABEL) + else: + INCOMING_COMMIT_SHA = GITHUB_SHA + print("") + print(f"Commit SHA passed: ${INCOMING_COMMIT_SHA}!") + print("") + + +def validate_github_default_branch(default_branch): + # === Validationg BRANCH === + if default_branch == "main": + ALL_TESTS = "Always API Core Other CLI Providers WWW Integration" + print(f'ALL_TESTS {ALL_TESTS}') + else: + # Skips Provider tests in case current default branch is not main + ALL_TESTS = "Always API Core Other CLI WWW Integration" + print(f'ALL_TESTS {ALL_TESTS}') + +# Validations +def check_if_any_py_files_changed(): + """" + Checking if Python files changed, if yes Build Image + """ + CHANGED_FILES = get_changed_files(INCOMING_COMMIT_SHA) + if ".py" in CHANGED_FILES: + print("check files changed and if there is python files BUILD IMAGE") + else: + print("No build image") \ No newline at end of file