From 9806ca39b0e35103cd8f0326630d6dcd370b1d63 Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Fri, 19 Aug 2022 12:54:59 +0700 Subject: [PATCH 1/4] 11815 New github action to test projects that depend on the project being changed --- .../report-connectors-dependency.yml | 42 ++++++++++ tools/bin/ci_check_dependency.py | 84 +++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 .github/workflows/report-connectors-dependency.yml create mode 100644 tools/bin/ci_check_dependency.py diff --git a/.github/workflows/report-connectors-dependency.yml b/.github/workflows/report-connectors-dependency.yml new file mode 100644 index 000000000000..6532199515d8 --- /dev/null +++ b/.github/workflows/report-connectors-dependency.yml @@ -0,0 +1,42 @@ +name: Report connectors dependency + +on: + pull_request: + +jobs: + report-affected-connectors: + name: "Report affected connectors" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # OR "2" -> To retrieve the preceding commit. + + - name: Extract current branch name + shell: bash + run: echo "##[set-output name=branch;]$(echo ${{ github.head_ref }})" + id: extract_branch + - name: Extract git-dif change file + run: | + git diff --name-only remotes/origin/master...origin/${{ steps.extract_branch.outputs.branch }} > ./changed_files.txt + id: extract_changed_files + - name: Install dependencies + run: | + python -m pip install --upgrade pip + - name: Create dependency report + run: | + output=$(python ./tools/bin/ci_check_dependency.py ./changed_files.txt) + output="${output//'%'/'%25'}" + output="${output//$'\n'/'%0A'}" + output="${output//$'\r'/'%0D'}" + echo "::set-output name=changed_files_report::$output" + id: dependency_report + - name: Comment report in PR + if: steps.dependency_report.outputs.changed_files_report != '' + uses: peter-evans/create-or-update-comment@v2 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + **NOTE** :warning: Changes in this PR affect the following connectors. Make sure to run corresponding integration tests: + ${{steps.dependency_report.outputs.changed_files_report}} + diff --git a/tools/bin/ci_check_dependency.py b/tools/bin/ci_check_dependency.py new file mode 100644 index 000000000000..b3676746c93a --- /dev/null +++ b/tools/bin/ci_check_dependency.py @@ -0,0 +1,84 @@ +import sys +import os +import os.path + +CONNECTOR_FOLDER = "airbyte-integrations/connectors/" +CONNECTOR_PATH = "./airbyte-integrations/connectors/" +WHITE_LIST = ["/src/main/", "build.gradle"] + + +def main(): + git_diff_file_path = ' '.join(sys.argv[1:]) + + # Get changed connectors files + connectors_changed_files = get_connectors_changed_files(git_diff_file_path) + # Get changed connectors + changed_connectors = get_changed_connectors(connectors_changed_files) + + # Get all existing connectors + all_connectors = get_all_connectors() + + # Getting all build.gradle file + all_build_gradle_files = get_connectors_gradle_files(all_connectors) + + # Try to find dependency in build.gradle file + depended_connectors = list(set(get_depended_connectors(changed_connectors, all_build_gradle_files))) + + write_report(depended_connectors) + + +def get_connectors_changed_files(path): + changed_connectors_files = [] + with open(path) as file: + for line in file: + if CONNECTOR_FOLDER in line: + changed_connectors_files.append(line) + return list(set(changed_connectors_files)) + + +def get_changed_connectors(changed_files): + changed_connectors = [] + for changed_file in changed_files: + # Check if this file should be ignored + if any(ignor in changed_file for ignor in WHITE_LIST): + changed_connectors.append(changed_file.split("/")[2]) + return list(set(changed_connectors)) + + +def get_all_connectors(): + walk = os.walk(CONNECTOR_PATH) + return [connector for connector in next(walk)[1]] + + +def get_connectors_gradle_files(all_connectors): + all_build_gradle_files = {} + for connector in all_connectors: + build_gradle_path = CONNECTOR_PATH + connector + "/" + build_gradle_file = find_file("build.gradle", build_gradle_path) + all_build_gradle_files[connector] = build_gradle_file + return all_build_gradle_files + + +def find_file(name, path): + for root, dirs, files in os.walk(path): + if name in files: + return os.path.join(root, name) + + +def get_depended_connectors(changed_connectors, all_build_gradle_files): + depended_connectors = [] + for changed_connector in changed_connectors: + for connector, gradle_file in all_build_gradle_files.items(): + with open(gradle_file) as f: + if changed_connector in f.read(): + depended_connectors.append(connector) + return depended_connectors + + +def write_report(depended_connectors): + for depended_connector in depended_connectors: + print("- " + depended_connector) + + +if __name__ == "__main__": + main() From 354010c0d00fc7aac61bcedab7d9b890bbd24b18 Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Fri, 19 Aug 2022 12:57:50 +0700 Subject: [PATCH 2/4] 11815 Dummy commit to destination-gcs --- .../integrations/destination/gcs/GcsStorageOperations.java | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java index 58881e4042e6..4fdf55a81479 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java @@ -51,3 +51,4 @@ protected Map getMetadataMapping() { } } + From d4f10f4f7e6a960a1fd5d8c60ce97a21beb289ba Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Tue, 23 Aug 2022 16:59:29 +0700 Subject: [PATCH 3/4] 11815 Updated git diff, Updated python script --- .../report-connectors-dependency.yml | 2 +- tools/bin/ci_check_dependency.py | 44 +++++++++++-------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.github/workflows/report-connectors-dependency.yml b/.github/workflows/report-connectors-dependency.yml index 6532199515d8..8e60d2c94b49 100644 --- a/.github/workflows/report-connectors-dependency.yml +++ b/.github/workflows/report-connectors-dependency.yml @@ -18,7 +18,7 @@ jobs: id: extract_branch - name: Extract git-dif change file run: | - git diff --name-only remotes/origin/master...origin/${{ steps.extract_branch.outputs.branch }} > ./changed_files.txt + git diff --name-only remotes/origin/master...origin/${{ steps.extract_branch.outputs.branch }} -- airbyte-integrations/ > ./changed_files.txt id: extract_changed_files - name: Install dependencies run: | diff --git a/tools/bin/ci_check_dependency.py b/tools/bin/ci_check_dependency.py index b3676746c93a..df983e65c311 100644 --- a/tools/bin/ci_check_dependency.py +++ b/tools/bin/ci_check_dependency.py @@ -4,16 +4,25 @@ CONNECTOR_FOLDER = "airbyte-integrations/connectors/" CONNECTOR_PATH = "./airbyte-integrations/connectors/" -WHITE_LIST = ["/src/main/", "build.gradle"] +IGNORE_LIST = [ + # Java + "/src/test/","/src/test-integration/", "/src/testFixtures/", + # Python + "/integration_tests/", "/unit_tests/", + # Common + "acceptance-test-config.yml", "acceptance-test-docker.sh", ".md", ".dockerignore", ".gitignore", "requirements.txt"] def main(): + # Used git diff checks airbyte-integrations/ folder only + # See .github/workflows/report-connectors-dependency.yml file git_diff_file_path = ' '.join(sys.argv[1:]) - # Get changed connectors files - connectors_changed_files = get_connectors_changed_files(git_diff_file_path) - # Get changed connectors - changed_connectors = get_changed_connectors(connectors_changed_files) + # Get changed files + changed_files = get_changed_files(git_diff_file_path) + # Get changed modules. e.g. source-acceptance-test from airbyte-integrations/bases/ + # or destination-mysql from airbyte-integrations/connectors/ + changed_modules = get_changed_modules(changed_files) # Get all existing connectors all_connectors = get_all_connectors() @@ -22,27 +31,26 @@ def main(): all_build_gradle_files = get_connectors_gradle_files(all_connectors) # Try to find dependency in build.gradle file - depended_connectors = list(set(get_depended_connectors(changed_connectors, all_build_gradle_files))) - + depended_connectors = list(set(get_depended_connectors(changed_modules, all_build_gradle_files))) write_report(depended_connectors) -def get_connectors_changed_files(path): +def get_changed_files(path): changed_connectors_files = [] with open(path) as file: for line in file: - if CONNECTOR_FOLDER in line: - changed_connectors_files.append(line) - return list(set(changed_connectors_files)) + changed_connectors_files.append(line) + return changed_connectors_files -def get_changed_connectors(changed_files): - changed_connectors = [] +def get_changed_modules(changed_files): + changed_modules = [] for changed_file in changed_files: # Check if this file should be ignored - if any(ignor in changed_file for ignor in WHITE_LIST): - changed_connectors.append(changed_file.split("/")[2]) - return list(set(changed_connectors)) + if not any(ignor in changed_file for ignor in IGNORE_LIST): + split_changed_file = changed_file.split("/") + changed_modules.append(split_changed_file[1] + ":" + split_changed_file[2]) + return list(set(changed_modules)) def get_all_connectors(): @@ -69,8 +77,8 @@ def get_depended_connectors(changed_connectors, all_build_gradle_files): depended_connectors = [] for changed_connector in changed_connectors: for connector, gradle_file in all_build_gradle_files.items(): - with open(gradle_file) as f: - if changed_connector in f.read(): + with open(gradle_file) as file: + if changed_connector in file.read(): depended_connectors.append(connector) return depended_connectors From c0ed398da978eadc0e734a75961fb14a5060018b Mon Sep 17 00:00:00 2001 From: Yevhen Sukhomud Date: Tue, 23 Aug 2022 17:32:07 +0700 Subject: [PATCH 4/4] 11815 Removed test change --- .../integrations/destination/gcs/GcsStorageOperations.java | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java index 4fdf55a81479..58881e4042e6 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java @@ -51,4 +51,3 @@ protected Map getMetadataMapping() { } } -