Skip to content

Commit

Permalink
restructuring files in selective check
Browse files Browse the repository at this point in the history
  • Loading branch information
edithturn committed Apr 14, 2022
1 parent 90a4c87 commit ad3edb8
Showing 6 changed files with 245 additions and 0 deletions.
15 changes: 15 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -55,6 +55,7 @@ concurrency:
jobs:

build-info:
timeout-minutes: 30
name: "Build info"
# The runs-on cannot refer to env. or secrets. context, so we have no
# option but to specify a hard-coded list here. This is "safe", as the list
@@ -121,6 +122,8 @@ jobs:
) && 'self-hosted' || 'ubuntu-20.04' }}
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
MOUNT_SELECTED_LOCAL_SOURCES: "true"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
outputs:
defaultBranch: ${{ steps.selective-checks.outputs.default-branch }}
cacheDirective: ${{ steps.dynamic-outputs.outputs.cacheDirective }}
@@ -198,6 +201,18 @@ jobs:
# Run all checks
./scripts/ci/selective_ci_checks.sh
fi
- name: "Setup python"
uses: actions/setup-python@v2
with:
python-version: ${{ needs.build-info.outputs.defaultPythonVersion }}
cache: 'pip'
cache-dependency-path: ./dev/breeze/setup*
- run: python -m pip install --user ./dev/breeze/
- name: Selective Checks Python
id: selective-checks-python
env:
PR_LABELS: "${{ steps.source-run-info.outputs.pullRequestLabels }}"
run: airflow-selective-checks build-image
# Avoid having to specify the runs-on logic every time. We use the custom
# env var AIRFLOW_SELF_HOSTED_RUNNER set only on our runners, but never
# on the public runners
1 change: 1 addition & 0 deletions dev/breeze/setup.cfg
Original file line number Diff line number Diff line change
@@ -73,6 +73,7 @@ console_scripts=
breeze=airflow_breeze.breeze:main
airflow-freespace=airflow_ci.freespace:main
airflow-find-newer-dependencies=airflow_ci.find_newer_dependencies:main
airflow-selective-checks=airflow_breeze.build_image.ci.selective_checks.selective_ci_checks:main

[bdist_wheel]
python-tag=py3
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Selective Check."""
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#!/usr/bin/env python3

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

import os
import os.path

import click

from airflow_breeze.build_image.ci.selective_checks.utilities import get_changed_files
from airflow_breeze.build_image.ci.selective_checks.validate_selective_checks import (
check_if_any_py_files_changed,
validate_github_default_branch,
validate_github_sha,
)

GITHUB_SHA = os.environ.get('GITHUB_SHA')
DEFAULT_BRANCH = os.environ.get('DEFAULT_BRANCH')
PR_LABELS = os.environ.get('PR_LABELS')
INCOMING_COMMIT_SHA = os.environ.get('GITHUB_SHA')


ANY_PY_FILES_CHANGED = r"\.py$"
AIRFLOW_SOURCES_TRIGGERING_TESTS = """ '^.pre-commit-config.yaml$'
'^airflow'
'^chart'
'^tests'
'^kubernetes_tests'"""
upgrade_to_newer_dependencies = "false"
print(AIRFLOW_SOURCES_TRIGGERING_TESTS)
print(ANY_PY_FILES_CHANGED)


@click.group()
def main():
validate_github_sha(GITHUB_SHA)
validate_github_default_branch(DEFAULT_BRANCH)
get_changed_files(INCOMING_COMMIT_SHA)
check_if_any_py_files_changed()


@main.command(name='build-image')
def build_image():
print("BUILD IMAGE")


@main.command(name='matrix-strategy')
def matrix_strategy():
print("PRINT python-versions")


if __name__ == '__main__':
main()
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#!/usr/bin/env python3

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.


import os
import os.path
import subprocess

INCOMING_COMMIT_SHA = os.environ.get('GITHUB_SHA')


def get_changed_files(INCOMING_COMMIT_SHA):
print("")
print(f"Incoming commit SHA: ${INCOMING_COMMIT_SHA}")
print("")
print(f"Changed files from ${INCOMING_COMMIT_SHA} vs it's first parent")
print("")
CHANGED_FILES = (
subprocess.check_output(
[
"git",
"diff-tree",
"--no-commit-id",
"--name-only",
"-r",
f"{INCOMING_COMMIT_SHA}^",
f"{INCOMING_COMMIT_SHA}",
]
)
.decode('utf-8')
.strip()
)

if not CHANGED_FILES:
print("")
# print(f"${COLOR_YELLOW}WARNING: Could not find any changed files ${COLOR_RESET}"
print("WARNING: Could not find any changed files")
print("Assuming that we should run all tests in this case")
print("")
print("set_outputs_run_everything_and_exit")
else:
print("")
print("Changed files:")
print("")
print(CHANGED_FILES)

return CHANGED_FILES
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.


import os
import os.path


from airflow_breeze.build_image.ci.selective_checks.utilities import (
get_changed_files
)

INCOMING_COMMIT_SHA = os.environ.get('GITHUB_SHA')

def validate_pull_request_label(PR_LABELS):
if "full tests needed" in PR_LABELS:
print(f"Found the right PR labels in ${PR_LABELS} : 'full tests needed'")
FULL_TESTS_NEEDED_LABEL = True
else:
print(f"Did not find the right PR labels in ${PR_LABELS}: 'full tests needed'")
FULL_TESTS_NEEDED_LABEL = False
return FULL_TESTS_NEEDED_LABEL


# Call it from Python Main
def validate_github_sha(GITHUB_SHA):
PR_LABELS = os.environ.get('PR_LABELS')
FULL_TESTS_NEEDED_LABEL = validate_pull_request_label(PR_LABELS)
global INCOMING_COMMIT_SHA
if not GITHUB_SHA:
print("")
print("No Commit SHA - running all tests (likely direct merge, or scheduled run)!")
print("")
INCOMING_COMMIT_SHA = ""
FULL_TESTS_NEEDED_LABEL = "true"
# output_all_basic_variables
# output_all_basic_variables()
# check_upgrade_to_newer_dependencies_needed
# set_outputs_run_everything_and_exit
print(FULL_TESTS_NEEDED_LABEL)
else:
INCOMING_COMMIT_SHA = GITHUB_SHA
print("")
print(f"Commit SHA passed: ${INCOMING_COMMIT_SHA}!")
print("")


def validate_github_default_branch(default_branch):
# === Validationg BRANCH ===
if default_branch == "main":
ALL_TESTS = "Always API Core Other CLI Providers WWW Integration"
print(f'ALL_TESTS {ALL_TESTS}')
else:
# Skips Provider tests in case current default branch is not main
ALL_TESTS = "Always API Core Other CLI WWW Integration"
print(f'ALL_TESTS {ALL_TESTS}')

# Validations
def check_if_any_py_files_changed():
""""
Checking if Python files changed, if yes Build Image
"""
CHANGED_FILES = get_changed_files(INCOMING_COMMIT_SHA)
if ".py" in CHANGED_FILES:
print("check files changed and if there is python files BUILD IMAGE")
else:
print("No build image")

0 comments on commit ad3edb8

Please sign in to comment.