Initial commit for new Breeze project

.github/workflows/ci.yml

@@ -237,6 +237,26 @@ jobs:
             echo "::set-output name=runsOn::\"self-hosted\""
           fi
 
+  run-new-breeze-tests:
+    timeout-minutes: 10
+    name: Breeze2 tests
+    runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }}
+    needs: [build-info]
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./dev/breeze
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+          cache: 'pip'
+      - run: pip install .
+      - run: python3 -m pytest -n auto --color=yes
+
   tests-ui:
     timeout-minutes: 10
     name: React UI tests

.pre-commit-config.yaml

@@ -404,6 +404,13 @@ repos:
           ^docs/apache-airflow-providers-apache-hive/commits\.rst$|
           git|
           ^CHANGELOG\.txt$
+      - id: capitalized-breeze
+        language: pygrep
+        name: Only capitalized Breeze used in Breeze2.
+        description: Please use capitalized "Breeze" in the new Breeze docs
+        entry: "breeze"
+        pass_filenames: true
+        files: ^dev/breeze/doc
       - id: base-operator
         language: pygrep
         name: Check BaseOperator[Link] core imports

BREEZE.rst

@@ -2195,14 +2195,14 @@ This is the current syntax for  `./breeze <./breeze>`_:
 
                  all airflow-config-yaml airflow-providers-available airflow-provider-yaml-files-ok
                  base-operator bats-tests bats-in-container-tests black blacken-docs boring-cyborg
-                 build build-providers-dependencies changelog-duplicates check-apache-license
-                 check-builtin-literals check-executables-have-shebangs check-extras-order
-                 check-hooks-apply check-integrations check-merge-conflict check-xml
-                 daysago-import-check debug-statements detect-private-key doctoc dont-use-safe-filter
-                 end-of-file-fixer fix-encoding-pragma flake8 flynt codespell forbid-tabs helm-lint
-                 identity incorrect-use-of-LoggingMixin insert-license isort json-schema
-                 language-matters lint-dockerfile lint-openapi markdownlint mermaid mixed-line-ending
-                 mypy mypy-helm no-providers-in-core-examples no-relative-imports
+                 build build-providers-dependencies capitalized-breeze changelog-duplicates
+                 check-apache-license check-builtin-literals check-executables-have-shebangs
+                 check-extras-order check-hooks-apply check-integrations check-merge-conflict
+                 check-xml daysago-import-check debug-statements detect-private-key doctoc
+                 dont-use-safe-filter end-of-file-fixer fix-encoding-pragma flake8 flynt codespell
+                 forbid-tabs helm-lint identity incorrect-use-of-LoggingMixin insert-license isort
+                 json-schema language-matters lint-dockerfile lint-openapi markdownlint mermaid
+                 mixed-line-ending mypy mypy-helm no-providers-in-core-examples no-relative-imports
                  pre-commit-descriptions pre-commit-hook-names pretty-format-json
                  provide-create-sessions providers-changelogs providers-init-file
                  providers-subpackages-init-file provider-yamls pydevd pydocstyle python-no-log-warn

Breeze2

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# isort: skip
+import os
+import sys
+
+# Python <3.4 does not have pathlib
+if sys.version_info.major != 3 or sys.version_info.minor < 7:
+    print("ERROR! Make sure you use Python 3.7+ !!")
+    sys.exit(1)
+
+import subprocess
+from os import execv
+from pathlib import Path
+
+AIRFLOW_SOURCES_DIR = Path(__file__).parent.resolve()
+BUILD_DIR = AIRFLOW_SOURCES_DIR / ".build"
+BUILD_BREEZE_DIR = BUILD_DIR / "breeze2"
+BUILD_BREEZE_CFG_SAVED = BUILD_BREEZE_DIR / "setup.cfg.saved"
+BUILD_BREEZE_VENV_DIR = BUILD_BREEZE_DIR / "venv"
+BUILD_BREEZE_VENV_BIN_DIR = BUILD_BREEZE_VENV_DIR / "bin"
+BUILD_BREEZE_VENV_PIP = BUILD_BREEZE_VENV_BIN_DIR / "pip"
+BUILD_BREEZE_VENV_BREEZE = BUILD_BREEZE_VENV_BIN_DIR / "Breeze2"
+
+BREEZE_SOURCE_PATH = AIRFLOW_SOURCES_DIR / "dev" / "breeze"
+BREEZE_SETUP_CFG_PATH = BREEZE_SOURCE_PATH / "setup.cfg"
+
+BUILD_BREEZE_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def needs_installation() -> bool:
+    """Returns true if Breeze's virtualenv needs (re)installation"""
+    if not BUILD_BREEZE_VENV_DIR.exists() or not BUILD_BREEZE_CFG_SAVED.exists():
+        return True
+    return BREEZE_SETUP_CFG_PATH.read_text() != BUILD_BREEZE_CFG_SAVED.read_text()
+
+
+def save_config():
+    """Saves cfg file to virtualenv to check if there is a need for reinstallation of the virtualenv"""
+    BUILD_BREEZE_CFG_SAVED.write_text(BREEZE_SETUP_CFG_PATH.read_text())
+
+
+if needs_installation():
+    print(f"(Re)Installing Breeze's virtualenv in {BUILD_BREEZE_VENV_DIR}")
+    BUILD_BREEZE_VENV_DIR.mkdir(parents=True, exist_ok=True)
+    subprocess.run([sys.executable, "-m", "venv", f"{BUILD_BREEZE_VENV_DIR}"], check=True)
+    subprocess.run(
+        [f"{BUILD_BREEZE_VENV_PIP}", "install", "--upgrade", "-e", "."], cwd=BREEZE_SOURCE_PATH, check=True
+    )
+    save_config()
+
+if os.name == 'nt':
+    # This is the best way of running it on Windows, though it leaves the original process hanging around
+    subprocess.run([f"{BUILD_BREEZE_VENV_BREEZE}.exe"] + sys.argv[1:], check=True)
+else:
+    execv(f"{BUILD_BREEZE_VENV_BREEZE}", [f"{BUILD_BREEZE_VENV_BREEZE}"] + sys.argv[1:])

STATIC_CODE_CHECKS.rst

@@ -148,6 +148,8 @@ require Breeze Docker images to be installed locally.
 ------------------------------------ ---------------------------------------------------------------- ------------
 ``build-providers-dependencies``       Regenerates the JSON file with cross-provider dependencies
 ------------------------------------ ---------------------------------------------------------------- ------------
+``capitalized-breeze``                 Breeze has to be Capitalized in Breeze2
+------------------------------------ ---------------------------------------------------------------- ------------
 ``changelog-duplicates``               Checks for duplicate changelog entries
 ------------------------------------ ---------------------------------------------------------------- ------------
 ``check-apache-license``               Checks compatibility with Apache License requirements

breeze-complete

@@ -85,6 +85,7 @@ blacken-docs
 boring-cyborg
 build
 build-providers-dependencies
+capitalized-breeze
 changelog-duplicates
 check-apache-license
 check-builtin-literals

dev/breeze/doc/adr/0001-record-architecture-decisions.md

@@ -0,0 +1,48 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+
+- [1. Record architecture decisions](#1-record-architecture-decisions)
+  - [Status](#status)
+  - [Context](#context)
+  - [Decision](#decision)
+  - [Consequences](#consequences)
+
+# 1. Record architecture decisions
+
+Date: 2021-11-28
+
+## Status
+
+Accepted
+
+## Context
+
+We need to record the architectural decisions made on this project.
+
+## Decision
+
+We will use Architecture Decision Records, as [described by Michael Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions).
+
+## Consequences
+
+See Michael Nygard's article, linked above. For a lightweight ADR toolset, see Nat Pryce's [adr-tools](https://github.com/npryce/adr-tools).

dev/breeze/doc/adr/0002-implement-standalone-python-command.md

@@ -0,0 +1,178 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+
+- [2. Implement standalone python command](#2-implement-standalone-python-command)
+  - [Status](#status)
+  - [Context](#context)
+  - [Decision](#decision)
+  - [Consequences](#consequences)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+# 2. Implement standalone python command
+
+Date: 2021-11-28
+
+## Status
+
+Draft
+
+## Context
+
+The [Breeze](https://github.com/apache/airflow/blob/main/BREEZE.rst) is
+a command line development environment for Apache Airflow that makes
+it easy to setup Airflow development and test environment easily
+(< 10 minutes is the goal) and enable contributors to run any subset
+of tests that are executed in our CI environment easily.
+
+The environment has proven to be very useful (it has successfully onboarded
+a number of new contributors, and it makes the development environment of
+even seasoned contributors much easier as it provides a very easy
+replication of the CI environment as well as very easy to setup test
+environment that can be used to run:
+
+* Unit tests
+* Integration tests
+* Kubernetes/Helm tests
+* System tests
+
+It also serves as a base for our CI execution environment. The same scripts and tools are used
+in our CI (based on GitHub actions). A lot of common code and function between CI and Breeze are
+shared between the CI and Breeze. All those tools are held in "ci" package.
+
+Unfortunately, Breeze is largely based on Bash code - for which very few people (except maybe the
+Breeze creator - Jarek Potiuk, the author of this document) have any other feeling that uneasiness,
+disgust and fear of it :).  Since Airflow is largely based on Python, the common consensus is that
+Breeze should be rewritten in Python.
+
+In November 2021, Outreachy sponsored two internship for two interns: @Bowrna and @edithturn were assigned to
+the projects:
+
+* Convert Airflow Local Development environment `Breeze` - from Bash-based to Python-based
+* Rewrite Github Action workflows to Python
+
+With @potiuk, @eladkal and @xurror as mentors.
+
+The long-standing issues about those two projects are (and we hope to close the projects during the
+three months internship - December 2021 - March 2022):
+
+* https://github.com/apache/airflow/issues/12282
+* https://github.com/apache/airflow/issues/13182
+
+There are a number of problems with Bash scripts:
+
+* They are difficult to understand, modify and debug as Bash "magic" is somewhat arcane
+* They are difficult to implement complex logic with
+* Navigating common code that is used from the scripts is cumbersome and lack IDE/tools support
+* Default Bash on MacOS is very old (from 3.* line) and it will not be updated to a newer version
+  which impacts cross-platform Breeze applicability
+* Bash only works well for Windows in WSL2 environment, which further undermines cross-platform
+  abilities of testing and running Airflow
+
+On the contrary Python - after dropping Python 2 end of life in January 2020, has become much more
+appealing as a common scripting language that can be cross-platform and ubiquitous.
+
+This is the current state of lines of code in the project (generated by `sloccount`):
+
+```
+SLOC    Directory         SLOC-by-Language (Sorted)
+
+144905  tests             python=144761,xml=132,sh=12
+130115  airflow           python=127249,javascript=2827,sh=39
+12052   docs              javascript=8977,python=2931,sh=144
+9073    scripts           sh=7457,python=1616
+6314    chart             python=6218,sh=96
+3665    top_dir           sh=2896,python=769
+3102    dev               python=2938,sh=164
+1723    kubernetes_tests  python=1723
+280     docker_tests      python=280
+140     metastore_browser python=140
+109     clients           sh=109
+28      images              sh=28
+
+Totals grouped by language (dominant language first):
+python:      288625 (92.65%)
+javascript:     11804 (3.79%)
+sh:           10945 (3.51%)
+xml:            132 (0.04%)
+```
+
+We have now >10K lines of shell code now. We'd announce the success of the project if the shell number is less
+than `300` lines of code or so, constituting less than `0.1%` of the code base.
+
+## Decision
+
+The main decision is:
+
+**Vast majority of both Breeze and our CI scripts should be Python-based**
+
+There are likely a number of scripts that will remain in Bash, but they should contain no sophisticated
+logic, they should not haave common code in form of libraries and only used to execute simple tasks inside
+Docker containers. No Bash should ever be used in the host environment.
+
+The "working" name of the new Breeze is "Breeze2". We might come up with a better name in the future. In
+order to distinguish from the Bash version of Breeze we will always use capitalized form of Breeze as opposed
+to lower-case often used for the Bash version.
+
+There are a few properties of Breeze/CI scripts that should be maintained though
+
+* It should be possible to start Breeze and run any of the CI scripts without having a specially prepared
+  virtualenv. If the virtualenv is needed - such environment should be prepared and maintained automatically
+  by the script being run. The idea is that new person starting their adventure with Airflow can simply
+  run a command and get everything done with the least number of prerequisites
+
+* The prerequisites for Breeze and CI are:
+   * Python 3.7+ (Python 3.6 end of life is December 2021)
+   * Docker (TBD which minimum version supported)
+   * Docker Compose (TBD which minimum version supported)
+   * No other tools and CLI commands should be needed
+   * The python requirements should be automatically installed when missing in a "Breeze" venv and updated
+     automatically when needed. The number of Python dependencies needed to run Breeze and CI scripts
+     should be minimal in order to make the virtualenv installation portable to Linux, MacOS and Windows
+     environment.
+
+* There are some basic assumptions that result from our common patterns across other components we use:
+   * we use `rich` library for colouring terminal output. Using wisely terminal colours
+     is an essential part of the developer experience. We will have to standardize color usage in
+     a follow-up adr
+   * we use `click` library to provide command line parsing and autocompletion (in the future). Click is
+     a comprehensive library with clean, decorator-based interface and provides rich customisation options
+   * we use `pytest` to run automated tests for our code
+   * until we are ready to share it with developers the new `Breeze` script resides in `dev/Breeze` folder,
+     without yet linking it from main directory of Airflow. Later we will link to it from the main directory
+     likely as `Breeze2` script (in some environments where filesystem is case-insensitive (MacOS) you cannot
+     really put two files differing only by case in the same folder.
+   * There is enough overlap between the CI and Breeze to reuse a lot of commands for building images and
+     other CI actions that they should be shared between Breeze and CI. Therefore `dev/Breeze` will
+     also become a home for all the CI scripts that will be used in GitHubActions in CI.
+
+## Consequences
+
+The consequences of the change should be largely invisible to the current users of Breeze. They should be
+able to perform the same actions and operations as in the Bash version (with possible later decision of
+deprecating or removing of some commands). The biggest consequence should be to the whole development
+community of Airflow - for them, modifying and extending and fixing Breeze and CI environment should
+become much more appealing.
+
+The old script should remain and be maintained until the most important functionality of the
+original Breeze script has been rewritten enough.

dev/breeze/pyproject.toml

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+[tool.black]
+line-length = 110
+target-version = ['py37', 'py37', 'py38', 'py39']
+skip-string-normalization = true