Remove warnings when starting breeze (apache#24183)

Breeze when started produced three warnings that were harmless,
but we should fix them to remove "false positives".

dev/breeze/src/airflow_breeze/breeze.py

@@ -17,9 +17,13 @@
 # under the License.
 from airflow_breeze.configure_rich_click import click  # isort: skip # noqa
 from airflow_breeze.commands.main_command import main
-from airflow_breeze.utils.path_utils import find_airflow_sources_root_to_operate_on
+from airflow_breeze.utils.path_utils import (
+    create_directories_and_files,
+    find_airflow_sources_root_to_operate_on,
+)
 
 find_airflow_sources_root_to_operate_on()
+create_directories_and_files()
 
 if __name__ == '__main__':
     main()

dev/breeze/src/airflow_breeze/commands/developer_commands.py

@@ -524,7 +524,7 @@ def stop(verbose: bool, dry_run: bool, preserve_volumes: bool):
     command_to_execute = ['docker-compose', 'down', "--remove-orphans"]
     if not preserve_volumes:
         command_to_execute.append("--volumes")
-    shell_params = ShellParams(verbose=verbose, backend="all")
+    shell_params = ShellParams(verbose=verbose, backend="all", include_mypy_volume=True)
     env_variables = get_env_variables_for_docker_commands(shell_params)
     run_command(command_to_execute, verbose=verbose, dry_run=dry_run, env=env_variables)
 
@@ -579,6 +579,7 @@ def enter_shell(**kwargs) -> RunCommandResult:
     if read_from_cache_file('suppress_cheatsheet') is None:
         get_console().print(CHEATSHEET, style=CHEATSHEET_STYLE)
     enter_shell_params = ShellParams(**filter_out_none(**kwargs))
+    enter_shell_params.include_mypy_volume = True
     rebuild_ci_image_if_needed(build_params=enter_shell_params, dry_run=dry_run, verbose=verbose)
     return run_shell(verbose, dry_run, enter_shell_params)
 

dev/breeze/src/airflow_breeze/commands/main_command.py

@@ -37,7 +37,6 @@
     option_use_packages_from_dist,
     option_verbose,
 )
-from airflow_breeze.utils.path_utils import create_directories_and_files
 
 
 @click.group(invoke_without_command=True, context_settings={'help_option_names': ['-h', '--help']})
@@ -64,6 +63,5 @@
 def main(ctx: click.Context, **kwargs):
     from airflow_breeze.commands.developer_commands import shell
 
-    create_directories_and_files()
     if not ctx.invoked_subcommand:
         ctx.forward(shell, extra_args={})

dev/breeze/src/airflow_breeze/params/build_ci_params.py

@@ -31,6 +31,7 @@ class BuildCiParams(CommonBuildParams):
     """
 
     airflow_constraints_mode: str = "constraints-source-providers"
+    default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
     airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
     airflow_extras: str = "devel_ci"
     airflow_pre_cached_pip_packages: bool = True

dev/breeze/src/airflow_breeze/params/build_prod_params.py

@@ -21,7 +21,7 @@
 from dataclasses import dataclass
 from typing import List
 
-from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
+from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
 from airflow_breeze.global_constants import (
     AIRFLOW_SOURCES_FROM,
     AIRFLOW_SOURCES_TO,
@@ -41,6 +41,7 @@ class BuildProdParams(CommonBuildParams):
     """
 
     airflow_constraints_mode: str = "constraints"
+    default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
     airflow_constraints_reference: str = ""
     airflow_is_in_context: bool = False
     cleanup_context: bool = False

dev/breeze/src/airflow_breeze/params/shell_params.py

@@ -48,6 +48,7 @@ class ShellParams:
     """
 
     airflow_branch: str = AIRFLOW_BRANCH
+    default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
     airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH
     airflow_extras: str = ""
     answer: Optional[str] = None
@@ -64,6 +65,7 @@ class ShellParams:
     github_repository: str = "apache/airflow"
     github_token: str = os.environ.get('GITHUB_TOKEN', "")
     image_tag: str = "latest"
+    include_mypy_volume: bool = False
     install_airflow_version: str = ""
     install_providers_from_sources: bool = True
     integration: Tuple[str, ...] = ()
@@ -191,6 +193,7 @@ def compose_files(self):
         local_all_sources_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/local-all-sources.yml"
         files_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/files.yml"
         remove_sources_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/remove-sources.yml"
+        mypy_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/mypy.yml"
         forward_credentials_docker_compose_file = (
             f"{str(SCRIPTS_CI_DIR)}/docker-compose/forward-credentials.yml"
         )
@@ -215,6 +218,8 @@ def compose_files(self):
             compose_ci_file.append(forward_credentials_docker_compose_file)
         if self.use_airflow_version is not None:
             compose_ci_file.append(remove_sources_docker_compose_file)
+        if self.include_mypy_volume:
+            compose_ci_file.append(mypy_docker_compose_file)
         if "all" in self.integration:
             integrations = AVAILABLE_INTEGRATIONS
         else:

dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

@@ -17,7 +17,6 @@
 """Various utils to prepare docker and docker compose commands."""
 import os
 import re
-import subprocess
 import sys
 from copy import deepcopy
 from random import randint
@@ -97,16 +96,6 @@
 ]
 
 
-def create_volume_if_missing(volume_name: str):
-    res_inspect = run_command(cmd=["docker", "inspect", volume_name], stdout=subprocess.DEVNULL, check=False)
-    if res_inspect.returncode != 0:
-        run_command(cmd=["docker", "volume", "create", volume_name], check=True)
-
-
-def create_static_check_volumes():
-    create_volume_if_missing("docker-compose_mypy-cache-volume")
-
-
 def get_extra_docker_flags(mount_sources: str) -> List[str]:
     """
     Returns extra docker flags based on the type of mounting we want to do for sources.
@@ -124,7 +113,7 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]:
                     ["--mount", f'type=bind,src={AIRFLOW_SOURCES_ROOT / src},dst={dst}']
                 )
         extra_docker_flags.extend(
-            ['--mount', "type=volume,src=docker-compose_mypy-cache-volume,dst=/opt/airflow/.mypy_cache"]
+            ['--mount', "type=volume,src=mypy-cache-volume,dst=/opt/airflow/.mypy_cache"]
         )
     else:  # none
         extra_docker_flags.extend(
@@ -512,6 +501,7 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None:
     :param env: environment variables to update with missing values if not set.
     """
     set_value_to_default_if_not_set(env, 'AIRFLOW_CONSTRAINTS_MODE', "constraints-source-providers")
+    set_value_to_default_if_not_set(env, 'AIRFLOW_CONSTRAINTS_REFERENCE', "constraints-source-providers")
     set_value_to_default_if_not_set(env, 'AIRFLOW_EXTRAS', "")
     set_value_to_default_if_not_set(env, 'ANSWER', "")
     set_value_to_default_if_not_set(env, 'BREEZE', "true")
@@ -558,7 +548,9 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None:
     "AIRFLOW_CI_IMAGE": "airflow_image_name",
     "AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_image_name_with_tag",
     "AIRFLOW_EXTRAS": "airflow_extras",
+    "DEFAULT_CONSTRAINTS_BRANCH": "default-constraints-branch",
     "AIRFLOW_CONSTRAINTS_MODE": "airflow_constraints_mode",
+    "AIRFLOW_CONSTRAINTS_REFERENCE": "airflow_constraints_reference",
     "AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes",
     "AIRFLOW_PROD_IMAGE": "airflow_image_name",
     "AIRFLOW_SOURCES": "airflow_sources",

dev/breeze/src/airflow_breeze/utils/path_utils.py

@@ -19,6 +19,7 @@
 """
 import hashlib
 import os
+import subprocess
 import sys
 import tempfile
 from functools import lru_cache
@@ -251,6 +252,18 @@ def find_airflow_sources_root_to_operate_on() -> Path:
 BREEZE_SOURCES_ROOT = AIRFLOW_SOURCES_ROOT / "dev" / "breeze"
 
 
+def create_volume_if_missing(volume_name: str):
+    from airflow_breeze.utils.run_utils import run_command
+
+    res_inspect = run_command(cmd=["docker", "inspect", volume_name], stdout=subprocess.DEVNULL, check=False)
+    if res_inspect.returncode != 0:
+        run_command(cmd=["docker", "volume", "create", volume_name], check=True)
+
+
+def create_static_check_volumes():
+    create_volume_if_missing("mypy-cache-volume")
+
+
 def create_directories_and_files() -> None:
     """
     Creates all directories and files that are needed for Breeze to work via docker-compose.
@@ -268,3 +281,4 @@ def create_directories_and_files() -> None:
     (AIRFLOW_SOURCES_ROOT / ".bash_aliases").touch()
     (AIRFLOW_SOURCES_ROOT / ".bash_history").touch()
     (AIRFLOW_SOURCES_ROOT / ".inputrc").touch()
+    create_static_check_volumes()

scripts/ci/docker-compose/local.yml

@@ -26,10 +26,7 @@ services:
     # or those that might be useful to see in the host as output of the
     # tests (such as logs)
     volumes:
-      - type: volume
-        source: mypy-cache-volume
-        target: /opt/airflow/.mypy_cache/
-        # START automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py
+      # START automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py
       - type: bind
         source: ../../../.bash_aliases
         target: /root/.bash_aliases
@@ -121,5 +118,3 @@ services:
         source: ../../../metastore_browser
         target: /opt/airflow/metastore_browser
         # END automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py
-volumes:
-  mypy-cache-volume:

scripts/ci/docker-compose/mypy.yml

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+version: "3.7"
+services:
+  airflow:
+    volumes:
+      - type: volume
+        source: mypy-cache-volume
+        target: /opt/airflow/.mypy_cache/
+volumes:
+  mypy-cache-volume:
+    external: true

scripts/ci/pre_commit/pre_commit_flake8.py

@@ -34,7 +34,8 @@
 if __name__ == '__main__':
     sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
     from airflow_breeze.global_constants import MOUNT_SELECTED
-    from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags
+    from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags
+    from airflow_breeze.utils.path_utils import create_static_check_volumes
     from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command
 
     airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN)

scripts/ci/pre_commit/pre_commit_migration_reference.py

@@ -34,7 +34,8 @@
 if __name__ == '__main__':
     sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
     from airflow_breeze.global_constants import MOUNT_SELECTED
-    from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags
+    from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags
+    from airflow_breeze.utils.path_utils import create_static_check_volumes
     from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command
 
     airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN)

scripts/ci/pre_commit/pre_commit_mypy.py

@@ -35,7 +35,8 @@
 if __name__ == '__main__':
     sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
     from airflow_breeze.global_constants import MOUNT_SELECTED
-    from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags
+    from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags
+    from airflow_breeze.utils.path_utils import create_static_check_volumes
     from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command
 
     airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN)

scripts/ci/pre_commit/pre_commit_ui_lint.py

@@ -33,7 +33,8 @@
 if __name__ == '__main__':
     sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
     from airflow_breeze.global_constants import MOUNT_SELECTED
-    from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags
+    from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags
+    from airflow_breeze.utils.path_utils import create_static_check_volumes
     from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command
 
     airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN)

scripts/ci/pre_commit/pre_commit_www_lint.py

@@ -34,7 +34,8 @@
 if __name__ == '__main__':
     sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
     from airflow_breeze.global_constants import MOUNT_SELECTED
-    from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags
+    from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags
+    from airflow_breeze.utils.path_utils import create_static_check_volumes
     from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command
 
     airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN)