From f62b57ba8c53f667ec31ec3ea791879b5a6c3a4e Mon Sep 17 00:00:00 2001 From: Jacob Lauzon Date: Thu, 21 Mar 2024 18:16:19 -0700 Subject: [PATCH] Attempt to fix replace_dev_req for extensions --- scripts/devops_tasks/tox_harness.py | 6 +-- tools/azure-sdk-tools/ci_tools/functions.py | 4 +- .../ci_tools/scenario/generation.py | 39 ++++++++++++------- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/scripts/devops_tasks/tox_harness.py b/scripts/devops_tasks/tox_harness.py index f8cfe763e5ea..78686f593e61 100644 --- a/scripts/devops_tasks/tox_harness.py +++ b/scripts/devops_tasks/tox_harness.py @@ -300,9 +300,9 @@ def prep_and_run_tox(targeted_packages: List[str], parsed_args: Namespace) -> No file.write("\n") if in_ci(): - replace_dev_reqs(destination_dev_req, package_dir) - replace_dev_reqs(test_tools_path, package_dir) - replace_dev_reqs(dependency_tools_path, package_dir) + replace_dev_reqs(destination_dev_req, package_dir, parsed_args.wheel_dir) + replace_dev_reqs(test_tools_path, package_dir, parsed_args.wheel_dir) + replace_dev_reqs(dependency_tools_path, package_dir, parsed_args.wheel_dir) os.environ["TOX_PARALLEL_NO_SPINNER"] = "1" inject_custom_reqs(destination_dev_req, parsed_args.injected_packages, package_dir) diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 400a06663312..6778730b27f7 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -558,8 +558,8 @@ def find_whl(whl_dir: str, pkg_name: str, pkg_version: str) -> str: whls = [os.path.relpath(w, whl_dir) for w in whls] if not whls: - logging.error("No whl is found in directory %s with package name format %s", whl_dir, pkg_name_format) - logging.info("List of whls in directory: %s", glob.glob(os.path.join(whl_dir, "*.whl"))) + logging.info(f"No whl is found in directory {whl_dir} with package name format {pkg_name_format}") + logging.info(f"List of whls in directory: {glob.glob(os.path.join(whl_dir, '*.whl'))}") return compatible_tags = get_interpreter_compatible_tags() diff --git a/tools/azure-sdk-tools/ci_tools/scenario/generation.py b/tools/azure-sdk-tools/ci_tools/scenario/generation.py index 4a2f0d9be5b8..db0e6fdd3dc8 100644 --- a/tools/azure-sdk-tools/ci_tools/scenario/generation.py +++ b/tools/azure-sdk-tools/ci_tools/scenario/generation.py @@ -4,6 +4,7 @@ import subprocess import shutil import logging +from typing import Optional from ci_tools.environment_exclusions import is_check_enabled from ci_tools.variables import in_ci @@ -172,7 +173,7 @@ def create_package_and_install( logging.info("Installed {w}".format(w=built_package)) -def replace_dev_reqs(file: str, pkg_root: str) -> None: +def replace_dev_reqs(file: str, pkg_root: str, wheel_dir: Optional[str]) -> None: """Takes a target requirements file, replaces all local relative install locations with wheels assembled from whatever that target path was. This is an extremely important step that runs on every dev_requirements.txt file before invoking any tox runs. @@ -181,6 +182,7 @@ def replace_dev_reqs(file: str, pkg_root: str) -> None: :param str file: the absolute path to the dev_requirements.txt file :param str pkg_root: the absolute path to the package's root + :param Optional[str] wheel_dir: the absolute path to the prebuilt wheel directory :return: None """ adjusted_req_lines = [] @@ -198,7 +200,7 @@ def replace_dev_reqs(file: str, pkg_root: str) -> None: if extras: extras = f"[{extras}" - adjusted_req_lines.append(f"{build_whl_for_req(amended_line, pkg_root)}{extras}") + adjusted_req_lines.append(f"{build_whl_for_req(amended_line, pkg_root, wheel_dir)}{extras}") req_file_name = os.path.basename(file) logging.info("Old {0}:{1}".format(req_file_name, original_req_lines)) @@ -262,7 +264,7 @@ def build_and_install_dev_reqs(file: str, pkg_root: str) -> None: adjusted_req_lines.append(amended_line) - adjusted_req_lines = list(map(lambda x: build_whl_for_req(x, pkg_root), adjusted_req_lines)) + adjusted_req_lines = list(map(lambda x: build_whl_for_req(x, pkg_root, None), adjusted_req_lines)) install_deps_commands = [ sys.executable, "-m", @@ -285,30 +287,41 @@ def is_relative_install_path(req: str, package_path: str) -> bool: return os.path.exists(possible_setup_path) -def build_whl_for_req(req: str, package_path: str) -> str: +def build_whl_for_req(req: str, package_path: str, wheel_dir: Optional[str]) -> str: """Builds a whl from the dev_requirements file. :param str req: a requirement from the dev_requirements.txt :param str package_path: the absolute path to the package's root + :param Optional[str] wheel_dir: the absolute path to the prebuilt wheel directory :return: The absolute path to the whl built or the requirement if a third-party package """ from ci_tools.build import create_package if is_relative_install_path(req, package_path): - # Create temp path if it doesn't exist - temp_dir = os.path.join(package_path, ".tmp_whl_dir") - if not os.path.exists(temp_dir): - os.mkdir(temp_dir) - req_pkg_path = os.path.abspath(os.path.join(package_path, req.replace("\n", ""))) parsed = ParsedSetup.from_path(req_pkg_path) - logging.info("Building wheel for package {}".format(parsed.name)) - create_package(req_pkg_path, temp_dir, enable_sdist=False) + # First check for prebuilt wheel + logging.info("Checking for prebuilt wheel for package {}".format(parsed.name)) + prebuilt_whl = None + if wheel_dir: + prebuilt_whl = find_whl(wheel_dir, parsed.name, parsed.version) + + if prebuilt_whl: + whl_path = os.path.join(wheel_dir, prebuilt_whl) + else: + # Create temp path if it doesn't exist + temp_dir = os.path.join(package_path, ".tmp_whl_dir") + if not os.path.exists(temp_dir): + os.mkdir(temp_dir) + + logging.info("Building wheel for package {}".format(parsed.name)) + create_package(req_pkg_path, temp_dir, enable_sdist=False) + + whl_path = os.path.join(temp_dir, find_whl(temp_dir, parsed.name, parsed.version)) - whl_path = os.path.join(temp_dir, find_whl(temp_dir, parsed.name, parsed.version)) logging.info("Wheel for package {0} is {1}".format(parsed.name, whl_path)) - logging.info("Replacing dev requirement. Old requirement:{0}, New requirement:{1}".format(req, whl_path)) + logging.info("Replacing dev requirement. Old requirement: {0}, New requirement: {1}".format(req, whl_path)) return whl_path else: return req