Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Attempt to fix replace_dev_reqs for Storage extensions #34888

Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Attempt to fix replace_dev_req for extensions
jalauzon-msft committed Mar 22, 2024
commit f62b57ba8c53f667ec31ec3ea791879b5a6c3a4e
6 changes: 3 additions & 3 deletions scripts/devops_tasks/tox_harness.py
Original file line number Diff line number Diff line change
@@ -300,9 +300,9 @@ def prep_and_run_tox(targeted_packages: List[str], parsed_args: Namespace) -> No
file.write("\n")

if in_ci():
replace_dev_reqs(destination_dev_req, package_dir)
replace_dev_reqs(test_tools_path, package_dir)
replace_dev_reqs(dependency_tools_path, package_dir)
replace_dev_reqs(destination_dev_req, package_dir, parsed_args.wheel_dir)
replace_dev_reqs(test_tools_path, package_dir, parsed_args.wheel_dir)
replace_dev_reqs(dependency_tools_path, package_dir, parsed_args.wheel_dir)
os.environ["TOX_PARALLEL_NO_SPINNER"] = "1"

inject_custom_reqs(destination_dev_req, parsed_args.injected_packages, package_dir)
4 changes: 2 additions & 2 deletions tools/azure-sdk-tools/ci_tools/functions.py
Original file line number Diff line number Diff line change
@@ -558,8 +558,8 @@ def find_whl(whl_dir: str, pkg_name: str, pkg_version: str) -> str:
whls = [os.path.relpath(w, whl_dir) for w in whls]

if not whls:
logging.error("No whl is found in directory %s with package name format %s", whl_dir, pkg_name_format)
logging.info("List of whls in directory: %s", glob.glob(os.path.join(whl_dir, "*.whl")))
logging.info(f"No whl is found in directory {whl_dir} with package name format {pkg_name_format}")
scbedd marked this conversation as resolved.
Show resolved Hide resolved
logging.info(f"List of whls in directory: {glob.glob(os.path.join(whl_dir, '*.whl'))}")
return

compatible_tags = get_interpreter_compatible_tags()
39 changes: 26 additions & 13 deletions tools/azure-sdk-tools/ci_tools/scenario/generation.py
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@
import subprocess
import shutil
import logging
from typing import Optional

from ci_tools.environment_exclusions import is_check_enabled
from ci_tools.variables import in_ci
@@ -172,7 +173,7 @@ def create_package_and_install(
logging.info("Installed {w}".format(w=built_package))


def replace_dev_reqs(file: str, pkg_root: str) -> None:
def replace_dev_reqs(file: str, pkg_root: str, wheel_dir: Optional[str]) -> None:
"""Takes a target requirements file, replaces all local relative install locations with wheels assembled from whatever that target path was.
This is an extremely important step that runs on every dev_requirements.txt file before invoking any tox runs.

@@ -181,6 +182,7 @@ def replace_dev_reqs(file: str, pkg_root: str) -> None:

:param str file: the absolute path to the dev_requirements.txt file
:param str pkg_root: the absolute path to the package's root
:param Optional[str] wheel_dir: the absolute path to the prebuilt wheel directory
:return: None
"""
adjusted_req_lines = []
@@ -198,7 +200,7 @@ def replace_dev_reqs(file: str, pkg_root: str) -> None:
if extras:
extras = f"[{extras}"

adjusted_req_lines.append(f"{build_whl_for_req(amended_line, pkg_root)}{extras}")
adjusted_req_lines.append(f"{build_whl_for_req(amended_line, pkg_root, wheel_dir)}{extras}")

req_file_name = os.path.basename(file)
logging.info("Old {0}:{1}".format(req_file_name, original_req_lines))
@@ -262,7 +264,7 @@ def build_and_install_dev_reqs(file: str, pkg_root: str) -> None:

adjusted_req_lines.append(amended_line)

adjusted_req_lines = list(map(lambda x: build_whl_for_req(x, pkg_root), adjusted_req_lines))
adjusted_req_lines = list(map(lambda x: build_whl_for_req(x, pkg_root, None), adjusted_req_lines))
scbedd marked this conversation as resolved.
Show resolved Hide resolved
install_deps_commands = [
sys.executable,
"-m",
@@ -285,30 +287,41 @@ def is_relative_install_path(req: str, package_path: str) -> bool:
return os.path.exists(possible_setup_path)


def build_whl_for_req(req: str, package_path: str) -> str:
def build_whl_for_req(req: str, package_path: str, wheel_dir: Optional[str]) -> str:
"""Builds a whl from the dev_requirements file.

:param str req: a requirement from the dev_requirements.txt
:param str package_path: the absolute path to the package's root
:param Optional[str] wheel_dir: the absolute path to the prebuilt wheel directory
:return: The absolute path to the whl built or the requirement if a third-party package
"""
from ci_tools.build import create_package

if is_relative_install_path(req, package_path):
# Create temp path if it doesn't exist
temp_dir = os.path.join(package_path, ".tmp_whl_dir")
if not os.path.exists(temp_dir):
os.mkdir(temp_dir)

req_pkg_path = os.path.abspath(os.path.join(package_path, req.replace("\n", "")))
parsed = ParsedSetup.from_path(req_pkg_path)

logging.info("Building wheel for package {}".format(parsed.name))
create_package(req_pkg_path, temp_dir, enable_sdist=False)
# First check for prebuilt wheel
logging.info("Checking for prebuilt wheel for package {}".format(parsed.name))
prebuilt_whl = None
if wheel_dir:
prebuilt_whl = find_whl(wheel_dir, parsed.name, parsed.version)

if prebuilt_whl:
whl_path = os.path.join(wheel_dir, prebuilt_whl)
else:
# Create temp path if it doesn't exist
temp_dir = os.path.join(package_path, ".tmp_whl_dir")
if not os.path.exists(temp_dir):
os.mkdir(temp_dir)

logging.info("Building wheel for package {}".format(parsed.name))
create_package(req_pkg_path, temp_dir, enable_sdist=False)

whl_path = os.path.join(temp_dir, find_whl(temp_dir, parsed.name, parsed.version))

whl_path = os.path.join(temp_dir, find_whl(temp_dir, parsed.name, parsed.version))
logging.info("Wheel for package {0} is {1}".format(parsed.name, whl_path))
logging.info("Replacing dev requirement. Old requirement:{0}, New requirement:{1}".format(req, whl_path))
logging.info("Replacing dev requirement. Old requirement: {0}, New requirement: {1}".format(req, whl_path))
return whl_path
else:
return req