Refactor Build, Versioning into azure-sdk-tools

.gitignore

@@ -45,6 +45,9 @@ coverage-*.xml
 stderr.txt
 stdout.txt
 _proxy_log_*.log
+_azci_*.log
+.artifacts
+.tmp_whl_dir
 
 # tox environment folders
 .tox/

.vscode/cspell.json

@@ -111,7 +111,9 @@
     "eng/tox/tox.ini",
     "eng/common/docgeneration/Generate-DocIndex.ps1",
     "eng/**/*.py",
-    ".gitignore"
+    ".gitignore",
+    "tools/azure-sdk-tools/packaging_tools/**",
+    "tools/azure-sdk-tools/setup.py"
   ],
   "words": [
     "pyyaml",
@@ -334,7 +336,20 @@
     "westcentralus",
     "westus",
     "wfile",
-    "westus"
+    "westus",
+    "BUILDID",
+    "documentdb",
+    "chdir",
+    "reqs",
+    "rgpy",
+    "swaggertosdk",
+    "rgname",
+    "mtype",
+    "venvtools",
+    "ispkg",
+    "openpyxl",
+    "deps",
+    "prnumber"
   ],
   "overrides": [
     {

doc/eng_sys_checks.md

@@ -1,8 +1,8 @@
 # Azure SDK for Python - Engineering System
 
 - [Azure SDK for Python - Engineering System](#azure-sdk-for-python---engineering-system)
-  - [Targeting a specific package at build time](#targeting-a-specific-package-at-build-time)
-  - [Skipping a tox test environment at queue time](#skipping-a-tox-test-environment-at-queue-time)
+  - [Targeting a specific package at build queue time](#targeting-a-specific-package-at-build-queue-time)
+  - [Skipping a tox test environment at build queue time](#skipping-a-tox-test-environment-at-build-queue-time)
   - [Skipping entire sections of builds](#skipping-entire-sections-of-builds)
   - [Environment variables important to CI](#environment-variables-important-to-ci)
   - [Analyze Checks](#analyze-checks)
@@ -29,10 +29,9 @@
   - [Nightly Live Checks](#nightly-live-checks)
     - [Running Samples](#running-samples)
 
-
 There are various tests currently enabled in Azure pipeline for Python SDK and some of them are enabled only for nightly CI checks. We also run some static analysis tool to verify code completeness, security and lint check.
 
-Check the [contributing guide](https://github.com/Azure/azure-sdk-for-python/blob/main/CONTRIBUTING.md#building-and-testing) for an intro to `tox`.
+Check the [contributing guide](https://github.com/Azure/azure-sdk-for-python/blob/main/CONTRIBUTING.md#building-and-testing) for an intro to `tox`. For a deeper dive into the tooling that enables the CI checks below and additional detail on reproducing builds locally please refer to the azure-sdk-tools README.md.
 
 As a contributor, you will see the build jobs run in two modes: `Nightly Scheduled` and `Pull Request`.
 
@@ -45,7 +44,7 @@ Example PR build:
 * `Analyze` tox envs run during the `Analyze job.
 * `Test <platform>_<pyversion>` runs PR/Nightly tox envs, depending on context.
 
-## Targeting a specific package at build time
+## Targeting a specific package at build queue time
 
 In both `public` and `internal` projects, all builds allow a filter to be introduced at build time to narrow the set of packages build/tested.
 
@@ -54,7 +53,7 @@ In both `public` and `internal` projects, all builds allow a filter to be introd
    1. For example, setting filter string `azure-mgmt-*` will filter a build to only management packages. A value of `azure-keyvault-secrets` will result in only building THAT specific package.
 3. Once it's set, run the build!
 
-## Skipping a tox test environment at queue time
+## Skipping a tox test environment at build queue time
 
 All build definitions allow choice at queue time as to which `tox` environments actually run during the test phase.
 

eng/ci_tools.txt

@@ -6,7 +6,7 @@ virtualenv==20.13.2
 wheel==0.37.0
 Jinja2==2.11.2
 packaging==20.4
-tox==3.15.0
+tox==3.25.1
 tox-monorepo==0.1.2
 twine==3.1.1; python_version >= '3.6'
 pathlib2==2.3.5
@@ -38,4 +38,4 @@ pytest-cov==2.8.1
 
 # local dev packages
 ./tools/azure-devtools
-./tools/azure-sdk-tools
+./tools/azure-sdk-tools[build]

eng/dependency_tools.txt

@@ -1,3 +1,3 @@
-../../../tools/azure-sdk-tools
+../../../tools/azure-sdk-tools[build]
 ../../../tools/azure-devtools
-aiohttp>=3.0; python_version >= '3.5'
+aiohttp>=3.0; python_version >= '3.5'

eng/pipelines/templates/jobs/ci.yml

@@ -191,15 +191,15 @@ jobs:
     parameters:
       JobTemplatePath: /eng/pipelines/templates/jobs/regression.yml
       GenerateJobName: generate_regression_matrix
-      SparseCheckoutPaths: [ "scripts/", "sdk/" ]
+      SparseCheckoutPaths: [ "scripts/", "sdk/", "tools/azure-sdk-tools/" ]
       MatrixConfigs:
           - Name: Python_regression_envs
             Path: eng/pipelines/templates/stages/regression-job-matrix.json
             Selection: sparse
             GenerateVMJobs: true
       PreGenerationSteps:
-        - script: |
-            pip install packaging==20.4
+        - pwsh: |
+            pip install "./tools/azure-sdk-tools[build]"
           displayName: 'Prep Environment'
         - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
           parameters:

eng/pipelines/templates/stages/archetype-python-release.yml

@@ -207,13 +207,11 @@ stages:
                       - checkout: self
                       - task: UsePythonVersion@0
                       - script: |
-                          pip install -r ./eng/versioning/requirements.txt
+                          pip install "./tools/azure-sdk-tools[build]"
                         displayName: Install versioning tool dependencies
-                      - task: PythonScript@0
+                      - pwsh: |
+                          sdk_increment_version --package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }}
                         displayName: Increment package version
-                        inputs:
-                          scriptPath: 'eng/versioning/version_increment.py'
-                          arguments: '--package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }}'
                       - template: /eng/common/pipelines/templates/steps/create-pull-request.yml
                         parameters:
                           RepoName: azure-sdk-for-python

eng/pipelines/templates/steps/analyze.yml

@@ -38,9 +38,8 @@ steps:
     condition: succeededOrFailed()
 
   - script: |
-      cd eng/versioning
-      pip install -r requirements.txt
-      python find_invalid_versions.py --always-succeed --service=${{parameters.ServiceDirectory}}
+      pip install "./tools/azure-sdk-tools[build]" -q -I
+      sdk_find_invalid_versions --always-succeed --service=${{parameters.ServiceDirectory}}
     displayName: Find Invalid Versions
     condition: succeededOrFailed()
 

eng/pipelines/templates/steps/build-artifacts.yml

@@ -62,11 +62,9 @@ steps:
     displayName: Update package properties with dev version
     condition: and(succeeded(),eq(variables['SetDevVersion'],'true'))
 
-  - task: PythonScript@0
+  - pwsh: |
+      sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}}
     displayName: 'Generate Packages'
-    inputs:
-      scriptPath: 'scripts/devops_tasks/build_packages.py'
-      arguments: '-d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --devbuild="$(SetDevVersion)"'
 
   - script: |
       twine check $(Build.ArtifactStagingDirectory)/**/*.whl

eng/pipelines/templates/steps/set-dev-build.yml

@@ -12,9 +12,7 @@ steps:
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
 
-  - task: PythonScript@0
-    condition: and(succeededOrFailed(), eq(variables['SetDevVersion'],'true'))
+  - pwsh: |
+      pip install "tools/azure-sdk-tools[build]"
+      sdk_set_dev_version "$(TargetingString)" --service="${{parameters.ServiceDirectory}}" --build-id="$(Build.BuildNumber)"
     displayName: "Update package versions for dev build"
-    inputs:
-      scriptPath: 'eng/versioning/version_set_dev.py'
-      arguments: '"$(TargetingString)" --service="${{parameters.ServiceDirectory}}" --build-id="$(Build.BuildNumber)"'

eng/regression_tools.txt

@@ -6,7 +6,7 @@ virtualenv==20.0.23
 wheel==0.34.2
 Jinja2==2.11.2
 packaging==20.4
-tox==3.15.0
+tox==3.25.1
 tox-monorepo==0.1.2
 twine==1.15.0; python_version == '2.7' or python_version == '3.5'
 twine==3.1.1; python_version >= '3.6'

eng/scripts/Language-Settings.ps1

@@ -22,7 +22,7 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory)
   try
   {
     Push-Location $RepoRoot
-    pip install packaging==20.4 setuptools==44.1.1 -q -I
+    pip install "./tools/azure-sdk-tools[build]" -q -I
     $allPkgPropLines = python (Join-path eng scripts get_package_properties.py) -s $searchPath
   }
   catch
@@ -523,8 +523,8 @@ function SetPackageVersion ($PackageName, $Version, $ServiceDirectory, $ReleaseD
   {
     $ReleaseDate = Get-Date -Format "yyyy-MM-dd"
   }
-  pip install -r "$EngDir/versioning/requirements.txt" -q -I
-  python "$EngDir/versioning/version_set.py" --package-name $PackageName --new-version $Version `
+  pip install "$RepoRoot/tools/azure-sdk-tools[build]" -q -I
+  sdk_set_version --package-name $PackageName --new-version $Version `
   --service $ServiceDirectory --release-date $ReleaseDate --replace-latest-entry-title $ReplaceLatestEntryTitle
 }
 

eng/scripts/get_package_properties.py

@@ -4,21 +4,24 @@
 import os
 import re
 
-sys.path.append(os.path.join('scripts', 'devops_tasks'))
-from common_tasks import get_package_properties
+from ci_tools.parsing import ParsedSetup
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Get package version details from the repo')
-    parser.add_argument('-s', '--search_path', required=True, help='The scope of the search')
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Get package version details from the repo")
+    parser.add_argument("-s", "--search_path", required=True, help="The scope of the search")
     args = parser.parse_args()
 
     # Use abspath for the os.walk because if setup parsing fails it often changes cwd which throws off the relative walk
     for root, dirs, files in os.walk(os.path.abspath(args.search_path)):
         if re.search(r"sdk[\\/][^\\/]+[\\/][^\\/]+$", root):
             if "setup.py" in files:
                 try:
-                    pkgName, version, is_new_sdk, setup_py_path = get_package_properties(root)
-                    print("{0} {1} {2} {3}".format(pkgName, version, is_new_sdk, setup_py_path))
+                    parsed = ParsedSetup.from_path(root)
+                    print(
+                        "{0} {1} {2} {3}".format(
+                            parsed.name, parsed.version, parsed.is_new_sdk, os.path.dirname(parsed.setup_filename)
+                        )
+                    )
                 except:
                     # Skip setup.py if the package cannot be parsed
-                    pass
+                    pass

eng/tox/create_package_and_install.py

@@ -16,16 +16,16 @@
 import sys
 import glob
 import shutil
+import pdb
 from pkg_resources import parse_version
 
-from tox_helper_tasks import find_whl, find_sdist, get_package_details, get_pip_list_output, parse_req
+from tox_helper_tasks import find_whl, find_sdist, get_pip_list_output
+from ci_tools.parsing import ParsedSetup, parse_require
+from ci_tools.build import create_package
 
 logging.getLogger().setLevel(logging.INFO)
 
-setup_parser_path = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "versioning"))
-sys.path.append(setup_parser_path)
-from setup_parser import get_install_requires
-
+from ci_tools.parsing import ParsedSetup
 
 def cleanup_build_artifacts(build_folder):
     # clean up egginfo
@@ -57,16 +57,16 @@ def discover_packages(setuppy_path, args):
 
 def discover_prebuilt_package(dist_directory, setuppy_path, package_type):
     packages = []
-    pkg_name, _, version, _, _ = get_package_details(setuppy_path)
+    pkg = ParsedSetup.from_path(setuppy_path)
     if package_type == "wheel":
-        prebuilt_package = find_whl(dist_directory, pkg_name, version)
+        prebuilt_package = find_whl(dist_directory, pkg.name, pkg.version)
     else:
-        prebuilt_package = find_sdist(dist_directory, pkg_name, version)
+        prebuilt_package = find_sdist(dist_directory, pkg.name, pkg.version)
 
     if prebuilt_package is None:
         logging.error(
             "Package is missing in prebuilt directory {0} for package {1} and version {2}".format(
-                dist_directory, pkg_name, version
+                dist_directory, pkg.name, pkg.version
             )
         )
         exit(1)
@@ -80,29 +80,9 @@ def in_ci():
 
 def build_and_discover_package(setuppy_path, dist_dir, target_setup, package_type):
     if package_type == "wheel":
-        check_call(
-            [
-                sys.executable,
-                setuppy_path,
-                "bdist_wheel",
-                "-d",
-                dist_dir,
-            ],
-            cwd = os.path.dirname(setuppy_path)
-        )
+        create_package(setuppy_path, dist_dir, enable_sdist=False)
     else:
-        check_call(
-            [
-                sys.executable,
-                setuppy_path,
-                "sdist",
-                "--format",
-                "zip",
-                "-d",
-                dist_dir,
-            ],
-            cwd = os.path.dirname(setuppy_path)
-        )
+        create_package(setuppy_path, dist_dir, enable_wheel=False)
 
     prebuilt_packages = [
         f for f in os.listdir(args.distribution_directory) if f.endswith(".whl" if package_type == "wheel" else ".zip")
@@ -217,7 +197,7 @@ def build_and_discover_package(setuppy_path, dist_dir, target_setup, package_typ
                 logging.info("Installing {w} from fresh built package.".format(w=built_package))
 
             if not args.pre_download_disabled:
-                requirements = get_install_requires(os.path.join(os.path.abspath(args.target_setup), "setup.py"))
+                requirements = ParsedSetup.from_path(os.path.join(os.path.abspath(args.target_setup), "setup.py")).requires
                 azure_requirements = [req.split(";")[0] for req in requirements if req.startswith("azure")]
 
                 if azure_requirements:
@@ -245,10 +225,14 @@ def build_and_discover_package(setuppy_path, dist_dir, target_setup, package_typ
                         installed_pkgs = get_pip_list_output()
 
                         # parse the specifier
-                        req_name, req_specifier = parse_req(req)
+                        req_name, req_specifier = parse_require(req)
 
                         # if we have the package already present...
                         if req_name in installed_pkgs:
+                            # if there is no specifier for the requirement, we can ignore it
+                            if req_specifier is None:
+                                addition_necessary = False
+
                             # ...do we need to install the new version? if the existing specifier matches, we're fine
                             if installed_pkgs[req_name] in req_specifier:
                                 addition_necessary = False