Optional Argument to dev_setup.py - Codifying Dev Dependencies (#4067)

* initial attempt + logic in dev_setup.py. * removing commented stuff * adding a couple example dev_requirements.txt * adding a comment to explain targeted_packages * adding requirements files for any package that has tests. * removing 'azure-common' as it is a runtime package that doesn't need to be explicitly added * making some progress. replaced all the dev_requirements files, now to see how it will react to my install actions * removing expandDependencies, swapping entirely to leverage only dev_requirements.txt * working * setting this up such that travis.ci won't get punished by the dev_requirements addition * updating yaml to take advantage of the new dev_setup stuff * we'll just rely on pip to rationalize that everything is installed
Azure · Dec 20, 2018 · 46b4b17 · 46b4b17
1 parent 92c969f
commit 46b4b17
Show file tree

Hide file tree

Showing 68 changed files with 140 additions and 24 deletions.
diff --git a/.azure-pipelines/generic.test.yml b/.azure-pipelines/generic.test.yml
@@ -29,7 +29,18 @@ jobs:
     inputs:
       versionSpec: '$(python.version)'
 
-  - script: 'python scripts/dev_setup.py'
+  - powershell: |
+     $inputFilterSet="$(build_targeting_string)"
+     
+     if ($inputFilterSet -eq "")
+     {
+        $inputFilterSet = "azure-*"
+     }
+     
+     foreach($inputFilter in $inputFilterSet.split(","))
+     {
+        python scripts/dev_setup.py -g $inputFilter
+     }
     displayName: 'Test Setup'
 
   - powershell: |

diff --git a/.azure-pipelines/specialcase.test.yml b/.azure-pipelines/specialcase.test.yml
@@ -17,16 +17,20 @@ jobs:
      $storedPath=$Env:PATH
      $Env:PATH="/home/vsts/pypy3-v6.0.0-linux64/bin:" + $Env:PATH
 
-     pypy3 -m ensurepip
-     pypy3 scripts/dev_setup.py
-     
      $inputFilterSet="$(build_targeting_string)"
       
      if ($inputFilterSet -eq "")
      {
         $inputFilterSet = "azure-*"
      }
-      
+
+     pypy3 -m ensurepip
+
+     foreach($inputFilter in $inputFilterSet.split(","))
+     {
+        pypy3 scripts/dev_setup.py -g $inputFilter
+     }
+     
      foreach($inputFilter in "$inputFilterSet".split(","))
      {
         $folderCombination = (gci -Directory -Name -Filter "$inputFilter"  | %{ "$(Build.SourcesDirectory)/$_/" }) -join " "
@@ -53,16 +57,20 @@ jobs:
      $storedPath=$Env:PATH     
      $Env:PATH="/home/vsts/bin:" + $Env:PATH
 
-     python3 -m ensurepip
-     python3 scripts/dev_setup.py
-     
      $inputFilterSet="$(build_targeting_string)"
       
      if ($inputFilterSet -eq "")
      {
         $inputFilterSet = "azure-*"
      }
-      
+
+     python3 -m ensurepip
+     
+     foreach($inputFilter in $inputFilterSet.split(","))
+     {
+        python3 scripts/dev_setup.py -g $inputFilter
+     }
+
      foreach($inputFilter in "$inputFilterSet".split(","))
      {
         $folderCombination = (gci -Directory -Name -Filter "$inputFilter"  | %{ "$(Build.SourcesDirectory)/$_/" }) -join " "

diff --git a/azure-applicationinsights/dev_requirements.txt b/azure-applicationinsights/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-batch/dev_requirements.txt b/azure-batch/dev_requirements.txt
@@ -0,0 +1,2 @@
+-e ../azure-sdk-tools
+-e ../azure-mgmt-batch
diff --git a/azure-cognitiveservices-language-spellcheck/dev_requirements.txt b/azure-cognitiveservices-language-spellcheck/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-cognitiveservices-language-textanalytics/dev_requirements.txt b/azure-cognitiveservices-language-textanalytics/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-cognitiveservices-search-entitysearch/dev_requirements.txt b/azure-cognitiveservices-search-entitysearch/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-cognitiveservices-search-videosearch/dev_requirements.txt b/azure-cognitiveservices-search-videosearch/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-cognitiveservices-vision-face/dev_requirements.txt b/azure-cognitiveservices-vision-face/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-common/dev_requirements.txt b/azure-common/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-eventgrid/dev_requirements.txt b/azure-eventgrid/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-graphrbac/dev_requirements.txt b/azure-graphrbac/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-keyvault/dev_requirements.txt b/azure-keyvault/dev_requirements.txt
@@ -0,0 +1,3 @@
+-e ../azure-mgmt-authorization
+-e ../azure-mgmt-keyvault
+-e ../azure-sdk-tools
diff --git a/azure-loganalytics/dev_requirements.txt b/azure-loganalytics/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-advisor/dev_requirements.txt b/azure-mgmt-advisor/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-authorization/dev_requirements.txt b/azure-mgmt-authorization/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-batch/dev_requirements.txt b/azure-mgmt-batch/dev_requirements.txt
@@ -0,0 +1,2 @@
+-e ../azure-sdk-tools
+-e ../azure-mgmt-keyvault
diff --git a/azure-mgmt-batchai/dev_requirements.txt b/azure-mgmt-batchai/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-billing/dev_requirements.txt b/azure-mgmt-billing/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-botservice/dev_requirements.txt b/azure-mgmt-botservice/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-cdn/dev_requirements.txt b/azure-mgmt-cdn/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-commerce/dev_requirements.txt b/azure-mgmt-commerce/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-compute/dev_requirements.txt b/azure-mgmt-compute/dev_requirements.txt
@@ -0,0 +1,4 @@
+-e ../azure-sdk-tools
+-e ../azure-mgmt-authorization
+-e ../azure-mgmt-storage
+-e ../azure-mgmt-network
diff --git a/azure-mgmt-consumption/dev_requirements.txt b/azure-mgmt-consumption/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-containerinstance/dev_requirements.txt b/azure-mgmt-containerinstance/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-containerregistry/dev_requirements.txt b/azure-mgmt-containerregistry/dev_requirements.txt
@@ -0,0 +1,2 @@
+-e ../azure-sdk-tools
+-e ../azure-mgmt-storage
diff --git a/azure-mgmt-containerservice/dev_requirements.txt b/azure-mgmt-containerservice/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-cosmosdb/dev_requirements.txt b/azure-mgmt-cosmosdb/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-datafactory/dev_requirements.txt b/azure-mgmt-datafactory/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-datalake-analytics/dev_requirements.txt b/azure-mgmt-datalake-analytics/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-datalake-store/dev_requirements.txt b/azure-mgmt-datalake-store/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-datamigration/dev_requirements.txt b/azure-mgmt-datamigration/dev_requirements.txt
@@ -0,0 +1,2 @@
+-e ../azure-sdk-tools
+-e ../azure-mgmt-network
diff --git a/azure-mgmt-devtestlabs/dev_requirements.txt b/azure-mgmt-devtestlabs/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-dns/dev_requirements.txt b/azure-mgmt-dns/dev_requirements.txt
@@ -0,0 +1,2 @@
+-e ../azure-sdk-tools
+-e ../azure-mgmt-network
diff --git a/azure-mgmt-documentdb/dev_requirements.txt b/azure-mgmt-documentdb/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-eventgrid/dev_requirements.txt b/azure-mgmt-eventgrid/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-eventhub/dev_requirements.txt b/azure-mgmt-eventhub/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-hanaonazure/dev_requirements.txt b/azure-mgmt-hanaonazure/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-iotcentral/dev_requirements.txt b/azure-mgmt-iotcentral/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-iothub/dev_requirements.txt b/azure-mgmt-iothub/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-keyvault/dev_requirements.txt b/azure-mgmt-keyvault/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-logic/dev_requirements.txt b/azure-mgmt-logic/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-machinelearningcompute/dev_requirements.txt b/azure-mgmt-machinelearningcompute/dev_requirements.txt
@@ -0,0 +1,2 @@
+-e ../azure-sdk-tools
+-e ../azure-mgmt-resource
diff --git a/azure-mgmt-managementpartner/dev_requirements.txt b/azure-mgmt-managementpartner/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-media/dev_requirements.txt b/azure-mgmt-media/dev_requirements.txt
@@ -0,0 +1,2 @@
+-e ../azure-sdk-tools
+-e ../azure-mgmt-storage
diff --git a/azure-mgmt-monitor/dev_requirements.txt b/azure-mgmt-monitor/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-network/dev_requirements.txt b/azure-mgmt-network/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-notificationhubs/dev_requirements.txt b/azure-mgmt-notificationhubs/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-recoveryservices/dev_requirements.txt b/azure-mgmt-recoveryservices/dev_requirements.txt
@@ -0,0 +1,2 @@
+-e ../azure-sdk-tools
+-e ../azure-mgmt-resource
diff --git a/azure-mgmt-recoveryservicesbackup/dev_requirements.txt b/azure-mgmt-recoveryservicesbackup/dev_requirements.txt
@@ -0,0 +1,2 @@
+-e ../azure-sdk-tools
+-e ../azure-mgmt-recoveryservices
diff --git a/azure-mgmt-redis/dev_requirements.txt b/azure-mgmt-redis/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-relay/dev_requirements.txt b/azure-mgmt-relay/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-reservations/dev_requirements.txt b/azure-mgmt-reservations/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-resource/dev_requirements.txt b/azure-mgmt-resource/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-resourcegraph/dev_requirements.txt b/azure-mgmt-resourcegraph/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-scheduler/dev_requirements.txt b/azure-mgmt-scheduler/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-search/dev_requirements.txt b/azure-mgmt-search/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-servermanager/dev_requirements.txt b/azure-mgmt-servermanager/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-servicebus/dev_requirements.txt b/azure-mgmt-servicebus/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-sql/dev_requirements.txt b/azure-mgmt-sql/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-storage/dev_requirements.txt b/azure-mgmt-storage/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-subscription/dev_requirements.txt b/azure-mgmt-subscription/dev_requirements.txt
@@ -0,0 +1,2 @@
+-e ../azure-sdk-tools
+-e ../azure-mgmt-billing
diff --git a/azure-mgmt-trafficmanager/dev_requirements.txt b/azure-mgmt-trafficmanager/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-mgmt-web/dev_requirements.txt b/azure-mgmt-web/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-sdk-tools/setup.py b/azure-sdk-tools/setup.py
@@ -17,12 +17,12 @@
     'pytest>=3.5.1',
     # 'azure-devtools>=0.4.1' override by packaging needs
     'readme_renderer',
-
-    # Should not be here, but split per package once they have test dependencies
-    'azure-storage-blob', # azure-servicemanagement-legacy azure-keyvault
-    'azure-storage-file', # azure-mgmt-batchai
-    'azure-storage-common', # azure-keyvault
-    'pyopenssl' # azure-servicemanagement-legacy
+    'azure-storage-blob',
+    'azure-storage-file',
+    'azure-storage-common',
+    'pyopenssl',
+    'azure-mgmt-resource',
+    'azure-mgmt-storage'
 ]
 
 setup(

diff --git a/azure-servicebus/dev_requirements.txt b/azure-servicebus/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/azure-servicemanagement-legacy/dev_requirements.txt b/azure-servicemanagement-legacy/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../azure-sdk-tools
diff --git a/scripts/dev_setup.py b/scripts/dev_setup.py
@@ -10,34 +10,49 @@
 import sys
 import glob
 import os
+import argparse
+from collections import Counter
 from subprocess import check_call, CalledProcessError
 
 root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), '..', '..'))
 
-
-def pip_command(command, error_ok=False):
+def pip_command(command, additionalDir='.', error_ok=False):
     try:
         print('Executing: ' + command)
-        check_call([sys.executable, '-m', 'pip'] + command.split(), cwd=root_dir)
+        check_call([sys.executable, '-m', 'pip'] + command.split(), cwd=os.path.join(root_dir, additionalDir))
         print()
     except CalledProcessError as err:
         print(err, file=sys.stderr)
         if not error_ok:
             sys.exit(1)
 
+# optional argument in a situation where we want to build a single package
+parser = argparse.ArgumentParser(description='Set up the dev environment for selected packages.')
+parser.add_argument('--globArg', '-g', dest='globArg', default='azure*', help='Defaulted to "azure*", used to limit the number of packages that dependencies will be installed for. ')
+args = parser.parse_args()
+
 packages = [os.path.dirname(p) for p in glob.glob('azure*/setup.py')]
 
+# keep targeted packages separate. python2 needs the nspkgs to work properly.
+targeted_packages = [os.path.dirname(p) for p in glob.glob('{0}/setup.py'.format(args.globArg))]
+
 # Extract nspkg and sort nspkg by number of "-"
-nspkg_packages = [p for p in packages if "nspkg" in p]
+nspkg_packages = [p for p in packages if 'nspkg' in p]
 nspkg_packages.sort(key = lambda x: len([c for c in x if c == '-']))
 
 # Manually push meta-packages at the end, in reverse dependency order
 meta_packages = ['azure-mgmt', 'azure']
 
-content_packages = [p for p in packages if p not in nspkg_packages+meta_packages]
+content_packages = [p for p in packages if p not in nspkg_packages+meta_packages and p in targeted_packages]
+
 # Put azure-common in front
-content_packages.remove("azure-common")
-content_packages.insert(0, "azure-common")
+if 'azure-common' in content_packages:
+    content_packages.remove('azure-common')
+content_packages.insert(0, 'azure-common')
+
+if 'azure-sdk-tools' in content_packages:
+    content_packages.remove('azure-sdk-tools')
+content_packages.insert(1, 'azure-sdk-tools')
 
 print('Running dev setup...')
 print('Root directory \'{}\'\n'.format(root_dir))
@@ -55,11 +70,14 @@ def pip_command(command, error_ok=False):
 
 # install packages
 for package_name in content_packages:
+    # if we are running dev_setup with no arguments. going after dev_requirements will be a pointless exercise
+    # and waste of cycles as all the dependencies will be installed regardless. 
+    if os.path.isfile('{}/dev_requirements.txt'.format(package_name)):
+        pip_command('install -r dev_requirements.txt', package_name)
     pip_command('install --ignore-requires-python -e {}'.format(package_name))
 
 # On Python 3, uninstall azure-nspkg if he got installed
 if sys.version_info >= (3, ):
     pip_command('uninstall -y azure-nspkg', error_ok=True)
 
-
-print('Finished dev setup.')
+print('Finished dev setup.')
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		-e ../azure-sdk-tools
		-e ../azure-mgmt-keyvault
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		-e ../azure-sdk-tools
		-e ../azure-mgmt-storage
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		-e ../azure-sdk-tools
		-e ../azure-mgmt-network
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		-e ../azure-sdk-tools
		-e ../azure-mgmt-resource
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		-e ../azure-sdk-tools
		-e ../azure-mgmt-recoveryservices
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		-e ../azure-sdk-tools
		-e ../azure-mgmt-billing