Build by service directory (part 2) (#3647)

* Setup CI pipelines to run at the service directory level.
Azure · Jun 12, 2019 · cdf431a · cdf431a
1 parent 0870a48
commit cdf431a
Show file tree

Hide file tree

Showing 42 changed files with 348 additions and 81 deletions.
diff --git a/.docsettings.yml b/.docsettings.yml
@@ -1,5 +1,6 @@
 omitted_paths:
   - documentation/ServicePrincipal/*
+  - eng/tools/select-packages/**
   - "sdk/*/arm-*"
   - "sdk/cognitiveservices/*"
 language: js
@@ -62,7 +63,7 @@ known_content_issues:
   - ["sdk/eventgrid/eventgrid/README.md", "#1583"]
   - ["sdk/core/abort-controller/README.md", "#1583"]
   - ["sdk/core/core-http/README.md", "#1583"]
-  - ['sdk/identity/identity/README.md', '#1583']
+  - ["sdk/identity/identity/README.md", "#1583"]
   - ["sdk/cosmosdb/cosmos/README.md", "#1583"]
   - ["sdk/cosmosdb/cosmos/src/test/readme.md", "#1583"]
   - ["sdk/cosmosdb/cosmos/samples/readme.md", "#1583"]

diff --git a/eng/pipelines/templates/jobs/archetype-sdk-client.yml b/eng/pipelines/templates/jobs/archetype-sdk-client.yml
@@ -1,7 +1,33 @@
 jobs:
+  - job: Prepare
+    variables:
+      - template: ../variables/globals.yml
+
+    pool:
+      vmImage: "ubuntu-16.04"
+
+    steps:
+      - task: NodeTool@0
+        inputs:
+          versionSpec: "$(NodeVersion)"
+        displayName: "Install Node.js $(NodeVersion)"
+
+      - script: |
+          npm install -g [email protected]
+        displayName: "Install npm version 6.9.0"
+
+      - template: ../steps/select-packages.yml
+        parameters:
+          ServiceDirectory: ${{parameters.ServiceDirectory}}
+
   - job: "Build"
     variables:
       - template: ../variables/globals.yml
+      - name: GeneratedPackageTargets
+        value: $[dependencies.Prepare.outputs['SelectPackages.GeneratedPackageTargets']]
+
+    dependsOn:
+      - Prepare
 
     pool:
       vmImage: "ubuntu-16.04"
@@ -20,21 +46,12 @@ jobs:
           node common/scripts/install-run-rush.js install
         displayName: "Install dependencies"
 
-      - ${{ if eq(parameters.PackageName,'')}}:
-          - script: |
-              node common/scripts/install-run-rush.js build --verbose
-            displayName: "Build libraries"
-          - script: |
-              node common/scripts/install-run-rush.js pack --verbose
-            displayName: "Pack libraries"
-
-      - ${{ if ne(parameters.PackageName,'')}}:
-          - script: |
-              node common/scripts/install-run-rush.js build --verbose --to ${{parameters.PackageName}}
-            displayName: "Build libraries"
-          - script: |
-              node common/scripts/install-run-rush.js pack --verbose --to ${{parameters.PackageName}}
-            displayName: "Pack libraries"
+      - script: |
+          node common/scripts/install-run-rush.js build --verbose $(GeneratedPackageTargets)
+        displayName: "Build libraries"
+      - script: |
+          node common/scripts/install-run-rush.js pack --verbose $(GeneratedPackageTargets)
+        displayName: "Pack libraries"
 
       - task: CopyFiles@2
         inputs:
@@ -51,8 +68,14 @@ jobs:
           targetPath: $(Build.ArtifactStagingDirectory)
 
   - job: "Analyze"
+
+    dependsOn:
+      - Prepare
+
     variables:
       - template: ../variables/globals.yml
+      - name: GeneratedPackageTargets
+        value: $[dependencies.Prepare.outputs['SelectPackages.GeneratedPackageTargets']]
 
     pool:
       vmImage: "ubuntu-16.04"
@@ -74,19 +97,11 @@ jobs:
           ward scan -d $(Build.SourcesDirectory) -c $(Build.SourcesDirectory)/.docsettings.yml
         displayName: "Verify Readmes"
 
-      - ${{ if eq(parameters.PackageName,'')}}:
-          - script: |
-              node common/scripts/install-run-rush.js install
-              node common/scripts/install-run-rush.js audit
-            condition: and(succeeded(), eq(variables['RunNpmAudit'], 'true'))
-            displayName: "Audit packages"
-
-      - ${{ if ne(parameters.PackageName,'')}}:
-          - script: |
-              node common/scripts/install-run-rush.js install
-              node common/scripts/install-run-rush.js audit --to ${{parameters.PackageName}}
-            condition: and(succeeded(), eq(variables['RunNpmAudit'], 'true'))
-            displayName: "Audit packages"
+      - script: |
+          node common/scripts/install-run-rush.js install
+          node common/scripts/install-run-rush.js audit $(GeneratedPackageTargets)
+        condition: and(succeeded(), eq(variables['RunNpmAudit'], 'true'))
+        displayName: "Audit packages"
 
       - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
         # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR
@@ -96,6 +111,9 @@ jobs:
 
   - job: "Test"
 
+    dependsOn:
+      - Prepare
+
     strategy:
       matrix:
         Linux_Node8:
@@ -138,6 +156,9 @@ jobs:
     pool:
       vmImage: "$(OSVmImage)"
 
+    variables:
+      GeneratedPackageTargets: $[dependencies.Prepare.outputs['SelectPackages.GeneratedPackageTargets']]
+
     steps:
       - task: NodeTool@0
         inputs:
@@ -148,29 +169,15 @@ jobs:
           node common/scripts/install-run-rush.js install
         displayName: "Install dependencies"
 
-      # If there is no package name, then don't use the --to option.
-      - ${{ if eq(parameters.PackageName,'')}}:
-          - script: |
-              node common/scripts/install-run-rush.js build --verbose
-            displayName: "Build libraries"
-          - script: |
-              node common/scripts/install-run-rush.js build:test --verbose
-            displayName: "Build test assets"
-          - script: |
-              node common/scripts/install-run-rush.js unit-test --verbose
-            displayName: "Test libraries"
-
-      # On the other hand, if there is a PackageName, supply the --to option.
-      - ${{ if ne(parameters.PackageName,'')}}:
-          - script: |
-              node common/scripts/install-run-rush.js build --verbose --to ${{parameters.PackageName}}
-            displayName: "Build libraries"
-          - script: |
-              node common/scripts/install-run-rush.js build:test --verbose --to ${{parameters.PackageName}}
-            displayName: "Build test assets"
-          - script: |
-              node common/scripts/install-run-rush.js unit-test --verbose --to ${{parameters.PackageName}}
-            displayName: "Test libraries"
+      - script: |
+          node common/scripts/install-run-rush.js build --verbose $(GeneratedPackageTargets)
+        displayName: "Build libraries"
+      - script: |
+          node common/scripts/install-run-rush.js build:test --verbose $(GeneratedPackageTargets)
+        displayName: "Build test assets"
+      - script: |
+          node common/scripts/install-run-rush.js unit-test --verbose $(GeneratedPackageTargets)
+        displayName: "Test libraries"
 
       - task: PublishTestResults@2
         inputs:

diff --git a/eng/pipelines/templates/steps/select-packages.yml b/eng/pipelines/templates/steps/select-packages.yml
@@ -0,0 +1,7 @@
+steps:
+  - script: |
+      npm ci # This causes us to download packages exclusively based on the lock file, faster and more secure.
+      npm run select-packages "$(System.DefaultWorkingDirectory)/sdk/${{parameters.ServiceDirectory}}/!(arm-*)/package.json"
+    workingDirectory: $(System.DefaultWorkingDirectory)/eng/tools/select-packages
+    name: SelectPackages
+    displayName: "Selecting packages to build"
diff --git a/eng/tools/select-packages/index.js b/eng/tools/select-packages/index.js
@@ -0,0 +1,78 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See the @microsoft/rush package's LICENSE file for license information.
+
+// This script searches for package.json files recursively under a given path
+// excluding packages under a node_modules folder and opens them to determine
+// whether they contain a "sdk-archetype" field with the value of "client".
+//
+// If they do, then the package name is recorded and then after all package.json files
+// have been scanned a single string is output using the Azure Pipelines Commands special
+// syntax which creates a variable for use within a later stage of the pipeine.
+var glob = require("glob");
+
+let logLineCounter = 0;
+const log = message => {
+  console.log(`select-packages(${logLineCounter++}): ${message}`);
+};
+
+log(`Working directory is "${process.cwd()}".`);
+
+let filter = process.argv[2];
+log(`Specified glob filter is "${filter}".`);
+
+// This code assumes that in the service directory, packages that we want to publish
+// exist only in directories that are immediate children of the service directory.
+glob(filter, (err, files) => {
+  if (err) {
+    log(`An error occured: ${err}`);
+    process.exit(1);
+  }
+
+  let packageTargets = "";
+
+  if (files) {
+    log(`Found ${files.length} packages under service directory.`);
+
+    for (let fileIndex in files) {
+      log(`Processing package file ${fileIndex} at "${files[fileIndex]}"`);
+      let packageContents = require(files[fileIndex]);
+
+      log(
+        `Package name for package ${fileIndex} is "${packageContents.name}".`
+      );
+
+      if (packageContents["sdk-type"] == "client") {
+        log(
+          `Package "${packageContents.name}" has sdk-type of "${
+            packageContents["sdk-type"]
+          }".`
+        );
+        packageTargets += `--to "${packageContents.name}" `;
+      } else {
+        log(
+          `Package "${
+            packageContents.name
+          }" DOES NOT have an sdk-type of "client".`
+        );
+      }
+    }
+
+    log(
+      `Finished processing packages. Emitting variable using: ${packageTargets}`
+    );
+
+    // Can't use regular logging here because the pattern for Azure Pipelines requires ##vso to be the first chars.
+    console.log(
+      `##vso[task.setvariable variable=GeneratedPackageTargets;isOutput=true;]${packageTargets}`
+    );
+
+    log(
+      `Emitted variable "GeneratedPackageTargets" with content: ${packageTargets}`
+    );
+  } else {
+    log("Did not find any packages under service directory.");
+    process.exit(2);
+  }
+
+  log("Done!");
+});
diff --git a/eng/tools/select-packages/package-lock.json b/eng/tools/select-packages/package-lock.json
diff --git a/eng/tools/select-packages/package.json b/eng/tools/select-packages/package.json
@@ -0,0 +1,14 @@
+{
+  "name": "select-packages",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "select-packages": "node index.js"
+  },
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "glob": "^7.1.4"
+  }
+}
diff --git a/sdk/core/abort-controller/package.json b/sdk/core/abort-controller/package.json
@@ -1,5 +1,6 @@
 {
   "name": "@azure/abort-controller",
+  "sdk-type": "client",
   "version": "1.0.0",
   "description": "Microsoft Azure SDK for JavaScript - Aborter",
   "main": "./dist/index.js",

diff --git a/sdk/core/amqp-common/ci.yml b/sdk/core/amqp-common/ci.yml
@@ -9,9 +9,9 @@ pr:
       - master
   paths:
     include:
-      - sdk/core/amqp-common/
+      - sdk/core/
 
 jobs:
   - template: ../../../eng/pipelines/templates/jobs/archetype-sdk-client.yml
     parameters:
-      PackageName: "@azure/amqp-common"
+      ServiceDirectory: core