feat: stop ADO extension reliance on pipeline-installed Node and Yarn versions (by migrating to yarn v3)

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
         node-version: 16.x
 
     - name: Install dependencies
-      run: yarn install --frozen-lockfile
+      run: yarn install --immutable
 
     - name: Check copyright headers
       run: yarn copyright:check
@@ -52,18 +52,18 @@ jobs:
         node-version: 16.x
 
     - name: Install dependencies
-      run: yarn install --frozen-lockfile
+      run: yarn install --immutable
 
     - name: Build
       run: yarn cbuild
 
     - name: Run tests
-      run: yarn test -- -- --ci --coverage
+      run: yarn test --ci --coverage
       env:
         NODE_OPTIONS: --max_old_space_size=4096
 
     - name: Publish code coverage to codecov
-      run: yarn publish-code-coverage -- -t ${{ secrets.CODECOV_TOKEN }}
+      run: yarn publish-code-coverage -t ${{ secrets.CODECOV_TOKEN }}
 
     - name: Run e2e tests
-      run: yarn test:e2e -- -- --ci
+      run: yarn test:e2e --ci

.gitignore

@@ -41,3 +41,12 @@ coverage
 
 # js map files
 *.js.map
+
+# Yarn config for non-zero-installs
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions

.prettierignore

@@ -4,6 +4,7 @@
 .gitattributes
 .github
 .prettierignore
+.yarn
 **/.DS_Store
 **/.funcignore
 **/.gitignore

.yarnrc.yml

@@ -0,0 +1,10 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+nodeLinker: node-modules
+
+plugins:
+    - path: .yarn/plugins/@yarnpkg/plugin-workspace-tools.cjs
+      spec: '@yarnpkg/plugin-workspace-tools'
+
+yarnPath: .yarn/releases/yarn-3.4.1.cjs

dev/README.md

@@ -10,7 +10,7 @@ This repository contains code for two packages:
 -   `packages/shared` contains the core logic that is not specific to running any specific CI solution. Any code that could be shared across additional CI solutions in the future such as a GitHub Action, should be put in the shared package.
 -   `packages/ado-extension` contains the code specifically relevant to the Azure DevOps extension.
 
-We use `lerna` and `yarn workspaces` to manage the monorepo. In most cases, running yarn scripts in the root directory should produce output from all packages.
+We use `yarn workspaces` to manage the monorepo. In most cases, running yarn scripts in the root directory should produce output from all packages.
 
 ## Development workflow
 

license-check-and-add-config.json

@@ -1,21 +1,22 @@
 {
     "ignore": [
+        ".dependabot",
         ".git",
         ".github",
-        "./.dependabot",
-        "**/yarn.lock",
-        "**/CODEOWNERS",
-        "package.json",
-        "cred-scan-suppressions.json",
-        "**/node_modules",
+        ".yarn",
+        "**/.DS_Store",
         "**/.gitattributes",
-        "**/test-results",
-        "**/dist",
         "**/*.baseline",
-        "**/*.snap",
         "**/*.snap.md",
+        "**/*.snap",
         "**/*.taskkey",
-        "**/.DS_Store"
+        "**/CODEOWNERS",
+        "**/dist",
+        "**/node_modules",
+        "**/test-results",
+        "**/yarn.lock",
+        "cred-scan-suppressions.json",
+        "package.json"
     ],
     "license": "./copyright-header.txt",
     "licenseFormats": {

package.json

@@ -7,18 +7,19 @@
     ],
     "description": "This project welcomes contributions and suggestions.  Most contributions require you to agree to a\r Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us\r the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.",
     "scripts": {
-        "build": "lerna run build --stream",
-        "clean": "lerna run clean --stream",
-        "cbuild": "lerna run cbuild --stream",
+        "packages": "yarn workspaces foreach --exclude accessibility-insights-action --verbose",
+        "build": "yarn packages --topological-dev run build",
+        "cbuild": "yarn packages --topological-dev run cbuild",
+        "clean": "yarn packages run clean",
         "format:check": "prettier --check \"**/*\"",
         "format:fix": "prettier --write \"**/*\"",
-        "lint:check": "lerna run lint:check --stream",
-        "lint:fix": "lerna run lint:fix --stream",
+        "lint:check": "yarn packages run lint:check",
+        "lint:fix": "yarn packages run lint:fix",
         "copyright:check": "license-check-and-add check -f ./license-check-and-add-config.json",
         "copyright:fix": "license-check-and-add add -f ./license-check-and-add-config.json",
-        "test": "lerna run test --stream",
-        "test:e2e": "lerna run test:e2e --stream",
-        "docs": "lerna run docs --stream",
+        "test": "yarn packages run test",
+        "test:e2e": "yarn packages run test:e2e",
+        "docs": "yarn packages run docs",
         "precheckin": "npm-run-all --serial copyright:check format:check cbuild lint:check test docs",
         "publish-code-coverage": "npx codecov"
     },
@@ -35,6 +36,7 @@
         "url": "https://github.com/microsoft/accessibility-insights-action/issues"
     },
     "homepage": "https://github.com/microsoft/accessibility-insights-action#readme",
+    "packageManager": "[email protected]",
     "devDependencies": {
         "@types/express": "^4.17.17",
         "@types/jest": "^29.4.0",
@@ -55,7 +57,7 @@
         "jest-extended": "^3.2.4",
         "jest-file-snapshot": "^0.5.0",
         "jest-junit": "^15.0.0",
-        "lerna": "^6.5.1",
+        "js-yaml": "^4.1.0",
         "license-check-and-add": "^4.0.5",
         "mockdate": "^3.0.5",
         "npm-run-all": "^4.1.5",

packages/ado-extension/package.json

@@ -27,13 +27,34 @@
         "url": "https://github.com/microsoft/accessibility-insights-action/issues"
     },
     "homepage": "https://github.com/microsoft/accessibility-insights-action#readme",
-    "devDependencies": {
-        "tfx-cli": "^0.14.0"
-    },
     "dependencies": {
-        "@accessibility-insights-action/shared": "^1.0.0",
+        "@accessibility-insights-action/shared": "workspace:*",
         "applicationinsights": "^2.4.2",
         "azure-pipelines-task-lib": "^4.2.0",
         "reflect-metadata": "^0.1.13"
+    },
+    "devDependencies": {
+        "@typescript-eslint/eslint-plugin": "^5.53.0",
+        "@typescript-eslint/parser": "^5.53.0",
+        "case-sensitive-paths-webpack-plugin": "^2.4.0",
+        "eslint": "^8.34.0",
+        "eslint-plugin-security": "^1.7.1",
+        "fork-ts-checker-webpack-plugin": "^7.3.0",
+        "jest": "^29.4.3",
+        "jest-extended": "^3.2.4",
+        "jest-file-snapshot": "^0.5.0",
+        "jest-junit": "^15.0.0",
+        "js-yaml": "^4.1.0",
+        "mockdate": "^3.0.5",
+        "npm-run-all": "^4.1.5",
+        "prettier": "^2.8.4",
+        "rimraf": "^4.1.2",
+        "tfx-cli": "^0.14.0",
+        "ts-jest": "^29.0.5",
+        "ts-loader": "^9.4.2",
+        "typemoq": "^2.1.0",
+        "typescript": "^4.9.5",
+        "webpack": "^5.75.0",
+        "webpack-cli": "^5.0.1"
     }
 }

packages/ado-extension/prepare-package-dir.js

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 const fs = require('fs');
 const path = require('path');
+const yaml = require('js-yaml');
 const packageJson = require(process.cwd() + '/package.json');
 const getWebpackConfig = require(process.cwd() + '/webpack.config');
 const { buildRuntimePackageMetadata } = require('../shared/build-runtime-package-metadata');
@@ -42,6 +43,14 @@ console.log('copied ado-extension-overview.md to dist/overview.md');
 fs.copyFileSync('../../icons/brand-blue-128px.png', 'dist/extension-icon.png');
 console.log('copied brand-blue-128px.png to dist/extension-icon.png');
 
+fs.cpSync('../../.yarn/releases', 'dist/pkg/.yarn/releases', { recursive: true });
+console.log('copied .yarn/releases to dist/pkg/.yarn/releases');
+
+const yarnrcYaml = yaml.load(fs.readFileSync('../../.yarnrc.yml'));
+delete yarnrcYaml['plugins'];
+fs.writeFileSync('dist/pkg/.yarnrc.yml', yaml.dump(yarnrcYaml));
+console.log('wrote plugin-free dist/pkg/.yarnrc.yml');
+
 // "icon.png" is the the icon image that will be shown in the task list for classic pipelines
 // While the documentation recommends 32x32, it works with 128x128 and prevents the image from appearing blurry
 // https://learn.microsoft.com/en-us/azure/devops/extend/develop/integrate-build-task?view=azure-devops#traditional-extension-layout

packages/ado-extension/src/install-runtime-dependencies.ts

@@ -1,17 +1,34 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-import { execSync } from 'child_process';
+import { execFileSync } from 'child_process';
+import { argv } from 'process';
+import { readdirSync } from 'fs';
+import { join } from 'path';
 
 export function installRuntimeDependencies(): void {
     console.log('##[group]Installing runtime dependencies');
 
-    // This intentionally uses execSync rather than the normally-preferred execFileSync
-    // because it relies on shell behavior to pick whether to invoke yarn, yarn.exe,
-    // yarn.bat, or yarn.cmd (any of these are possible depending on OS + how Yarn is
-    // installed). This doesn't create a shell injection concern because the command
-    // is a fixed string.
-    execSync('yarn install --prod --ignore-engines --frozen-lockfile', {
+    // It's very important that we use the same node binary here as
+    // we're already executing under. Some transitive dependencies
+    // involve native build steps that are node-version-dependent
+    // and will throw runtime errors if "yarn install" is run with
+    // a different node version than we execute under.
+    //
+    // This is why we don't just invoke "yarn" - that would use whatever
+    // node version the user's pipeline happens to have installed globally,
+    // which might differ from the version that the build agent is
+    // running our task with.
+    const nodePath = argv[0];
+
+    const yarnReleasesPath = join(__dirname, '.yarn', 'releases');
+    const yarnFilename = readdirSync(yarnReleasesPath)[0];
+    const yarnPath = join(yarnReleasesPath, yarnFilename);
+
+    console.log(`##[debug]Using node from ${nodePath}`);
+    console.log(`##[debug]Using bundled yarn from ${yarnPath}`);
+
+    execFileSync(nodePath, [yarnPath, 'install', '--immutable'], {
         stdio: 'inherit',
         cwd: __dirname,
     });

packages/shared/build-runtime-package-metadata.js

@@ -57,14 +57,22 @@ function buildRuntimePackageMetadata({ packageJson, webpackConfig, outputDirecto
     // yarn.bat, or yarn.cmd (any of these are possible depending on OS + how Yarn is
     // installed). This doesn't create a shell injection concern because the command
     // is a fixed string.
-    execSync('yarn install --prod --ignore-engines --ignore-scripts', {
+    execSync('yarn install --mode=update-lockfile', {
         stdio: 'inherit',
         cwd: outputDirectory,
     });
 
     const outNodeModulesPath = path.join(outputDirectory, 'node_modules');
     console.log(`removing ${outNodeModulesPath} left behind by yarn.lock update`);
-    rmdirSync(outNodeModulesPath, { recursive: true });
+    rmdirSync(outNodeModulesPath, { recursive: true, force: true });
+
+    const outYarnCache = path.join(outputDirectory, '.yarn', 'cache');
+    console.log(`removing ${outYarnCache} left behind by yarn.lock update`);
+    rmdirSync(outYarnCache, { recursive: true, force: true });
+
+    const outYarnInstallState = path.join(outputDirectory, '.yarn', 'install-state.gz');
+    console.log(`removing ${outYarnInstallState} left behind by yarn.lock update`);
+    rmdirSync(outYarnInstallState, { recursive: true, force: true });
 
     console.log('building runtime package metadata complete');
 }

packages/shared/package.json

@@ -26,7 +26,6 @@
         "url": "https://github.com/microsoft/accessibility-insights-action/issues"
     },
     "homepage": "https://github.com/microsoft/accessibility-insights-action#readme",
-    "devDependencies": {},
     "dependencies": {
         "@types/react": "^16.14",
         "@types/react-dom": "^16.9",
@@ -45,5 +44,32 @@
         "reflect-metadata": "^0.1.13",
         "serialize-error": "^11.0.0",
         "serve-static": "^1.15.0"
+    },
+    "devDependencies": {
+        "@types/express": "^4.17.17",
+        "@types/jest": "^29.4.0",
+        "@types/lodash": "^4.14.191",
+        "@types/marked": "^4.0.8",
+        "@types/marked-terminal": "^3.1.3",
+        "@types/normalize-path": "^3.0.0",
+        "@types/puppeteer-core": "^7.0.4",
+        "@types/serve-static": "^1.15.0",
+        "@typescript-eslint/eslint-plugin": "^5.53.0",
+        "@typescript-eslint/parser": "^5.53.0",
+        "eslint": "^8.34.0",
+        "eslint-plugin-security": "^1.7.1",
+        "fork-ts-checker-webpack-plugin": "^7.3.0",
+        "jest": "^29.4.3",
+        "jest-extended": "^3.2.4",
+        "jest-file-snapshot": "^0.5.0",
+        "jest-junit": "^15.0.0",
+        "mockdate": "^3.0.5",
+        "npm-run-all": "^4.1.5",
+        "prettier": "^2.8.4",
+        "rimraf": "^4.1.2",
+        "ts-jest": "^29.0.5",
+        "ts-loader": "^9.4.2",
+        "typemoq": "^2.1.0",
+        "typescript": "^4.9.5"
     }
 }

pipelines/build.yaml

@@ -21,7 +21,7 @@ jobs:
                 versionSpec: '16.x'
             displayName: use node 16.x
 
-          - script: yarn install --frozen-lockfile
+          - script: yarn install --immutable
             displayName: Install dependencies
 
           - script: yarn copyright:check
@@ -36,7 +36,7 @@ jobs:
           - script: yarn lint:check
             displayName: Check for lint errors
 
-          - script: yarn test -- -- --ci --coverage
+          - script: yarn test --ci --coverage
             displayName: Run tests
             env:
                 NODE_OPTIONS: --max_old_space_size=4096
@@ -115,7 +115,7 @@ jobs:
                 versionSpec: '16.x'
             displayName: use node 16.x
 
-          - script: yarn install --frozen-lockfile
+          - script: yarn install --immutable
             displayName: Install dependencies
 
           - script: yarn cbuild