PNPM workspaces

.vscode/launch.json

@@ -5,11 +5,47 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "type": "pwa-chrome",
+      "name": "dtslint-runner local all",
+      "program": "${workspaceFolder}/packages/dtslint-runner/dist/index.js",
       "request": "launch",
-      "name": "Launch Chrome against localhost",
-      "url": "http://localhost:8080",
-      "webRoot": "${workspaceFolder}"
-    }
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "args": [
+          "--path", "../DefinitelyTyped", "--selection", "all", "--localTypeScriptPath", "../../ts/built/local",
+      ],
+      "type": "node"
+    },
+    {
+      "name": "jest",
+      "program": "${workspaceFolder}/node_modules/.bin/jest",
+      "request": "launch",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "args": [ "definitions-parser"],
+      "type": "node"
+    },
+    {
+      "name": "dts-critic test",
+      "program": "${workspaceFolder}/packages/dts-critic/dist/index.js",
+      "request": "launch",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "args": [ "--dts", "packages/dts-critic/testsource/tslib/index.d.ts", ],
+      "type": "node"
+    },
+    {
+      "name": "dtslint aframe",
+      "program": "${workspaceFolder}/packages/dtslint/dist/index.js",
+      "request": "launch",
+      "skipFiles": [
+        "<node_internals>/**"
+      ],
+      "args": [ "../DefinitelyTyped/types/aframe", ],
+
+      "type": "node"
+    },
   ]
 }

README.md

@@ -3,12 +3,13 @@
 A monorepo for formerly disparate DefinitelyTyped-related tools:
 
 - [definitions-parser](packages/definitions-parser): the part of [microsoft/types-publisher](https://github.com/microsoft/types-publisher) that reads DefinitelyTyped repository data
-- [dtslint](packages/dtslint): [microsoft/dtslint](https://github.com/microsoft/dtslint)
-- [dtslint-runner](packages/dtslint-runner): [DefinitelyTyped/dtslint-runner](https://github.com/DefinitelyTyped/dtslint-runner)
-- [dts-critic](packages/dts-critic): [DefinitelyTyped/dts-critic](https://github.com/DefinitelyTyped/dts-critic)
-- [header-parser](packages/header-parser): [microsoft/definitelytyped-header-parser](https://github.com/microsoft/definitelytyped-header-parser)
+- [dtslint](packages/dtslint): [microsoft/dtslint](https://github.com/microsoft/dtslint), make sure a package is correct for DT
+- [dtslint-runner](packages/dtslint-runner): [DefinitelyTyped/dtslint-runner](https://github.com/DefinitelyTyped/dtslint-runner), test all packages, or all changed packages, on DT with dtslint
+- [eslint-plugin](packages/eslint-plugin): provides DT-specific lint rules (except for the few remaining tslint rules, still in dtslint)
+- [dts-critic](packages/dts-critic): [DefinitelyTyped/dts-critic](https://github.com/DefinitelyTyped/dts-critic), issue errors when a types packages mismatches its original Javascript package.
+- [header-parser](packages/header-parser): [microsoft/definitelytyped-header-parser](https://github.com/microsoft/definitelytyped-header-parser), check and extract DT-related info from package.json
 - [publisher](packages/publisher): the rest of [microsoft/types-publisher](https://github.com/microsoft/types-publisher)
-- [retag](packages/retag): [DefinitelyTyped/dt-retag](https://github.com/DefinitelyTyped/dt-retag)
+- [retag](packages/retag): [DefinitelyTyped/dt-retag](https://github.com/DefinitelyTyped/dt-retag), script to add ATA tags to `@types` packages. Run weekly.
 - [typescript-versions](packages/typescript-versions): the part of [definitelytyped-header-parser](https://github.com/microsoft/definitelytyped-header-parser) that tracked which TypeScript versions are published to npm and supported on DefinitelyTyped
 - [utils](packages/utils): shared utilities, mostly extracted from [microsoft/types-publisher](https://github.com/microsoft/types-publisher)
 

package.json

@@ -18,7 +18,7 @@
     "@typescript-eslint/eslint-plugin": "^5.55.0",
     "@typescript-eslint/parser": "^5.55.0",
     "eslint": "^7.31.0",
-    "eslint-plugin-import": "^2.22.1",
+    "eslint-plugin-import": "^2.27.5",
     "eslint-plugin-jsdoc": "^30.7.8",
     "jest": "^29.5.0",
     "prettier": "^2.6.2",

packages/definitions-parser/allowedPackageJsonDependencies.txt

@@ -605,6 +605,7 @@ bl
 bookshelf
 boxen
 broadcast-channel
+broccoli-node-api
 broccoli-plugin
 bson
 buffer
@@ -724,6 +725,7 @@ log4js
 logform
 loglevel
 logrocket
+long
 lottie-web
 magic-string
 markdownlint

packages/definitions-parser/src/get-affected-packages.ts

@@ -1,109 +1,82 @@
-import { mapDefined, mapIterable, sort } from "@definitelytyped/utils";
-import {
-  TypingsData,
-  AllPackages,
-  PackageId,
-  PackageBase,
-  getMangledNameForScopedPackage,
-  formatDependencyVersion,
-} from "./packages";
-
-export interface Affected {
-  readonly changedPackages: readonly TypingsData[];
-  readonly dependentPackages: readonly TypingsData[];
-  allPackages: AllPackages;
+import { assertDefined, execAndThrowErrors, mapDefined, withoutStart } from "@definitelytyped/utils";
+import { sourceBranch, sourceRemote } from "./lib/settings";
+import { AllPackages, PackageId, formatTypingVersion, getDependencyFromFile } from "./packages";
+import { resolve } from "path";
+import { satisfies } from "semver";
+export interface PreparePackagesResult {
+  readonly packageNames: Set<string>;
+  readonly dependents: Set<string>;
 }
 
 /** Gets all packages that have changed on this branch, plus all packages affected by the change. */
-export function getAffectedPackages(allPackages: AllPackages, changedPackageIds: PackageId[]): Affected {
-  const resolved = changedPackageIds.map((id) => allPackages.tryResolve(id));
-  // If a package doesn't exist, that's because it was deleted.
-  const changed = mapDefined(resolved, (id) => allPackages.tryGetTypingsData(id));
-  const dependent = mapIterable(collectDependers(resolved, getReverseDependencies(allPackages, resolved)), (p) =>
-    allPackages.getTypingsData(p)
-  );
-  return { changedPackages: changed, dependentPackages: sortPackages(dependent), allPackages };
-}
-
-/** Every package name in the original list, plus their dependencies (incl. dependencies' dependencies). */
-export function allDependencies(allPackages: AllPackages, packages: Iterable<TypingsData>): TypingsData[] {
-  return sortPackages(transitiveClosure(packages, (pkg) => allPackages.allDependencyTypings(pkg)));
-}
-
-/** Collect all packages that depend on changed packages, and all that depend on those, etc. */
-function collectDependers(
-  changedPackages: PackageId[],
-  reverseDependencies: Map<PackageId, Set<PackageId>>
-): Set<PackageId> {
-  const dependers = transitiveClosure(changedPackages, (pkg) => reverseDependencies.get(pkg) || []);
-  // Don't include the original changed packages, just their dependers
-  for (const original of changedPackages) {
-    dependers.delete(original);
-  }
-  return dependers;
-}
-
-function sortPackages(packages: Iterable<TypingsData>): TypingsData[] {
-  return sort<TypingsData>(packages, PackageBase.compare); // tslint:disable-line no-unbound-method
-}
-
-function transitiveClosure<T>(initialItems: Iterable<T>, getRelatedItems: (item: T) => Iterable<T>): Set<T> {
-  const all = new Set<T>();
-  const workList: T[] = [];
-
-  function add(item: T): void {
-    if (!all.has(item)) {
-      all.add(item);
-      workList.push(item);
+export async function getAffectedPackages(
+  allPackages: AllPackages,
+  deletions: PackageId[],
+  definitelyTypedPath: string
+): Promise<PreparePackagesResult> {
+  const allDependents = [];
+  const filters = [`--filter '...[${sourceRemote}/${sourceBranch}]'`];
+  for (const d of deletions) {
+    for (const dep of allPackages.allTypings()) {
+      for (const [name, version] of dep.allPackageJsonDependencies()) {
+        if (
+          "@types/" + d.typesDirectoryName === name &&
+          (d.version === "*" || satisfies(formatTypingVersion(d.version), version))
+        ) {
+          filters.push(`--filter '...${dep.name}'`);
+          break;
+        }
+      }
     }
   }
-
-  for (const item of initialItems) {
-    add(item);
-  }
-
-  while (workList.length) {
-    const item = workList.pop()!;
-    for (const newItem of getRelatedItems(item)) {
-      add(newItem);
-    }
+  const changedPackageNames = await execAndThrowErrors(
+    `pnpm ls -r --depth -1 --parseable --filter '[${sourceRemote}/${sourceBranch}]'`,
+    definitelyTypedPath
+  );
+  // Chunk into 100-package chunks because of CMD.COM's command-line length limit
+  for (let i = 0; i < filters.length; i += 100) {
+    allDependents.push(
+      await execAndThrowErrors(
+        `pnpm ls -r --depth -1 --parseable ${filters.slice(i, i + 100).join(" ")}`,
+        definitelyTypedPath
+      )
+    );
   }
-
-  return all;
+  return getAffectedPackagesWorker(allPackages, changedPackageNames, allDependents, definitelyTypedPath);
 }
-
-/** Generate a map from a package to packages that depend on it. */
-function getReverseDependencies(
+/** This function is exported for testing, since it's determined entirely by its inputs. */
+export function getAffectedPackagesWorker(
   allPackages: AllPackages,
-  changedPackages: PackageId[]
-): Map<PackageId, Set<PackageId>> {
-  const map = new Map<string, [PackageId, Set<PackageId>]>();
-  for (const changed of changedPackages) {
-    map.set(packageIdToKey(changed), [changed, new Set()]);
-  }
-  for (const typing of allPackages.allTypings()) {
-    if (!map.has(packageIdToKey(typing.id))) {
-      map.set(packageIdToKey(typing.id), [typing.id, new Set()]);
-    }
-  }
-  for (const typing of allPackages.allTypings()) {
-    for (const [name, version] of Object.entries(typing.dependencies)) {
-      const dependencies = map.get(packageIdToKey(allPackages.tryResolve({ name, version })));
-      if (dependencies) {
-        dependencies[1].add(typing.id);
-      }
-    }
-    for (const dependencyName of typing.testDependencies) {
-      const version = typing.pathMappings[dependencyName] || "*";
-      const dependencies = map.get(packageIdToKey(allPackages.tryResolve({ name: dependencyName, version })));
-      if (dependencies) {
-        dependencies[1].add(typing.id);
-      }
-    }
-  }
-  return new Map(map.values());
+  changedOutput: string,
+  dependentOutputs: string[],
+  definitelyTypedPath: string
+): PreparePackagesResult {
+  const dt = resolve(definitelyTypedPath);
+  const changedDirs = mapDefined(changedOutput.split("\n"), getDirectoryName(dt));
+  const dependentDirs = mapDefined(dependentOutputs.join("\n").split("\n"), getDirectoryName(dt));
+  const packageNames = new Set(
+    changedDirs.map(
+      (c) =>
+        assertDefined(
+          allPackages.tryGetTypingsData(assertDefined(getDependencyFromFile(c + "/index.d.ts"), "bad path " + c))
+        ).subDirectoryPath
+    )
+  );
+  const dependents = new Set(
+    dependentDirs
+      .map(
+        (d) =>
+          assertDefined(
+            allPackages.tryGetTypingsData(assertDefined(getDependencyFromFile(d + "/index.d.ts"), "bad path " + d)),
+            d + " package not found"
+          ).subDirectoryPath
+      )
+      .filter((d) => !packageNames.has(d))
+  );
+  return { packageNames, dependents };
 }
 
-function packageIdToKey(pkg: PackageId): string {
-  return getMangledNameForScopedPackage(pkg.name) + "/v" + formatDependencyVersion(pkg.version);
+function getDirectoryName(dt: string): (line: string) => string | undefined {
+  return (line) =>
+    line && line !== dt ? assertDefined(withoutStart(line, dt + "/"), line + " is missing prefix " + dt) : undefined;
 }