From 1b98dd9feb91679c0339b237896d6764ee8dae89 Mon Sep 17 00:00:00 2001 From: Victor Savkin Date: Sat, 22 Sep 2018 08:23:01 -0400 Subject: [PATCH] fix: not all schematic collections get picked up and displayed --- .../src/integration/extensions.spec.ts | 20 ++-- .../src/integration/generate.spec.ts | 2 +- .../src/integration/tasks.spec.ts | 4 +- .../src/integration/utils.ts | 13 ++- .../lib/schematics/schematics.component.ts | 9 +- libs/ui/src/lib/flags/flags.component.ts | 1 - package.json | 3 +- server/package.json | 1 + server/src/api/commands.ts | 13 +-- server/src/api/read-projects.ts | 2 +- server/src/api/read-schematic-collections.ts | 77 +++++++++------- server/src/schema/resolvers.ts | 19 ++-- server/src/server.ts | 2 +- server/src/utils.ts | 92 +++++++++++++++++-- 14 files changed, 164 insertions(+), 94 deletions(-) diff --git a/apps/angular-console-e2e/src/integration/extensions.spec.ts b/apps/angular-console-e2e/src/integration/extensions.spec.ts index 2b027d1e67..7d0ab8635d 100644 --- a/apps/angular-console-e2e/src/integration/extensions.spec.ts +++ b/apps/angular-console-e2e/src/integration/extensions.spec.ts @@ -3,11 +3,13 @@ import { clickOnTask, goBack, goToExtensions, + goToGenerate, openProject, projectPath, taskListHeaders, tasks, - texts + texts, + waitForActionToComplete } from './utils'; describe('Extensions', () => { @@ -33,18 +35,16 @@ describe('Extensions', () => { }); it('adds an extension', () => { - clickOnTask('Available Extensions', '@progress/kendo-angular-menu', false); - cy.get('div.context-title').contains( - '@progress/kendo-angular-menu extension' - ); + clickOnTask('Available Extensions', '@angular/material', false); + cy.get('div.context-title').contains('@angular/material'); cy.get('button') .contains('Add') .click(); - cy.wait(100); + checkDisplayedCommand(`$ ng add @angular/material`); - checkDisplayedCommand(`$ ng add @progress/kendo-angular-menu`); + waitForActionToComplete(); goBack(); @@ -52,5 +52,11 @@ describe('Extensions', () => { taskListHeaders($p => { expect(texts($p)[0]).to.equal('Available Extensions'); }); + + // check that the schematics added by angular material are available + goToGenerate(); + taskListHeaders($p => { + expect(texts($p)[0]).to.equal('@angular/material'); + }); }); }); diff --git a/apps/angular-console-e2e/src/integration/generate.spec.ts b/apps/angular-console-e2e/src/integration/generate.spec.ts index ea17a01800..0e0d175c71 100644 --- a/apps/angular-console-e2e/src/integration/generate.spec.ts +++ b/apps/angular-console-e2e/src/integration/generate.spec.ts @@ -22,7 +22,7 @@ describe('Generate', () => { it('filters schematics', () => { taskListHeaders($p => { - expect($p.length).to.equal(1); + expect($p.length).to.equal(2); expect(texts($p)[0]).to.equal('@schematics/angular'); }); diff --git a/apps/angular-console-e2e/src/integration/tasks.spec.ts b/apps/angular-console-e2e/src/integration/tasks.spec.ts index 4588240893..a6bd77cc6a 100644 --- a/apps/angular-console-e2e/src/integration/tasks.spec.ts +++ b/apps/angular-console-e2e/src/integration/tasks.spec.ts @@ -10,7 +10,7 @@ import { taskListHeaders, tasks, texts, - waitForBuild + waitForActionToComplete } from './utils'; describe('Tasks', () => { @@ -74,7 +74,7 @@ describe('Tasks', () => { .contains('Run') .click(); - waitForBuild(); + waitForActionToComplete(); checkFileExists(`dist/proj/main.js`); goBack(); diff --git a/apps/angular-console-e2e/src/integration/utils.ts b/apps/angular-console-e2e/src/integration/utils.ts index f379f8c974..e149abfbd3 100644 --- a/apps/angular-console-e2e/src/integration/utils.ts +++ b/apps/angular-console-e2e/src/integration/utils.ts @@ -175,14 +175,19 @@ export function waitForAutocomplete() { cy.wait(700); } -export function waitForBuild() { - cy.wait(35000); -} - export function waitForNgNew() { cy.wait(120000); } +export function waitForActionToComplete() { + cy.wait(100); // this is to give the app time ot disable the button first + cy.get('button.action-button:enabled[color="primary"]', { + timeout: 120000 + }).should($p => { + expect($p.length).to.equal(1); + }); +} + export function autocompletion(callback: (s: any) => void) { cy.get('div.mat-autocomplete-panel').within(() => { cy.root() diff --git a/libs/feature-generate/src/lib/schematics/schematics.component.ts b/libs/feature-generate/src/lib/schematics/schematics.component.ts index e2ce1aed63..0f74983955 100644 --- a/libs/feature-generate/src/lib/schematics/schematics.component.ts +++ b/libs/feature-generate/src/lib/schematics/schematics.component.ts @@ -55,14 +55,7 @@ export class SchematicsComponent { map(r => { const collections: Array = (r as any).data.workspace .schematicCollections; - return collections - .map(c => { - const s = [...c.schematics].sort((a, b) => - a.name.localeCompare(b.name) - ); - return { ...c, schematics: s }; - }) - .filter(c => c.schematics.length > 0); + return collections.filter(c => c.schematics.length > 0); }) ); diff --git a/libs/ui/src/lib/flags/flags.component.ts b/libs/ui/src/lib/flags/flags.component.ts index c66b947f2a..a3a55a964a 100644 --- a/libs/ui/src/lib/flags/flags.component.ts +++ b/libs/ui/src/lib/flags/flags.component.ts @@ -217,7 +217,6 @@ export class FlagsComponent { this.subscription = this.formGroup.valueChanges .pipe(startWith(this.formGroup.value)) .subscribe(value => { - console.log('changes', value); this.emitNext(value); }); } diff --git a/package.json b/package.json index fe79134452..7995e8ba41 100644 --- a/package.json +++ b/package.json @@ -138,7 +138,8 @@ "universal-analytics": "^0.4.17", "uuid": "^3.3.2", "xterm": "3.4.1", - "zone.js": "^0.8.26" + "zone.js": "^0.8.26", + "strip-json-comments": "2.0.1" }, "jest": { "modulePathIgnorePatterns": [ diff --git a/server/package.json b/server/package.json index 379fc0ad38..be860ee90d 100644 --- a/server/package.json +++ b/server/package.json @@ -24,6 +24,7 @@ "universal-analytics": "^0.4.17", "uuid": "^3.3.2", "apollo-server-express": "^2.0.4", + "strip-json-comments": "2.0.1", "electron-store": "2.0.0" }, "devDependencies": { diff --git a/server/src/api/commands.ts b/server/src/api/commands.ts index 277662e794..cfc4a88e38 100644 --- a/server/src/api/commands.ts +++ b/server/src/api/commands.ts @@ -1,6 +1,5 @@ -import { listFilesRec } from '../utils'; - import * as os from 'os'; + const spawn = require('node-pty-prebuilt').spawn; interface CommandResult { @@ -12,7 +11,6 @@ interface CommandResult { let commandRunIndex = 0; export let commandInProgress: CommandResult | null; -export const files: { [path: string]: string[] } = {}; export function runCommand(cwd: string, program: string, cmds: string[]) { stopAllCommands(); @@ -49,12 +47,3 @@ export function stopAllCommands() { } commandInProgress = null; } - -export function listFiles(path: string) { - setTimeout(() => { - files[path] = listFilesRec(path); - setTimeout(() => { - listFiles(path); - }, 60000); - }, 0); -} diff --git a/server/src/api/read-projects.ts b/server/src/api/read-projects.ts index a4cd1de8a5..02943897bc 100644 --- a/server/src/api/read-projects.ts +++ b/server/src/api/read-projects.ts @@ -61,7 +61,7 @@ export function readSchema(basedir: string, builder: string) { function readBuildersFile(basedir: string, npmPackage: string): any { const packageJson = readJsonFile( path.join(npmPackage, 'package.json'), - basedir + path.join(basedir, 'node_modules') ); const b = packageJson.json.builders; const buildersPath = b.startsWith('.') ? b : `./${b}`; diff --git a/server/src/api/read-schematic-collections.ts b/server/src/api/read-schematic-collections.ts index e87844144a..c44b88c95a 100644 --- a/server/src/api/read-schematic-collections.ts +++ b/server/src/api/read-schematic-collections.ts @@ -1,5 +1,10 @@ import * as path from 'path'; -import { normalizeSchema, readJsonFile } from '../utils'; +import { + fileExistsSync, + listOfUnnestedNpmPackages, + normalizeSchema, + readJsonFile +} from '../utils'; interface SchematicCollection { name: string; @@ -22,10 +27,34 @@ interface Schematic { }[]; } -export function readSchematicCollections( +export function readAllSchematicCollections(basedir: string) { + const nodeModulesDir = path.join(basedir, 'node_modules'); + const packages = listOfUnnestedNpmPackages(nodeModulesDir); + const schematicCollections = packages.filter(p => { + try { + return !!readJsonFile(path.join(p, 'package.json'), nodeModulesDir).json + .schematics; + } catch (e) { + if ( + e.message && + (e.message.indexOf('no such file') > -1 || + e.message.indexOf('not a directory') > -1) + ) { + return false; + } else { + throw e; + } + } + }); + return schematicCollections.map(c => + readSchematicCollections(nodeModulesDir, c) + ); +} + +function readSchematicCollections( basedir: string, collectionName: string -): SchematicCollection[] { +): SchematicCollection { const packageJson = readJsonFile( path.join(collectionName, 'package.json'), basedir @@ -34,42 +63,24 @@ export function readSchematicCollections( packageJson.json.schematics, path.dirname(packageJson.path) ); - const collectionSchematics = []; - let ex = [] as any[]; - if (collection.json.extends) { - const e = Array.isArray(collection.json.extends) - ? collection.json.extends - : [collection.json.extends]; - ex = [...ex, ...e]; - } - const schematicCollection = { name: collectionName, schematics: [] as Schematic[] }; Object.entries(collection.json.schematics).forEach(([k, v]: [any, any]) => { - if (!v.hidden) { - if (v.extends) { - ex.push(v.extends.split(':')[0]); - } else { - const schematicSchema = readJsonFile( - v.schema, - path.dirname(collection.path) - ); + if (!v.hidden && !v.extends) { + const schematicSchema = readJsonFile( + v.schema, + path.dirname(collection.path) + ); - schematicCollection.schematics.push({ - name: k, - collection: collectionName, - schema: normalizeSchema(schematicSchema.json), - description: v.description - }); - } + schematicCollection.schematics.push({ + name: k, + collection: collectionName, + schema: normalizeSchema(schematicSchema.json), + description: v.description + }); } }); - - let res = [schematicCollection]; - new Set(ex).forEach(e => { - res = [...res, ...readSchematicCollections(basedir, e)]; - }); - return res; + return schematicCollection; } diff --git a/server/src/schema/resolvers.ts b/server/src/schema/resolvers.ts index a34432ee0d..1e25c40544 100644 --- a/server/src/schema/resolvers.ts +++ b/server/src/schema/resolvers.ts @@ -4,7 +4,9 @@ import { filterByName, findClosestNg, findExecutable, - readJsonFile + readJsonFile, + files, + cacheFiles } from '../utils'; import { completeFiles, @@ -13,7 +15,7 @@ import { completeProjects } from '../api/completions'; -import { readSchematicCollections } from '../api/read-schematic-collections'; +import { readAllSchematicCollections } from '../api/read-schematic-collections'; import { readDescription, readProjects, @@ -26,8 +28,6 @@ import { openInEditor, readEditors } from '../api/read-editors'; import { readNpmScripts, readNpmScriptSchema } from '../api/read-npm-scripts'; import { readDirectory } from '../api/read-directory'; import { - listFiles, - files, commandInProgress, runCommand, stopAllCommands @@ -76,14 +76,7 @@ const Workspace = { if (!directoryExists(path.join(p, 'node_modules'))) { throw new Error(`node_modules is not found`); } - - const angularJson = readJsonFile('./angular.json', p).json; - const collectionName = - angularJson.cli && angularJson.cli.defaultCollection - ? angularJson.cli.defaultCollection - : '@schematics/angular'; - - return filterByName(readSchematicCollections(p, collectionName), args); + return filterByName(readAllSchematicCollections(p), args); }, npmScripts(workspace: any, args: any) { return filterByName(workspace.npmScripts, args); @@ -128,7 +121,7 @@ const Database = { workspace(_root, args: any) { try { if (!files[args.path]) { - listFiles(args.path); + cacheFiles(args.path); } const packageJson = readJsonFile('./package.json', args.path).json; const angularJson = readJsonFile('./angular.json', args.path).json; diff --git a/server/src/server.ts b/server/src/server.ts index b196e6e7a7..3ba2c01e56 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -6,7 +6,7 @@ import { filterByName, findClosestNg, findExecutable, - listFilesRec, + listFiles, readJsonFile } from './utils'; import { schema } from './schema'; diff --git a/server/src/utils.ts b/server/src/utils.ts index ba241efd8c..0400a74752 100644 --- a/server/src/utils.ts +++ b/server/src/utils.ts @@ -4,8 +4,10 @@ import { stat, statSync } from 'fs'; import { platform } from 'os'; import { bindNodeCallback, Observable } from 'rxjs'; import { map } from 'rxjs/operators'; +import * as stripJsonComments from 'strip-json-comments'; -const resolve = require('resolve'); +export const files: { [path: string]: string[] } = {}; +export let fileContents: { [path: string]: any } = {}; export function findExecutable(command: string, cwd: string): string { const paths = (process.env.PATH as string).split(path.delimiter); @@ -68,7 +70,23 @@ export function findClosestNg(dir: string): string { } } -export function listFilesRec(dirName: string): string[] { +export function listOfUnnestedNpmPackages(nodeModulesDir: string): string[] { + const res: string[] = []; + fs.readdirSync(nodeModulesDir).forEach(npmPackageOrScope => { + if (npmPackageOrScope.startsWith('@')) { + fs.readdirSync(path.join(nodeModulesDir, npmPackageOrScope)).forEach( + p => { + res.push(`${npmPackageOrScope}/${p}`); + } + ); + } else { + res.push(npmPackageOrScope); + } + }); + return res; +} + +export function listFiles(dirName: string): string[] { // TODO use .gitignore to skip files if (dirName.indexOf('node_modules') > -1) return []; if (dirName.indexOf('dist') > -1) return []; @@ -83,7 +101,7 @@ export function listFilesRec(dirName: string): string[] { if (!fs.statSync(child).isDirectory()) { res.push(child); } else if (fs.statSync(child).isDirectory()) { - res.push(...listFilesRec(child)); + res.push(...listFiles(child)); } } catch (e) {} }); @@ -91,6 +109,25 @@ export function listFilesRec(dirName: string): string[] { return res; } +function cacheJsonFiles(basedir: string) { + try { + const nodeModulesDir = path.join(basedir, 'node_modules'); + const packages = listOfUnnestedNpmPackages(nodeModulesDir); + + const res: any = {}; + const schematicCollections = packages.forEach(p => { + const filePath = path.join(nodeModulesDir, p, 'package.json'); + if (!fileExistsSync(filePath)) return; + res[filePath] = readAndParseJson( + path.join(nodeModulesDir, p, 'package.json') + ); + }); + return res; + } catch (e) { + return {}; + } +} + export function directoryExists(filePath: string): boolean { try { return statSync(filePath).isDirectory(); @@ -112,15 +149,32 @@ export function fileExists(filePath: string): Observable { return observableStat(filePath).pipe(map(stat => stat.isFile())); } +function readAndParseJson(fullFilePath: string): any { + return JSON.parse( + stripJsonComments(fs.readFileSync(fullFilePath).toString()) + ); +} + export function readJsonFile( - path: string, + filePath: string, basedir: string -): { [k: string]: any } { - const fullFilePath = resolve.sync(path, { basedir }); - return { - path: fullFilePath, - json: JSON.parse(fs.readFileSync(fullFilePath).toString()) - }; +): { path: string; json: any } { + const fullFilePath = path.join(basedir, filePath); + + // we can try to retrieve node_modules files from the cache because + // they don't change very often + const cache = basedir.endsWith('node_modules'); + if (cache && fileContents[fullFilePath]) { + return { + path: fullFilePath, + json: fileContents[fullFilePath] + }; + } else { + return { + path: fullFilePath, + json: readAndParseJson(fullFilePath) + }; + } } export function normalizeSchema(p: { @@ -182,3 +236,21 @@ export function normalizePath(value: string): string { .filter(r => !!r) .join('\\'); } + +/** + * To improve performance angularconsole preprocesses + * + * * the list of local files + * * json files from node_modules we are likely to read + * + * both the data sets get updated every 30 seconds. + */ +export function cacheFiles(path: string) { + setTimeout(() => { + files[path] = listFiles(path); + fileContents = cacheJsonFiles(path); + setTimeout(() => { + cacheFiles(path); + }, 30000); + }, 0); +}