feat(DefinitelyTyped): add process with ui to run createMock on every type of DefinitelyTyped repository

.github/workflows/definitelyTyped.yml

@@ -1,15 +1,11 @@
 name: DefinitelyTyped Tests
 
 on:
-  push:
-    branches:
-      - 'feature/definitelytype'
-    paths:
-      - 'src/**'
-      - 'definitelyTyped/**'
+  schedule:
+    - cron: '* * */1 * *'
 
 jobs:
-  performance:
+  definitely-typed:
     runs-on: ubuntu-latest
 
     strategy:
@@ -26,14 +22,11 @@ jobs:
         run: |
           npm install
           npm run build
-        env:
-          CI: true
       - name: definitelyTyped
         run: |
-          cd definitelyTyped
+          cd definitelyTypedTests
           npm install
-          npm run install-types
-          npm run process-types TYPES=2
+          npm run start-chunk
       - name: create environment variable date
         run: echo "::set-env name=DATE::$(date +%Y-%m-%dT%H-%M-%S)"
       - name: definitelyTyped push changes
@@ -44,13 +37,13 @@ jobs:
         run: |
           git add data/*
           git checkout .
-          git config submodule.DefinitelyTyped.active false
       - name: Create Pull Request
         uses: peter-evans/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN_TYPESCRIPTTDD }}
         with:
           commit-message: Add DefinitelyTyped Tests data
           committer: typescripttdd <[email protected]>
           author: typescripttdd <[email protected]>
           title: DefinitelyTyped Tests Data on ${{ env.DATE }}
-          branch: definitely-typed-tests-${{ env.DATE }}
-          token: ${{ secrets.GITHUB_TOKEN }}
+          branch: definitely-typed-tests

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 dist
 .idea
+/definitelyTyped/DefinitelyTyped/

config/karma/karma.config.base.js

@@ -1,5 +1,5 @@
 const webpackConfig = require('../test/webpack.js');
-const ProcessService = require('../process/process');
+const ProcessService = require('../../utils/process/process');
 process.env.CHROME_BIN = require('puppeteer').executablePath();
 
 module.exports = function(config, url) {

data/definitelyTyped/list.json

@@ -0,0 +1 @@
+[{"id":"756c5f8d-be26-43fc-8831-0a12df4c972e.json","initialDate":"2020-01-11T17:17:58.333Z","lastUpdatedDate":"2020-01-11T19:29:38.377Z","typesProcessed":512}]

definitelyTypedTests/.env

@@ -0,0 +1 @@
+DEFINITELY_TYPED_DATA_URL=../data/definitelyTyped

definitelyTypedTests/index.js

@@ -0,0 +1,141 @@
+process.on('unhandledRejection', error => {
+    console.log('unhandledRejection', error);
+    process.exit(1);
+});
+
+require('dotenv').config();
+const fs = require('fs');
+const processService = require('../utils/process/process')(process);
+const execPromise = require('../utils/exec/execPromise');
+const definitelyTyped = require('./src/definitelyTyped')();
+const config = require('./src/config');
+const output = require('./src/multiProcessOutput');
+const dataFileSystem = require('../utils/dataFileSystem/dataFileSystemWriter')(process.env.DEFINITELY_TYPED_DATA_URL);
+
+try {
+    processService.ensureArgumentsValidity(['TYPES', 'PROCESS_COUNT']);
+} catch(e) {
+    console.error(e.message);
+    return;
+}
+
+const outputService = output.createNew();
+
+(async function runApp() {
+    const typesDirectories = definitelyTyped.getTypes();
+    const runConfig = await config();
+
+    logConfig(runConfig);
+    let startDirectoryIndex = runConfig.offsetType;
+    const allRuns = [];
+
+    for(let i = 0; i < runConfig.processes.length; i++) {
+        allRuns.push(
+            runAllDir(typesDirectories.slice(startDirectoryIndex, startDirectoryIndex + runConfig.processes[i].items), i)
+        );
+        startDirectoryIndex += runConfig.processes[i].items;
+    }
+
+    Promise.all(allRuns).then(() => {
+        const generatedOutput = outputService.generateOutput();
+        const date = new Date().toISOString();
+
+        if (runConfig.entryToUpdate) {
+            dataFileSystem.updateData(runConfig.entryToUpdate.id, {
+                lastUpdatedDate: date,
+                typesProcessed: generatedOutput.length + runConfig.offsetType
+            }, generatedOutput);
+        } else {
+            dataFileSystem.addData({
+                initialDate: date,
+                lastUpdatedDate: date,
+                typesProcessed: generatedOutput.length
+            }, generatedOutput);
+        }
+    });
+})();
+
+function logConfig(processesConfig) {
+    console.log(`Total types: ${processesConfig.totalTypesCount}`);
+    console.log(`Processes: ${processesConfig.processes.length}`);
+    console.log(`Average types per process: ${processesConfig.averageTypesCountPerProcess}`);
+    console.log();
+}
+
+function runAllDir(dirs, processId) {
+    fs.writeFileSync(`${processId}.index.ts`, '');
+    fs.writeFileSync(`tsconfig.types.${processId}.json`, '');
+
+    return dirs.reduce((promise, dir) => promise.then(() => run(dir, processId)), Promise.resolve())
+        .then(() => {
+            fs.unlinkSync(`tsconfig.types.${processId}.json`);
+            fs.unlinkSync(`${processId}.index.ts`);
+            if (fs.existsSync(`${processId}.index.js`)) {
+                fs.unlinkSync(`${processId}.index.js`);
+            }
+        });
+}
+
+async function run(dir, processId) {
+    const config = {
+        'compilerOptions': {
+            'lib': [
+                'es6',
+                'dom'
+            ],
+            'noEmit': false,
+            'plugins': [
+                {
+                    'transform': '../dist/transformer',
+                    'debug': true
+                }
+            ],
+            'typeRoots': [
+                './DefinitelyTyped/types/'
+            ],
+            "types": [],
+            "baseUrl": "./DefinitelyTyped/types/"
+        },
+        'files': [
+            `./${processId}.index.ts`
+        ]
+    };
+
+    fs.writeFileSync(`tsconfig.types.${processId}.json`, JSON.stringify(config));
+    fs.writeFileSync(`${processId}.index.ts`, `import pak = require('./${definitelyTyped.folder}/types/${dir}/'); import { createMock } from '../dist'; createMock<typeof pak>();`);
+
+    return execPromise(`npx ttsc --project tsconfig.types.${processId}.json`)
+        .then((response) => {
+            if (response) {
+                process.stdout.write(`TYPE: ${dir} P${processId} `);
+                console.warn('☐');
+                outputService.addData(processId, dir, {
+                    response: 'warning',
+                    message: response.toString()
+                });
+            } else {
+                process.stdout.write(`TYPE: ${dir} P${processId} `);
+                console.info('✔');
+                outputService.addData(processId, dir, {
+                    response: 'success'
+                });
+            }
+        })
+        .catch((error) => {
+            process.stdout.write(`TYPE: ${dir} P${processId} `);
+            console.error('✘');
+
+            let errorData = error.error.toString();
+            console.error(error.error);
+
+            if (error.stdout.trim()) {
+                errorData += "\n" + error.stdout;
+                console.error(error.stdout);
+            }
+
+            outputService.addData(processId, dir, {
+                response: 'error',
+                message: errorData
+            });
+        });
+}

definitelyTypedTests/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "definitelytyped-tests",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "start": "npm run install-types && npm run process-all-types && npm run post-process-types",
+    "start-chunk": "npm run install-types && npm run process-chunk && npm run post-process-types",
+    "process-chunk": "npm run process-types TYPES=500 PROCESS_COUNT=20",
+    "install-types": "node src/installTypes.js",
+    "process-all-types": "node index.js TYPES=all PROCESS_COUNT=20",
+    "process-types": "node index.js",
+    "post-process-types": "node src/postProcessTypes.js"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "dotenv": "^8.2.0",
+    "ttypescript": "^1.5.8",
+    "typescript": "^3.7.4"
+  }
+}

definitelyTypedTests/src/config.js

@@ -0,0 +1,56 @@
+const processService = require('../../utils/process/process')(process);
+const maximiseParallelRun = require('./maximiseParallel');
+const definitelyTyped = require('./definitelyTyped')();
+const nodeReader = require('../../utils/dataFileSystem/nodeFileReader')();
+const dataFileSystemReader = require("../../utils/dataFileSystem/dataFileSystemReader");
+const dataReader = dataFileSystemReader(process.env.DEFINITELY_TYPED_DATA_URL, nodeReader);
+
+function getLatestEntry(latestListEntry) {
+    return latestListEntry.sort((a, b) => {
+        return a.lastUpdatedDate > b.lastUpdatedDate ? -1 : 1;
+    })[0];
+}
+
+async function getRunConfig() {
+    const listEntry = await dataReader.getDataIds();
+    const latestEntry = getLatestEntry(listEntry);
+    const totalTypes = definitelyTyped.getTypes().length;
+    const entryToUpdate = latestEntry && latestEntry.typesProcessed >= totalTypes ? null : latestEntry;
+    const offsetType = entryToUpdate ? entryToUpdate.typesProcessed : 0;
+
+    const totalTypesCount = getTotalTypesCount(offsetType, totalTypes);
+    const processesMaximized = maximiseParallelRun(getProcessesCount(), totalTypesCount);
+    const sum = processesMaximized.reduce((previous, current) => previous + current.items, 0);
+    const avg = sum / processesMaximized.length;
+
+    return {
+        totalTypesCount: totalTypesCount,
+        processes: processesMaximized,
+        averageTypesCountPerProcess: avg,
+        entryToUpdate: entryToUpdate,
+        offsetType: offsetType
+    };
+}
+
+function getTotalTypesCount(offsetType, totalTypes) {
+    const typesDirectoriesLength = totalTypes - offsetType;
+    const typesToProcess = processService.getArgument('TYPES');
+
+    if (typesToProcess) {
+        const maybeCount = parseInt(typesToProcess);
+
+        if (!Number.isNaN(maybeCount)) {
+            return Math.min(typesDirectoriesLength, maybeCount);
+        } else if (typesToProcess === "all") {
+            return typesDirectoriesLength;
+        }
+    }
+
+    return 50;
+}
+
+function getProcessesCount() {
+    return processService.getArgument('PROCESS_COUNT') || 1;
+}
+
+module.exports = getRunConfig;

definitelyTypedTests/src/definitelyTyped.js

@@ -0,0 +1,18 @@
+const path = require('path');
+const fs = require('fs');
+
+function definitelyTyped() {
+  const definitelyTypedFolder = path.join('..', 'DefinitelyTyped');
+
+  return {
+    folder: definitelyTypedFolder,
+    typesFolder: path.join(definitelyTypedFolder, 'types'),
+    getTypes() {
+      return this._types || (this._types = fs.readdirSync(this.typesFolder, {withFileTypes: true})
+        .filter(dirent => dirent.isDirectory())
+        .map(dirent => dirent.name));
+    }
+  }
+}
+
+module.exports = definitelyTyped;