Merge pull request #2003 from Azure/ant/linters

Add linter validation

.github/dependabot.yml

@@ -0,0 +1,19 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+  - package-ecosystem: "npm"
+    directory: "/generator"
+    schedule:
+      interval: "monthly"
+  - package-ecosystem: "npm"
+    directory: "/tools"
+    schedule:
+      interval: "monthly"

.github/workflows/generate-types.yml

@@ -35,9 +35,9 @@ jobs:
         run: |
           if [ -z "${{ github.event.inputs.single_path }}" ]
           then
-            npm run generate-all -- '{"localPath": "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs"}'
+            npm run generate-all -- --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs"
           else
-            npm run generate-single -- '${{ github.event.inputs.single_path }}/resource-manager' '$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs'
+            npm run generate-single -- --base-path '${{ github.event.inputs.single_path }}/resource-manager' --output-path '$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs'
           fi
         working-directory: generator
 

.github/workflows/main.yml

@@ -4,7 +4,8 @@ on:
   pull_request:
     branches: [ main ]
 jobs:
-  build:
+  schema-tests:
+    name: Schema Tests
     runs-on: ubuntu-latest
 
     steps:
@@ -21,6 +22,10 @@ jobs:
       - name: Install NPM modules
         run: npm ci
         working-directory: ./tools
+
+      - name: Lint
+        run: npm run lint
+        working-directory: ./tools
 
       - name: Run CI tests
         run: npm test
@@ -30,9 +35,33 @@ jobs:
         run: dotnet test DeploymentsSchemaTests/DeploymentsSchemaTests.csproj
         working-directory: ./tools
 
+  build-generator:
+    name: Build Generator
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/[email protected]
+
+      - name: Setup Node.js
+        uses: actions/[email protected]
+        with:
+          node-version: 14.x
+
+      - name: Install
+        run: npm ci
+        working-directory: ./generator
+
+      - name: Lint
+        run: npm run lint
+        working-directory: ./generator
+
+      - name: Build
+        run: npm run build
+        working-directory: ./generator
+
   automerge:
     runs-on: ubuntu-latest
-    needs: build
+    needs: schema-tests
     env:
       PR_PREFIX: sdkAuto/
     if: ${{ github.event.pull_request.user.login == 'openapi-sdkautomation[bot]' }}

.gitignore

@@ -9,6 +9,7 @@ Debug
 Release
 obj
 bin
+out
 
 # Temporary directory for GitHub workflows to use
 /workflow-temp

.sdkauto/generateScript.sh

@@ -2,10 +2,13 @@
 
 set -e
 
-PARAMS=$(cat $1 | jq '{localPath: .specFolder, readmeFiles: .relatedReadmeMdFiles, $outputPath}' -c --arg outputPath $2)
-
-echo $PARAMS
+LOCAL_PATH=$(cat $1 | jq -r '.specFolder')
+README_FILES=$(cat $1 | jq -r '[.relatedReadmeMdFiles[]] | join(",")')
+OUTPUT_PATH=$2
 
 pushd generator
 
-npm run generate-all $PARAMS
+echo "npm run generate-all -- --local-path $LOCAL_PATH --readme-files $README_FILES --output-path $OUTPUT_PATH"
+#npm run generate-all -- --local-path $LOCAL_PATH --readme-files $README_FILES --output-path $OUTPUT_PATH
+
+popd

README.md

@@ -24,7 +24,7 @@ You can use the generator in this repo to automatically generate a schema from a
 ```bash
 cd generator
 npm install
-npm run generate-single myprovider/resource-manager
+npm run generate-single -- --base-path myprovider/resource-manager
 ```
 4. Review the generator logs to ensure no errors, and review the changes generated.
 5. Ensure that you have reviewed the guidelines under [Submitting a PR](#submitting-a-pr).

azure-pipelines-autogen.yml

@@ -62,7 +62,7 @@ jobs:
   - script: |
       set -Eeuxo pipefail
       cd generator
-      npm run generate-all $(generateParams)
+      npm run generate-all -- --batch-count $(batchCount) --batch-index $(batchIndex)
     displayName: 'Generate schemas'
 
   - script: |

generator/.eslintrc.js

@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+module.exports = {
+  root: true,
+  parser: "@typescript-eslint/parser",
+  parserOptions: {
+    ecmaVersion: 2020,
+    sourceType: "module",
+  },
+  plugins: ["header"],
+  extends: ["eslint:recommended"],
+  ignorePatterns: ["/out/**/*"],
+  rules: {
+    "header/header": [
+      2,
+      "line",
+      [
+        " Copyright (c) Microsoft Corporation.",
+        " Licensed under the MIT License.",
+      ],
+    ],
+  },
+  overrides: [
+    {
+      files: ["*.ts"],
+      extends: [
+        "plugin:@typescript-eslint/recommended",
+      ],
+      rules: {
+      },
+    },
+    {
+      files: ["*.js"],
+      env: { node: true },
+    },
+  ],
+};

generator/README.md

@@ -9,8 +9,8 @@
 * `npm run clean` - Resets the local schemas directory to a clean state.
 * `npm run list-basepaths` - Lists all the valid base paths from the specs repo for generation.
 * `npm run list-resources` - Lists all the schema-declared types and API versions referenced in [schemas/2019-04-01/deploymentTemplate.json](/schemas/2019-04-01/deploymentTemplate.json).
-* `npm run generate-all` - Re-generates all schemas for providers which have been added to [autogenlist.ts](./autogenlist.ts).
-* `npm run generate-single {basePath} [{localPath}]` - Re-generates schemas for all resource types and API versions found under a given base path. Use optional `{localPath}` to generate from local specs repo.
+* `npm run generate-all` - Re-generates all schemas for providers which have been added to [autogenlist.ts](./autogenlist.ts). Run `npm run generate-all -- --help` to see available options.
+* `npm run generate-single` - Re-generates schemas for all resource types and API versions found under a given base path. Run `npm run generate-single -- --help` to see available options.
 * See [package.json](./package.json) for the full set of commands.
 
 ### Daily autogeneration
@@ -39,7 +39,7 @@ If your team has been opted out, and you would like to opt in, please raise an i
 > The following steps are no longer necessary for onboarding, but listed here for reference:
 > 1. Follow the steps listed under [Getting Started](#getting-started). The following commands assume you are running in the > `generator` directory.
 > 3. Run `npm run list-basepaths` to discover the specs repo path containing the swagger definitions.
-> 4. Run `npm run generate-single {basePath}` to generate schemas for them. Save the console output to file.
+> 4. Run `npm run generate-single -- --base-path {basePath}` to generate schemas for them. Save the console output to file.
 > 5. Manually remove any existing references to your provider namespace (if any) from [schemas/2019-04-01/deploymentTemplate.> json](/schemas/2019-04-01/deploymentTemplate.json).
 > 6. Add the {basePath} to the autogenlist array in [autogenlist.ts](./autogenlist.ts).
 > 7. Commit all the changed files, and submit a PR to this repo with title "Onboarding {provider} for autogeneration". Include > the full output saved in step 2 as a comment.

generator/autogenlist.ts

@@ -1,3 +1,5 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
 import { ScopeType, AutoGenConfig } from './models';
 import { postProcessor as insightsApplicationPostProcessor } from './processors/Microsoft.Insights.Application';
 import { postProcessor as resourcesPostProcessor } from './processors/Microsoft.Resources';
@@ -371,9 +373,9 @@ const autoGenList: AutoGenConfig[] = [
         namespace: 'Microsoft.HDInsight',
     },
     {
-        basePath: 'resourcehealth/resource-manager',
-        namespace: 'Microsoft.ResourceHealth',
-    }, 
+        basePath: 'resourcehealth/resource-manager',
+        namespace: 'Microsoft.ResourceHealth',
+    },
     {
         basePath: 'EnterpriseKnowledgeGraph/resource-manager',
         namespace: 'Microsoft.EnterpriseKnowledgeGraph',

generator/cmd/clean.ts

@@ -1,3 +1,5 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
 import * as constants from '../constants';
 import { resetGitDirectory } from '../git';
 import { executeSynchronous } from '../utils';

generator/cmd/findbasepath.ts

@@ -1,3 +1,5 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
 import { executeSynchronous } from '../utils';
 import { findAutogenEntries } from '../autogenlist';
 

generator/cmd/generateall.ts

@@ -1,33 +1,28 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, generateBasePaths, getPackageString, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs';
 import { SchemaConfiguration, generateSchemas, clearAutoGeneratedSchemaRefs, saveAutoGeneratedSchemaRefs, getApiVersionsByNamespace } from '../generate';
 import { findOrGenerateAutogenEntries } from '../autogenlist';
 import chalk from 'chalk';
-import { flatten, keys, partition } from 'lodash';
-import { executeSynchronous, chunker, writeJsonFile, lowerCaseEquals } from '../utils';
-import { AutoGenConfig, Package } from '../models';
+import { flatten, keys } from 'lodash';
+import { executeSynchronous, chunker, writeJsonFile } from '../utils';
+import { Package } from '../models';
+import yargs from 'yargs';
 
-interface GenerateAllParams {
-    batchCount?: number,
-    batchIndex?: number,
-    localPath?: string,
-    readmeFiles?: string[],
-    outputPath?: string,
-}
-
-function parseParams(): GenerateAllParams {
-    if (!process.argv[2]) {
-        return {};
-    }
-
-    return JSON.parse(process.argv[2]);
-}
+const argsConfig = yargs
+  .strict()
+  .option('batch-count', { type: 'number', desc: 'If running in batch mode, the total number of batch jobs running' })
+  .option('batch-index', { type: 'number', desc: 'If running in batch mode, the index of this batch job' })
+  .option('local-path', { type: 'string', desc: 'The local path to the azure-rest-api-specs repo' })
+  .option('readme-files', { type: 'array', desc: 'The list of readme.md files to generate schemas for' })
+  .option('output-path', { type: 'string', desc: 'The base path to save schema output' });
 
 executeSynchronous(async () => {
-    const params = parseParams();
+    const args = await argsConfig.parseAsync();
 
     let basePaths;
-    let localPath = params.localPath;
+    let localPath = args['local-path'];
     if (!localPath) {
         localPath = constants.specsRepoPath;
         basePaths = await cloneAndGenerateBasePaths(localPath, constants.specsRepoUri, constants.specsRepoCommitHash);
@@ -36,8 +31,8 @@ executeSynchronous(async () => {
         basePaths = await generateBasePaths(localPath);
     }
 
-    if (params.batchCount !== undefined && params.batchIndex !== undefined) {
-        basePaths = chunker(basePaths, params.batchCount)[params.batchIndex];
+    if (args['batch-count'] !== undefined && args['batch-index'] !== undefined) {
+        basePaths = chunker(basePaths, args['batch-count'])[args['batch-index']];
     }
 
     const schemaConfigs: SchemaConfiguration[] = [];
@@ -50,10 +45,11 @@ executeSynchronous(async () => {
         let filteredAutoGenList = findOrGenerateAutogenEntries(basePath, namespaces)
             .filter(x => x.disabledForAutogen !== true);
 
-        if (!!params.readmeFiles) {
+        if (args['readme-files']) {
             filteredAutoGenList = filteredAutoGenList.filter(c => {
-                let r = params.readmeFiles?.find(f => f.startsWith('specification/' + c.basePath));
-                if (!!r) {
+                const readmeFiles = args['readme-files']?.map(x => x.toString());
+                const r = readmeFiles?.find(f => f.startsWith('specification/' + c.basePath));
+                if (r) {
                     c.readmeFile = r;
                     return true;
                 }
@@ -64,7 +60,7 @@ executeSynchronous(async () => {
         await clearAutoGeneratedSchemaRefs(filteredAutoGenList);
 
         for (const autoGenConfig of filteredAutoGenList) {
-            let pkg = {
+            const pkg = {
                 path: ['schemas']
             } as Package;
             try {
@@ -88,8 +84,8 @@ executeSynchronous(async () => {
 
     await saveAutoGeneratedSchemaRefs(flatten(schemaConfigs));
 
-    if (!!params.outputPath) {
-        const outputPath = await resolveAbsolutePath(params.outputPath);
+    if (args['output-path']) {
+        const outputPath = await resolveAbsolutePath(args['output-path']);
         await writeJsonFile(outputPath, { packages });
     } else {
         if (errors.length > 0) {

generator/cmd/generateonboardedreport.ts

@@ -1,16 +1,17 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs';
-import chalk from 'chalk';
 import { findOrGenerateAutogenEntries } from '../autogenlist';
-import { executeSynchronous, lowerCaseEquals, writeJsonFile, safeMkdir } from '../utils';
+import { executeSynchronous, writeJsonFile, safeMkdir } from '../utils';
 import { getApiVersionsByNamespace } from '../generate';
 import { keys, partition } from 'lodash';
 import path from 'path';
 
 executeSynchronous(async () => {
     const basePaths = await cloneAndGenerateBasePaths(constants.specsRepoPath, constants.specsRepoUri, constants.specsRepoCommitHash);
 
-    let allBasePaths = [];
+    const allBasePaths = [];
 
     for (const basePath of basePaths) {
         const readme = await validateAndReturnReadmePath(constants.specsRepoPath, basePath);

generator/cmd/generatesingle.ts

@@ -1,15 +1,24 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs';
 import { generateSchemas, saveAutoGeneratedSchemaRefs, getApiVersionsByNamespace } from '../generate';
-import process from 'process';
 import { findOrGenerateAutogenEntries } from '../autogenlist';
 import chalk from 'chalk';
 import { keys } from 'lodash';
 import { executeSynchronous } from '../utils';
+import yargs from 'yargs';
+
+const argsConfig = yargs
+  .strict()
+  .option('base-path', { type: 'string', demandOption: true, desc: 'The swagger base path in the specs repo (e.g. "compute/resource-manager")' })
+  .option('local-path', { type: 'string', desc: 'The local path to the azure-rest-api-specs repo' });
 
 executeSynchronous(async () => {
-    const basePath = process.argv[2];
-    let localPath = process.argv[3];
+    const args = await argsConfig.parseAsync();
+
+    const basePath = args['base-path'];
+    let localPath = args['local-path'];
     if (!localPath) {
         localPath = constants.specsRepoPath;
         await cloneAndGenerateBasePaths(localPath, constants.specsRepoUri, constants.specsRepoCommitHash);

generator/cmd/listbasepaths.ts

@@ -1,8 +1,10 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs';
 import chalk from 'chalk';
 import { findOrGenerateAutogenEntries } from '../autogenlist';
-import { executeSynchronous, lowerCaseEquals } from '../utils';
+import { executeSynchronous } from '../utils';
 import { getApiVersionsByNamespace } from '../generate';
 import { keys, partition } from 'lodash';