feat(cli): import a single module per api group (#402)

Follow-on to #378 Emit a single module per API group for all imports: ```sh cdk8s import github:account/repo[@Version] ``` you can use a single line import statement for multiple API types: ```ts import { CompositeResourceDefinition, Composition } from './imports/apiextensions.crossplane.io'; ``` Here's some sample output: https://github.com/prasek/crossplane-cdk/tree/cross-pkg/test/imports/provider-aws Resolves #401 *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
cdk8s-team · Dec 3, 2020 · ac295fe · ac295fe
1 parent 48a417c
commit ac295fe
Show file tree

Hide file tree

Showing 41 changed files with 59,756 additions and 6,181 deletions.
diff --git a/examples/python/crd/main.py b/examples/python/crd/main.py
@@ -2,8 +2,8 @@
 from constructs import Construct
 from cdk8s import App, Chart
 
-from imports.jenkins.io import jenkins
-from imports.mattermost.com import clusterinstallation
+from imports.io import jenkins
+from imports.com import mattermost
 
 class MyChart(Chart):
     def __init__(self, scope: Construct, ns: str):
@@ -23,15 +23,15 @@ def __init__(self, scope: Construct, ns: str):
           )
         )
 
-        clusterinstallation.ClusterInstallation(self, "foo",
-          spec=clusterinstallation.ClusterInstallationSpec(
+        mattermost.ClusterInstallation(self, "foo",
+          spec=mattermost.ClusterInstallationSpec(
             ingress_name="example.mattermost-example.dev",
             replicas=2,
-            minio=clusterinstallation.ClusterInstallationSpecMinio(
+            minio=mattermost.ClusterInstallationSpecMinio(
               storage_size="10Gi",
               replicas=4
             ),
-            database=clusterinstallation.ClusterInstallationSpecDatabase(
+            database=mattermost.ClusterInstallationSpecDatabase(
               storage_size="10Gi",
               replicas=4
             ),

diff --git a/examples/typescript/cdk8s-plus-elasticsearch-query/cdk8s.yaml b/examples/typescript/cdk8s-plus-elasticsearch-query/cdk8s.yaml
@@ -1,2 +1,4 @@
 language: typescript
 app: node main.js
+imports:
+  - elastic.yaml
diff --git a/examples/typescript/cdk8s-plus-elasticsearch-query/elastic.yaml b/examples/typescript/cdk8s-plus-elasticsearch-query/elastic.yaml
diff --git a/...ticsearch.k8s.elastic.co/elasticsearch.ts → ...y/imports/elasticsearch.k8s.elastic.co.ts b/...ticsearch.k8s.elastic.co/elasticsearch.ts → ...y/imports/elasticsearch.k8s.elastic.co.ts
@@ -11,12 +11,12 @@ export class Elasticsearch extends ApiObject {
   /**
    * Defines a "Elasticsearch" API object
    * @param scope the scope in which to define this object
-   * @param name a scope-local name for the object
-   * @param options configuration options
+   * @param id a scope-local name for the object
+   * @param props initialiation props
    */
-  public constructor(scope: Construct, name: string, options: ElasticsearchOptions = {}) {
-    super(scope, name, {
-      ...options,
+  public constructor(scope: Construct, id: string, props: ElasticsearchProps = {}) {
+    super(scope, id, {
+      ...props,
       kind: 'Elasticsearch',
       apiVersion: 'elasticsearch.k8s.elastic.co/v1',
     });
@@ -28,7 +28,7 @@ export class Elasticsearch extends ApiObject {
  *
  * @schema Elasticsearch
  */
-export interface ElasticsearchOptions {
+export interface ElasticsearchProps {
   /**
    * @schema Elasticsearch#metadata
    */

diff --git a/examples/typescript/cdk8s-plus-elasticsearch-query/main.ts b/examples/typescript/cdk8s-plus-elasticsearch-query/main.ts
@@ -1,6 +1,6 @@
 import { Construct } from 'constructs';
 import { App, Chart } from 'cdk8s';
-import { Elasticsearch } from './imports/elasticsearch.k8s.elastic.co/elasticsearch';
+import { Elasticsearch } from './imports/elasticsearch.k8s.elastic.co';
 import * as kplus from 'cdk8s-plus-17';
 
 export class MyChart extends Chart {

diff --git a/examples/typescript/crd/index.ts b/examples/typescript/crd/index.ts
@@ -1,8 +1,8 @@
 import { Construct } from 'constructs';
 import { App, Chart } from 'cdk8s';
 
-import { Jenkins } from './imports/jenkins.io/jenkins';
-import { ClusterInstallation } from './imports/mattermost-mattermost.com/clusterinstallation';
+import { Jenkins } from './imports/jenkins.io';
+import { ClusterInstallation } from './imports/mattermost-mattermost.com';
 
 export class HelloKube extends Chart {
   constructor(scope: Construct, id: string) {

diff --git a/packages/cdk8s-cli/src/import/base.ts b/packages/cdk8s-cli/src/import/base.ts
@@ -52,14 +52,31 @@ export abstract class ImportBase {
       console.error('warning: no definitions to import');
     }
 
-    for (const name of this.moduleNames) {
+    const mapFunc = ( origName: any ) => {
+      let name = origName;
+      switch (options.targetLanguage) {
+        case Language.PYTHON:
+        case Language.JAVA:
+          name = name.split('.').reverse().join('.');
+          break;
+      }
+      return {
+        origName: origName,
+        name: name,
+      };
+    };
+
+    // sort to ensure python writes parent packages first, so children are not deleted
+    const modules = this.moduleNames.map(mapFunc).sort((a: any, b: any) => a.name.localeCompare(b.name));
+
+    for (const module of modules) {
       // output the name of the imported resource
-      console.log(name);
+      console.log(module.origName);
 
-      const fileName = moduleNamePrefix ? `${moduleNamePrefix}-${name}.ts` : `${name}.ts`;
+      const fileName = moduleNamePrefix ? `${moduleNamePrefix}-${module.name}.ts` : `${module.name}.ts`;
       code.openFile(fileName);
       code.indentation = 2;
-      await this.generateTypeScript(code, name, {
+      await this.generateTypeScript(code, module.origName, {
         classNamePrefix: options.classNamePrefix,
       });
 
@@ -82,7 +99,7 @@ export abstract class ImportBase {
 
           const opts: srcmak.Options = {
             entrypoint: fileName,
-            moduleKey: name,
+            moduleKey: module.name,
             deps: deps.map(dep => path.dirname(require.resolve(`${dep}/package.json`))),
           };
 
@@ -93,7 +110,7 @@ export abstract class ImportBase {
 
           // python!
           if (options.targetLanguage === Language.PYTHON) {
-            const moduleName = `${moduleNamePrefix ? `${moduleNamePrefix}.${name}` : name}`.replace(/-/g, '_');
+            const moduleName = `${moduleNamePrefix ? `${moduleNamePrefix}.${module.name}` : module.name}`.replace(/-/g, '_');
             opts.python = {
               outdir: outdir,
               moduleName,
@@ -102,7 +119,7 @@ export abstract class ImportBase {
 
           // java!
           if (options.targetLanguage === Language.JAVA) {
-            const javaName = name.replace(/\//g, '.').replace(/-/g, '_');
+            const javaName = module.name.replace(/\//g, '.').replace(/-/g, '_');
             opts.java = {
               outdir: '.',
               package: `imports.${moduleNamePrefix ? moduleNamePrefix + '.' + javaName : javaName}`,

diff --git a/packages/cdk8s-cli/src/import/crd.ts b/packages/cdk8s-cli/src/import/crd.ts
@@ -5,6 +5,7 @@ import { ImportSpec } from '../config';
 import { download } from '../util';
 import { GenerateOptions, ImportBase } from './base';
 import { generateConstruct } from './codegen';
+import { GroupVersionKind } from './k8s';
 
 const CRD_KIND = 'CustomResourceDefinition';
 
@@ -71,10 +72,18 @@ export class CustomResourceDefinition {
     this.fqn = this.kind;
   }
 
-  public get moduleName() {
+  public get key() {
     return `${this.group}/${this.kind.toLocaleLowerCase()}`;
   }
 
+  public get gvk(): GroupVersionKind {
+    return {
+      group: this.group,
+      version: this.version,
+      kind: this.kind,
+    };
+  }
+
   public async generateTypeScript(code: CodeMaker, options: GenerateOptions) {
     const types = new TypeGenerator();
 
@@ -88,10 +97,6 @@ export class CustomResourceDefinition {
       prefix: options.classNamePrefix,
     });
 
-    code.line('// generated by cdk8s');
-    code.line('import { ApiObject } from \'cdk8s\';');
-    code.line('import { Construct } from \'constructs\';');
-    code.line();
     code.line(types.render());
   }
 }
@@ -103,12 +108,13 @@ export class ImportCustomResourceDefinition extends ImportBase {
     return yaml.parseAllDocuments(manifest).map((doc: yaml.Document) => doc.toJSON());
   }
 
-  private readonly defs: CustomResourceDefinition[] = [];
+  private readonly groups: Record<string, CustomResourceDefinition[]> = { };
 
   constructor(manifest: ManifestObjectDefinition[]) {
     super();
 
-    const defs: Record<string, CustomResourceDefinition> = { };
+    const crds: Record<string, CustomResourceDefinition> = { };
+    const groups: Record<string, CustomResourceDefinition[]> = { };
 
     const extractCRDs = (objects: ManifestObjectDefinition[] = []) => {
       for (const obj of objects) {
@@ -120,13 +126,13 @@ export class ImportCustomResourceDefinition extends ImportBase {
         // found a crd, yey!
         if (obj.kind === CRD_KIND) {
           const crd = new CustomResourceDefinition(obj);
-          const key = crd.moduleName;
+          const key = crd.key;
 
-          if (key in defs) {
+          if (key in crds) {
             throw new Error(`${key} already exists`);
           }
+          crds[key] = crd;
 
-          defs[key] = crd;
           continue;
         }
 
@@ -140,15 +146,34 @@ export class ImportCustomResourceDefinition extends ImportBase {
 
     extractCRDs(manifest);
 
-    this.defs = Object.values(defs);
+    //sort to ensure consistent ordering for snapshot compare
+    const sortedCrds = Object.values(crds).sort((a: CustomResourceDefinition, b: CustomResourceDefinition) => a.key.localeCompare(b.key));
+
+    for (const crd of sortedCrds) {
+      const g = crd.gvk.group;
+      if ( !(g in groups) ) {
+        groups[g] = new Array<CustomResourceDefinition>();
+      }
+      groups[g].push(crd);
+    }
+
+    this.groups = groups;
   }
 
   public get moduleNames() {
-    return this.defs.map(crd => crd.moduleName);
+    return Object.keys(this.groups);
   }
 
   protected async generateTypeScript(code: CodeMaker, moduleName: string, options: GenerateOptions) {
-    for (const crd of this.defs.filter(c => moduleName === c.moduleName)) {
+    const crds = this.groups[moduleName];
+
+    code.line('// generated by cdk8s');
+    code.line('import { ApiObject } from \'cdk8s\';');
+    code.line('import { Construct } from \'constructs\';');
+    code.line();
+
+    for (const crd of crds) {
+      console.log(`  ${crd.key}`);
       await crd.generateTypeScript(code, options);
     }
   }

diff --git a/packages/cdk8s-cli/src/import/crds-dev.ts b/packages/cdk8s-cli/src/import/crds-dev.ts
@@ -1,3 +1,4 @@
+import { ImportSpec } from '../config';
 import { ImportCustomResourceDefinition, ManifestObjectDefinition } from './crd';
 
 /**
@@ -21,7 +22,8 @@ import { ImportCustomResourceDefinition, ManifestObjectDefinition } from './crd'
  *
  * @param source
  */
-export async function importCrdsDevRepoMatch(source: string): Promise<undefined | ManifestObjectDefinition[]> {
+export async function importCrdsDevRepoMatch(importSpec: ImportSpec): Promise<undefined | ManifestObjectDefinition[]> {
+  const { source } = importSpec;
   const url = crdsDevUrl(source);
   if (url) {
     const crd = await ImportCustomResourceDefinition.match({

diff --git a/packages/cdk8s-cli/src/import/dispatch.ts b/packages/cdk8s-cli/src/import/dispatch.ts
@@ -25,7 +25,7 @@ async function matchImporter(importSpec: ImportSpec, argv: any) {
     return new ImportKubernetesApi(k8s);
   }
 
-  const crdsDev = await importCrdsDevRepoMatch(importSpec.source);
+  const crdsDev = await importCrdsDevRepoMatch(importSpec);
   if (crdsDev) {
     return new ImportCustomResourceDefinition(crdsDev);
   }

diff --git a/packages/cdk8s-cli/src/import/k8s.ts b/packages/cdk8s-cli/src/import/k8s.ts
@@ -144,7 +144,7 @@ function tryGetObjectName(def: JSONSchema4): GroupVersionKind | undefined {
   return objectName;
 }
 
-interface GroupVersionKind {
+export interface GroupVersionKind {
   readonly group: string;
   readonly kind: string;
   readonly version: string;