diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000000..7ea9c48f99 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 88 +exclude = *.egg,*/interfaces.py,node_modules,.state +ignore = W503,E203 +select = E,W,F,N diff --git a/.gitignore b/.gitignore index 12d027be13..d569e6ab65 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ node_modules/ .BUILD_COMPLETED lerna-debug.log .DS_Store -.idea \ No newline at end of file +.idea diff --git a/packages/jsii-build-tools/bin/package-python b/packages/jsii-build-tools/bin/package-python new file mode 100755 index 0000000000..8ea2d10df6 --- /dev/null +++ b/packages/jsii-build-tools/bin/package-python @@ -0,0 +1,7 @@ +#!/bin/bash +set -euo pipefail + +rm -rf dist/python +mkdir -p dist/python +mv *.whl dist/python +mv *.tar.gz dist/python diff --git a/packages/jsii-build-tools/package.json b/packages/jsii-build-tools/package.json index ce37888ca5..2171289788 100644 --- a/packages/jsii-build-tools/package.json +++ b/packages/jsii-build-tools/package.json @@ -8,7 +8,8 @@ "package-js": "bin/package-js", "package-java": "bin/package-java", "package-dotnet": "bin/package-dotnet", - "package-ruby": "bin/package-ruby" + "package-ruby": "bin/package-ruby", + "package-python": "bin/package-python" }, "scripts": { "build": "chmod +x bin/*" diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts new file mode 100644 index 0000000000..95631979c6 --- /dev/null +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -0,0 +1,1500 @@ +import path = require('path'); +import util = require('util'); + +import { CodeMaker, toSnakeCase } from 'codemaker'; +import * as escapeStringRegexp from 'escape-string-regexp'; +import * as spec from 'jsii-spec'; +import { Generator, GeneratorOptions } from '../generator'; +import { Target, TargetOptions } from '../target'; +import { shell } from '../util'; + +export default class Python extends Target { + protected readonly generator = new PythonGenerator(); + + constructor(options: TargetOptions) { + super(options); + } + + public async build(sourceDir: string, outDir: string): Promise { + // Format our code to make it easier to read, we do this here instead of trying + // to do it in the code generation phase, because attempting to mix style and + // function makes the code generation harder to maintain and read, while doing + // this here is easy. + await shell("black", ["--py36", sourceDir], {}); + + // Actually package up our code, both as a sdist and a wheel for publishing. + await shell("python3", ["setup.py", "sdist", "--dist-dir", outDir], { cwd: sourceDir }); + await shell("python3", ["setup.py", "bdist_wheel", "--dist-dir", outDir], { cwd: sourceDir }); + } +} + +// ################## +// # CODE GENERATOR # +// ################## +export const debug = (o: any) => { + // tslint:disable-next-line:no-console + console.log(util.inspect(o, false, null, true)); +}; + +const PYTHON_BUILTIN_TYPES = ["bool", "str", "None"]; + +const PYTHON_KEYWORDS = [ + "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", + "continue", "def", "del", "elif", "else", "except", "finally", "for", "from", + "global", "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", + "raise", "return", "try", "while", "with", "yield" +]; + +const toPythonModuleName = (name: string): string => { + if (name.match(/^@[^/]+\/[^/]+$/)) { + name = name.replace(/^@/g, ""); + name = name.replace(/\//g, "."); + } + + name = toSnakeCase(name.replace(/-/g, "_")); + + return name; +}; + +const pythonModuleNameToFilename = (name: string): string => { + return name.replace(/\./g, "/"); +}; + +const toPythonPackageName = (name: string): string => { + return toPythonModuleName(name).replace(/_/g, "-"); +}; + +const toPythonIdentifier = (name: string): string => { + if (PYTHON_KEYWORDS.indexOf(name) > -1) { + return name + "_"; + } + + return name; +}; + +const toPythonMethodName = (name: string): string => { + return toPythonIdentifier(toSnakeCase(name)); +}; + +const toPythonPropertyName = (name: string): string => { + return toPythonIdentifier(toSnakeCase(name)); +}; + +const setDifference = (setA: Set, setB: Set): Set => { + const difference = new Set(setA); + for (const elem of setB) { + difference.delete(elem); + } + return difference; +}; + +const sortMembers = (sortable: PythonBase[], resolver: TypeResolver): PythonBase[] => { + const sorted: PythonBase[] = []; + const seen: Set = new Set(); + + // We're going to take a copy of our sortable item, because it'll make it easier if + // this method doesn't have side effects. + sortable = sortable.slice(); + + // The first thing we want to do, is push any item which is not sortable to the very + // front of the list. This will be things like methods, properties, etc. + for (const item of sortable) { + if (!isSortableType(item)) { + sorted.push(item); + seen.add(item); + } + } + sortable = sortable.filter(i => !seen.has(i)); + + // Now that we've pulled out everything that couldn't possibly have dependencies, + // we will go through the remaining items, and pull off any items which have no + // dependencies that we haven't already sorted. + while (sortable.length > 0) { + for (const item of (sortable as Array)) { + const itemDeps: Set = new Set(item.dependsOn(resolver)); + if (setDifference(itemDeps, seen).size === 0) { + sorted.push(item); + seen.add(item); + + break; + } + } + + const leftover = sortable.filter(i => !seen.has(i)); + if (leftover.length === sortable.length) { + throw new Error("Could not sort members (circular dependency?)."); + } else { + sortable = leftover; + } + } + + return sorted; +}; + +const recurseForNamedTypeReferences = (typeRef: spec.TypeReference): spec.NamedTypeReference[] => { + if (spec.isPrimitiveTypeReference(typeRef)) { + return []; + } else if (spec.isCollectionTypeReference(typeRef)) { + return recurseForNamedTypeReferences(typeRef.collection.elementtype); + } else if (spec.isNamedTypeReference(typeRef)) { + return [typeRef]; + } else if (typeRef.union) { + const types: spec.NamedTypeReference[] = []; + for (const type of typeRef.union.types) { + types.push(...recurseForNamedTypeReferences(type)); + } + return types; + } else { + throw new Error("Invalid type reference: " + JSON.stringify(typeRef)); + } +}; + +interface PythonBase { + readonly name: string; + + emit(code: CodeMaker, resolver: TypeResolver, opts?: any): void; +} + +interface PythonType extends PythonBase { + // The JSII FQN for this item, if this item doesn't exist as a JSII type, then it + // doesn't have a FQN and it should be null; + readonly fqn: string | null; + + addMember(member: PythonBase): void; +} + +interface ISortableType { + dependsOn(resolver: TypeResolver): PythonType[]; +} + +function isSortableType(arg: any): arg is ISortableType { + return arg.dependsOn !== undefined; +} + +interface PythonTypeOpts { + bases?: spec.TypeReference[]; +} + +abstract class BasePythonClassType implements PythonType, ISortableType { + + public readonly name: string; + public readonly fqn: string | null; + + protected bases: spec.TypeReference[]; + protected members: PythonBase[]; + + constructor(name: string, fqn: string, opts: PythonTypeOpts) { + const { + bases = [], + } = opts; + + this.name = name; + this.fqn = fqn; + this.bases = bases; + this.members = []; + } + + public dependsOn(resolver: TypeResolver): PythonType[] { + const dependencies: PythonType[] = []; + const parent = resolver.getParent(this.fqn!); + + // We need to return any bases that are in the same module at the same level of + // nesting. + const seen: Set = new Set(); + for (const base of this.bases) { + if (spec.isNamedTypeReference(base)) { + if (resolver.isInModule(base)) { + // Given a base, we need to locate the base's parent that is the same as + // our parent, because we only care about dependencies that are at the + // same level of our own. + // TODO: We might need to recurse into our members to also find their + // dependencies. + let baseItem = resolver.getType(base); + let baseParent = resolver.getParent(base); + while (baseParent !== parent) { + baseItem = baseParent; + baseParent = resolver.getParent(baseItem.fqn!); + } + + if (!seen.has(baseItem.fqn!)) { + dependencies.push(baseItem); + seen.add(baseItem.fqn!); + } + } + } + } + + return dependencies; + } + + public addMember(member: PythonBase) { + this.members.push(member); + } + + public emit(code: CodeMaker, resolver: TypeResolver) { + resolver = this.fqn ? resolver.bind(this.fqn) : resolver; + + const classParams = this.getClassParams(resolver); + const bases = classParams.length > 0 ? `(${classParams.join(", ")})` : ""; + + code.openBlock(`class ${this.name}${bases}`); + + this.emitPreamble(code, resolver); + + if (this.members.length > 0) { + for (const member of sortMembers(this.members, resolver)) { + member.emit(code, resolver); + } + } else { + code.line("pass"); + } + + code.closeBlock(); + } + + protected abstract getClassParams(resolver: TypeResolver): string[]; + + protected emitPreamble(_code: CodeMaker, _resolver: TypeResolver) { return; } +} + +interface BaseMethodOpts { + abstract?: boolean; + liftedProp?: spec.InterfaceType, + parent?: spec.NamedTypeReference, +} + +interface BaseMethodEmitOpts { + renderAbstract?: boolean; +} + +abstract class BaseMethod implements PythonBase { + + public readonly name: string; + public readonly abstract: boolean; + + protected readonly abstract implicitParameter: string; + protected readonly jsiiMethod?: string; + protected readonly decorator?: string; + protected readonly classAsFirstParameter: boolean = false; + protected readonly returnFromJSIIMethod: boolean = true; + + private readonly jsName?: string; + private readonly parameters: spec.Parameter[]; + private readonly returns?: spec.TypeReference; + private readonly liftedProp?: spec.InterfaceType; + private readonly parent?: spec.NamedTypeReference; + + constructor(name: string, + jsName: string | undefined, + parameters: spec.Parameter[], + returns?: spec.TypeReference, + opts: BaseMethodOpts = {}) { + const { abstract = false } = opts; + + this.name = name; + this.abstract = abstract; + this.jsName = jsName; + this.parameters = parameters; + this.returns = returns; + this.liftedProp = opts.liftedProp; + this.parent = opts.parent; + } + + public emit(code: CodeMaker, resolver: TypeResolver, opts?: BaseMethodEmitOpts) { + const { renderAbstract = true } = opts || {}; + + let returnType: string; + if (this.returns !== undefined) { + returnType = resolver.resolve(this.returns, { forwardReferences: false }); + } else { + returnType = "None"; + } + + // We cannot (currently?) trust the JSII to accurately tell us whether a + // parameter is truly optional or not. Because of that, we have to selectively + // choose when we're going to respect the optional flag and emit a default value + // to only be at the tail end of the method signature. + // See: https://github.com/awslabs/jsii/issues/284 + let optionalStartsAt: number | undefined; + for (const [idx, param] of this.parameters.entries()) { + if (param.type.optional && optionalStartsAt === undefined) { + optionalStartsAt = idx; + } else if (!param.type.optional) { + optionalStartsAt = undefined; + } + } + + // We cannot (currently?) blindly use the names given to us by the JSII for + // initializers, because our keyword lifting will allow two names to clash. + // This can hopefully be removed once we get https://github.com/awslabs/jsii/issues/288 + // resolved, so build up a list of all of the prop names so we can check against + // them later. + const liftedPropNames: Set = new Set(); + if (this.liftedProp !== undefined + && this.liftedProp.properties !== undefined + && this.liftedProp.properties.length >= 1) { + for (const prop of this.liftedProp.properties) { + liftedPropNames.add(toPythonIdentifier(prop.name)); + } + } + + // We need to turn a list of JSII parameters, into Python style arguments with + // gradual typing, so we'll have to iterate over the list of parameters, and + // build the list, converting as we go. + const pythonParams: string[] = [this.implicitParameter]; + for (const [idx, param] of this.parameters.entries()) { + // We cannot (currently?) blindly use the names given to us by the JSII for + // initializers, because our keyword lifting will allow two names to clash. + // This can hopefully be removed once we get https://github.com/awslabs/jsii/issues/288 + // resolved. + let paramName: string = toPythonIdentifier(param.name); + while (liftedPropNames.has(paramName)) { + paramName = `${paramName}_`; + } + + const paramType = resolver.resolve(param.type, { forwardReferences: false}); + const paramDefault = optionalStartsAt !== undefined && idx >= optionalStartsAt ? "=None" : ""; + + pythonParams.push(`${paramName}: ${paramType}${paramDefault}`); + } + + // If we have a lifted parameter, then we'll drop the last argument to our params + // and then we'll lift all of the params of the lifted type as keyword arguments + // to the function. + if (this.liftedProp !== undefined) { + // Remove our last item. + pythonParams.pop(); + + if (this.liftedProp.properties !== undefined && this.liftedProp.properties.length >= 1) { + // All of these parameters are keyword only arguments, so we'll mark them + // as such. + pythonParams.push("*"); + + // Iterate over all of our props, and reflect them into our params. + for (const prop of this.liftedProp.properties) { + const paramName = toPythonIdentifier(prop.name); + const paramType = resolver.resolve(prop.type, { forwardReferences: false }); + const paramDefault = prop.type.optional ? "=None" : ""; + + pythonParams.push(`${paramName}: ${paramType}${paramDefault}`); + } + } + } else if (this.parameters.length >= 1 && this.parameters.slice(-1)[0].variadic) { + // Another situation we could be in, is that instead of having a plain parameter + // we have a variadic parameter where we need to expand the last parameter as a + // *args. + pythonParams.pop(); + + const lastParameter = this.parameters.slice(-1)[0]; + const paramName = toPythonIdentifier(lastParameter.name); + const paramType = resolver.resolve( + lastParameter.type, + { forwardReferences: false, ignoreOptional: true }, + ); + + pythonParams.push(`*${paramName}: ${paramType}`); + } + + if (this.jsName !== undefined) { + code.line(`@jsii.member(jsii_name="${this.jsName}")`); + } + + if (this.decorator !== undefined) { + code.line(`@${this.decorator}`); + } + + if (renderAbstract && this.abstract) { + code.line("@abc.abstractmethod"); + } + + code.openBlock(`def ${this.name}(${pythonParams.join(", ")}) -> ${returnType}`); + this.emitBody(code, resolver, renderAbstract); + code.closeBlock(); + } + + private emitBody(code: CodeMaker, resolver: TypeResolver, renderAbstract: boolean) { + if (this.jsiiMethod === undefined || (renderAbstract && this.abstract)) { + code.line("..."); + } else { + if (this.liftedProp !== undefined) { + this.emitAutoProps(code, resolver); + } + + this.emitJsiiMethodCall(code, resolver); + } + } + + private emitAutoProps(code: CodeMaker, resolver: TypeResolver) { + const lastParameter = this.parameters.slice(-1)[0]; + const argName = toPythonIdentifier(lastParameter.name); + const typeName = resolver.resolve(lastParameter.type, {ignoreOptional: true }); + + // We need to build up a list of properties, which are mandatory, these are the + // ones we will specifiy to start with in our dictionary literal. + const mandatoryPropMembers: string[] = []; + for (const prop of this.liftedProp!.properties || []) { + if (prop.type.optional) { + continue; + } + + mandatoryPropMembers.push(`"${toPythonIdentifier(prop.name)}": ${toPythonIdentifier(prop.name)}`); + } + code.line(`${argName}: ${typeName} = {${mandatoryPropMembers.join(", ")}}`); + code.line(); + + // Now we'll go through our optional properties, and if they haven't been set + // we'll add them to our dictionary. + for (const prop of this.liftedProp!.properties || []) { + if (!prop.type.optional) { + continue; + } + + code.openBlock(`if ${toPythonIdentifier(prop.name)} is not None`); + code.line(`${argName}["${toPythonIdentifier(prop.name)}"] = ${toPythonIdentifier(prop.name)}`); + code.closeBlock(); + } + } + + private emitJsiiMethodCall(code: CodeMaker, resolver: TypeResolver) { + const methodPrefix: string = this.returnFromJSIIMethod ? "return " : ""; + + const jsiiMethodParams: string[] = []; + if (this.classAsFirstParameter) { + if (this.parent === undefined) { + throw new Error("Parent not known."); + } + jsiiMethodParams.push(resolver.resolve(this.parent)); + } + jsiiMethodParams.push(this.implicitParameter); + if (this.jsName !== undefined) { + jsiiMethodParams.push(`"${this.jsName}"`); + } + + const paramNames: string[] = []; + for (const param of this.parameters) { + paramNames.push(toPythonIdentifier(param.name)); + } + + code.line(`${methodPrefix}jsii.${this.jsiiMethod}(${jsiiMethodParams.join(", ")}, [${paramNames.join(", ")}])`); + } +} + +interface BasePropertyOpts { + abstract?: boolean; + immutable?: boolean; +} + +interface BasePropertyEmitOpts { + renderAbstract?: boolean; +} + +abstract class BaseProperty implements PythonBase { + + public readonly name: string; + public readonly abstract: boolean; + + protected readonly abstract decorator: string; + protected readonly abstract implicitParameter: string; + protected readonly jsiiGetMethod?: string; + protected readonly jsiiSetMethod?: string; + + private readonly jsName: string; + private readonly type: spec.TypeReference; + private readonly immutable: boolean; + + constructor(name: string, jsName: string, type: spec.TypeReference, opts: BasePropertyOpts = {}) { + const { + abstract = false, + immutable = false, + } = opts; + + this.name = name; + this.abstract = abstract; + this.jsName = jsName; + this.type = type; + this.immutable = immutable; + } + + public emit(code: CodeMaker, resolver: TypeResolver, opts?: BasePropertyEmitOpts) { + const { renderAbstract = true } = opts || {}; + const pythonType = resolver.resolve(this.type, { forwardReferences: false }); + + code.line(`@${this.decorator}`); + code.line(`@jsii.member(jsii_name="${this.jsName}")`); + if (renderAbstract && this.abstract) { + code.line("@abc.abstractmethod"); + } + code.openBlock(`def ${this.name}(${this.implicitParameter}) -> ${pythonType}`); + if (this.jsiiGetMethod !== undefined && (!renderAbstract || !this.abstract)) { + code.line(`return jsii.${this.jsiiGetMethod}(${this.implicitParameter}, "${this.jsName}")`); + } else { + code.line("..."); + } + code.closeBlock(); + + if (!this.immutable) { + code.line(`@${this.name}.setter`); + if (renderAbstract && this.abstract) { + code.line("@abc.abstractmethod"); + } + code.openBlock(`def ${this.name}(${this.implicitParameter}, value: ${pythonType})`); + if (this.jsiiSetMethod !== undefined && (!renderAbstract || !this.abstract)) { + code.line(`return jsii.${this.jsiiSetMethod}(${this.implicitParameter}, "${this.jsName}", value)`); + } else { + code.line("..."); + } + code.closeBlock(); + } + } +} + +class Interface extends BasePythonClassType { + + protected getClassParams(resolver: TypeResolver): string[] { + const params: string[] = this.bases.map(b => resolver.resolve(b)); + + params.push("jsii.compat.Protocol"); + + return params; + } + +} + +class InterfaceMethod extends BaseMethod { + protected readonly implicitParameter: string = "self"; +} + +class InterfaceProperty extends BaseProperty { + protected readonly decorator: string = "property"; + protected readonly implicitParameter: string = "self"; +} + +class TypedDict extends BasePythonClassType { + + public emit(code: CodeMaker, resolver: TypeResolver) { + resolver = this.fqn ? resolver.bind(this.fqn) : resolver; + + // MyPy doesn't let us mark some keys as optional, and some keys as mandatory, + // we can either mark either the entire class as mandatory or the entire class + // as optional. However, we can make two classes, one with all mandatory keys + // and one with all optional keys in order to emulate this. So we'll go ahead + // and implement this "split" class logic. + + const classParams = this.getClassParams(resolver); + + const mandatoryMembers = this.members.filter( + item => item instanceof TypedDictProperty ? !item.optional : true + ); + const optionalMembers = this.members.filter( + item => item instanceof TypedDictProperty ? item.optional : false + ); + + if (mandatoryMembers.length >= 1 && optionalMembers.length >= 1) { + // In this case, we have both mandatory *and* optional members, so we'll + // do our split class logic. + + // We'll emit the optional members first, just because it's a little nicer + // for the final class in the chain to have the mandatory members. + code.openBlock(`class _${this.name}(${classParams.concat(["total=False"]).join(", ")})`); + for (const member of optionalMembers) { + member.emit(code, resolver); + } + code.closeBlock(); + + // Now we'll emit the mandatory members. + code.line(`@jsii.data_type(jsii_type="${this.fqn}")`); + code.openBlock(`class ${this.name}(_${this.name})`); + for (const member of sortMembers(mandatoryMembers, resolver)) { + member.emit(code, resolver); + } + code.closeBlock(); + } else { + code.line(`@jsii.data_type(jsii_type="${this.fqn}")`); + + // In this case we either have no members, or we have all of one type, so + // we'll see if we have any optional members, if we don't then we'll use + // total=True instead of total=False for the class. + if (optionalMembers.length >= 1) { + code.openBlock(`class ${this.name}(${classParams.concat(["total=False"]).join(", ")})`); + } else { + code.openBlock(`class ${this.name}(${classParams.join(", ")})`); + } + + // Finally we'll just iterate over and emit all of our members. + if (this.members.length > 0) { + for (const member of sortMembers(this.members, resolver)) { + member.emit(code, resolver); + } + } else { + code.line("pass"); + } + + code.closeBlock(); + } + } + + protected getClassParams(resolver: TypeResolver): string[] { + const params: string[] = this.bases.map(b => resolver.resolve(b)); + + params.push("jsii.compat.TypedDict"); + + return params; + } + +} + +class TypedDictProperty implements PythonBase { + + public readonly name: string; + + private readonly type: spec.TypeReference; + + constructor(name: string, type: spec.TypeReference) { + this.name = name; + this.type = type; + } + + get optional(): boolean { + return this.type.optional !== undefined ? this.type.optional : false; + } + + public emit(code: CodeMaker, resolver: TypeResolver) { + const resolvedType = resolver.resolve( + this.type, + { forwardReferences: false, ignoreOptional: true } + ); + code.line(`${this.name}: ${resolvedType}`); + } +} + +interface ClassOpts extends PythonTypeOpts { + abstract?: boolean; +} + +class Class extends BasePythonClassType { + + private abstract: boolean; + + constructor(name: string, fqn: string, opts: ClassOpts) { + super(name, fqn, opts); + + const { abstract = false } = opts; + + this.abstract = abstract; + } + + public emit(code: CodeMaker, resolver: TypeResolver) { + // First we do our normal class logic for emitting our members. + super.emit(code, resolver); + + // Then, if our class is Abstract, we have to go through and redo all of + // this logic, except only emiting abstract methods and properties as non + // abstract, and subclassing our initial class. + if (this.abstract) { + resolver = this.fqn ? resolver.bind(this.fqn) : resolver; + code.openBlock(`class ${this.getProxyClassName()}(${this.name})`); + + // Filter our list of members to *only* be abstract members, and not any + // other types. + const abstractMembers = this.members.filter( + m => (m instanceof BaseMethod || m instanceof BaseProperty) && m.abstract + ); + if (abstractMembers.length > 0) { + for (const member of abstractMembers) { + member.emit(code, resolver, { renderAbstract: false }); + } + } else { + code.line("pass"); + } + + code.closeBlock(); + } + } + + protected emitPreamble(code: CodeMaker, _resolver: TypeResolver) { + if (this.abstract) { + code.line("@staticmethod"); + code.openBlock("def __jsii_proxy_class__()"); + code.line(`return ${this.getProxyClassName()}`); + code.closeBlock(); + } + } + + protected getClassParams(resolver: TypeResolver): string[] { + const params: string[] = this.bases.map(b => resolver.resolve(b)); + const metaclass: string = this.abstract ? "JSIIAbstractClass" : "JSIIMeta"; + + params.push(`metaclass=jsii.${metaclass}`); + params.push(`jsii_type="${this.fqn}"`); + + return params; + } + + private getProxyClassName(): string { + return `_${this.name}Proxy`; + } +} + +class StaticMethod extends BaseMethod { + protected readonly decorator?: string = "classmethod"; + protected readonly implicitParameter: string = "cls"; + protected readonly jsiiMethod: string = "sinvoke"; +} + +class Initializer extends BaseMethod { + protected readonly implicitParameter: string = "self"; + protected readonly jsiiMethod: string = "create"; + protected readonly classAsFirstParameter: boolean = true; + protected readonly returnFromJSIIMethod: boolean = false; +} + +class Method extends BaseMethod { + protected readonly implicitParameter: string = "self"; + protected readonly jsiiMethod: string = "invoke"; +} + +class StaticProperty extends BaseProperty { + protected readonly decorator: string = "classproperty"; + protected readonly implicitParameter: string = "cls"; + protected readonly jsiiGetMethod: string = "sget"; + protected readonly jsiiSetMethod: string = "sset"; +} + +class Property extends BaseProperty { + protected readonly decorator: string = "property"; + protected readonly implicitParameter: string = "self"; + protected readonly jsiiGetMethod: string = "get"; + protected readonly jsiiSetMethod: string = "set"; +} + +class Enum extends BasePythonClassType { + + protected getClassParams(_resolver: TypeResolver): string[] { + return ["enum.Enum"]; + } + +} + +class EnumMember implements PythonBase { + + public readonly name: string; + + private readonly value: string; + + constructor(name: string, value: string) { + this.name = name; + this.value = value; + } + + public emit(code: CodeMaker, _resolver: TypeResolver) { + code.line(`${this.name} = "${this.value}"`); + } +} + +class Namespace extends BasePythonClassType { + protected getClassParams(_resolver: TypeResolver): string[] { + return []; + } +} + +interface ModuleOpts { + assembly: spec.Assembly, + assemblyFilename: string; + loadAssembly: boolean; +} + +class Module implements PythonType { + + public readonly name: string; + public readonly fqn: string | null; + + private assembly: spec.Assembly; + private assemblyFilename: string; + private loadAssembly: boolean; + private members: PythonBase[]; + + constructor(name: string, fqn: string | null, opts: ModuleOpts) { + this.name = name; + this.fqn = fqn; + + this.assembly = opts.assembly; + this.assemblyFilename = opts.assemblyFilename; + this.loadAssembly = opts.loadAssembly; + this.members = []; + } + + public addMember(member: PythonBase) { + this.members.push(member); + } + + public emit(code: CodeMaker, resolver: TypeResolver) { + resolver = this.fqn ? resolver.bind(this.fqn, this.name) : resolver; + + // Before we write anything else, we need to write out our module headers, this + // is where we handle stuff like imports, any required initialization, etc. + code.line("import abc"); + code.line("import datetime"); + code.line("import enum"); + code.line("import typing"); + code.line(); + code.line("import jsii"); + code.line("import jsii.compat"); + code.line("import publication"); + code.line(); + code.line("from jsii.python import classproperty"); + + // Go over all of the modules that we need to import, and import them. + this.emitDependencyImports(code, resolver); + + // Determine if we need to write out the kernel load line. + if (this.loadAssembly) { + code.line( + `__jsii_assembly__ = jsii.JSIIAssembly.load(` + + `"${this.assembly.name}", ` + + `"${this.assembly.version}", ` + + `__name__, ` + + `"${this.assemblyFilename}")` + ); + } + + // Emit all of our members. + for (const member of sortMembers(this.members, resolver)) { + member.emit(code, resolver); + } + + // Whatever names we've exported, we'll write out our __all__ that lists them. + const exportedMembers = this.members.map(m => `"${m.name}"`); + if (this.loadAssembly) { + exportedMembers.push(`"__jsii_assembly__"`); + } + code.line(`__all__ = [${exportedMembers.sort().join(", ")}]`); + + // Finally, we'll use publication to ensure that all of the non-public names + // get hidden from dir(), tab-complete, etc. + code.line(); + code.line("publication.publish()"); + } + + private emitDependencyImports(code: CodeMaker, _resolver: TypeResolver) { + const deps = Array.from( + new Set([ + ...Object.keys(this.assembly.dependencies || {}).map(d => toPythonModuleName(d)), + ]) + ); + + for (const [idx, moduleName] of deps.sort().entries()) { + // If this our first dependency, add a blank line to format our imports + // slightly nicer. + if (idx === 0) { + code.line(); + } + + code.line(`import ${moduleName}`); + } + } +} + +interface PackageMetadata { + summary?: string; + readme?: string; + url?: string; +} + +interface PackageData { + filename: string; + data: string | null; +} + +class Package { + + public readonly name: string; + public readonly version: string; + public readonly metadata: PackageMetadata; + + private modules: Map; + private data: Map; + + constructor(name: string, version: string, metadata: PackageMetadata) { + this.name = name; + this.version = version; + this.metadata = metadata; + + this.modules = new Map(); + this.data = new Map(); + } + + public addModule(module: Module) { + this.modules.set(module.name, module); + } + + public addData(module: Module, filename: string, data: string | null) { + if (!this.data.has(module.name)) { + this.data.set(module.name, new Array()); + } + + this.data.get(module.name)!.push({filename, data}); + } + + public write(code: CodeMaker, resolver: TypeResolver) { + const modules = [...this.modules.values()].sort((a, b) => a.name.localeCompare(b.name)); + + // Iterate over all of our modules, and write them out to disk. + for (const mod of modules) { + const filename = path.join("src", pythonModuleNameToFilename(mod.name), "__init__.py"); + + code.openFile(filename); + mod.emit(code, resolver); + code.closeFile(filename); + } + + // Handle our package data. + const packageData: {[key: string]: string[]} = {}; + for (const [mod, pdata] of this.data) { + for (const data of pdata) { + if (data.data != null) { + const filepath = path.join("src", pythonModuleNameToFilename(mod), data.filename); + + code.openFile(filepath); + code.line(data.data); + code.closeFile(filepath); + } + } + + packageData[mod] = pdata.map(pd => pd.filename); + } + + const setupKwargs = { + name: this.name, + version: this.version, + description: this.metadata.summary, + url: this.metadata.url, + package_dir: {"": "src"}, + packages: modules.map(m => m.name), + package_data: packageData, + python_requires: ">=3.6", + install_requires: ["publication"], + }; + + // We Need a setup.py to make this Package, actually a Package. + // TODO: + // - Author + // - License + // - Classifiers + code.openFile("setup.py"); + code.line("import json"); + code.line("import setuptools"); + code.line(); + code.line('kwargs = json.loads("""'); + code.line(JSON.stringify(setupKwargs, null, 4)); + code.line('""")'); + code.line(); + code.line("setuptools.setup(**kwargs)"); + code.closeFile("setup.py"); + + // Because we're good citizens, we're going to go ahead and support pyproject.toml + // as well. + // TODO: Might be easier to just use a TOML library to write this out. + code.openFile("pyproject.toml"); + code.line("[build-system]"); + code.line('requires = ["setuptools", "wheel"]'); + code.closeFile("pyproject.toml"); + + // We also need to write out a MANIFEST.in to ensure that all of our required + // files are included. + code.openFile("MANIFEST.in"); + code.line("include pyproject.toml"); + code.closeFile("MANIFEST.in"); + } +} + +interface TypeResolverOpts { + forwardReferences?: boolean; + ignoreOptional?: boolean; +} + +class TypeResolver { + + private readonly types: Map; + private boundTo?: string; + private readonly stdTypesRe = new RegExp("^(datetime\\.datetime|typing\\.[A-Z][a-z]+|jsii\\.Number)$"); + private readonly boundRe: RegExp; + private readonly moduleName?: string; + private readonly moduleRe: RegExp; + + constructor(types: Map, boundTo?: string, moduleName?: string) { + this.types = types; + this.moduleName = moduleName; + this.boundTo = boundTo !== undefined ? this.toPythonFQN(boundTo) : boundTo; + + if (this.moduleName !== undefined) { + this.moduleRe = new RegExp(`^(${escapeStringRegexp(this.moduleName)})\\.(.+)$`); + } + + if (this.boundTo !== undefined) { + this.boundRe = new RegExp(`^(${escapeStringRegexp(this.boundTo)})\\.(.+)$`); + } + } + + public bind(fqn: string, moduleName?: string): TypeResolver { + return new TypeResolver( + this.types, + fqn, + moduleName !== undefined ? moduleName : this.moduleName, + ); + } + + public isInModule(typeRef: spec.NamedTypeReference | string): boolean { + const pythonType = typeof typeRef !== "string" ? this.toPythonFQN(typeRef.fqn) : typeRef; + return this.moduleRe.test(pythonType); + } + + public isInNamespace(typeRef: spec.NamedTypeReference | string): boolean { + const pythonType = typeof typeRef !== "string" ? this.toPythonFQN(typeRef.fqn) : typeRef; + return this.boundRe.test(pythonType); + } + + public getParent(typeRef: spec.NamedTypeReference | string): PythonType { + const fqn = typeof typeRef !== "string" ? typeRef.fqn : typeRef; + const [, parentFQN] = fqn.match(/^(.+)\.[^\.]+$/) as string[]; + const parent = this.types.get(parentFQN); + + if (parent === undefined) { + throw new Error(`Could not find parent: ${parentFQN}`); + } + + return parent; + } + + public getType(typeRef: spec.NamedTypeReference): PythonType { + const type = this.types.get(typeRef.fqn); + + if (type === undefined) { + throw new Error(`Could not locate type: "${typeRef.fqn}"`); + } + + return type; + } + + public resolve( + typeRef: spec.TypeReference, + opts: TypeResolverOpts = { forwardReferences: true, ignoreOptional: false }): string { + const { + forwardReferences = true, + } = opts; + + // First, we need to resolve our given type reference into the Python type. + let pythonType = this.toPythonType(typeRef, opts.ignoreOptional); + + // If we split our types by any of the "special" characters that can't appear in + // identifiers (like "[],") then we will get a list of all of the identifiers, + // no matter how nested they are. The downside is we might get trailing/leading + // spaces or empty items so we'll need to trim and filter this list. + const types = pythonType.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s !== ""); + + for (const innerType of types) { + // Built in types do not need formatted in any particular way. + if (PYTHON_BUILTIN_TYPES.indexOf(innerType) > -1) { + continue; + } + + // These are not exactly built in types, but they're also not types that + // this resolver has to worry about. + if (this.stdTypesRe.test(innerType)) { + continue; + } + + // If our resolver is bound to the same module as the type we're trying to + // resolve, then we'll implement the needed logic to use module relative naming + // and to handle forward references (if needed). + if (this.isInModule(innerType)) { + // If our type is part of the same namespace, then we'll return a namespace + // relative name, otherwise a module relative name. + let typeName: string; + if (this.isInNamespace(innerType)) { + [, , typeName] = innerType.match(this.boundRe) as string[]; + } else { + [, , typeName] = innerType.match(this.moduleRe) as string[]; + } + + // This re will look for the entire type, boxed by either the start/end of + // a string, a comma, a space, a quote, or open/closing brackets. This will + // ensure that we only match whole type names, and not partial ones. + const re = new RegExp('((?:^|[[,\\s])"?)' + innerType + '("?(?:$|[\\],\\s]))'); + + // We need to handle forward references, our caller knows if we're able to + // use them in the current context or not, so if not, we'll wrap our forward + // reference in quotes. + // We have special logic here for checking if our thing is actually *in* + // our module, behond what we've already done, because our other logic will + // work for submodules, but this can't. + if (!forwardReferences && this.isInModule(innerType)) { + pythonType = pythonType.replace(re, `$1"${innerType}"$2`); + } + + // Now that we've gotten forward references out of the way, we will want + // to replace the entire type string, with just the type portion. + pythonType = pythonType.replace(re, `$1${typeName}$2`); + } + } + + return pythonType; + } + + private toPythonType(typeRef: spec.TypeReference, ignoreOptional?: boolean): string { + let pythonType: string; + + // Get the underlying python type. + if (spec.isPrimitiveTypeReference(typeRef)) { + pythonType = this.toPythonPrimitive(typeRef.primitive); + } else if (spec.isCollectionTypeReference(typeRef)) { + pythonType = this.toPythonCollection(typeRef); + } else if (spec.isNamedTypeReference(typeRef)) { + pythonType = this.toPythonFQN(typeRef.fqn); + } else if (typeRef.union) { + const types = new Array(); + for (const subtype of typeRef.union.types) { + types.push(this.toPythonType(subtype)); + } + pythonType = `typing.Union[${types.join(", ")}]`; + } else { + throw new Error("Invalid type reference: " + JSON.stringify(typeRef)); + } + + // If our type is Optional, then we'll wrap our underlying type with typing.Optional + // However, if we're not respecting optionals, then we'll just skip over this. + // We explicitly don't emit this when our type is typing.Any, because typing.Any + // already implied that None is an accepted type. + // See: https://github.com/awslabs/jsii/issues/284 + if (!ignoreOptional && typeRef.optional && pythonType !== "typing.Any") { + pythonType = `typing.Optional[${pythonType}]`; + } + + return pythonType; + } + + private toPythonPrimitive(primitive: spec.PrimitiveType): string { + switch (primitive) { + case spec.PrimitiveType.Boolean: return "bool"; + case spec.PrimitiveType.Date: return "datetime.datetime"; + case spec.PrimitiveType.Json: return "typing.Mapping[typing.Any, typing.Any]"; + case spec.PrimitiveType.Number: return "jsii.Number"; + case spec.PrimitiveType.String: return "str"; + case spec.PrimitiveType.Any: return "typing.Any"; + default: + throw new Error("Unknown primitive type: " + primitive); + } + } + + private toPythonCollection(ref: spec.CollectionTypeReference): string { + const elementPythonType = this.toPythonType(ref.collection.elementtype); + switch (ref.collection.kind) { + case spec.CollectionKind.Array: return `typing.List[${elementPythonType}]`; + case spec.CollectionKind.Map: return `typing.Mapping[str,${elementPythonType}]`; + default: + throw new Error(`Unsupported collection kind: ${ref.collection.kind}`); + } + } + + private toPythonFQN(fqn: string): string { + const [, modulePart, typePart] = fqn.match(/^((?:[^A-Z\.][^\.]+\.?)+)(?:\.([A-Z].+))?$/) as string[]; + const fqnParts: string[] = [toPythonModuleName(modulePart)]; + + if (typePart) { + fqnParts.push(typePart.split(".").map(cur => toPythonIdentifier(cur)).join(".")); + } + + return fqnParts.join("."); + } +} + +class PythonGenerator extends Generator { + + private package: Package; + private types: Map; + + constructor(options = new GeneratorOptions()) { + super(options); + + this.code.openBlockFormatter = s => `${s}:`; + this.code.closeBlockFormatter = _s => ""; + + this.types = new Map(); + } + + protected getAssemblyOutputDir(assm: spec.Assembly) { + return path.join("src", pythonModuleNameToFilename(this.getAssemblyModuleName(assm))); + } + + protected onBeginAssembly(assm: spec.Assembly, _fingerprint: boolean) { + this.package = new Package( + toPythonPackageName(assm.name), + assm.version, + { + summary: assm.description, + readme: assm.readme !== undefined ? assm.readme.markdown : "", + url: assm.homepage, + }, + ); + + const assemblyModule = new Module( + this.getAssemblyModuleName(assm), + null, + { assembly: assm, + assemblyFilename: this.getAssemblyFileName(), + loadAssembly: false }, + ); + + this.package.addModule(assemblyModule); + this.package.addData(assemblyModule, this.getAssemblyFileName(), null); + } + + protected onEndAssembly(_assm: spec.Assembly, _fingerprint: boolean) { + this.package.write(this.code, new TypeResolver(this.types)); + } + + protected onBeginNamespace(ns: string) { + // If we're generating the Namespace that matches our assembly, then we'll + // actually be generating a module, otherwise we'll generate a class within + // that module. + if (ns === this.assembly.name) { + const module = new Module( + toPythonModuleName(ns), + ns, + { assembly: this.assembly, + assemblyFilename: this.getAssemblyFileName(), + loadAssembly: ns === this.assembly.name }, + ); + + this.package.addModule(module); + // Add our py.typed marker to ensure that gradual typing works for this + // package. + this.package.addData(module, "py.typed", ""); + + this.types.set(ns, module); + } else { + // This should be temporary code, which can be removed and turned into an + // error case once https://github.com/awslabs/jsii/issues/270 and + // https://github.com/awslabs/jsii/issues/283 are solved. + this.addPythonType( + new Namespace( + toPythonIdentifier(ns.replace(/^.+\.([^\.]+)$/, "$1")), + ns, + {}, + ), + ); + } + } + + protected onBeginClass(cls: spec.ClassType, abstract: boolean | undefined) { + const klass = new Class( + toPythonIdentifier(cls.name), + cls.fqn, + { abstract, bases: cls.base !== undefined ? [cls.base] : [] } + ); + + if (cls.initializer !== undefined) { + const { parameters = [] } = cls.initializer; + + klass.addMember( + new Initializer( + "__init__", + undefined, + parameters, + cls.initializer.returns, + { liftedProp: this.getliftedProp(cls.initializer), parent: cls }, + ) + ); + } + + this.addPythonType(klass); + } + + protected onStaticMethod(cls: spec.ClassType, method: spec.Method) { + const { parameters = [] } = method; + + this.getPythonType(cls.fqn).addMember( + new StaticMethod( + toPythonMethodName(method.name!), + method.name, + parameters, + method.returns, + { abstract: method.abstract, liftedProp: this.getliftedProp(method) }, + ) + ); + } + + protected onStaticProperty(cls: spec.ClassType, prop: spec.Property) { + this.getPythonType(cls.fqn).addMember( + new StaticProperty( + toPythonPropertyName(prop.name), + prop.name, + prop.type, + { abstract: prop.abstract, immutable: prop.immutable }, + ) + ); + } + + protected onMethod(cls: spec.ClassType, method: spec.Method) { + const { parameters = [] } = method; + + this.getPythonType(cls.fqn).addMember( + new Method( + toPythonMethodName(method.name!), + method.name, + parameters, + method.returns, + { abstract: method.abstract, liftedProp: this.getliftedProp(method) }, + ) + ); + } + + protected onProperty(cls: spec.ClassType, prop: spec.Property) { + this.getPythonType(cls.fqn).addMember( + new Property( + toPythonPropertyName(prop.name), + prop.name, + prop.type, + { abstract: prop.abstract, immutable: prop.immutable }, + ) + ); + } + + protected onBeginInterface(ifc: spec.InterfaceType) { + let iface: Interface | TypedDict; + + if (ifc.datatype) { + iface = new TypedDict( + toPythonIdentifier(ifc.name), + ifc.fqn, + { bases: ifc.interfaces }, + ); + } else { + iface = new Interface( + toPythonIdentifier(ifc.name), + ifc.fqn, + { bases: ifc.interfaces }, + ); + } + + this.addPythonType(iface); + } + + protected onEndInterface(_ifc: spec.InterfaceType) { return; } + + protected onInterfaceMethod(ifc: spec.InterfaceType, method: spec.Method) { + const { parameters = [] } = method; + + this.getPythonType(ifc.fqn).addMember( + new InterfaceMethod( + toPythonMethodName(method.name!), + method.name, + parameters, + method.returns, + { liftedProp: this.getliftedProp(method) }, + ) + ); + } + + protected onInterfaceProperty(ifc: spec.InterfaceType, prop: spec.Property) { + let ifaceProperty: InterfaceProperty | TypedDictProperty; + + if (ifc.datatype) { + ifaceProperty = new TypedDictProperty( + toPythonIdentifier(prop.name), + prop.type, + ); + } else { + ifaceProperty = new InterfaceProperty( + toPythonPropertyName(prop.name), + prop.name, + prop.type, + { immutable: prop.immutable }, + ); + } + + this.getPythonType(ifc.fqn).addMember(ifaceProperty); + } + + protected onBeginEnum(enm: spec.EnumType) { + this.addPythonType(new Enum(toPythonIdentifier(enm.name), enm.fqn, {})); + } + + protected onEnumMember(enm: spec.EnumType, member: spec.EnumMember) { + this.getPythonType(enm.fqn).addMember( + new EnumMember( + toPythonIdentifier(member.name), + member.name, + ) + ); + } + + protected onInterfaceMethodOverload(_ifc: spec.InterfaceType, _overload: spec.Method, _originalMethod: spec.Method) { + throw new Error("Unhandled Type: InterfaceMethodOverload"); + } + + protected onUnionProperty(_cls: spec.ClassType, _prop: spec.Property, _union: spec.UnionTypeReference) { + throw new Error("Unhandled Type: UnionProperty"); + } + + protected onMethodOverload(_cls: spec.ClassType, _overload: spec.Method, _originalMethod: spec.Method) { + throw new Error("Unhandled Type: MethodOverload"); + } + + protected onStaticMethodOverload(_cls: spec.ClassType, _overload: spec.Method, _originalMethod: spec.Method) { + throw new Error("Unhandled Type: StaticMethodOverload"); + } + + private getAssemblyModuleName(assm: spec.Assembly): string { + return `${toPythonModuleName(assm.name)}._jsii`; + } + + private getParentFQN(fqn: string): string { + const m = fqn.match(/^(.+)\.[^\.]+$/); + + if (m === null) { + throw new Error(`Could not determine parent FQN of: ${fqn}`); + } + + return m[1]; + } + + private getParent(fqn: string): PythonType { + return this.getPythonType(this.getParentFQN(fqn)); + } + + private getPythonType(fqn: string): PythonType { + const type = this.types.get(fqn); + + if (type === undefined) { + throw new Error(`Could not locate type: "${fqn}"`); + } + + return type; + } + + private addPythonType(type: PythonType) { + if (type.fqn === null) { + throw new Error("Cannot add a Python type without a FQN."); + } + + this.getParent(type.fqn).addMember(type); + this.types.set(type.fqn, type); + } + + private getliftedProp(method: spec.Method): spec.InterfaceType | undefined { + // If there are parameters to this method, and if the last parameter's type is + // a datatype interface, then we want to lift the members of that last paramter + // as keyword arguments to this function. + if (method.parameters !== undefined && method.parameters.length >= 1) { + const lastParameter = method.parameters.slice(-1)[0]; + if (spec.isNamedTypeReference(lastParameter.type)) { + const lastParameterType = this.findType(lastParameter.type.fqn); + if (spec.isInterfaceType(lastParameterType) && lastParameterType.datatype) { + return lastParameterType; + } + } + } + + return undefined; + } +} diff --git a/packages/jsii-pacmak/package.json b/packages/jsii-pacmak/package.json index 2503871f94..e35fdd7c5c 100644 --- a/packages/jsii-pacmak/package.json +++ b/packages/jsii-pacmak/package.json @@ -22,6 +22,7 @@ "dependencies": { "clone": "^2.1.1", "codemaker": "^0.7.10", + "escape-string-regexp": "^1.0.5", "fs-extra": "^7.0.0", "jsii-spec": "^0.7.10", "spdx-license-list": "^4.1.0", @@ -31,6 +32,7 @@ "devDependencies": { "@scope/jsii-calc-lib": "^0.7.10", "@types/clone": "^0.1.30", + "@types/escape-string-regexp": "^1.0.0", "@types/fs-extra": "^5.0.4", "@types/node": "^8.10.37", "@types/nodeunit": "0.0.30", diff --git a/packages/jsii-python-runtime/.gitignore b/packages/jsii-python-runtime/.gitignore new file mode 100644 index 0000000000..14eb494328 --- /dev/null +++ b/packages/jsii-python-runtime/.gitignore @@ -0,0 +1,5 @@ +src/jsii/_embedded +src/jsii/_metadata.json + +*.egg-info +dist/ diff --git a/packages/jsii-python-runtime/MANIFEST.in b/packages/jsii-python-runtime/MANIFEST.in new file mode 100644 index 0000000000..8ab5288007 --- /dev/null +++ b/packages/jsii-python-runtime/MANIFEST.in @@ -0,0 +1,2 @@ +include pyproject.toml +include src/jsii/_metadata.json diff --git a/packages/jsii-python-runtime/bin/generate b/packages/jsii-python-runtime/bin/generate new file mode 100755 index 0000000000..cc901c3bf8 --- /dev/null +++ b/packages/jsii-python-runtime/bin/generate @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +import json +import os +import os.path +import shutil + +EMBEDDED_SOURCE = "node_modules/jsii-runtime/webpack/" + + +# Copy metadata over into the Python package +with open("package.json") as fp: + data = json.load(fp) + +with open("src/jsii/_metadata.json", "w") as fp: + json.dump({"version": data["version"]}, fp) + + +# Embed the JSII runtime into the Python Package. +for filename in os.listdir(EMBEDDED_SOURCE): + filepath = os.path.join(EMBEDDED_SOURCE, filename) + shutil.copy2(filepath, "src/jsii/_embedded/jsii") diff --git a/packages/jsii-python-runtime/package.json b/packages/jsii-python-runtime/package.json new file mode 100644 index 0000000000..d99b992e6f --- /dev/null +++ b/packages/jsii-python-runtime/package.json @@ -0,0 +1,27 @@ +{ + "name": "jsii-python-runtime", + "version": "0.0.0", + "description": "Python client for jsii runtime", + "main": "index.js", + "scripts": { + "generate": "bin/generate", + "build": "npm run generate && python setup.py sdist -d . bdist_wheel -d . && rm -rf build", + "package": "package-python", + "prepack": "echo ok", + "test": "echo ok" + }, + "dependencies": { + "jsii-build-tools": "^0.7.4", + "jsii-runtime": "^0.7.1" + }, + "repository": { + "type": "git", + "url": "git://github.com/awslabs/jsii" + }, + "author": { + "name": "Amazon Web Services", + "url": "https://aws.amazon.com", + "organization": true + }, + "license": "Apache-2.0" +} diff --git a/packages/jsii-python-runtime/pyproject.toml b/packages/jsii-python-runtime/pyproject.toml new file mode 100644 index 0000000000..d1e6ae6e56 --- /dev/null +++ b/packages/jsii-python-runtime/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools", "wheel"] diff --git a/packages/jsii-python-runtime/setup.py b/packages/jsii-python-runtime/setup.py new file mode 100644 index 0000000000..d3d7396128 --- /dev/null +++ b/packages/jsii-python-runtime/setup.py @@ -0,0 +1,26 @@ +import json +import setuptools + + +with open("src/jsii/_metadata.json") as fp: + metadata = json.load(fp) + + +setuptools.setup( + name="jsii", + version=metadata["version"], + package_dir={"": "src"}, + packages=setuptools.find_packages(where="src"), + package_data={ + "jsii": ["_metadata.json", "py.typed"], + "jsii._embedded.jsii": ["*.js", "*.js.map", "*.wasm"], + }, + install_requires=[ + "attrs", + "cattrs", + "importlib_resources ; python_version < '3.7'", + "typing_extensions>=3.6.4", + "mypy_extensions>=0.4.0", + ], + python_requires=">=3.6", +) diff --git a/packages/jsii-python-runtime/src/jsii/__init__.py b/packages/jsii-python-runtime/src/jsii/__init__.py new file mode 100644 index 0000000000..cddd95e6fa --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/__init__.py @@ -0,0 +1,50 @@ +from typing import Union + +from ._runtime import ( + JSIIAssembly, + JSIIMeta, + JSIIAbstractClass, + data_type, + member, + kernel, +) + + +# JS doesn't have distinct float or integer types, but we do. So we'll define our own +# type alias that will allow either. +Number = Union[int, float] + + +# Alias our Kernel methods here, so that jsii. works. This will hide the fact +# that there is a kernel at all from our callers. +load = kernel.load +create = kernel.create +delete = kernel.delete +get = kernel.get +set = kernel.set +sget = kernel.sget +sset = kernel.sset +invoke = kernel.invoke +sinvoke = kernel.sinvoke +stats = kernel.stats + + +__all__ = [ + "JSIIAssembly", + "JSIIMeta", + "JSIIAbstractClass", + "Number", + "data_type", + "member", + "kernel", + "load", + "create", + "delete", + "get", + "set", + "sget", + "sset", + "invoke", + "sinvoke", + "stats", +] diff --git a/packages/jsii-python-runtime/src/jsii/_compat.py b/packages/jsii-python-runtime/src/jsii/_compat.py new file mode 100644 index 0000000000..27798ed192 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_compat.py @@ -0,0 +1,11 @@ +# Internal Compatability Shims +import sys + + +if sys.version_info >= (3, 7): + import importlib.resources as importlib_resources +else: + import importlib_resources + + +__all__ = ["importlib_resources"] diff --git a/packages/jsii-python-runtime/src/jsii/_embedded/__init__.py b/packages/jsii-python-runtime/src/jsii/_embedded/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/jsii-python-runtime/src/jsii/_embedded/jsii/__init__.py b/packages/jsii-python-runtime/src/jsii/_embedded/jsii/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py new file mode 100644 index 0000000000..59d20604e6 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py @@ -0,0 +1,168 @@ +import inspect + +from typing import Any, List, Optional, Type + +import collections.abc +import functools + +import attr + +from jsii import _reference_map +from jsii._utils import Singleton +from jsii._kernel.providers import BaseKernel, ProcessKernel +from jsii._kernel.types import JSClass, Referenceable +from jsii._kernel.types import ( + LoadRequest, + CreateRequest, + DeleteRequest, + GetRequest, + InvokeRequest, + SetRequest, + StaticGetRequest, + StaticInvokeRequest, + StaticSetRequest, + StatsRequest, + ObjRef, + Override, +) + + +_nothing = object() + + +def _get_overides(klass: JSClass, obj: Any) -> List[Override]: + overrides = [] + + # We need to inspect each item in the MRO, until we get to our JSClass, at that + # point we'll bail, because those methods are not the overriden methods, but the + # "real" methods. + for mro_klass in type(obj).mro(): + if mro_klass is klass: + break + + for name, item in mro_klass.__dict__.items(): + # We're only interested in things that also exist on the JSII class, and + # which are themselves, jsii members. + original = getattr(klass, name, _nothing) + if original is not _nothing: + if inspect.isfunction(item) and hasattr(original, "__jsii_name__"): + overrides.append( + Override(method=original.__jsii_name__, cookie=name) + ) + elif inspect.isdatadescriptor(item) and hasattr( + original.fget, "__jsii_name__" + ): + overrides.append( + Override(property=original.fget.__jsii_name__, cookie=name) + ) + + return overrides + + +def _recursize_dereference(kernel, d): + if isinstance(d, collections.abc.Mapping): + return {k: _recursize_dereference(kernel, v) for k, v in d.items()} + elif isinstance(d, ObjRef): + return _reference_map.resolve_reference(kernel, d) + else: + return d + + +def _dereferenced(fn): + @functools.wraps(fn) + def wrapped(kernel, *args, **kwargs): + return _recursize_dereference(kernel, fn(kernel, *args, **kwargs)) + + return wrapped + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class Statistics: + + object_count: int + + +class Kernel(metaclass=Singleton): + + # This class translates between the Pythonic interface for the kernel, and the + # Kernel Provider interface that maps more directly to the JSII Kernel interface. + # It currently only supports the idea of a process kernel provider, however it + # should be possible to move to other providers in the future. + + # TODO: We don't currently have any error handling, but we need to. This should + # probably live at the provider layer though, maybe with something catching + # them at this layer to translate it to something more Pythonic, depending + # on what the provider layer looks like. + + def __init__(self, provider_class: Type[BaseKernel] = ProcessKernel) -> None: + self.provider = provider_class() + + # TODO: Do we want to return anything from this method? Is the return value useful + # to anyone? + def load(self, name: str, version: str, tarball: str) -> None: + self.provider.load(LoadRequest(name=name, version=version, tarball=tarball)) + + # TODO: Is there a way to say that obj has to be an instance of klass? + def create( + self, klass: JSClass, obj: Any, args: Optional[List[Any]] = None + ) -> ObjRef: + if args is None: + args = [] + + overrides = _get_overides(klass, obj) + + obj.__jsii_ref__ = self.provider.create( + CreateRequest(fqn=klass.__jsii_type__, args=args, overrides=overrides) + ) + + def delete(self, ref: ObjRef) -> None: + self.provider.delete(DeleteRequest(objref=ref)) + + @_dereferenced + def get(self, obj: Referenceable, property: str) -> Any: + return self.provider.get( + GetRequest(objref=obj.__jsii_ref__, property_=property) + ).value + + def set(self, obj: Referenceable, property: str, value: Any) -> None: + self.provider.set( + SetRequest(objref=obj.__jsii_ref__, property_=property, value=value) + ) + + @_dereferenced + def sget(self, klass: JSClass, property: str) -> Any: + return self.provider.sget( + StaticGetRequest(fqn=klass.__jsii_type__, property_=property) + ).value + + def sset(self, klass: JSClass, property: str, value: Any) -> None: + self.provider.sset( + StaticSetRequest(fqn=klass.__jsii_type__, property_=property, value=value) + ) + + @_dereferenced + def invoke( + self, obj: Referenceable, method: str, args: Optional[List[Any]] = None + ) -> Any: + if args is None: + args = [] + + return self.provider.invoke( + InvokeRequest(objref=obj.__jsii_ref__, method=method, args=args) + ).result + + @_dereferenced + def sinvoke( + self, klass: JSClass, method: str, args: Optional[List[Any]] = None + ) -> Any: + if args is None: + args = [] + + return self.provider.sinvoke( + StaticInvokeRequest(fqn=klass.__jsii_type__, method=method, args=args) + ).result + + def stats(self): + resp = self.provider.stats(StatsRequest()) + + return Statistics(object_count=resp.objectCount) diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/providers/__init__.py b/packages/jsii-python-runtime/src/jsii/_kernel/providers/__init__.py new file mode 100644 index 0000000000..26ec99a6a2 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_kernel/providers/__init__.py @@ -0,0 +1,5 @@ +from jsii._kernel.providers.base import BaseKernel +from jsii._kernel.providers.process import ProcessKernel + + +__all__ = ["BaseKernel", "ProcessKernel"] diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/providers/base.py b/packages/jsii-python-runtime/src/jsii/_kernel/providers/base.py new file mode 100644 index 0000000000..27699379b4 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_kernel/providers/base.py @@ -0,0 +1,71 @@ +import abc + +from typing import Optional + +from jsii._kernel.types import ( + LoadRequest, + LoadResponse, + CreateRequest, + CreateResponse, + GetRequest, + GetResponse, + InvokeRequest, + InvokeResponse, + DeleteRequest, + DeleteResponse, + SetRequest, + SetResponse, + StaticGetRequest, + StaticInvokeRequest, + StaticSetRequest, + StatsRequest, + StatsResponse, +) + + +class BaseKernel(metaclass=abc.ABCMeta): + + # The API provided by this Kernel is not very pythonic, however it is done to map + # this API as closely to the JSII runtime as possible. Higher level abstractions + # that layer ontop of the Kernel will provide a translation layer that make this + # much more Pythonic. + + @abc.abstractmethod + def load(self, request: LoadRequest) -> LoadResponse: + ... + + @abc.abstractmethod + def create(self, request: CreateRequest) -> CreateResponse: + ... + + @abc.abstractmethod + def get(self, request: GetRequest) -> GetResponse: + ... + + @abc.abstractmethod + def set(self, request: SetRequest) -> SetResponse: + ... + + @abc.abstractmethod + def sget(self, request: StaticGetRequest) -> GetResponse: + ... + + @abc.abstractmethod + def sset(self, request: StaticSetRequest) -> SetResponse: + ... + + @abc.abstractmethod + def invoke(self, request: InvokeRequest) -> InvokeResponse: + ... + + @abc.abstractmethod + def sinvoke(self, request: StaticInvokeRequest) -> InvokeResponse: + ... + + @abc.abstractmethod + def delete(self, request: DeleteRequest) -> DeleteResponse: + ... + + @abc.abstractmethod + def stats(self, request: Optional[StatsRequest] = None) -> StatsResponse: + ... diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py b/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py new file mode 100644 index 0000000000..718de28e7c --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py @@ -0,0 +1,287 @@ +import contextlib +import importlib.machinery +import json +import os.path +import subprocess +import tempfile + +from typing import Type, Union, Mapping, Any, Optional + +import attr +import cattr # type: ignore + +import jsii._embedded.jsii + +from jsii._compat import importlib_resources +from jsii._utils import memoized_property +from jsii._kernel.providers.base import BaseKernel +from jsii._kernel.types import ( + ObjRef, + KernelRequest, + KernelResponse, + LoadRequest, + LoadResponse, + CreateRequest, + CreateResponse, + DeleteRequest, + DeleteResponse, + GetRequest, + GetResponse, + InvokeRequest, + InvokeResponse, + SetRequest, + SetResponse, + StaticGetRequest, + StaticInvokeRequest, + StaticSetRequest, + StatsRequest, + StatsResponse, +) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class _HelloResponse: + + hello: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class _OkayResponse: + + # We could technically mark this as KernelResponse, because we know that + # it is going to be one of those. However, we can't disambiguate the different + # types because some of them have the same keys as each other, so the only way + # to know what type the result is expected to be, is to know what method is + # being called. Thus we'll expect Any here, and structure this value separately. + ok: Any + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class _ErrorRespose: + + error: str + stack: str + + +_ProcessResponse = Union[_OkayResponse, _ErrorRespose] + + +def _with_api_key(api_name, asdict): + def unstructurer(value): + unstructured = asdict(value) + unstructured["api"] = api_name + + return unstructured + + return unstructurer + + +def _with_reference(data, type_): + if not isinstance(data, type_): + return type_(ref=data.ref) + return data + + +def _unstructure_ref(value): + return {"$jsii.byref": value.ref} + + +def ohook(d): + if d.keys() == {"$jsii.byref"}: + return ObjRef(ref=d["$jsii.byref"]) + return d + + +def jdefault(obj): + if hasattr(obj, "__jsii_ref__"): + return _unstructure_ref(obj.__jsii_ref__) + raise TypeError + + +class _NodeProcess: + def __init__(self): + self._serializer = cattr.Converter() + self._serializer.register_unstructure_hook( + LoadRequest, + _with_api_key("load", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + CreateRequest, + _with_api_key("create", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + DeleteRequest, + _with_api_key("del", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + GetRequest, _with_api_key("get", self._serializer.unstructure_attrs_asdict) + ) + self._serializer.register_unstructure_hook( + StaticGetRequest, + _with_api_key("sget", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + SetRequest, _with_api_key("set", self._serializer.unstructure_attrs_asdict) + ) + self._serializer.register_unstructure_hook( + StaticSetRequest, + _with_api_key("sset", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + InvokeRequest, + _with_api_key("invoke", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + StaticInvokeRequest, + _with_api_key("sinvoke", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + StatsRequest, + _with_api_key("stats", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook(ObjRef, _unstructure_ref) + self._serializer.register_structure_hook(ObjRef, _with_reference) + + self._ctx_stack = contextlib.ExitStack() + + def __del__(self): + self.stop() + + def _jsii_runtime(self): + # We have the JSII Runtime bundled with our package and we want to extract it, + # however if we just blindly use importlib.resources for this, we're going to + # have our jsii-runtime.js existing in a *different* temporary directory from + # the jsii-runtime.js.map, which we don't want. We can manually set up a + # temporary directory and extract our resources to there, but we don't want to + # pay the case of setting up a a temporary directory and shuffling bytes around + # in the common case where these files already exist on disk side by side. So + # we will check what loader the embedded package used, if it's a + # SourceFileLoader then we'll assume it's going to be on the filesystem and + # just use importlib.resources.path. + + # jsii-runtime.js MUST be the first item in this list. + filenames = ["jsii-runtime.js", "jsii-runtime.js.map", "mappings.wasm"] + + if isinstance( + jsii._embedded.jsii.__loader__, importlib.machinery.SourceFileLoader + ): + paths = [ + self._ctx_stack.enter_context( + importlib_resources.path(jsii._embedded.jsii, f) + ) + for f in filenames + ] + else: + tmpdir = self._ctx_stack.enter_context(tempfile.TemporaryDirectory()) + paths = [os.path.join(tmpdir, filename) for filename in filenames] + + for path, filename in zip(paths, filenames): + with open(path, "wb") as fp: + fp.write( + importlib_resources.read_binary(jsii._embedded.jsii, filename) + ) + + # Ensure that our jsii-runtime.js is the first entry in our paths, and that all + # of our paths, are in a commmon directory, and we didn't get them split into + # multiple directories somehow. + assert os.path.basename(paths[0]) == filenames[0] + assert os.path.commonpath(paths) == os.path.dirname(paths[0]) + + # Return our first path, which should be the path for jsii-runtime.js + return paths[0] + + def _next_message(self) -> Mapping[Any, Any]: + return json.loads(self._process.stdout.readline(), object_hook=ohook) + + def start(self): + self._process = subprocess.Popen( + ["node", self._jsii_runtime()], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + self.handshake() + + def stop(self): + # TODO: We can write an empty string here instead? + self._process.terminate() + + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() + + self._ctx_stack.close() + + def handshake(self): + resp: _HelloResponse = self._serializer.structure( + self._next_message(), _HelloResponse + ) + + # TODO: Replace with proper error. + assert ( + resp.hello == "jsii-runtime@0.7.7" + ), f"Invalid JSII Runtime Version: {resp.hello!r}" + + def send( + self, request: KernelRequest, response_type: Type[KernelResponse] + ) -> KernelResponse: + req_dict = self._serializer.unstructure(request) + # TODO: We need a cleaner solution to this, ideally we'll get + # #python-attrs/attrs#429 fixed. + if "property_" in req_dict: + req_dict["property"] = req_dict.pop("property_") + data = json.dumps(req_dict, default=jdefault).encode("utf8") + + # Send our data, ensure that it is framed with a trailing \n + self._process.stdin.write(b"%b\n" % (data,)) + self._process.stdin.flush() + + resp: _ProcessResponse = self._serializer.structure( + self._next_message(), _ProcessResponse + ) + + if isinstance(resp, _OkayResponse): + return self._serializer.structure(resp.ok, response_type) + else: + raise NotImplementedError + + +class ProcessKernel(BaseKernel): + @memoized_property + def _process(self) -> _NodeProcess: + process = _NodeProcess() + process.start() + + return process + + def load(self, request: LoadRequest) -> LoadResponse: + return self._process.send(request, LoadResponse) + + def create(self, request: CreateRequest) -> CreateResponse: + return self._process.send(request, CreateResponse) + + def get(self, request: GetRequest) -> GetResponse: + return self._process.send(request, GetResponse) + + def set(self, request: SetRequest) -> SetResponse: + return self._process.send(request, SetResponse) + + def sget(self, request: StaticGetRequest) -> GetResponse: + return self._process.send(request, GetResponse) + + def sset(self, request: StaticSetRequest) -> SetResponse: + return self._process.send(request, SetResponse) + + def invoke(self, request: InvokeRequest) -> InvokeResponse: + return self._process.send(request, InvokeResponse) + + def sinvoke(self, request: StaticInvokeRequest) -> InvokeResponse: + return self._process.send(request, InvokeResponse) + + def delete(self, request: DeleteRequest) -> DeleteResponse: + return self._process.send(request, DeleteResponse) + + def stats(self, request: Optional[StatsRequest] = None) -> StatsResponse: + if request is None: + request = StatsRequest() + return self._process.send(request, StatsResponse) diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/types.py b/packages/jsii-python-runtime/src/jsii/_kernel/types.py new file mode 100644 index 0000000000..46b100f861 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_kernel/types.py @@ -0,0 +1,250 @@ +from typing import Union, List, Any, Optional, Mapping + +import attr + +from jsii.compat import Protocol + + +# TODO: +# - HelloResponse +# - OkayResponse +# - ErrorResponse + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ObjRef: + + ref: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class Override: + + method: Optional[str] = None + property_: Optional[str] = None + cookie: Optional[str] = None + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class LoadRequest: + + name: str + version: str + tarball: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class LoadResponse: + + assembly: str + types: int + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class CreateRequest: + + fqn: str + args: List[Any] = attr.Factory(list) + overrides: List[Override] = attr.Factory(list) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class CreateResponse(ObjRef): + ... + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class DeleteRequest: + + objref: ObjRef + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class DeleteResponse: + ... + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class GetRequest: + + objref: ObjRef + property_: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class StaticGetRequest: + + fqn: str + property_: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class GetResponse: + + value: Any = None + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class StaticSetRequest: + + fqn: str + property_: str + value: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class SetRequest: + + objref: ObjRef + property_: str + value: Any + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class SetResponse: + ... + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class StaticInvokeRequest: + + fqn: str + method: str + args: Optional[List[Any]] = attr.Factory(list) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class InvokeRequest: + + objref: ObjRef + method: str + args: Optional[List[Any]] = attr.Factory(list) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class InvokeResponse: + + result: Any = None + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class BeginRequest: + + objref: ObjRef + method: str + args: Optional[List[Any]] = attr.Factory(list) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class BeginResponse: + + promiseid: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class EndRequest: + + promiseid: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class EndResponse: + + result: Any + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class Callback: + + cbid: str + cookie: Optional[str] + invoke: Optional[InvokeRequest] + get: Optional[GetRequest] + set: Optional[SetRequest] + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class CallbacksRequest: + ... + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class CallbacksResponse: + + callbacks: List[Callback] + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class CompleteRequest: + + cbid: str + err: Optional[str] = None + result: Optional[Any] = None + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class CompleteResponse: + + cbid: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class NamingRequest: + + assembly: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class NamingResponse: + + naming: Mapping[str, Mapping[str, Optional[Any]]] + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class StatsRequest: + ... + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class StatsResponse: + + objectCount: int + + +KernelRequest = Union[ + LoadRequest, + CreateRequest, + DeleteRequest, + GetRequest, + StaticGetRequest, + InvokeRequest, + StaticInvokeRequest, + StatsRequest, +] + +KernelResponse = Union[ + LoadResponse, + CreateResponse, + DeleteResponse, + GetResponse, + InvokeResponse, + StatsResponse, +] + + +class JSClass(Protocol): + + @property + def __jsii_type__(self) -> str: + """ + Returns a str that points to this class inside of the Javascript runtime. + """ + + +class Referenceable(Protocol): + + @property + def __jsii_ref__(self) -> ObjRef: + """ + Returns an ObjRef that points to this object on the JS side. + """ diff --git a/packages/jsii-python-runtime/src/jsii/_reference_map.py b/packages/jsii-python-runtime/src/jsii/_reference_map.py new file mode 100644 index 0000000000..960bc860f6 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_reference_map.py @@ -0,0 +1,80 @@ +# This module exists to break an import cycle between jsii.runtime and jsii.kernel +import inspect +import weakref + +from .compat import TypedDict +from ._kernel.types import JSClass, Referenceable + + +_types = {} +_data_types = {} + + +def register_type(klass: JSClass): + _types[klass.__jsii_type__] = klass + + +def register_data_type(data_type: TypedDict): + _data_types[data_type.__jsii_type__] = data_type + + +class _FakeReference: + def __init__(self, ref: str) -> None: + self.__jsii_ref__ = ref + + +class _ReferenceMap: + def __init__(self, types): + self._refs = weakref.WeakValueDictionary() + self._types = types + + def register(self, inst: Referenceable): + self._refs[inst.__jsii_ref__.ref] = inst + + def resolve(self, kernel, ref): + # First we need to check our reference map to see if we have any instance that + # already matches this reference. + try: + return self._refs[ref.ref] + except KeyError: + pass + + # If we got to this point, then we didn't have a referene for this, in that case + # we want to create a new instance, but we need to create it in such a way that + # we don't try to recreate the type inside of the JSII interface. + class_fqn = ref.ref.rsplit("@", 1)[0] + if class_fqn in _types: + klass = _types[class_fqn] + + # If this class is an abstract class, then we'll use the generated proxy + # class instead of the abstract class to handle return values for this type. + if inspect.isabstract(klass): + klass = klass.__jsii_proxy_class__() + + # Create our instance, bypassing __init__ by directly calling __new__, and + # then assign our reference to __jsii_ref__ + inst = klass.__new__(klass) + inst.__jsii_ref__ = ref + elif class_fqn in _data_types: + data_type = _data_types[class_fqn] + + # A Data type is nothing more than a dictionary, however we need to iterate + # over all of it's properties, and ask the kernel for the values of each of + # then in order to constitute our dict + inst = {} + + for name in data_type.__annotations__.keys(): + # This is a hack, because our kernel expects an object that has a + # __jsii_ref__ attached to it, and we don't have one of those. + inst[name] = kernel.get(_FakeReference(ref), name) + else: + raise ValueError(f"Unknown type: {class_fqn}") + + return inst + + +_refs = _ReferenceMap(_types) + + +register_reference = _refs.register +resolve_reference = _refs.resolve diff --git a/packages/jsii-python-runtime/src/jsii/_runtime.py b/packages/jsii-python-runtime/src/jsii/_runtime.py new file mode 100644 index 0000000000..96ec90a698 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_runtime.py @@ -0,0 +1,100 @@ +import abc +import weakref +import os + +import attr + +from jsii import _reference_map +from jsii._compat import importlib_resources +from jsii._kernel import Kernel +from jsii.python import _ClassPropertyMeta + + +# Yea, a global here is kind of gross, however, there's not really a better way of +# handling this. Fundamentally this is a global value, since we can only reasonably +# have a single kernel active at any one time in a real program. +kernel = Kernel() + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class JSIIAssembly: + + name: str + version: str + module: str + filename: str + + @classmethod + def load(cls, *args, _kernel=kernel, **kwargs): + # Our object here really just acts as a record for our JSIIAssembly, it doesn't + # offer any functionality itself, besides this class method that will trigger + # the loading of the given assembly in the JSII Kernel. + assembly = cls(*args, **kwargs) + + # Actually load the assembly into the kernel, we're using the + # importlib.resources API here isntead of manually constructing the path, in + # the hopes that this will make JSII modules able to be used with zipimport + # instead of only on the FS. + with importlib_resources.path( + f"{assembly.module}._jsii", assembly.filename + ) as assembly_path: + _kernel.load(assembly.name, assembly.version, os.fspath(assembly_path)) + + # Give our record of the assembly back to the caller. + return assembly + + +class JSIIMeta(_ClassPropertyMeta, type): + def __new__(cls, name, bases, attrs, *, jsii_type=None): + # We want to ensure that subclasses of a JSII class do not require setting the + # jsii_type keyword argument. They should be able to subclass it as normal. + # Since their parent class will have the __jsii_type__ variable defined, they + # will as well anyways. + if jsii_type is not None: + attrs["__jsii_type__"] = jsii_type + + obj = super().__new__(cls, name, bases, attrs) + + # Now that we've created the class, we'll need to register it with our reference + # mapper. We only do this for types that are actually jsii types, and not any + # subclasses of them. + if jsii_type is not None: + _reference_map.register_type(obj) + + return obj + + def __call__(cls, *args, **kwargs): + inst = super().__call__(*args, **kwargs) + + # Register this instance with our reference map. + _reference_map.register_reference(inst) + + # Whenever the object we're creating gets garbage collected, then we want to + # delete it from the JS runtime as well. + # TODO: Figure out if this is *really* true, what happens if something goes + # out of scope at the Python level, but something is holding onto it + # at the JS level? What mechanics are in place for this if any? + weakref.finalize(inst, kernel.delete, inst.__jsii_ref__) + + return inst + + +class JSIIAbstractClass(abc.ABCMeta, JSIIMeta): + pass + + +def data_type(*, jsii_type): + def deco(cls): + cls.__jsii_type__ = jsii_type + _reference_map.register_data_type(cls) + return cls + + return deco + + +def member(*, jsii_name): + def deco(fn): + fn.__jsii_name__ = jsii_name + return fn + + return deco diff --git a/packages/jsii-python-runtime/src/jsii/_utils.py b/packages/jsii-python-runtime/src/jsii/_utils.py new file mode 100644 index 0000000000..08e69ba035 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_utils.py @@ -0,0 +1,27 @@ +import functools + +from typing import Any, Mapping, Type + + +class Singleton(type): + + _instances: Mapping[Type[Any], Any] = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + + return cls._instances[cls] + + +def memoized_property(fgetter): + stored = [] + + @functools.wraps(fgetter) + def wrapped(self): + nonlocal stored + if not stored: + stored.append(fgetter(self)) + return stored[0] + + return property(wrapped) diff --git a/packages/jsii-python-runtime/src/jsii/compat.py b/packages/jsii-python-runtime/src/jsii/compat.py new file mode 100644 index 0000000000..ea8b6c8d87 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/compat.py @@ -0,0 +1,7 @@ +# External Compatability Shims + +from mypy_extensions import TypedDict +from typing_extensions import Protocol + + +__all__ = ["Protocol", "TypedDict"] diff --git a/packages/jsii-python-runtime/src/jsii/py.typed b/packages/jsii-python-runtime/src/jsii/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/jsii-python-runtime/src/jsii/python.py b/packages/jsii-python-runtime/src/jsii/python.py new file mode 100644 index 0000000000..d8d2d5396f --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/python.py @@ -0,0 +1,38 @@ +class _ClassProperty: + def __init__(self, fget, fset=None): + self.fget = fget + self.fset = fset + + def __get__(self, obj, klass=None): + if klass is None: + klass = type(obj) + return self.fget.__get__(obj, klass)(klass) + + def __set__(self, obj, value): + if not self.fset: + raise AttributeError("Can't set attribute.") + + klass = type(obj) + return self.fset.__get__(obj, klass)(value) + + def setter(self, func): + if not isinstance(func, (classmethod, staticmethod)): + func = classmethod(func) + + self.fset = func + + return self + + +def classproperty(func): + return _ClassProperty(func) + + +class _ClassPropertyMeta(type): + + def __setattr__(self, key, value): + obj = getattr(self, key, None) + if isinstance(obj, _ClassProperty): + return obj.__set__(self, value) + + return super().__setattr__(key, value)