From 4b8af4939389ef146a37e048ee6e67d79b2562c1 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Mon, 20 Aug 2018 21:39:36 -0400 Subject: [PATCH 01/88] Initial Python Runtime Support --- .flake8 | 5 + packages/jsii-python-runtime/package.json | 32 +++ .../jsii-python-runtime/src/jsii/__init__.py | 0 .../jsii-python-runtime/src/jsii/_utils.py | 27 ++ .../src/jsii/kernel/__init__.py | 92 +++++++ .../src/jsii/kernel/providers/__init__.py | 5 + .../src/jsii/kernel/providers/base.py | 71 ++++++ .../src/jsii/kernel/providers/process.py | 217 +++++++++++++++++ .../src/jsii/kernel/types.py | 230 ++++++++++++++++++ .../jsii-python-runtime/src/jsii/runtime.py | 193 +++++++++++++++ 10 files changed, 872 insertions(+) create mode 100644 .flake8 create mode 100644 packages/jsii-python-runtime/package.json create mode 100644 packages/jsii-python-runtime/src/jsii/__init__.py create mode 100644 packages/jsii-python-runtime/src/jsii/_utils.py create mode 100644 packages/jsii-python-runtime/src/jsii/kernel/__init__.py create mode 100644 packages/jsii-python-runtime/src/jsii/kernel/providers/__init__.py create mode 100644 packages/jsii-python-runtime/src/jsii/kernel/providers/base.py create mode 100644 packages/jsii-python-runtime/src/jsii/kernel/providers/process.py create mode 100644 packages/jsii-python-runtime/src/jsii/kernel/types.py create mode 100644 packages/jsii-python-runtime/src/jsii/runtime.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000000..7ea9c48f99 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 88 +exclude = *.egg,*/interfaces.py,node_modules,.state +ignore = W503,E203 +select = E,W,F,N diff --git a/packages/jsii-python-runtime/package.json b/packages/jsii-python-runtime/package.json new file mode 100644 index 0000000000..bce2c58736 --- /dev/null +++ b/packages/jsii-python-runtime/package.json @@ -0,0 +1,32 @@ +{ + "name": "jsii-python-runtime", + "version": "0.0.0", + "description": "Python client for jsii runtime", + "main": "index.js", + "scripts": { + "build": "echo ok", + "prepack": "echo ok", + "test": "echo ok" + }, + "dependencies": { + "jsii-runtime": "^0.6.4" + }, + "repository": { + "type": "git", + "url": "git://github.com/awslabs/jsii" + }, + "author": { + "name": "Amazon Web Services", + "url": "https://aws.amazon.com" + }, + "author": { + "name": "Amazon Web Services", + "url": "https://aws.amazon.com", + "organization": true + }, + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/awslabs/jsii.git" + } +} diff --git a/packages/jsii-python-runtime/src/jsii/__init__.py b/packages/jsii-python-runtime/src/jsii/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/jsii-python-runtime/src/jsii/_utils.py b/packages/jsii-python-runtime/src/jsii/_utils.py new file mode 100644 index 0000000000..08e69ba035 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_utils.py @@ -0,0 +1,27 @@ +import functools + +from typing import Any, Mapping, Type + + +class Singleton(type): + + _instances: Mapping[Type[Any], Any] = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + + return cls._instances[cls] + + +def memoized_property(fgetter): + stored = [] + + @functools.wraps(fgetter) + def wrapped(self): + nonlocal stored + if not stored: + stored.append(fgetter(self)) + return stored[0] + + return property(wrapped) diff --git a/packages/jsii-python-runtime/src/jsii/kernel/__init__.py b/packages/jsii-python-runtime/src/jsii/kernel/__init__.py new file mode 100644 index 0000000000..82d489a0e4 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/kernel/__init__.py @@ -0,0 +1,92 @@ +from typing import Any, List, Optional, Type + +import attr + +from jsii._utils import Singleton +from jsii.kernel.providers import BaseKernel, ProcessKernel +from jsii.kernel.types import ( + LoadRequest, + CreateRequest, + DeleteRequest, + GetRequest, + InvokeRequest, + SetRequest, + StaticGetRequest, + StaticInvokeRequest, + StaticSetRequest, + StatsRequest, + ObjRef, +) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class Statistics: + + object_count: int + + +class Kernel(metaclass=Singleton): + + # This class translates between the Pythonic interface for the kernel, and the + # Kernel Provider interface that maps more directly to the JSII Kernel interface. + # It currently only supports the idea of a process kernel provider, however it + # should be possible to move to other providers in the future. + + # TODO: We don't currently have any error handling, but we need to. This should + # probably live at the provider layer though, maybe with something catching + # them at this layer to translate it to something more Pythonic, depending + # on what the provider layer looks like. + + def __init__(self, provider_class: Type[BaseKernel] = ProcessKernel) -> None: + self.provider = provider_class() + + # TODO: Do we want to return anything from this method? Is the return value useful + # to anyone? + def load(self, name: str, version: str, tarball: str) -> None: + self.provider.load(LoadRequest(name=name, version=version, tarball=tarball)) + + # TODO: Can we do protocols in typing? + def create(self, klass: Any) -> ObjRef: + return self.provider.create(CreateRequest(fqn=klass.__jsii_type__)) + + def delete(self, ref: ObjRef) -> None: + self.provider.delete(DeleteRequest(objref=ref)) + + def get(self, ref: ObjRef, property: str) -> Any: + return self.provider.get(GetRequest(objref=ref, property_=property)).value + + def set(self, ref: ObjRef, property: str, value: Any) -> None: + self.provider.set( + SetRequest(objref=ref, property_=property, value=value) + ) + + def sget(self, klass: Any, property: str) -> Any: + return self.provider.sget( + StaticGetRequest(fqn=klass.__jsii_type__, property_=property) + ).value + + def sset(self, klass: Any, property: str, value: Any) -> None: + return self.provider.sset( + StaticSetRequest(fqn=klass.__jsii_type__, property_=property, value=value) + ) + + def invoke(self, ref: ObjRef, method: str, args: Optional[List[Any]] = None) -> Any: + if args is None: + args = [] + + return self.provider.invoke( + InvokeRequest(objref=ref, method=method, args=args) + ).result + + def sinvoke(self, klass: Any, method: str, args: Optional[List[Any]] = None) -> Any: + if args is None: + args = [] + + return self.provider.sinvoke( + StaticInvokeRequest(fqn=klass.__jsii_type__, method=method, args=args) + ).result + + def stats(self): + resp = self.provider.stats(StatsRequest()) + + return Statistics(object_count=resp.objectCount) diff --git a/packages/jsii-python-runtime/src/jsii/kernel/providers/__init__.py b/packages/jsii-python-runtime/src/jsii/kernel/providers/__init__.py new file mode 100644 index 0000000000..ba922882f9 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/kernel/providers/__init__.py @@ -0,0 +1,5 @@ +from jsii.kernel.providers.base import BaseKernel +from jsii.kernel.providers.process import ProcessKernel + + +__all__ = ["BaseKernel", "ProcessKernel"] diff --git a/packages/jsii-python-runtime/src/jsii/kernel/providers/base.py b/packages/jsii-python-runtime/src/jsii/kernel/providers/base.py new file mode 100644 index 0000000000..4fd28cdbb3 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/kernel/providers/base.py @@ -0,0 +1,71 @@ +import abc + +from typing import Optional + +from jsii.kernel.types import ( + LoadRequest, + LoadResponse, + CreateRequest, + CreateResponse, + GetRequest, + GetResponse, + InvokeRequest, + InvokeResponse, + DeleteRequest, + DeleteResponse, + SetRequest, + SetResponse, + StaticGetRequest, + StaticInvokeRequest, + StaticSetRequest, + StatsRequest, + StatsResponse, +) + + +class BaseKernel(metaclass=abc.ABCMeta): + + # The API provided by this Kernel is not very pythonic, however it is done to map + # this API as closely to the JSII runtime as possible. Higher level abstractions + # that layer ontop of the Kernel will provide a translation layer that make this + # much more Pythonic. + + @abc.abstractmethod + def load(self, request: LoadRequest) -> LoadResponse: + ... + + @abc.abstractmethod + def create(self, request: CreateRequest) -> CreateResponse: + ... + + @abc.abstractmethod + def get(self, request: GetRequest) -> GetResponse: + ... + + @abc.abstractmethod + def set(self, request: SetRequest) -> SetResponse: + ... + + @abc.abstractmethod + def sget(self, request: StaticGetRequest) -> GetResponse: + ... + + @abc.abstractmethod + def sset(self, request: StaticSetRequest) -> SetResponse: + ... + + @abc.abstractmethod + def invoke(self, request: InvokeRequest) -> InvokeResponse: + ... + + @abc.abstractmethod + def sinvoke(self, request: StaticInvokeRequest) -> InvokeResponse: + ... + + @abc.abstractmethod + def delete(self, request: DeleteRequest) -> DeleteResponse: + ... + + @abc.abstractmethod + def stats(self, request: Optional[StatsRequest] = None) -> StatsResponse: + ... diff --git a/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py b/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py new file mode 100644 index 0000000000..2354fd8278 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py @@ -0,0 +1,217 @@ +import json +import subprocess + +from typing import Type, Union, Mapping, Any, Optional + +import attr +import cattr # type: ignore + +from jsii._utils import memoized_property +from jsii.kernel.providers.base import BaseKernel +from jsii.kernel.types import ( + ObjRef, + KernelRequest, + KernelResponse, + LoadRequest, + LoadResponse, + CreateRequest, + CreateResponse, + DeleteRequest, + DeleteResponse, + GetRequest, + GetResponse, + InvokeRequest, + InvokeResponse, + SetRequest, + SetResponse, + StaticGetRequest, + StaticInvokeRequest, + StaticSetRequest, + StatsRequest, + StatsResponse, +) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class _HelloResponse: + + hello: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class _OkayResponse: + + # We could technically mark this as KernelResponse, because we know that + # it is going to be one of those. However, we can't disambiguate the different + # types because some of them have the same keys as each other, so the only way + # to know what type the result is expected to be, is to know what method is + # being called. Thus we'll expect Any here, and structure this value separately. + ok: Any + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class _ErrorRespose: + + error: str + stack: str + + +_ProcessResponse = Union[_OkayResponse, _ErrorRespose] + + +def _with_api_key(api_name, asdict): + def unstructurer(value): + unstructured = asdict(value) + unstructured["api"] = api_name + + return unstructured + + return unstructurer + + +def _with_reference(data, type_): + return type_(data["$jsii.byref"]) + + +def _unstructure_ref(value): + return {"$jsii.byref": value.ref} + + +class _NodeProcess: + def __init__(self): + self._serializer = cattr.Converter() + self._serializer.register_unstructure_hook( + LoadRequest, + _with_api_key("load", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + CreateRequest, + _with_api_key("create", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + DeleteRequest, + _with_api_key("del", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + GetRequest, _with_api_key("get", self._serializer.unstructure_attrs_asdict) + ) + self._serializer.register_unstructure_hook( + StaticGetRequest, + _with_api_key("sget", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + SetRequest, _with_api_key("set", self._serializer.unstructure_attrs_asdict) + ) + self._serializer.register_unstructure_hook( + StaticSetRequest, + _with_api_key("sset", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + InvokeRequest, + _with_api_key("invoke", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + StaticInvokeRequest, + _with_api_key("sinvoke", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook( + StatsRequest, + _with_api_key("stats", self._serializer.unstructure_attrs_asdict), + ) + self._serializer.register_unstructure_hook(ObjRef, _unstructure_ref) + self._serializer.register_structure_hook(ObjRef, _with_reference) + + def __del__(self): + self.stop() + + def _next_message(self) -> Mapping[Any, Any]: + return json.loads(self._process.stdout.readline()) + + def start(self): + self._process = subprocess.Popen( + "jsii-runtime", shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE + ) + self.handshake() + + def stop(self): + # TODO: We can write an empty string here instead? + self._process.terminate() + + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() + + def handshake(self): + resp: _HelloResponse = self._serializer.structure( + self._next_message(), _HelloResponse + ) + + # TODO: Replace with proper error. + assert ( + resp.hello == "jsii-runtime@0.6.4" + ), f"Invalid JSII Runtime Version: {resp.hello!r}" + + def send( + self, request: KernelRequest, response_type: Type[KernelResponse] + ) -> KernelResponse: + req_dict = self._serializer.unstructure(request) + # TODO: We need a cleaner solution to this, ideally we'll get + # #python-attrs/attrs#429 fixed. + if "property_" in req_dict: + req_dict["property"] = req_dict.pop("property_") + data = json.dumps(req_dict).encode("utf8") + + # Send our data, ensure that it is framed with a trailing \n + self._process.stdin.write(b"%b\n" % (data,)) + self._process.stdin.flush() + + resp: _ProcessResponse = self._serializer.structure( + self._next_message(), _ProcessResponse + ) + + if isinstance(resp, _OkayResponse): + return self._serializer.structure(resp.ok, response_type) + else: + raise NotImplementedError + + +class ProcessKernel(BaseKernel): + @memoized_property + def _process(self) -> _NodeProcess: + process = _NodeProcess() + process.start() + + return process + + def load(self, request: LoadRequest) -> LoadResponse: + return self._process.send(request, LoadResponse) + + def create(self, request: CreateRequest) -> CreateResponse: + return self._process.send(request, CreateResponse) + + def get(self, request: GetRequest) -> GetResponse: + return self._process.send(request, GetResponse) + + def set(self, request: SetRequest) -> SetResponse: + return self._process.send(request, SetResponse) + + def sget(self, request: StaticGetRequest) -> GetResponse: + return self._process.send(request, GetResponse) + + def sset(self, request: StaticSetRequest) -> SetResponse: + return self._process.send(request, SetResponse) + + def invoke(self, request: InvokeRequest) -> InvokeResponse: + return self._process.send(request, InvokeResponse) + + def sinvoke(self, request: StaticInvokeRequest) -> InvokeResponse: + return self._process.send(request, InvokeResponse) + + def delete(self, request: DeleteRequest) -> DeleteResponse: + return self._process.send(request, DeleteResponse) + + def stats(self, request: Optional[StatsRequest] = None) -> StatsResponse: + if request is None: + request = StatsRequest() + return self._process.send(request, StatsResponse) diff --git a/packages/jsii-python-runtime/src/jsii/kernel/types.py b/packages/jsii-python-runtime/src/jsii/kernel/types.py new file mode 100644 index 0000000000..e58f2a84a2 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/kernel/types.py @@ -0,0 +1,230 @@ +from typing import Union, List, Any, Optional, Mapping + +import attr + + +# TODO: +# - HelloResponse +# - OkayResponse +# - ErrorResponse + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class ObjRef: + + ref: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class Override: + + method: Optional[str] = None + property_: Optional[str] = None + cookie: Optional[str] = None + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class LoadRequest: + + name: str + version: str + tarball: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class LoadResponse: + + assembly: str + types: int + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class CreateRequest: + + fqn: str + args: Optional[List[Any]] = attr.Factory(list) + overrides: Optional[List[Override]] = attr.Factory(list) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class CreateResponse(ObjRef): + ... + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class DeleteRequest: + + objref: ObjRef + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class DeleteResponse: + ... + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class GetRequest: + + objref: ObjRef + property_: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class StaticGetRequest: + + fqn: str + property_: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class GetResponse: + + value: Any + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class StaticSetRequest: + + fqn: str + property_: str + value: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class SetRequest: + + objref: ObjRef + property_: str + value: Any + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class SetResponse: + ... + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class StaticInvokeRequest: + + fqn: str + method: str + args: Optional[List[Any]] = attr.Factory(list) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class InvokeRequest: + + objref: ObjRef + method: str + args: Optional[List[Any]] = attr.Factory(list) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class InvokeResponse: + + result: Any + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class BeginRequest: + + objref: ObjRef + method: str + args: Optional[List[Any]] = attr.Factory(list) + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class BeginResponse: + + promiseid: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class EndRequest: + + promiseid: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class EndResponse: + + result: Any + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class Callback: + + cbid: str + cookie: Optional[str] + invoke: Optional[InvokeRequest] + get: Optional[GetRequest] + set: Optional[SetRequest] + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class CallbacksRequest: + ... + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class CallbacksResponse: + + callbacks: List[Callback] + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class CompleteRequest: + + cbid: str + err: Optional[str] = None + result: Optional[Any] = None + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class CompleteResponse: + + cbid: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class NamingRequest: + + assembly: str + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class NamingResponse: + + naming: Mapping[str, Mapping[str, Optional[Any]]] + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class StatsRequest: + ... + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class StatsResponse: + + objectCount: int + + +KernelRequest = Union[ + LoadRequest, + CreateRequest, + DeleteRequest, + GetRequest, + StaticGetRequest, + InvokeRequest, + StaticInvokeRequest, + StatsRequest, +] + +KernelResponse = Union[ + LoadResponse, + CreateResponse, + DeleteResponse, + GetResponse, + InvokeResponse, + StatsResponse, +] diff --git a/packages/jsii-python-runtime/src/jsii/runtime.py b/packages/jsii-python-runtime/src/jsii/runtime.py new file mode 100644 index 0000000000..4d14021045 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/runtime.py @@ -0,0 +1,193 @@ +from typing import Any, Callable, TypeVar + +import typing +import weakref + + +class JSIIMeta(type): + def __new__(cls, name, bases, attrs, *, jsii_type, jsii_kernel): + # TODO: We need to figure out exactly what we're going to do this the kernel + # here. Right now we're "creating" a new one per type, and relying on + # the fact it's a singleton so that everything is in the same kernel. + # That might not make complete sense though, particularly if we ever + # want to have multiple kernels in a single process (does that even + # make sense?). Perhaps we should pass it in like we pass the type, and + # have the autogenerated code either create it once per library or once + # per process. + attrs["__jsii_kernel__"] = jsii_kernel + attrs["__jsii_type__"] = jsii_type + + # We need to lift all of the classproperty instances out of the class, because + # they need to actually be set *on* the metaclass.. which is us. However we + # don't want to have to derive a separate metaclass for each class, so instead + # we're going to dynamically handle these. + class_properties, new_attrs = {}, {} + for key, value in attrs.items(): + if isinstance(value, jsii_classproperty): + class_properties[key] = value + else: + new_attrs[key] = value + new_attrs["__jsii_class_properties__"] = class_properties + + obj = type.__new__(cls, name, bases, new_attrs) + + # We need to go through an implement the __set_name__ portion of our API. + for key, value in obj.__jsii_class_properties__.items(): + value.__set_name__(obj, key) + + return obj + + def __call__(cls, *args, **kwargs): + # Create our object at the Python level. + obj = cls.__new__(cls, *args, **kwargs) + + # Create our object at the JS level. + # TODO: Handle args/kwargs + # TODO: Handle Overrides + obj.__jsii_ref__ = cls.__jsii_kernel__.create(cls) + + # Whenever the object we're creating gets garbage collected, then we want to + # delete it from the JS runtime as well. + # TODO: Figure out if this is *really* true, what happens if something goes + # out of scope at the Python level, but something is holding onto it + # at the JS level? What mechanics are in place for this if any? + weakref.finalize(obj, cls.__jsii_kernel__.delete, obj.__jsii_ref__) + + # Instatiate our object at the PYthon level. + if isinstance(obj, cls): + obj.__init__(*args, **kwargs) + + return obj + + def __getattr__(obj, name): + if name in obj.__jsii_class_properties__: + return obj.__jsii_class_properties__[name].__get__(obj, None) + + raise AttributeError(f"type object {obj.__name__!r} has no attribute {name!r}") + + def __setattr__(obj, name, value): + if name in obj.__jsii_class_properties__: + return obj.__jsii_class_properties__[name].__set__(obj, value) + + return super().__setattr__(name, value) + + +# NOTE: We do this goofy thing so that typing works on our generated stub classes, +# because MyPy does not currently understand the decorator protocol very well. +# Something like https://github.com/python/peps/pull/242 should make this no +# longer required. +if typing.TYPE_CHECKING: + jsii_property = property +else: + # The naming is a little funky on this, since it's a class but named like a + # function. This is done to better mimic other decorators like @proeprty. + class jsii_property: + + # We throw away the getter function here, because it doesn't really matter or + # provide us with anything useful. It exists only to provide a way to pass + # naming into us, and so that consumers of this library can "look at the + # source" and at least see something that resembles the structure of the + # library they're using, even though it won't have any of the body of the code. + def __init__(self, getter): + pass + + # TODO: Figure out a way to let the caller of this override the name. This might + # be useful in cases where the name that the JS level code is using isn't + # a valid python identifier, but we still want to be able to bind it, and + # doing so would require giving it a different name at the Python level. + def __set_name__(self, owner, name): + self.name = name + + def __get__(self, instance, owner): + return instance.__jsii_kernel__.get(instance.__jsii_ref__, self.name) + + def __set__(self, instance, value): + instance.__jsii_kernel__.set(instance.__jsii_ref__, self.name, value) + + +class _JSIIMethod: + def __init__(self, obj, method_name): + self.obj = obj + self.method_name = method_name + + def __call__(self, *args): + kernel = self.obj.__jsii_kernel__ + return kernel.invoke(self.obj.__jsii_ref__, self.method_name, args) + + +# NOTE: We do this goofy thing so that typing works on our generated stub classes, +# because MyPy does not currently understand the decorator protocol very well. +# Something like https://github.com/python/peps/pull/242 should make this no +# longer required. +if typing.TYPE_CHECKING: + F = TypeVar("F", bound=Callable[..., Any]) + + def jsii_method(func: F) -> F: + ... + + +else: + # Again, the naming is a little funky on this, since it's a class but named like a + # function. This is done to better mimic other decorators like @classmethod. + class jsii_method: + + # Again, we're throwing away the actual function body, because it exists only + # to provide the structure of the library for people who read the code, and a + # way to pass the name/typing signatures through. + def __init__(self, meth): + pass + + # TODO: Figure out a way to let the caller of this override the name. This might + # be useful in cases where the name that the JS level code is using isn't + # a valid python identifier, but we still want to be able to bind it, and + # doing so would require giving it a different name at the Python level. + def __set_name__(self, owner, name): + self.name = name + + def __get__(self, instance, owner): + return _JSIIMethod(instance, self.name) + + +class _JSIIClassMethod: + def __init__(self, klass, method_name): + self.klass = klass + self.method_name = method_name + + def __call__(self, *args): + kernel = self.klass.__jsii_kernel__ + return kernel.sinvoke(self.klass, self.method_name, args) + + +if typing.TYPE_CHECKING: + jsii_classmethod = classmethod +else: + + class jsii_classmethod: + def __init__(self, meth): + pass + + def __set_name__(self, owner, name): + self.name = name + + def __get__(self, instance, owner): + return _JSIIClassMethod(owner, self.name) + + +if typing.TYPE_CHECKING: + # TODO: Figure out if this type checks correctly, if not how do we make it type + # check correctly... or is it even possible at all? + jsii_classproperty = property +else: + + class jsii_classproperty: + def __init__(self, meth): + pass + + def __set_name__(self, owner, name): + self.name = name + + def __get__(self, instance, owner): + return instance.__jsii_kernel__.sget(instance, self.name) + + def __set__(self, instance, value): + instance.__jsii_kernel__.sset(instance, self.name, value) From fa291d77f9fecf88a904bc11bbbf111424d4aabe Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Wed, 5 Sep 2018 16:55:47 -0400 Subject: [PATCH 02/88] Bump versions --- packages/jsii-python-runtime/package.json | 12 ++---------- .../src/jsii/kernel/providers/process.py | 2 +- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/jsii-python-runtime/package.json b/packages/jsii-python-runtime/package.json index bce2c58736..69f6b2eaa9 100644 --- a/packages/jsii-python-runtime/package.json +++ b/packages/jsii-python-runtime/package.json @@ -9,24 +9,16 @@ "test": "echo ok" }, "dependencies": { - "jsii-runtime": "^0.6.4" + "jsii-runtime": "^0.7.1" }, "repository": { "type": "git", "url": "git://github.com/awslabs/jsii" }, - "author": { - "name": "Amazon Web Services", - "url": "https://aws.amazon.com" - }, "author": { "name": "Amazon Web Services", "url": "https://aws.amazon.com", "organization": true }, - "license": "Apache-2.0", - "repository": { - "type": "git", - "url": "https://github.com/awslabs/jsii.git" - } + "license": "Apache-2.0" } diff --git a/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py b/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py index 2354fd8278..3658c1a6a0 100644 --- a/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py +++ b/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py @@ -149,7 +149,7 @@ def handshake(self): # TODO: Replace with proper error. assert ( - resp.hello == "jsii-runtime@0.6.4" + resp.hello == "jsii-runtime@0.7.1" ), f"Invalid JSII Runtime Version: {resp.hello!r}" def send( From 3180d972eecd5c78dfc3e519e29c4fa8bd7270db Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Tue, 11 Sep 2018 12:58:18 -0400 Subject: [PATCH 03/88] Upgrade for latest JSII --- .../jsii-python-runtime/src/jsii/kernel/providers/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py b/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py index 3658c1a6a0..d291564edd 100644 --- a/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py +++ b/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py @@ -149,7 +149,7 @@ def handshake(self): # TODO: Replace with proper error. assert ( - resp.hello == "jsii-runtime@0.7.1" + resp.hello == "jsii-runtime@0.7.4" ), f"Invalid JSII Runtime Version: {resp.hello!r}" def send( From 1a9297c926e282b7db91d2c7b6cdbdccefb9fb48 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Tue, 11 Sep 2018 15:57:49 -0400 Subject: [PATCH 04/88] Handle non-primitive types being returned from JSII runtime --- .../src/jsii/kernel/__init__.py | 30 +++++++++-- .../src/jsii/kernel/providers/process.py | 12 ++++- .../jsii-python-runtime/src/jsii/runtime.py | 52 +++++++++++++++++-- 3 files changed, 84 insertions(+), 10 deletions(-) diff --git a/packages/jsii-python-runtime/src/jsii/kernel/__init__.py b/packages/jsii-python-runtime/src/jsii/kernel/__init__.py index 82d489a0e4..bf6250e56a 100644 --- a/packages/jsii-python-runtime/src/jsii/kernel/__init__.py +++ b/packages/jsii-python-runtime/src/jsii/kernel/__init__.py @@ -1,7 +1,12 @@ from typing import Any, List, Optional, Type +import collections.abc +import functools + import attr +import jsii.runtime + from jsii._utils import Singleton from jsii.kernel.providers import BaseKernel, ProcessKernel from jsii.kernel.types import ( @@ -19,6 +24,23 @@ ) +def _recursize_dereference(d): + if isinstance(d, collections.abc.Mapping): + return {k: _recursize_dereference(v) for k, v in d.items()} + elif isinstance(d, ObjRef): + return jsii.runtime.resolve_reference(d) + else: + return d + + +def _dereferenced(fn): + @functools.wraps(fn) + def wrapped(*args, **kwargs): + return _recursize_dereference(fn(*args, **kwargs)) + + return wrapped + + @attr.s(auto_attribs=True, frozen=True, slots=True) class Statistics: @@ -52,14 +74,14 @@ def create(self, klass: Any) -> ObjRef: def delete(self, ref: ObjRef) -> None: self.provider.delete(DeleteRequest(objref=ref)) + @_dereferenced def get(self, ref: ObjRef, property: str) -> Any: return self.provider.get(GetRequest(objref=ref, property_=property)).value def set(self, ref: ObjRef, property: str, value: Any) -> None: - self.provider.set( - SetRequest(objref=ref, property_=property, value=value) - ) + self.provider.set(SetRequest(objref=ref, property_=property, value=value)) + @_dereferenced def sget(self, klass: Any, property: str) -> Any: return self.provider.sget( StaticGetRequest(fqn=klass.__jsii_type__, property_=property) @@ -70,6 +92,7 @@ def sset(self, klass: Any, property: str, value: Any) -> None: StaticSetRequest(fqn=klass.__jsii_type__, property_=property, value=value) ) + @_dereferenced def invoke(self, ref: ObjRef, method: str, args: Optional[List[Any]] = None) -> Any: if args is None: args = [] @@ -78,6 +101,7 @@ def invoke(self, ref: ObjRef, method: str, args: Optional[List[Any]] = None) -> InvokeRequest(objref=ref, method=method, args=args) ).result + @_dereferenced def sinvoke(self, klass: Any, method: str, args: Optional[List[Any]] = None) -> Any: if args is None: args = [] diff --git a/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py b/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py index d291564edd..3e5451a525 100644 --- a/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py +++ b/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py @@ -70,13 +70,21 @@ def unstructurer(value): def _with_reference(data, type_): - return type_(data["$jsii.byref"]) + if not isinstance(data, type_): + return type_(ref=data.ref) + return data def _unstructure_ref(value): return {"$jsii.byref": value.ref} +def ohook(d): + if d.keys() == {"$jsii.byref"}: + return ObjRef(ref=d["$jsii.byref"]) + return d + + class _NodeProcess: def __init__(self): self._serializer = cattr.Converter() @@ -125,7 +133,7 @@ def __del__(self): self.stop() def _next_message(self) -> Mapping[Any, Any]: - return json.loads(self._process.stdout.readline()) + return json.loads(self._process.stdout.readline(), object_hook=ohook) def start(self): self._process = subprocess.Popen( diff --git a/packages/jsii-python-runtime/src/jsii/runtime.py b/packages/jsii-python-runtime/src/jsii/runtime.py index 4d14021045..ee28a8c17c 100644 --- a/packages/jsii-python-runtime/src/jsii/runtime.py +++ b/packages/jsii-python-runtime/src/jsii/runtime.py @@ -4,6 +4,38 @@ import weakref +_types = {} + + +class _ReferenceMap: + def __init__(self, types): + self._refs = weakref.WeakValueDictionary() + self._types = types + + def register(self, ref, inst): + self._refs[ref] = inst + + def resolve(self, ref): + # First we need to check our reference map to see if we have any instance that + # already matches this reference. + try: + return self._refs[ref.ref] + except KeyError: + pass + + # If we got to this point, then we didn't have a referene for this, in that case + # we want to create a new instance, but we need to create it in such a way that + # we don't try to recreate the type inside of the JSII interface. + class_ = _types[ref.ref.split("@", 1)[0]] + return class_.__class__.from_reference(class_, ref) + + +_refs = _ReferenceMap(_types) + + +resolve_reference = _refs.resolve + + class JSIIMeta(type): def __new__(cls, name, bases, attrs, *, jsii_type, jsii_kernel): # TODO: We need to figure out exactly what we're going to do this the kernel @@ -35,16 +67,18 @@ def __new__(cls, name, bases, attrs, *, jsii_type, jsii_kernel): for key, value in obj.__jsii_class_properties__.items(): value.__set_name__(obj, key) + _types[obj.__jsii_type__] = obj + return obj def __call__(cls, *args, **kwargs): - # Create our object at the Python level. - obj = cls.__new__(cls, *args, **kwargs) - # Create our object at the JS level. # TODO: Handle args/kwargs # TODO: Handle Overrides - obj.__jsii_ref__ = cls.__jsii_kernel__.create(cls) + ref = cls.__jsii_kernel__.create(cls) + + # Create our object at the Python level. + obj = cls.__class__.from_reference(cls, ref, *args, **kwargs) # Whenever the object we're creating gets garbage collected, then we want to # delete it from the JS runtime as well. @@ -53,12 +87,20 @@ def __call__(cls, *args, **kwargs): # at the JS level? What mechanics are in place for this if any? weakref.finalize(obj, cls.__jsii_kernel__.delete, obj.__jsii_ref__) - # Instatiate our object at the PYthon level. + # Instatiate our object at the Python level. if isinstance(obj, cls): obj.__init__(*args, **kwargs) return obj + def from_reference(cls, ref, *args, **kwargs): + obj = cls.__new__(cls, *args, **kwargs) + obj.__jsii_ref__ = ref + + _refs.register(obj.__jsii_ref__.ref, obj) + + return obj + def __getattr__(obj, name): if name in obj.__jsii_class_properties__: return obj.__jsii_class_properties__[name].__get__(obj, None) From c4c652ddbccd75f82152022ab1d4be8f1f4786c0 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 20 Sep 2018 01:14:19 -0400 Subject: [PATCH 05/88] Switch to using a single, global Kernel --- packages/jsii-python-runtime/src/jsii/runtime.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/jsii-python-runtime/src/jsii/runtime.py b/packages/jsii-python-runtime/src/jsii/runtime.py index ee28a8c17c..048f5dc643 100644 --- a/packages/jsii-python-runtime/src/jsii/runtime.py +++ b/packages/jsii-python-runtime/src/jsii/runtime.py @@ -3,6 +3,14 @@ import typing import weakref +from jsii.kernel import Kernel + + +# Yea, a global here is kind of gross, however, there's not really a better way of +# handling this. Fundamentally this is a global value, since we can only reasonably +# have a single kernel active at any one time in a real program. +kernel = Kernel() + _types = {} @@ -37,7 +45,7 @@ def resolve(self, ref): class JSIIMeta(type): - def __new__(cls, name, bases, attrs, *, jsii_type, jsii_kernel): + def __new__(cls, name, bases, attrs, *, jsii_type): # TODO: We need to figure out exactly what we're going to do this the kernel # here. Right now we're "creating" a new one per type, and relying on # the fact it's a singleton so that everything is in the same kernel. @@ -46,7 +54,7 @@ def __new__(cls, name, bases, attrs, *, jsii_type, jsii_kernel): # make sense?). Perhaps we should pass it in like we pass the type, and # have the autogenerated code either create it once per library or once # per process. - attrs["__jsii_kernel__"] = jsii_kernel + attrs["__jsii_kernel__"] = kernel attrs["__jsii_type__"] = jsii_type # We need to lift all of the classproperty instances out of the class, because From a162358509a3cc2ad50c1d9dd0246ff47027e9ee Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 20 Sep 2018 01:53:40 -0400 Subject: [PATCH 06/88] Rearrange imports to deal with import cycles --- .../src/jsii/{kernel => _kernel}/__init__.py | 9 ++-- .../src/jsii/_kernel/providers/__init__.py | 5 +++ .../{kernel => _kernel}/providers/base.py | 2 +- .../{kernel => _kernel}/providers/process.py | 4 +- .../src/jsii/{kernel => _kernel}/types.py | 0 .../src/jsii/_reference_map.py | 39 ++++++++++++++++++ .../src/jsii/kernel/providers/__init__.py | 5 --- .../jsii-python-runtime/src/jsii/runtime.py | 41 +++---------------- 8 files changed, 57 insertions(+), 48 deletions(-) rename packages/jsii-python-runtime/src/jsii/{kernel => _kernel}/__init__.py (95%) create mode 100644 packages/jsii-python-runtime/src/jsii/_kernel/providers/__init__.py rename packages/jsii-python-runtime/src/jsii/{kernel => _kernel}/providers/base.py (98%) rename packages/jsii-python-runtime/src/jsii/{kernel => _kernel}/providers/process.py (98%) rename packages/jsii-python-runtime/src/jsii/{kernel => _kernel}/types.py (100%) create mode 100644 packages/jsii-python-runtime/src/jsii/_reference_map.py delete mode 100644 packages/jsii-python-runtime/src/jsii/kernel/providers/__init__.py diff --git a/packages/jsii-python-runtime/src/jsii/kernel/__init__.py b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py similarity index 95% rename from packages/jsii-python-runtime/src/jsii/kernel/__init__.py rename to packages/jsii-python-runtime/src/jsii/_kernel/__init__.py index bf6250e56a..b61868ac66 100644 --- a/packages/jsii-python-runtime/src/jsii/kernel/__init__.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py @@ -5,11 +5,10 @@ import attr -import jsii.runtime - +from jsii import _reference_map from jsii._utils import Singleton -from jsii.kernel.providers import BaseKernel, ProcessKernel -from jsii.kernel.types import ( +from jsii._kernel.providers import BaseKernel, ProcessKernel +from jsii._kernel.types import ( LoadRequest, CreateRequest, DeleteRequest, @@ -28,7 +27,7 @@ def _recursize_dereference(d): if isinstance(d, collections.abc.Mapping): return {k: _recursize_dereference(v) for k, v in d.items()} elif isinstance(d, ObjRef): - return jsii.runtime.resolve_reference(d) + return _reference_map.resolve_reference(d) else: return d diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/providers/__init__.py b/packages/jsii-python-runtime/src/jsii/_kernel/providers/__init__.py new file mode 100644 index 0000000000..26ec99a6a2 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_kernel/providers/__init__.py @@ -0,0 +1,5 @@ +from jsii._kernel.providers.base import BaseKernel +from jsii._kernel.providers.process import ProcessKernel + + +__all__ = ["BaseKernel", "ProcessKernel"] diff --git a/packages/jsii-python-runtime/src/jsii/kernel/providers/base.py b/packages/jsii-python-runtime/src/jsii/_kernel/providers/base.py similarity index 98% rename from packages/jsii-python-runtime/src/jsii/kernel/providers/base.py rename to packages/jsii-python-runtime/src/jsii/_kernel/providers/base.py index 4fd28cdbb3..27699379b4 100644 --- a/packages/jsii-python-runtime/src/jsii/kernel/providers/base.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/providers/base.py @@ -2,7 +2,7 @@ from typing import Optional -from jsii.kernel.types import ( +from jsii._kernel.types import ( LoadRequest, LoadResponse, CreateRequest, diff --git a/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py b/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py similarity index 98% rename from packages/jsii-python-runtime/src/jsii/kernel/providers/process.py rename to packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py index 3e5451a525..7bae4e4144 100644 --- a/packages/jsii-python-runtime/src/jsii/kernel/providers/process.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py @@ -7,8 +7,8 @@ import cattr # type: ignore from jsii._utils import memoized_property -from jsii.kernel.providers.base import BaseKernel -from jsii.kernel.types import ( +from jsii._kernel.providers.base import BaseKernel +from jsii._kernel.types import ( ObjRef, KernelRequest, KernelResponse, diff --git a/packages/jsii-python-runtime/src/jsii/kernel/types.py b/packages/jsii-python-runtime/src/jsii/_kernel/types.py similarity index 100% rename from packages/jsii-python-runtime/src/jsii/kernel/types.py rename to packages/jsii-python-runtime/src/jsii/_kernel/types.py diff --git a/packages/jsii-python-runtime/src/jsii/_reference_map.py b/packages/jsii-python-runtime/src/jsii/_reference_map.py new file mode 100644 index 0000000000..c04d74eeaa --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_reference_map.py @@ -0,0 +1,39 @@ +# This module exists to break an import cycle between jsii.runtime and jsii.kernel +import weakref + + +_types = {} + + +def register_type(type_, obj): + _types[type_] = obj + + +class _ReferenceMap: + def __init__(self, types): + self._refs = weakref.WeakValueDictionary() + self._types = types + + def register(self, ref, inst): + self._refs[ref] = inst + + def resolve(self, ref): + # First we need to check our reference map to see if we have any instance that + # already matches this reference. + try: + return self._refs[ref.ref] + except KeyError: + pass + + # If we got to this point, then we didn't have a referene for this, in that case + # we want to create a new instance, but we need to create it in such a way that + # we don't try to recreate the type inside of the JSII interface. + class_ = _types[ref.ref.split("@", 1)[0]] + return class_.__class__.from_reference(class_, ref) + + +_refs = _ReferenceMap(_types) + + +register_reference = _refs.register +resolve_reference = _refs.resolve diff --git a/packages/jsii-python-runtime/src/jsii/kernel/providers/__init__.py b/packages/jsii-python-runtime/src/jsii/kernel/providers/__init__.py deleted file mode 100644 index ba922882f9..0000000000 --- a/packages/jsii-python-runtime/src/jsii/kernel/providers/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from jsii.kernel.providers.base import BaseKernel -from jsii.kernel.providers.process import ProcessKernel - - -__all__ = ["BaseKernel", "ProcessKernel"] diff --git a/packages/jsii-python-runtime/src/jsii/runtime.py b/packages/jsii-python-runtime/src/jsii/runtime.py index 048f5dc643..5b048e5277 100644 --- a/packages/jsii-python-runtime/src/jsii/runtime.py +++ b/packages/jsii-python-runtime/src/jsii/runtime.py @@ -2,8 +2,11 @@ import typing import weakref +import os -from jsii.kernel import Kernel +from jsii import _reference_map +from jsii._compat import importlib_resources +from jsii._kernel import Kernel # Yea, a global here is kind of gross, however, there's not really a better way of @@ -12,38 +15,6 @@ kernel = Kernel() -_types = {} - - -class _ReferenceMap: - def __init__(self, types): - self._refs = weakref.WeakValueDictionary() - self._types = types - - def register(self, ref, inst): - self._refs[ref] = inst - - def resolve(self, ref): - # First we need to check our reference map to see if we have any instance that - # already matches this reference. - try: - return self._refs[ref.ref] - except KeyError: - pass - - # If we got to this point, then we didn't have a referene for this, in that case - # we want to create a new instance, but we need to create it in such a way that - # we don't try to recreate the type inside of the JSII interface. - class_ = _types[ref.ref.split("@", 1)[0]] - return class_.__class__.from_reference(class_, ref) - - -_refs = _ReferenceMap(_types) - - -resolve_reference = _refs.resolve - - class JSIIMeta(type): def __new__(cls, name, bases, attrs, *, jsii_type): # TODO: We need to figure out exactly what we're going to do this the kernel @@ -75,7 +46,7 @@ def __new__(cls, name, bases, attrs, *, jsii_type): for key, value in obj.__jsii_class_properties__.items(): value.__set_name__(obj, key) - _types[obj.__jsii_type__] = obj + _reference_map.register_type(obj.__jsii_type__, obj) return obj @@ -105,7 +76,7 @@ def from_reference(cls, ref, *args, **kwargs): obj = cls.__new__(cls, *args, **kwargs) obj.__jsii_ref__ = ref - _refs.register(obj.__jsii_ref__.ref, obj) + _reference_map.register_reference(obj.__jsii_ref__.ref, obj) return obj From 59c7876df6688ebb643caa7865fcfe62e1944ff8 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 20 Sep 2018 01:54:38 -0400 Subject: [PATCH 07/88] Provide a mechanism to load JSII Assemblies --- .../jsii-python-runtime/src/jsii/_compat.py | 7 +++++ .../jsii-python-runtime/src/jsii/runtime.py | 30 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 packages/jsii-python-runtime/src/jsii/_compat.py diff --git a/packages/jsii-python-runtime/src/jsii/_compat.py b/packages/jsii-python-runtime/src/jsii/_compat.py new file mode 100644 index 0000000000..0d2cfa5e96 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_compat.py @@ -0,0 +1,7 @@ +try: + import importlib.resources as importlib_resources +except ImportError: + import importlib_resources + + +__all__ = ["importlib_resources"] diff --git a/packages/jsii-python-runtime/src/jsii/runtime.py b/packages/jsii-python-runtime/src/jsii/runtime.py index 5b048e5277..f31e7e2a45 100644 --- a/packages/jsii-python-runtime/src/jsii/runtime.py +++ b/packages/jsii-python-runtime/src/jsii/runtime.py @@ -4,6 +4,8 @@ import weakref import os +import attr + from jsii import _reference_map from jsii._compat import importlib_resources from jsii._kernel import Kernel @@ -15,6 +17,34 @@ kernel = Kernel() +@attr.s(auto_attribs=True, frozen=True, slots=True) +class JSIIAssembly: + + name: str + version: str + module: str + filename: str + + @classmethod + def load(cls, *args, _kernel=kernel, **kwargs): + # Our object here really just acts as a record for our JSIIAssembly, it doesn't + # offer any functionality itself, besides this class method that will trigger + # the loading of the given assembly in the JSII Kernel. + assembly = cls(*args, **kwargs) + + # Actually load the assembly into the kernel, we're using the + # importlib.resources API here isntead of manually constructing the path, in + # the hopes that this will make JSII modules able to be used with zipimport + # instead of only on the FS. + with importlib_resources.path( + f"{assembly.module}._jsii", assembly.filename + ) as assembly_path: + _kernel.load(assembly.name, assembly.version, os.fspath(assembly_path)) + + # Give our record of the assembly back to the caller. + return assembly + + class JSIIMeta(type): def __new__(cls, name, bases, attrs, *, jsii_type): # TODO: We need to figure out exactly what we're going to do this the kernel From ad57330e8e3b303ac687be81d1a8a5a2ff3afffb Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 20 Sep 2018 01:58:23 -0400 Subject: [PATCH 08/88] Add the WIP Python target for pacmak --- packages/jsii-pacmak/lib/targets/python.ts | 344 +++++++++++++++++++++ 1 file changed, 344 insertions(+) create mode 100644 packages/jsii-pacmak/lib/targets/python.ts diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts new file mode 100644 index 0000000000..199ab0dab5 --- /dev/null +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -0,0 +1,344 @@ +/* tslint:disable */ +import path = require('path'); +import util = require('util'); + +import * as spec from 'jsii-spec'; +import { Generator, GeneratorOptions } from '../generator'; +import { Target, TargetOptions } from '../target'; +import { CodeMaker } from 'codemaker'; +import { shell } from '../util'; + +export default class Python extends Target { + protected readonly generator = new PythonGenerator(); + + constructor(options: TargetOptions) { + super(options); + } + + public async build(sourceDir: string, outDir: string): Promise { + // Format our code to make it easier to read, we do this here instead of trying + // to do it in the code generation phase, because attempting to mix style and + // function makes the code generation harder to maintain and read, while doing + // this here is easy. + await shell("black", ["--py36", sourceDir], {}); + + return this.copyFiles(sourceDir, outDir); + } +} + +// ################## +// # CODE GENERATOR # +// ################## +const debug = function(o: any) { + console.log(util.inspect(o, false, null, true)); +} + + +class Module { + + readonly name: string; + readonly assembly?: spec.Assembly; + readonly assemblyFilename?: string; + + private buffer: object[]; + + constructor(ns: string, assembly?: [spec.Assembly, string]) { + this.name = ns; + + if (assembly != undefined) { + this.assembly = assembly[0]; + this.assemblyFilename = assembly[1]; + } + + this.buffer = []; + } + + // We're purposely replicating the API of CodeMaker here, because CodeMaker cannot + // Operate on more than one file at a time, so we have to buffer our calls to + // CodeMaker, otherwise we can end up in inconsistent state when we get things like: + // - onBeginNamespace(foo) + // - onBeginNamespace(foo.bar) + // - OnEndNamespace(foo.bar) + // - Inconsitent State, where we're now back in the scope of foo, but due to + // the fact that if we had opened a file in onBeginNamespace(foo), we would + // have had to close it for onBeginNamespace(foo.bar), and re-opening it + // would overwrite the contents. + // - OnEndNamespace(foo) + // To solve this, we buffer all of the things we *would* have written out via + // CodeMaker via this API, and then we will just iterate over it in the + // onEndNamespace event and write it out then. + + public line(...args: any[]) { + this.buffer.push({method: "line", args: args}); + } + + public indent(...args: any[]) { + this.buffer.push({method: "indent", args: args}); + } + + public unindent(...args: any[]) { + this.buffer.push({method: "unindent", args: args}); + } + + public open(...args: any[]) { + this.buffer.push({method: "open", args: args}); + } + + public close(...args: any[]) { + this.buffer.push({method: "close", args: args}); + } + + public openBlock(...args: any[]) { + this.buffer.push({method: "openBlock", args: args}); + } + + public closeBlock(...args: any[]) { + this.buffer.push({method: "closeBlock", args: args}); + } + + public write(code: CodeMaker) { + // Before we do Anything, we need to write out our module headers, this is where + // we handle stuff like imports, any required initialization, etc. + code.line("from jsii.runtime import JSIIAssembly, JSIIMeta, jsii_method, jsii_property, jsii_classmethod, jsii_classproperty") + + // Determine if we need to write out the kernel load line. + if (this.assembly && this.assemblyFilename) { + code.line(`__jsii_assembly__ = JSIIAssembly.load("${this.assembly.name}", "${this.assembly.version}", __name__, "${this.assemblyFilename}")`); + } + + // Now that we've gotten all of the module header stuff done, we need to go + // through and actually write out the meat of our module. + for (let buffered of this.buffer) { + let methodName = (buffered as any)["method"] as string; + let args = (buffered as any)["args"] as any[]; + + (code as any)[methodName](...args); + } + } +} + +class PythonGenerator extends Generator { + + private moduleStack: Module[]; + + constructor(options = new GeneratorOptions()) { + super(options); + + this.code.openBlockFormatter = s => `${s}:`; + this.code.closeBlockFormatter = _s => ""; + + this.moduleStack = []; + } + + protected getAssemblyOutputDir(mod: spec.Assembly) { + return path.join("src", this.toPythonModuleName(mod.name), "_jsii"); + } + + protected onBeginAssembly(assm: spec.Assembly, _fingerprint: boolean) { + debug("onBeginAssembly"); + + // We need to write out an __init__.py for our _jsii package so that + // importlib.resources will be able to load our assembly from it. + const assemblyInitFilename = path.join(this.getAssemblyOutputDir(assm), "__init__.py"); + + this.code.openFile(assemblyInitFilename); + this.code.closeFile(assemblyInitFilename); + } + + protected onBeginNamespace(ns: string) { + debug(`onBeginNamespace: ${ns}`); + + const moduleName = this.toPythonModuleName(ns); + const loadAssembly = this.assembly.name == ns ? true : false; + + let moduleArgs: any[] = []; + + if (loadAssembly) { + moduleArgs.push([this.assembly, this.getAssemblyFileName()]); + } + + this.moduleStack.push(new Module(moduleName, ...moduleArgs)); + } + + protected onEndNamespace(ns: string) { + debug(`onEndNamespace: ${ns}`); + + let module = this.moduleStack.pop() as Module; + let moduleFilename = path.join("src", this.toPythonModuleFilename(module.name), "__init__.py"); + + this.code.openFile(moduleFilename); + module.write(this.code); + this.code.closeFile(moduleFilename); + } + + protected onBeginClass(cls: spec.ClassType, abstract: boolean | undefined) { + debug("onBeginClass"); + + // TODO: Figure out what to do with abstract here. + abstract; + + this.currentModule().openBlock(`class ${cls.name}(metaclass=JSIIMeta, jsii_type="${cls.fqn}")`); + } + + protected onEndClass(_cls: spec.ClassType) { + debug("onEndClass"); + + this.currentModule().closeBlock(); + } + + protected onStaticMethod(_cls: spec.ClassType, method: spec.Method) { + debug("onStaticMethod"); + + // TODO: Handle the case where the Python name and the JSII name differ. + this.currentModule().line("@jsii_classmethod"); + this.emitPythonMethod(method.name, "cls", method.parameters, method.returns); + } + + protected onMethod(_cls: spec.ClassType, method: spec.Method) { + debug("onMethod"); + + this.currentModule().line("@jsii_method"); + this.emitPythonMethod(method.name, "self", method.parameters, method.returns); + } + + protected onStaticProperty(_cls: spec.ClassType, prop: spec.Property) { + debug("onStaticProperty"); + + // TODO: Properties have a bunch of states, they can have getters and setters + // we need to better handle all of these cases. + this.currentModule().line("@jsii_classproperty"); + this.emitPythonMethod(prop.name, "self", [], prop.type); + } + + protected onProperty(_cls: spec.ClassType, prop: spec.Property) { + debug("onProperty"); + + this.currentModule().line("@jsii_property"); + this.emitPythonMethod(prop.name, "self", [], prop.type); + } + + private emitPythonMethod(name?: string, implicitParam?: string, params: spec.Parameter[] = [], returns?: spec.TypeReference) { + // TODO: Handle imports (if needed) for type. + const returnType = returns ? this.toPythonType(returns) : "None"; + + // We need to turn a list of JSII parameters, into Python style arguments with + // gradual typing, so we'll have to iterate over the list of parameters, and + // build the list, converting as we go. + // TODO: Handle imports (if needed) for all of these types. + + let pythonParams: string[] = implicitParam ? [implicitParam] : []; + for (let param of params) { + pythonParams.push(`${param.name}: ${this.toPythonType(param.type)}`); + } + + let module = this.currentModule(); + + module.openBlock(`def ${name}(${pythonParams.join(", ")}) -> ${returnType}`); + module.line("..."); + module.closeBlock(); + } + + private toPythonType(typeref: spec.TypeReference): string { + if (spec.isPrimitiveTypeReference(typeref)) { + return this.toPythonPrimitive(typeref.primitive); + } else if (spec.isNamedTypeReference(typeref)) { + // TODO: We need to actually handle this, isntead of just returning the FQN + // as a string. + return `"${typeref.fqn}"`; + } else { + throw new Error("Invalid type reference: " + JSON.stringify(typeref)); + } + + /* + if (spec.isPrimitiveTypeReference(typeref)) { + return [ this.toJavaPrimitive(typeref.primitive) ]; + } else if (spec.isCollectionTypeReference(typeref)) { + return [ this.toJavaCollection(typeref, forMarshalling) ]; + } else if (spec.isNamedTypeReference(typeref)) { + return [ this.toNativeFqn(typeref.fqn) ]; + } else if (typeref.union) { + const types = new Array(); + for (const subtype of typeref.union.types) { + for (const t of this.toJavaTypes(subtype, forMarshalling)) { + types.push(t); + } + } + return types; + } else { + throw new Error('Invalid type reference: ' + JSON.stringify(typeref)); + } + */ + + /* + switch (primitive) { + case spec.PrimitiveType.Boolean: return 'java.lang.Boolean'; + case spec.PrimitiveType.Date: return 'java.time.Instant'; + case spec.PrimitiveType.Json: return 'com.fasterxml.jackson.databind.node.ObjectNode'; + case spec.PrimitiveType.Number: return 'java.lang.Number'; + case spec.PrimitiveType.String: return 'java.lang.String'; + case spec.PrimitiveType.Any: return 'java.lang.Object'; + default: + throw new Error('Unknown primitive type: ' + primitive); + } + */ + } + + private toPythonPrimitive(primitive: spec.PrimitiveType): string { + switch (primitive) { + case spec.PrimitiveType.String: + return "str"; + default: + throw new Error("Unknown primitive type: " + primitive); + } + } + + private currentModule(): Module { + return this.moduleStack.slice(-1)[0]; + } + + private toPythonModuleName(name: string): string { + return this.code.toSnakeCase(name.replace(/-/g, "_")); + } + + private toPythonModuleFilename(name: string): string { + return name.replace(/\./g, "/"); + } + + // Not Currently Used + + protected onEndAssembly(_assm: spec.Assembly, _fingerprint: boolean) { + debug("onEndAssembly"); + } + + protected onBeginInterface(_ifc: spec.InterfaceType) { + debug("onBeginInterface"); + } + + protected onEndInterface(_ifc: spec.InterfaceType) { + debug("onEndInterface"); + } + + protected onInterfaceMethod(_ifc: spec.InterfaceType, _method: spec.Method) { + debug("onInterfaceMethod"); + } + + protected onInterfaceMethodOverload(_ifc: spec.InterfaceType, _overload: spec.Method, _originalMethod: spec.Method) { + debug("onInterfaceMethodOverload"); + } + + protected onInterfaceProperty(_ifc: spec.InterfaceType, _prop: spec.Property) { + debug("onInterfaceProperty"); + } + + protected onUnionProperty(_cls: spec.ClassType, _prop: spec.Property, _union: spec.UnionTypeReference) { + debug("onUnionProperty"); + } + + protected onMethodOverload(_cls: spec.ClassType, _overload: spec.Method, _originalMethod: spec.Method) { + debug("onMethodOverload"); + } + + protected onStaticMethodOverload(_cls: spec.ClassType, _overload: spec.Method, _originalMethod: spec.Method) { + debug("onStaticMethodOverload"); + } +} From 0e4225fe3e6ef546090ff9e4acd90fe4281c9d2c Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 21 Sep 2018 12:00:30 -0400 Subject: [PATCH 09/88] Prefix all of our interal names with _ --- packages/jsii-pacmak/lib/targets/python.ts | 35 +++++++++++++++++----- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 199ab0dab5..8a95e0fa88 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -99,11 +99,23 @@ class Module { public write(code: CodeMaker) { // Before we do Anything, we need to write out our module headers, this is where // we handle stuff like imports, any required initialization, etc. - code.line("from jsii.runtime import JSIIAssembly, JSIIMeta, jsii_method, jsii_property, jsii_classmethod, jsii_classproperty") + code.line( + this.generateImportFrom( + "jsii.runtime", + [ + "JSIIAssembly", + "JSIIMeta", + "jsii_method", + "jsii_property", + "jsii_classmethod", + "jsii_classproperty", + ] + ) + ) // Determine if we need to write out the kernel load line. if (this.assembly && this.assemblyFilename) { - code.line(`__jsii_assembly__ = JSIIAssembly.load("${this.assembly.name}", "${this.assembly.version}", __name__, "${this.assemblyFilename}")`); + code.line(`__jsii_assembly__ = _runtime.JSIIAssembly.load("${this.assembly.name}", "${this.assembly.version}", __name__, "${this.assemblyFilename}")`); } // Now that we've gotten all of the module header stuff done, we need to go @@ -115,6 +127,15 @@ class Module { (code as any)[methodName](...args); } } + + private generateImportFrom(from: string, names: string[]): string { + // Whenever we import something, we want to prefix all of the names we're + // importing with an underscore to indicate that these names are private. We + // do this, because otherwise we could get clashes in the names we use, and the + // names of exported classes. + const importNames = names.map(n => `${n} as _${n}`); + return `from ${from} import ${importNames.join(", ")}`; + } } class PythonGenerator extends Generator { @@ -177,7 +198,7 @@ class PythonGenerator extends Generator { // TODO: Figure out what to do with abstract here. abstract; - this.currentModule().openBlock(`class ${cls.name}(metaclass=JSIIMeta, jsii_type="${cls.fqn}")`); + this.currentModule().openBlock(`class ${cls.name}(metaclass=_JSIIMeta, jsii_type="${cls.fqn}")`); } protected onEndClass(_cls: spec.ClassType) { @@ -190,14 +211,14 @@ class PythonGenerator extends Generator { debug("onStaticMethod"); // TODO: Handle the case where the Python name and the JSII name differ. - this.currentModule().line("@jsii_classmethod"); + this.currentModule().line("@_jsii_classmethod"); this.emitPythonMethod(method.name, "cls", method.parameters, method.returns); } protected onMethod(_cls: spec.ClassType, method: spec.Method) { debug("onMethod"); - this.currentModule().line("@jsii_method"); + this.currentModule().line("@_jsii_method"); this.emitPythonMethod(method.name, "self", method.parameters, method.returns); } @@ -206,14 +227,14 @@ class PythonGenerator extends Generator { // TODO: Properties have a bunch of states, they can have getters and setters // we need to better handle all of these cases. - this.currentModule().line("@jsii_classproperty"); + this.currentModule().line("@_jsii_classproperty"); this.emitPythonMethod(prop.name, "self", [], prop.type); } protected onProperty(_cls: spec.ClassType, prop: spec.Property) { debug("onProperty"); - this.currentModule().line("@jsii_property"); + this.currentModule().line("@_jsii_property"); this.emitPythonMethod(prop.name, "self", [], prop.type); } From 89e4422585d62fe3f592e4ccbcd0c85b6cf2dc0a Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 21 Sep 2018 12:11:32 -0400 Subject: [PATCH 10/88] Write out an __all__ that lists our exported names --- packages/jsii-pacmak/lib/targets/python.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 8a95e0fa88..c1e684c867 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -41,6 +41,7 @@ class Module { readonly assemblyFilename?: string; private buffer: object[]; + private exportedNames: string[]; constructor(ns: string, assembly?: [spec.Assembly, string]) { this.name = ns; @@ -51,6 +52,13 @@ class Module { } this.buffer = []; + this.exportedNames = []; + } + + // Adds a name to the list of names that this module will export, this will control + // things like what is listed inside of __all__. + public exportName(name: string) { + this.exportedNames.push(name); } // We're purposely replicating the API of CodeMaker here, because CodeMaker cannot @@ -115,6 +123,7 @@ class Module { // Determine if we need to write out the kernel load line. if (this.assembly && this.assemblyFilename) { + this.exportName("__jsii_assembly__"); code.line(`__jsii_assembly__ = _runtime.JSIIAssembly.load("${this.assembly.name}", "${this.assembly.version}", __name__, "${this.assemblyFilename}")`); } @@ -126,6 +135,10 @@ class Module { (code as any)[methodName](...args); } + + // Whatever names we've exported, we'll write out our __all__ that lists them. + const stringifiedExportedNames = this.exportedNames.sort().map(s => `"${s}"`); + code.line(`__all__ = [${stringifiedExportedNames.join(", ")}]`); } private generateImportFrom(from: string, names: string[]): string { @@ -198,6 +211,7 @@ class PythonGenerator extends Generator { // TODO: Figure out what to do with abstract here. abstract; + this.currentModule().exportName(cls.name); this.currentModule().openBlock(`class ${cls.name}(metaclass=_JSIIMeta, jsii_type="${cls.fqn}")`); } From 15546d74f4aa30b55cbfca616feef9bd4814eb32 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 21 Sep 2018 13:18:30 -0400 Subject: [PATCH 11/88] Add support for more types --- packages/jsii-pacmak/lib/targets/python.ts | 100 +++++++++++++++++++-- 1 file changed, 93 insertions(+), 7 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index c1e684c867..9ec60c40d2 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -34,6 +34,9 @@ const debug = function(o: any) { } +const PYTHON_BUILTIN_TYPES = ["bool", "str", "None"] + + class Module { readonly name: string; @@ -42,6 +45,7 @@ class Module { private buffer: object[]; private exportedNames: string[]; + private importedModules: string[]; constructor(ns: string, assembly?: [spec.Assembly, string]) { this.name = ns; @@ -53,6 +57,7 @@ class Module { this.buffer = []; this.exportedNames = []; + this.importedModules = []; } // Adds a name to the list of names that this module will export, this will control @@ -61,6 +66,38 @@ class Module { this.exportedNames.push(name); } + // Adds a name to the list of modules that should be imported at the top of this + // file. + public importModule(name: string) { + this.importedModules.push(name); + } + + public maybeImportType(type: string) { + if (PYTHON_BUILTIN_TYPES.indexOf(type) > -1) { + // For built in types, we don't need to do anything, and can just return now + return; + } + + const [, typeModule] = type.match(/(.*)\.(.*)/) as any[]; + + // Given a name like foo.bar.Frob, we want to import the module that Frob exists + // in. Given that all classes exported by JSII have to be pascal cased, and all + // of our imports are snake cases, this should be safe. We're going to double + // check this though, to ensure that our importable here is safe. + if (typeModule != typeModule.toLowerCase()) { + // If we ever get to this point, we'll need to implment aliasing for our + // imports. + throw new Error("Type module is not lower case.") + } + + // We only want to actually import the type for this module, if it isn't the + // module that we're currently in, otherwise we'll jus rely on the module scope + // to make the name available to us. + if (typeModule != this.name) { + this.importModule(typeModule); + } + } + // We're purposely replicating the API of CodeMaker here, because CodeMaker cannot // Operate on more than one file at a time, so we have to buffer our calls to // CodeMaker, otherwise we can end up in inconsistent state when we get things like: @@ -121,6 +158,15 @@ class Module { ) ) + // Go over all of the modules that we need to import, and import them. + for (let [idx, modName] of this.importedModules.sort().entries()) { + if (idx == 0) { + code.line(); + } + + code.line(`import ${modName}`); + } + // Determine if we need to write out the kernel load line. if (this.assembly && this.assemblyFilename) { this.exportName("__jsii_assembly__"); @@ -253,8 +299,12 @@ class PythonGenerator extends Generator { } private emitPythonMethod(name?: string, implicitParam?: string, params: spec.Parameter[] = [], returns?: spec.TypeReference) { + let module = this.currentModule(); + // TODO: Handle imports (if needed) for type. const returnType = returns ? this.toPythonType(returns) : "None"; + module.maybeImportType(returnType); + // We need to turn a list of JSII parameters, into Python style arguments with // gradual typing, so we'll have to iterate over the list of parameters, and @@ -263,12 +313,12 @@ class PythonGenerator extends Generator { let pythonParams: string[] = implicitParam ? [implicitParam] : []; for (let param of params) { - pythonParams.push(`${param.name}: ${this.toPythonType(param.type)}`); + let paramType = this.toPythonType(param.type); + module.maybeImportType(paramType); + pythonParams.push(`${param.name}: ${this.formatPythonType(paramType)}`); } - let module = this.currentModule(); - - module.openBlock(`def ${name}(${pythonParams.join(", ")}) -> ${returnType}`); + module.openBlock(`def ${name}(${pythonParams.join(", ")}) -> ${this.formatPythonType(returnType)}`); module.line("..."); module.closeBlock(); } @@ -279,7 +329,7 @@ class PythonGenerator extends Generator { } else if (spec.isNamedTypeReference(typeref)) { // TODO: We need to actually handle this, isntead of just returning the FQN // as a string. - return `"${typeref.fqn}"`; + return this.toPythonFQN(typeref.fqn); } else { throw new Error("Invalid type reference: " + JSON.stringify(typeref)); } @@ -320,13 +370,49 @@ class PythonGenerator extends Generator { private toPythonPrimitive(primitive: spec.PrimitiveType): string { switch (primitive) { - case spec.PrimitiveType.String: - return "str"; + case spec.PrimitiveType.Boolean: return "bool"; + case spec.PrimitiveType.Date: return "dateetime.datetime"; + case spec.PrimitiveType.Json: return "typing.Mapping[typing.Any, typing.Any]"; + case spec.PrimitiveType.Number: return "numbers.Number"; + case spec.PrimitiveType.String: return "str"; + case spec.PrimitiveType.Any: return "typing.Any"; default: throw new Error("Unknown primitive type: " + primitive); } } + private toPythonFQN(name: string): string { + return name.split(".").map((cur, idx, arr) => { + if (idx == arr.length - 1) { + return cur; + } else { + return this.toPythonModuleName(cur); + } + }).join("."); + } + + private formatPythonType(type: string) { + // Built in types do not need formatted in any particular way. + if(PYTHON_BUILTIN_TYPES.indexOf(type) > -1) { + return type; + } + + const [, typeModule, typeName] = type.match(/(.*)\.(.*)/) as any[]; + + // Types whose module is different than our current module, can also just be + // formatted as they are. + if (this.currentModule().name != typeModule) { + return type; + } + + // Otherwise, this is a type that exists in this module, and we can jsut emit + // the name. + // TODO: We currently emit this as a string, because that's how forward + // references used to work prior to 3.7. Ideally we will move to using 3.7 + // and can just use native forward references. + return `"${typeName}"`; + } + private currentModule(): Module { return this.moduleStack.slice(-1)[0]; } From 6b1cbd61fe7d69dab7871fe235f4a1020e387557 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 21 Sep 2018 13:37:18 -0400 Subject: [PATCH 12/88] Fix some MyPy errors --- packages/jsii-python-runtime/src/jsii/_compat.py | 7 +++++-- packages/jsii-python-runtime/src/jsii/_kernel/__init__.py | 2 +- packages/jsii-python-runtime/src/jsii/_kernel/types.py | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/jsii-python-runtime/src/jsii/_compat.py b/packages/jsii-python-runtime/src/jsii/_compat.py index 0d2cfa5e96..5a50288ef0 100644 --- a/packages/jsii-python-runtime/src/jsii/_compat.py +++ b/packages/jsii-python-runtime/src/jsii/_compat.py @@ -1,6 +1,9 @@ -try: +import sys + + +if sys.version_info >= (3, 7): import importlib.resources as importlib_resources -except ImportError: +else: import importlib_resources diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py index b61868ac66..4bf1270923 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py @@ -87,7 +87,7 @@ def sget(self, klass: Any, property: str) -> Any: ).value def sset(self, klass: Any, property: str, value: Any) -> None: - return self.provider.sset( + self.provider.sset( StaticSetRequest(fqn=klass.__jsii_type__, property_=property, value=value) ) diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/types.py b/packages/jsii-python-runtime/src/jsii/_kernel/types.py index e58f2a84a2..095153ac8c 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/types.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/types.py @@ -42,8 +42,8 @@ class LoadResponse: class CreateRequest: fqn: str - args: Optional[List[Any]] = attr.Factory(list) - overrides: Optional[List[Override]] = attr.Factory(list) + args: List[Any] = attr.Factory(list) + overrides: List[Override] = attr.Factory(list) @attr.s(auto_attribs=True, frozen=True, slots=True) From 882e5c70b32d0c6da478305a4085d8822b41bb39 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 21 Sep 2018 15:11:21 -0400 Subject: [PATCH 13/88] Generate actual Python packages --- packages/jsii-pacmak/lib/targets/python.ts | 48 +++++++++++++++++++--- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 9ec60c40d2..be1cdab211 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -22,7 +22,9 @@ export default class Python extends Target { // this here is easy. await shell("black", ["--py36", sourceDir], {}); - return this.copyFiles(sourceDir, outDir); + // Actually package up our code, both as a sdist and a wheel for publishing. + await shell("python", ["setup.py", "sdist", "--dist-dir", outDir], { cwd: sourceDir }); + await shell("python", ["setup.py", "bdist_wheel", "--dist-dir", outDir], { cwd: sourceDir }); } } @@ -225,6 +227,46 @@ class PythonGenerator extends Generator { this.code.closeFile(assemblyInitFilename); } + protected onEndAssembly(assm: spec.Assembly, _fingerprint: boolean) { + debug("onEndAssembly"); + debug(assm); + + // We need to write out our packaging for the Python ecosystem here. + // TODO: + // - Author + // - README + // - License + // - Classifiers + // - install_requires + this.code.openFile("setup.py"); + this.code.line("import setuptools"); + this.code.indent("setuptools.setup("); + this.code.line(`name="${assm.name}",`); + this.code.line(`version="${assm.version}",`); + this.code.line(`description="${assm.description}",`); + this.code.line(`url="${assm.homepage}",`); + this.code.line('package_dir={"": "src"},'); + this.code.line('packages=setuptools.find_packages(where="src"),'); + this.code.line(`package_data={"${this.toPythonModuleName(assm.name)}._jsii": ["*.jsii.tgz"]},`); + this.code.line('python_requires=">=3.6",'); + this.code.unindent(")"); + this.code.closeFile("setup.py"); + + // Because we're good citizens, we're going to go ahead and support pyproject.toml + // as well. + // TODO: Might be easier to just use a TOML library to write this out. + this.code.openFile("pyproject.toml"); + this.code.line("[build-system]"); + this.code.line('requires = ["setuptools", "wheel"]'); + this.code.closeFile("pyproject.toml"); + + // We also need to write out a MANIFEST.in to ensure that all of our required + // files are included. + this.code.openFile("MANIFEST.in") + this.code.line("include pyproject.toml") + this.code.closeFile("MANIFEST.in") + } + protected onBeginNamespace(ns: string) { debug(`onBeginNamespace: ${ns}`); @@ -427,10 +469,6 @@ class PythonGenerator extends Generator { // Not Currently Used - protected onEndAssembly(_assm: spec.Assembly, _fingerprint: boolean) { - debug("onEndAssembly"); - } - protected onBeginInterface(_ifc: spec.InterfaceType) { debug("onBeginInterface"); } From 0e34a80842a2eae8a10c10f4f997c771a16c366b Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 21 Sep 2018 15:11:36 -0400 Subject: [PATCH 14/88] Correct the name of the import JSIIAssembly --- packages/jsii-pacmak/lib/targets/python.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index be1cdab211..61d4bd2634 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -172,7 +172,7 @@ class Module { // Determine if we need to write out the kernel load line. if (this.assembly && this.assemblyFilename) { this.exportName("__jsii_assembly__"); - code.line(`__jsii_assembly__ = _runtime.JSIIAssembly.load("${this.assembly.name}", "${this.assembly.version}", __name__, "${this.assemblyFilename}")`); + code.line(`__jsii_assembly__ = _JSIIAssembly.load("${this.assembly.name}", "${this.assembly.version}", __name__, "${this.assemblyFilename}")`); } // Now that we've gotten all of the module header stuff done, we need to go From 5ea945de40b6110fa02f275a3442ea5bb0f24d11 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 21 Sep 2018 19:01:19 -0400 Subject: [PATCH 15/88] Produce packages for the JSII Python Runtime --- .gitignore | 7 +++++++ packages/jsii-build-tools/bin/package-python | 7 +++++++ packages/jsii-build-tools/package.json | 3 ++- packages/jsii-python-runtime/.gitignore | 1 + packages/jsii-python-runtime/MANIFEST.in | 2 ++ packages/jsii-python-runtime/bin/generate | 9 +++++++++ packages/jsii-python-runtime/package.json | 5 ++++- packages/jsii-python-runtime/pyproject.toml | 2 ++ packages/jsii-python-runtime/setup.py | 21 ++++++++++++++++++++ 9 files changed, 55 insertions(+), 2 deletions(-) create mode 100755 packages/jsii-build-tools/bin/package-python create mode 100644 packages/jsii-python-runtime/.gitignore create mode 100644 packages/jsii-python-runtime/MANIFEST.in create mode 100755 packages/jsii-python-runtime/bin/generate create mode 100644 packages/jsii-python-runtime/pyproject.toml create mode 100644 packages/jsii-python-runtime/setup.py diff --git a/.gitignore b/.gitignore index b7615a96eb..6042ea8813 100644 --- a/.gitignore +++ b/.gitignore @@ -368,3 +368,10 @@ jspm_packages/ # that NuGet uses for external packages. We don't want to ignore the # lerna packages. !/packages/* + + +# Python packages generate this while building them. +*.egg-info + +# MyPy cache +.mypy_cache/ diff --git a/packages/jsii-build-tools/bin/package-python b/packages/jsii-build-tools/bin/package-python new file mode 100755 index 0000000000..8ea2d10df6 --- /dev/null +++ b/packages/jsii-build-tools/bin/package-python @@ -0,0 +1,7 @@ +#!/bin/bash +set -euo pipefail + +rm -rf dist/python +mkdir -p dist/python +mv *.whl dist/python +mv *.tar.gz dist/python diff --git a/packages/jsii-build-tools/package.json b/packages/jsii-build-tools/package.json index a50346cb96..7085182067 100644 --- a/packages/jsii-build-tools/package.json +++ b/packages/jsii-build-tools/package.json @@ -8,7 +8,8 @@ "package-js": "bin/package-js", "package-java": "bin/package-java", "package-dotnet": "bin/package-dotnet", - "package-ruby": "bin/package-ruby" + "package-ruby": "bin/package-ruby", + "package-python": "bin/package-python" }, "scripts": { "build": "chmod +x bin/*" diff --git a/packages/jsii-python-runtime/.gitignore b/packages/jsii-python-runtime/.gitignore new file mode 100644 index 0000000000..21719fccd5 --- /dev/null +++ b/packages/jsii-python-runtime/.gitignore @@ -0,0 +1 @@ +src/jsii/_metadata.json diff --git a/packages/jsii-python-runtime/MANIFEST.in b/packages/jsii-python-runtime/MANIFEST.in new file mode 100644 index 0000000000..8ab5288007 --- /dev/null +++ b/packages/jsii-python-runtime/MANIFEST.in @@ -0,0 +1,2 @@ +include pyproject.toml +include src/jsii/_metadata.json diff --git a/packages/jsii-python-runtime/bin/generate b/packages/jsii-python-runtime/bin/generate new file mode 100755 index 0000000000..07e029f998 --- /dev/null +++ b/packages/jsii-python-runtime/bin/generate @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +import json + +with open("package.json") as fp: + data = json.load(fp) + + +with open("src/jsii/_metadata.json", "w") as fp: + json.dump({"version": data["version"]}, fp) diff --git a/packages/jsii-python-runtime/package.json b/packages/jsii-python-runtime/package.json index 69f6b2eaa9..d99b992e6f 100644 --- a/packages/jsii-python-runtime/package.json +++ b/packages/jsii-python-runtime/package.json @@ -4,11 +4,14 @@ "description": "Python client for jsii runtime", "main": "index.js", "scripts": { - "build": "echo ok", + "generate": "bin/generate", + "build": "npm run generate && python setup.py sdist -d . bdist_wheel -d . && rm -rf build", + "package": "package-python", "prepack": "echo ok", "test": "echo ok" }, "dependencies": { + "jsii-build-tools": "^0.7.4", "jsii-runtime": "^0.7.1" }, "repository": { diff --git a/packages/jsii-python-runtime/pyproject.toml b/packages/jsii-python-runtime/pyproject.toml new file mode 100644 index 0000000000..d1e6ae6e56 --- /dev/null +++ b/packages/jsii-python-runtime/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools", "wheel"] diff --git a/packages/jsii-python-runtime/setup.py b/packages/jsii-python-runtime/setup.py new file mode 100644 index 0000000000..96eb147875 --- /dev/null +++ b/packages/jsii-python-runtime/setup.py @@ -0,0 +1,21 @@ +import json +import setuptools + + +with open("src/jsii/_metadata.json") as fp: + metadata = json.load(fp) + + +setuptools.setup( + name="jsii", + version=metadata["version"], + package_dir={"": "src"}, + packages=setuptools.find_packages(where="src"), + package_data={"jsii": ["_metadata.json"]}, + install_requires=[ + "attrs", + "cattrs", + "importlib_resources ; python_version < '3.7'", + ], + python_requires=">=3.6", +) From 82759444033df18dcabf73b04265a8f6253b7adf Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 21 Sep 2018 19:49:07 -0400 Subject: [PATCH 16/88] Embed the jsii-runtime within the jsii-python-runtime --- packages/jsii-python-runtime/.gitignore | 1 + packages/jsii-python-runtime/bin/generate | 14 ++++- packages/jsii-python-runtime/setup.py | 5 +- .../src/jsii/_embedded/__init__.py | 0 .../src/jsii/_embedded/jsii/__init__.py | 0 .../src/jsii/_kernel/providers/process.py | 58 ++++++++++++++++++- 6 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 packages/jsii-python-runtime/src/jsii/_embedded/__init__.py create mode 100644 packages/jsii-python-runtime/src/jsii/_embedded/jsii/__init__.py diff --git a/packages/jsii-python-runtime/.gitignore b/packages/jsii-python-runtime/.gitignore index 21719fccd5..4c8711ebb6 100644 --- a/packages/jsii-python-runtime/.gitignore +++ b/packages/jsii-python-runtime/.gitignore @@ -1 +1,2 @@ +src/jsii/_embedded src/jsii/_metadata.json diff --git a/packages/jsii-python-runtime/bin/generate b/packages/jsii-python-runtime/bin/generate index 07e029f998..cc901c3bf8 100755 --- a/packages/jsii-python-runtime/bin/generate +++ b/packages/jsii-python-runtime/bin/generate @@ -1,9 +1,21 @@ #!/usr/bin/env python3 import json +import os +import os.path +import shutil +EMBEDDED_SOURCE = "node_modules/jsii-runtime/webpack/" + + +# Copy metadata over into the Python package with open("package.json") as fp: data = json.load(fp) - with open("src/jsii/_metadata.json", "w") as fp: json.dump({"version": data["version"]}, fp) + + +# Embed the JSII runtime into the Python Package. +for filename in os.listdir(EMBEDDED_SOURCE): + filepath = os.path.join(EMBEDDED_SOURCE, filename) + shutil.copy2(filepath, "src/jsii/_embedded/jsii") diff --git a/packages/jsii-python-runtime/setup.py b/packages/jsii-python-runtime/setup.py index 96eb147875..dd45451e06 100644 --- a/packages/jsii-python-runtime/setup.py +++ b/packages/jsii-python-runtime/setup.py @@ -11,7 +11,10 @@ version=metadata["version"], package_dir={"": "src"}, packages=setuptools.find_packages(where="src"), - package_data={"jsii": ["_metadata.json"]}, + package_data={ + "jsii": ["_metadata.json"], + "jsii._embedded.jsii": ["*.js", "*.js.map", "*.wasm"], + }, install_requires=[ "attrs", "cattrs", diff --git a/packages/jsii-python-runtime/src/jsii/_embedded/__init__.py b/packages/jsii-python-runtime/src/jsii/_embedded/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/jsii-python-runtime/src/jsii/_embedded/jsii/__init__.py b/packages/jsii-python-runtime/src/jsii/_embedded/jsii/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py b/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py index 7bae4e4144..11d7fe62c3 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py @@ -1,11 +1,18 @@ +import contextlib +import importlib.machinery import json +import os.path import subprocess +import tempfile from typing import Type, Union, Mapping, Any, Optional import attr import cattr # type: ignore +import jsii._embedded.jsii + +from jsii._compat import importlib_resources from jsii._utils import memoized_property from jsii._kernel.providers.base import BaseKernel from jsii._kernel.types import ( @@ -129,15 +136,62 @@ def __init__(self): self._serializer.register_unstructure_hook(ObjRef, _unstructure_ref) self._serializer.register_structure_hook(ObjRef, _with_reference) + self._ctx_stack = contextlib.ExitStack() + def __del__(self): self.stop() + def _jsii_runtime(self): + # We have the JSII Runtime bundled with our package and we want to extract it, + # however if we just blindly use importlib.resources for this, we're going to + # have our jsii-runtime.js existing in a *different* temporary directory from + # the jsii-runtime.js.map, which we don't want. We can manually set up a + # temporary directory and extract our resources to there, but we don't want to + # pay the case of setting up a a temporary directory and shuffling bytes around + # in the common case where these files already exist on disk side by side. So + # we will check what loader the embedded package used, if it's a + # SourceFileLoader then we'll assume it's going to be on the filesystem and + # just use importlib.resources.path. + + # jsii-runtime.js MUST be the first item in this list. + filenames = ["jsii-runtime.js", "jsii-runtime.js.map", "mappings.wasm"] + + if isinstance( + jsii._embedded.jsii.__loader__, importlib.machinery.SourceFileLoader + ): + paths = [ + self._ctx_stack.enter_context( + importlib_resources.path(jsii._embedded.jsii, f) + ) + for f in filenames + ] + else: + tmpdir = self._ctx_stack.enter_context(tempfile.TemporaryDirectory()) + paths = [os.path.join(tmpdir, filename) for filename in filenames] + + for path, filename in zip(paths, filenames): + with open(path, "wb") as fp: + fp.write( + importlib_resources.read_binary(jsii._embedded.jsii, filename) + ) + + # Ensure that our jsii-runtime.js is the first entry in our paths, and that all + # of our paths, are in a commmon directory, and we didn't get them split into + # multiple directories somehow. + assert os.path.basename(paths[0]) == filenames[0] + assert os.path.commonpath(paths) == os.path.dirname(paths[0]) + + # Return our first path, which should be the path for jsii-runtime.js + return paths[0] + def _next_message(self) -> Mapping[Any, Any]: return json.loads(self._process.stdout.readline(), object_hook=ohook) def start(self): self._process = subprocess.Popen( - "jsii-runtime", shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE + ["node", self._jsii_runtime()], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, ) self.handshake() @@ -150,6 +204,8 @@ def stop(self): except subprocess.TimeoutExpired: self._process.kill() + self._ctx_stack.close() + def handshake(self): resp: _HelloResponse = self._serializer.structure( self._next_message(), _HelloResponse From 21f7ecb9bb5ea1085fa8885afc4bb1a0b0fc9e7e Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 27 Sep 2018 20:16:26 -0400 Subject: [PATCH 17/88] Handle collection types in JSII --- packages/jsii-pacmak/lib/targets/python.ts | 73 +++++++++++++--------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 61d4bd2634..6983f8fda7 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -75,28 +75,44 @@ class Module { } public maybeImportType(type: string) { - if (PYTHON_BUILTIN_TYPES.indexOf(type) > -1) { - // For built in types, we don't need to do anything, and can just return now - return; - } + let types: string[] = []; - const [, typeModule] = type.match(/(.*)\.(.*)/) as any[]; + // Before we do anything else, we need to split apart any collections, these + // always have the syntax of something[something, maybesomething], so we'll + // check for [] first. + if (type.match(/[^\[]*\[.+\]/)) { + const [, genericType, innerTypes] = type.match(/([^\[]*)\[(.+)\]/) as any[]; - // Given a name like foo.bar.Frob, we want to import the module that Frob exists - // in. Given that all classes exported by JSII have to be pascal cased, and all - // of our imports are snake cases, this should be safe. We're going to double - // check this though, to ensure that our importable here is safe. - if (typeModule != typeModule.toLowerCase()) { - // If we ever get to this point, we'll need to implment aliasing for our - // imports. - throw new Error("Type module is not lower case.") + types.push(genericType); + types.push(...innerTypes.split(",")); + } else { + types.push(type); } - // We only want to actually import the type for this module, if it isn't the - // module that we're currently in, otherwise we'll jus rely on the module scope - // to make the name available to us. - if (typeModule != this.name) { - this.importModule(typeModule); + // Loop over all of the types we've discovered, and check them for being + // importable + for (let type of types) { + // For built in types, we don't need to do anything, and can just move on. + if (PYTHON_BUILTIN_TYPES.indexOf(type) > -1) { continue; } + + let [, typeModule] = type.match(/(.*)\.(.*)/) as any[]; + + // Given a name like foo.bar.Frob, we want to import the module that Frob exists + // in. Given that all classes exported by JSII have to be pascal cased, and all + // of our imports are snake cases, this should be safe. We're going to double + // check this though, to ensure that our importable here is safe. + if (typeModule != typeModule.toLowerCase()) { + // If we ever get to this point, we'll need to implment aliasing for our + // imports. + throw new Error(`Type module is not lower case: '${typeModule}'`); + } + + // We only want to actually import the type for this module, if it isn't the + // module that we're currently in, otherwise we'll jus rely on the module scope + // to make the name available to us. + if (typeModule != this.name) { + this.importModule(typeModule); + } } } @@ -368,9 +384,9 @@ class PythonGenerator extends Generator { private toPythonType(typeref: spec.TypeReference): string { if (spec.isPrimitiveTypeReference(typeref)) { return this.toPythonPrimitive(typeref.primitive); + } else if (spec.isCollectionTypeReference(typeref)) { + return this.toPythonCollection(typeref); } else if (spec.isNamedTypeReference(typeref)) { - // TODO: We need to actually handle this, isntead of just returning the FQN - // as a string. return this.toPythonFQN(typeref.fqn); } else { throw new Error("Invalid type reference: " + JSON.stringify(typeref)); @@ -395,19 +411,16 @@ class PythonGenerator extends Generator { throw new Error('Invalid type reference: ' + JSON.stringify(typeref)); } */ + } - /* - switch (primitive) { - case spec.PrimitiveType.Boolean: return 'java.lang.Boolean'; - case spec.PrimitiveType.Date: return 'java.time.Instant'; - case spec.PrimitiveType.Json: return 'com.fasterxml.jackson.databind.node.ObjectNode'; - case spec.PrimitiveType.Number: return 'java.lang.Number'; - case spec.PrimitiveType.String: return 'java.lang.String'; - case spec.PrimitiveType.Any: return 'java.lang.Object'; + private toPythonCollection(ref: spec.CollectionTypeReference) { + const elementPythonType = this.toPythonType(ref.collection.elementtype); + switch (ref.collection.kind) { + case spec.CollectionKind.Array: return `typing.List[${elementPythonType}]`; + case spec.CollectionKind.Map: return `typing.Mapping[str,${elementPythonType}]`; default: - throw new Error('Unknown primitive type: ' + primitive); + throw new Error(`Unsupported collection kind: ${ref.collection.kind}`); } - */ } private toPythonPrimitive(primitive: spec.PrimitiveType): string { From e2708e0e412171c1f46ef460d0adb92fcc059191 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 27 Sep 2018 20:29:36 -0400 Subject: [PATCH 18/88] Handle scoped packages --- packages/jsii-pacmak/lib/targets/python.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 6983f8fda7..b7a360e71a 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -473,11 +473,25 @@ class PythonGenerator extends Generator { } private toPythonModuleName(name: string): string { - return this.code.toSnakeCase(name.replace(/-/g, "_")); + if (name.match(/^@[^/]+\/[^/]+$/)) { + name = name.replace(/^@/g, ""); + name = name.replace(/\//g, "."); + } + + name = this.code.toSnakeCase(name.replace(/-/g, "_")); + + return name; } private toPythonModuleFilename(name: string): string { - return name.replace(/\./g, "/"); + if (name.match(/^@[^/]+\/[^/]+$/)) { + name = name.replace(/^@/g, ""); + name = name.replace(/\//g, "."); + } + + name = name.replace(/\./g, "/"); + + return name; } // Not Currently Used From 340f8e9c3a0a4f4af1bd69402df1ea759c705473 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 27 Sep 2018 20:39:01 -0400 Subject: [PATCH 19/88] Handle the case where a JSII name conflicts with a Python keyword --- packages/jsii-pacmak/lib/targets/python.ts | 47 ++++++++++++++++++---- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index b7a360e71a..3ab0786e4a 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -38,6 +38,13 @@ const debug = function(o: any) { const PYTHON_BUILTIN_TYPES = ["bool", "str", "None"] +const PYTHON_KEYWORDS = [ + "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", + "continue", "def", "del", "elif", "else", "except", "finally", "for", "from", + "global", "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", + "raise", "return", "try", "while", "with", "yield" +] + class Module { @@ -312,11 +319,13 @@ class PythonGenerator extends Generator { protected onBeginClass(cls: spec.ClassType, abstract: boolean | undefined) { debug("onBeginClass"); + const clsName = this.toPythonIdentifier(cls.name); + // TODO: Figure out what to do with abstract here. abstract; - this.currentModule().exportName(cls.name); - this.currentModule().openBlock(`class ${cls.name}(metaclass=_JSIIMeta, jsii_type="${cls.fqn}")`); + this.currentModule().exportName(clsName); + this.currentModule().openBlock(`class ${clsName}(metaclass=_JSIIMeta, jsii_type="${cls.fqn}")`); } protected onEndClass(_cls: spec.ClassType) { @@ -329,31 +338,42 @@ class PythonGenerator extends Generator { debug("onStaticMethod"); // TODO: Handle the case where the Python name and the JSII name differ. + const methodName = this.toPythonIdentifier(method.name!); + this.currentModule().line("@_jsii_classmethod"); - this.emitPythonMethod(method.name, "cls", method.parameters, method.returns); + this.emitPythonMethod(methodName, "cls", method.parameters, method.returns); } protected onMethod(_cls: spec.ClassType, method: spec.Method) { debug("onMethod"); + // TODO: Handle the case where the Python name and the JSII name differ. + const methodName = this.toPythonIdentifier(method.name!); + this.currentModule().line("@_jsii_method"); - this.emitPythonMethod(method.name, "self", method.parameters, method.returns); + this.emitPythonMethod(methodName, "self", method.parameters, method.returns); } protected onStaticProperty(_cls: spec.ClassType, prop: spec.Property) { debug("onStaticProperty"); + // TODO: Handle the case where the Python name and the JSII name differ. + const propertyName = this.toPythonIdentifier(prop.name!); + // TODO: Properties have a bunch of states, they can have getters and setters // we need to better handle all of these cases. this.currentModule().line("@_jsii_classproperty"); - this.emitPythonMethod(prop.name, "self", [], prop.type); + this.emitPythonMethod(propertyName, "self", [], prop.type); } protected onProperty(_cls: spec.ClassType, prop: spec.Property) { debug("onProperty"); + // TODO: Handle the case where the Python name and the JSII name differ. + const propertyName = this.toPythonIdentifier(prop.name!); + this.currentModule().line("@_jsii_property"); - this.emitPythonMethod(prop.name, "self", [], prop.type); + this.emitPythonMethod(propertyName, "self", [], prop.type); } private emitPythonMethod(name?: string, implicitParam?: string, params: spec.Parameter[] = [], returns?: spec.TypeReference) { @@ -371,9 +391,12 @@ class PythonGenerator extends Generator { let pythonParams: string[] = implicitParam ? [implicitParam] : []; for (let param of params) { + let paramName = this.toPythonIdentifier(param.name); let paramType = this.toPythonType(param.type); + module.maybeImportType(paramType); - pythonParams.push(`${param.name}: ${this.formatPythonType(paramType)}`); + + pythonParams.push(`${paramName}: ${this.formatPythonType(paramType)}`); } module.openBlock(`def ${name}(${pythonParams.join(", ")}) -> ${this.formatPythonType(returnType)}`); @@ -381,6 +404,14 @@ class PythonGenerator extends Generator { module.closeBlock(); } + private toPythonIdentifier(name: string): string { + if (PYTHON_KEYWORDS.indexOf(name) > -1) { + return name + "_"; + } + + return name; + } + private toPythonType(typeref: spec.TypeReference): string { if (spec.isPrimitiveTypeReference(typeref)) { return this.toPythonPrimitive(typeref.primitive); @@ -439,7 +470,7 @@ class PythonGenerator extends Generator { private toPythonFQN(name: string): string { return name.split(".").map((cur, idx, arr) => { if (idx == arr.length - 1) { - return cur; + return this.toPythonIdentifier(cur); } else { return this.toPythonModuleName(cur); } From ea14b3cbed001cd78cab6584847516cd23454091 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 27 Sep 2018 23:35:04 -0400 Subject: [PATCH 20/88] Work with PEP 420 style namespace pacages --- packages/jsii-pacmak/lib/targets/python.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 3ab0786e4a..9db0942cf4 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -224,6 +224,7 @@ class Module { class PythonGenerator extends Generator { + private modules: Module[]; private moduleStack: Module[]; constructor(options = new GeneratorOptions()) { @@ -232,6 +233,7 @@ class PythonGenerator extends Generator { this.code.openBlockFormatter = s => `${s}:`; this.code.closeBlockFormatter = _s => ""; + this.modules = []; this.moduleStack = []; } @@ -252,7 +254,9 @@ class PythonGenerator extends Generator { protected onEndAssembly(assm: spec.Assembly, _fingerprint: boolean) { debug("onEndAssembly"); - debug(assm); + + const packageName = this.toPythonPackageName(assm.name); + const moduleNames = this.modules.map(m => m.name); // We need to write out our packaging for the Python ecosystem here. // TODO: @@ -264,12 +268,12 @@ class PythonGenerator extends Generator { this.code.openFile("setup.py"); this.code.line("import setuptools"); this.code.indent("setuptools.setup("); - this.code.line(`name="${assm.name}",`); + this.code.line(`name="${packageName}",`); this.code.line(`version="${assm.version}",`); this.code.line(`description="${assm.description}",`); this.code.line(`url="${assm.homepage}",`); this.code.line('package_dir={"": "src"},'); - this.code.line('packages=setuptools.find_packages(where="src"),'); + this.code.line(`packages=[${moduleNames.map(m => `"${m}"`).join(",")}],`) this.code.line(`package_data={"${this.toPythonModuleName(assm.name)}._jsii": ["*.jsii.tgz"]},`); this.code.line('python_requires=">=3.6",'); this.code.unindent(")"); @@ -302,7 +306,10 @@ class PythonGenerator extends Generator { moduleArgs.push([this.assembly, this.getAssemblyFileName()]); } - this.moduleStack.push(new Module(moduleName, ...moduleArgs)); + let mod = new Module(moduleName, ...moduleArgs); + + this.modules.push(mod); + this.moduleStack.push(mod); } protected onEndNamespace(ns: string) { @@ -404,6 +411,10 @@ class PythonGenerator extends Generator { module.closeBlock(); } + private toPythonPackageName(name: string): string { + return this.toPythonModuleName(name).replace(/_/g, "-"); + } + private toPythonIdentifier(name: string): string { if (PYTHON_KEYWORDS.indexOf(name) > -1) { return name + "_"; From 2f930776458fabd07178a185680825e6fedf3bc5 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 27 Sep 2018 23:58:53 -0400 Subject: [PATCH 21/88] Deduplicate imports --- packages/jsii-pacmak/lib/targets/python.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 9db0942cf4..5327fbadf1 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -78,7 +78,9 @@ class Module { // Adds a name to the list of modules that should be imported at the top of this // file. public importModule(name: string) { - this.importedModules.push(name); + if (!this.importedModules.includes(name)) { + this.importedModules.push(name); + } } public maybeImportType(type: string) { @@ -90,10 +92,10 @@ class Module { if (type.match(/[^\[]*\[.+\]/)) { const [, genericType, innerTypes] = type.match(/([^\[]*)\[(.+)\]/) as any[]; - types.push(genericType); - types.push(...innerTypes.split(",")); + types.push(genericType.trim()); + types.push(...innerTypes.split(",").map((s: string) => s.trim())); } else { - types.push(type); + types.push(type.trim()); } // Loop over all of the types we've discovered, and check them for being From 9d6362f31a36246c644e9c01412a44df06e26648 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 28 Sep 2018 00:02:04 -0400 Subject: [PATCH 22/88] Remove some debug output --- packages/jsii-pacmak/lib/targets/python.ts | 30 +++++++--------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 5327fbadf1..3c47d489c2 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -244,8 +244,6 @@ class PythonGenerator extends Generator { } protected onBeginAssembly(assm: spec.Assembly, _fingerprint: boolean) { - debug("onBeginAssembly"); - // We need to write out an __init__.py for our _jsii package so that // importlib.resources will be able to load our assembly from it. const assemblyInitFilename = path.join(this.getAssemblyOutputDir(assm), "__init__.py"); @@ -255,8 +253,6 @@ class PythonGenerator extends Generator { } protected onEndAssembly(assm: spec.Assembly, _fingerprint: boolean) { - debug("onEndAssembly"); - const packageName = this.toPythonPackageName(assm.name); const moduleNames = this.modules.map(m => m.name); @@ -297,8 +293,6 @@ class PythonGenerator extends Generator { } protected onBeginNamespace(ns: string) { - debug(`onBeginNamespace: ${ns}`); - const moduleName = this.toPythonModuleName(ns); const loadAssembly = this.assembly.name == ns ? true : false; @@ -314,9 +308,7 @@ class PythonGenerator extends Generator { this.moduleStack.push(mod); } - protected onEndNamespace(ns: string) { - debug(`onEndNamespace: ${ns}`); - + protected onEndNamespace(_ns: string) { let module = this.moduleStack.pop() as Module; let moduleFilename = path.join("src", this.toPythonModuleFilename(module.name), "__init__.py"); @@ -326,8 +318,6 @@ class PythonGenerator extends Generator { } protected onBeginClass(cls: spec.ClassType, abstract: boolean | undefined) { - debug("onBeginClass"); - const clsName = this.toPythonIdentifier(cls.name); // TODO: Figure out what to do with abstract here. @@ -338,14 +328,10 @@ class PythonGenerator extends Generator { } protected onEndClass(_cls: spec.ClassType) { - debug("onEndClass"); - this.currentModule().closeBlock(); } protected onStaticMethod(_cls: spec.ClassType, method: spec.Method) { - debug("onStaticMethod"); - // TODO: Handle the case where the Python name and the JSII name differ. const methodName = this.toPythonIdentifier(method.name!); @@ -354,8 +340,6 @@ class PythonGenerator extends Generator { } protected onMethod(_cls: spec.ClassType, method: spec.Method) { - debug("onMethod"); - // TODO: Handle the case where the Python name and the JSII name differ. const methodName = this.toPythonIdentifier(method.name!); @@ -364,8 +348,6 @@ class PythonGenerator extends Generator { } protected onStaticProperty(_cls: spec.ClassType, prop: spec.Property) { - debug("onStaticProperty"); - // TODO: Handle the case where the Python name and the JSII name differ. const propertyName = this.toPythonIdentifier(prop.name!); @@ -376,8 +358,6 @@ class PythonGenerator extends Generator { } protected onProperty(_cls: spec.ClassType, prop: spec.Property) { - debug("onProperty"); - // TODO: Handle the case where the Python name and the JSII name differ. const propertyName = this.toPythonIdentifier(prop.name!); @@ -542,33 +522,41 @@ class PythonGenerator extends Generator { protected onBeginInterface(_ifc: spec.InterfaceType) { debug("onBeginInterface"); + throw new Error("Unhandled Type: Interface"); } protected onEndInterface(_ifc: spec.InterfaceType) { debug("onEndInterface"); + throw new Error("Unhandled Type: Interface"); } protected onInterfaceMethod(_ifc: spec.InterfaceType, _method: spec.Method) { debug("onInterfaceMethod"); + throw new Error("Unhandled Type: InterfaceMethod"); } protected onInterfaceMethodOverload(_ifc: spec.InterfaceType, _overload: spec.Method, _originalMethod: spec.Method) { debug("onInterfaceMethodOverload"); + throw new Error("Unhandled Type: InterfaceMethodOverload"); } protected onInterfaceProperty(_ifc: spec.InterfaceType, _prop: spec.Property) { debug("onInterfaceProperty"); + throw new Error("Unhandled Type: InterfaceProperty"); } protected onUnionProperty(_cls: spec.ClassType, _prop: spec.Property, _union: spec.UnionTypeReference) { debug("onUnionProperty"); + throw new Error("Unhandled Type: UnionProperty"); } protected onMethodOverload(_cls: spec.ClassType, _overload: spec.Method, _originalMethod: spec.Method) { debug("onMethodOverload"); + throw new Error("Unhandled Type: MethodOverload"); } protected onStaticMethodOverload(_cls: spec.ClassType, _overload: spec.Method, _originalMethod: spec.Method) { debug("onStaticMethodOverload"); + throw new Error("Unhandled Type: StaticMethodOverload"); } } From 0cb0ad5a4189939461cd167bdec333a9d0ad5fb3 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 28 Sep 2018 01:33:17 -0400 Subject: [PATCH 23/88] Add support for Union types --- packages/jsii-pacmak/lib/targets/python.ts | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 3c47d489c2..9df643c624 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -412,29 +412,15 @@ class PythonGenerator extends Generator { return this.toPythonCollection(typeref); } else if (spec.isNamedTypeReference(typeref)) { return this.toPythonFQN(typeref.fqn); - } else { - throw new Error("Invalid type reference: " + JSON.stringify(typeref)); - } - - /* - if (spec.isPrimitiveTypeReference(typeref)) { - return [ this.toJavaPrimitive(typeref.primitive) ]; - } else if (spec.isCollectionTypeReference(typeref)) { - return [ this.toJavaCollection(typeref, forMarshalling) ]; - } else if (spec.isNamedTypeReference(typeref)) { - return [ this.toNativeFqn(typeref.fqn) ]; } else if (typeref.union) { const types = new Array(); for (const subtype of typeref.union.types) { - for (const t of this.toJavaTypes(subtype, forMarshalling)) { - types.push(t); - } + types.push(this.toPythonType(subtype)); } - return types; + return `typing.Union[${types.join(", ")}]`; } else { - throw new Error('Invalid type reference: ' + JSON.stringify(typeref)); + throw new Error("Invalid type reference: " + JSON.stringify(typeref)); } - */ } private toPythonCollection(ref: spec.CollectionTypeReference) { From 90b8ebe0d406c4db329974ae5480e7796295e969 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 28 Sep 2018 02:10:09 -0400 Subject: [PATCH 24/88] Add support for interfaces --- packages/jsii-pacmak/lib/targets/python.ts | 62 ++++++++++++------- packages/jsii-python-runtime/setup.py | 1 + .../jsii-python-runtime/src/jsii/_compat.py | 1 + .../jsii-python-runtime/src/jsii/compat.py | 6 ++ 4 files changed, 47 insertions(+), 23 deletions(-) create mode 100644 packages/jsii-python-runtime/src/jsii/compat.py diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 9df643c624..2852b6e779 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -171,6 +171,7 @@ class Module { public write(code: CodeMaker) { // Before we do Anything, we need to write out our module headers, this is where // we handle stuff like imports, any required initialization, etc. + code.line(this.generateImportFrom("jsii.compat", ["Protocol"])); code.line( this.generateImportFrom( "jsii.runtime", @@ -183,7 +184,7 @@ class Module { "jsii_classproperty", ] ) - ) + ); // Go over all of the modules that we need to import, and import them. for (let [idx, modName] of this.importedModules.sort().entries()) { @@ -365,6 +366,41 @@ class PythonGenerator extends Generator { this.emitPythonMethod(propertyName, "self", [], prop.type); } + protected onBeginInterface(ifc: spec.InterfaceType) { + const currentModule = this.currentModule(); + const interfaceName = this.toPythonIdentifier(ifc.name); + + let interfaceBases: string[] = []; + for (let interfaceBase of (ifc.interfaces || [])) { + let interfaceBaseType = this.toPythonType(interfaceBase); + interfaceBases.push(this.formatPythonType(interfaceBaseType, true)); + currentModule.maybeImportType(interfaceBaseType); + } + interfaceBases.push("_Protocol"); + + // TODO: Data Type + + currentModule.exportName(interfaceName); + currentModule.openBlock(`class ${interfaceName}(${interfaceBases.join(",")})`); + } + + protected onEndInterface(_ifc: spec.InterfaceType) { + this.currentModule().closeBlock(); + } + + protected onInterfaceMethod(_ifc: spec.InterfaceType, method: spec.Method) { + const methodName = this.toPythonIdentifier(method.name!); + + this.emitPythonMethod(methodName, "self", method.parameters, method.returns); + } + + protected onInterfaceProperty(_ifc: spec.InterfaceType, prop: spec.Property) { + const propertyName = this.toPythonIdentifier(prop.name!); + + this.currentModule().line("@property"); + this.emitPythonMethod(propertyName, "self", [], prop.type); + } + private emitPythonMethod(name?: string, implicitParam?: string, params: spec.Parameter[] = [], returns?: spec.TypeReference) { let module = this.currentModule(); @@ -456,7 +492,7 @@ class PythonGenerator extends Generator { }).join("."); } - private formatPythonType(type: string) { + private formatPythonType(type: string, fowardReference: boolean = false) { // Built in types do not need formatted in any particular way. if(PYTHON_BUILTIN_TYPES.indexOf(type) > -1) { return type; @@ -475,7 +511,7 @@ class PythonGenerator extends Generator { // TODO: We currently emit this as a string, because that's how forward // references used to work prior to 3.7. Ideally we will move to using 3.7 // and can just use native forward references. - return `"${typeName}"`; + return fowardReference ? typeName : `"${typeName}"`; } private currentModule(): Module { @@ -506,31 +542,11 @@ class PythonGenerator extends Generator { // Not Currently Used - protected onBeginInterface(_ifc: spec.InterfaceType) { - debug("onBeginInterface"); - throw new Error("Unhandled Type: Interface"); - } - - protected onEndInterface(_ifc: spec.InterfaceType) { - debug("onEndInterface"); - throw new Error("Unhandled Type: Interface"); - } - - protected onInterfaceMethod(_ifc: spec.InterfaceType, _method: spec.Method) { - debug("onInterfaceMethod"); - throw new Error("Unhandled Type: InterfaceMethod"); - } - protected onInterfaceMethodOverload(_ifc: spec.InterfaceType, _overload: spec.Method, _originalMethod: spec.Method) { debug("onInterfaceMethodOverload"); throw new Error("Unhandled Type: InterfaceMethodOverload"); } - protected onInterfaceProperty(_ifc: spec.InterfaceType, _prop: spec.Property) { - debug("onInterfaceProperty"); - throw new Error("Unhandled Type: InterfaceProperty"); - } - protected onUnionProperty(_cls: spec.ClassType, _prop: spec.Property, _union: spec.UnionTypeReference) { debug("onUnionProperty"); throw new Error("Unhandled Type: UnionProperty"); diff --git a/packages/jsii-python-runtime/setup.py b/packages/jsii-python-runtime/setup.py index dd45451e06..03c115f664 100644 --- a/packages/jsii-python-runtime/setup.py +++ b/packages/jsii-python-runtime/setup.py @@ -19,6 +19,7 @@ "attrs", "cattrs", "importlib_resources ; python_version < '3.7'", + "typing_extensions>=3.6.4", ], python_requires=">=3.6", ) diff --git a/packages/jsii-python-runtime/src/jsii/_compat.py b/packages/jsii-python-runtime/src/jsii/_compat.py index 5a50288ef0..27798ed192 100644 --- a/packages/jsii-python-runtime/src/jsii/_compat.py +++ b/packages/jsii-python-runtime/src/jsii/_compat.py @@ -1,3 +1,4 @@ +# Internal Compatability Shims import sys diff --git a/packages/jsii-python-runtime/src/jsii/compat.py b/packages/jsii-python-runtime/src/jsii/compat.py new file mode 100644 index 0000000000..1c1e037c51 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/compat.py @@ -0,0 +1,6 @@ +# External Compatability Shims + +from typing_extensions import Protocol + + +__all__ = ["Protocol"] From 507f787b87c7cb93ac9fbb71eeba70c42168d54c Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 28 Sep 2018 21:55:00 -0400 Subject: [PATCH 25/88] Fix setup.py to always include JSII assembly --- packages/jsii-pacmak/lib/targets/python.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 2852b6e779..6d32f3f133 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -241,7 +241,7 @@ class PythonGenerator extends Generator { } protected getAssemblyOutputDir(mod: spec.Assembly) { - return path.join("src", this.toPythonModuleName(mod.name), "_jsii"); + return path.join("src", this.toPythonModuleFilename(this.toPythonModuleName(mod.name)), "_jsii"); } protected onBeginAssembly(assm: spec.Assembly, _fingerprint: boolean) { @@ -255,8 +255,12 @@ class PythonGenerator extends Generator { protected onEndAssembly(assm: spec.Assembly, _fingerprint: boolean) { const packageName = this.toPythonPackageName(assm.name); + const topLevelModuleName = this.toPythonModuleName(packageName); const moduleNames = this.modules.map(m => m.name); + moduleNames.push(`${topLevelModuleName}._jsii`); + moduleNames.sort(); + // We need to write out our packaging for the Python ecosystem here. // TODO: // - Author @@ -273,7 +277,7 @@ class PythonGenerator extends Generator { this.code.line(`url="${assm.homepage}",`); this.code.line('package_dir={"": "src"},'); this.code.line(`packages=[${moduleNames.map(m => `"${m}"`).join(",")}],`) - this.code.line(`package_data={"${this.toPythonModuleName(assm.name)}._jsii": ["*.jsii.tgz"]},`); + this.code.line(`package_data={"${topLevelModuleName}._jsii": ["*.jsii.tgz"]},`); this.code.line('python_requires=">=3.6",'); this.code.unindent(")"); this.code.closeFile("setup.py"); From 653d774ac8c6067a76bbb8db1e1809a1a3303f8f Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 28 Sep 2018 23:02:28 -0400 Subject: [PATCH 26/88] Handle recurisve generic collection types --- packages/jsii-pacmak/lib/targets/python.ts | 42 +++++++++++++++------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 6d32f3f133..f58f5a2616 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -84,19 +84,7 @@ class Module { } public maybeImportType(type: string) { - let types: string[] = []; - - // Before we do anything else, we need to split apart any collections, these - // always have the syntax of something[something, maybesomething], so we'll - // check for [] first. - if (type.match(/[^\[]*\[.+\]/)) { - const [, genericType, innerTypes] = type.match(/([^\[]*)\[(.+)\]/) as any[]; - - types.push(genericType.trim()); - types.push(...innerTypes.split(",").map((s: string) => s.trim())); - } else { - types.push(type.trim()); - } + const types = this.extractTypes(type); // Loop over all of the types we've discovered, and check them for being // importable @@ -125,6 +113,34 @@ class Module { } } + private extractTypes(type: string): string[] { + let types: string[] = []; + + // Before we do anything else, we need to split apart any collections, these + // always have the syntax of something[something, maybesomething], so we'll + // check for [] first. + if (type.match(/[^\[]*\[.+\]/)) { + let [, genericType, parsedTypes] = type.match(/([^\[]*)\[(.+)\]/) as any[]; + parsedTypes = parsedTypes.split(",").map((s: string) => s.trim()); + + const innerTypes: string[] = []; + for (let innerType of parsedTypes) { + if (innerType.match(/\[/)) { + innerTypes.push(...this.extractTypes(innerType)); + } else { + innerTypes.push(innerType); + } + } + + types.push(genericType.trim()); + types.push(...innerTypes); + } else { + types.push(type.trim()); + } + + return types; + } + // We're purposely replicating the API of CodeMaker here, because CodeMaker cannot // Operate on more than one file at a time, so we have to buffer our calls to // CodeMaker, otherwise we can end up in inconsistent state when we get things like: From 52f62d888adfe264a827a9a5737cb0c8af3789dc Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 28 Sep 2018 23:02:53 -0400 Subject: [PATCH 27/88] Handle classes and interfaces without bodies --- packages/jsii-pacmak/lib/targets/python.ts | 24 ++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index f58f5a2616..7a2e1d0483 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -348,8 +348,16 @@ class PythonGenerator extends Generator { this.currentModule().openBlock(`class ${clsName}(metaclass=_JSIIMeta, jsii_type="${cls.fqn}")`); } - protected onEndClass(_cls: spec.ClassType) { - this.currentModule().closeBlock(); + protected onEndClass(cls: spec.ClassType) { + const currentModule = this.currentModule(); + + // If our class does not have any members, then we need to emit a pass statement + // to give it *some* kind of a body. + if (cls.properties == undefined && cls.methods == undefined) { + currentModule.line("pass"); + } + + currentModule.closeBlock(); } protected onStaticMethod(_cls: spec.ClassType, method: spec.Method) { @@ -404,8 +412,16 @@ class PythonGenerator extends Generator { currentModule.openBlock(`class ${interfaceName}(${interfaceBases.join(",")})`); } - protected onEndInterface(_ifc: spec.InterfaceType) { - this.currentModule().closeBlock(); + protected onEndInterface(ifc: spec.InterfaceType) { + const currentModule = this.currentModule(); + + // If our interface does not have any members, then we need to emit a pass + // statement to give it *some* kind of a body. + if (ifc.properties == undefined && ifc.methods == undefined) { + currentModule.line("pass"); + } + + currentModule.closeBlock(); } protected onInterfaceMethod(_ifc: spec.InterfaceType, method: spec.Method) { From 0408cb3ea32779b893832839b798e6499a925667 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 29 Sep 2018 00:20:39 -0400 Subject: [PATCH 28/88] Use a better mechanism of extracting a list of types --- packages/jsii-pacmak/lib/targets/python.ts | 34 ++++------------------ 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 7a2e1d0483..17fb2205b8 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -84,7 +84,11 @@ class Module { } public maybeImportType(type: string) { - const types = this.extractTypes(type); + // If we split our types by any of the "special" characters that can't appear in + // identifiers (like "[],") then we will get a list of all of the identifiers, + // no matter how nested they are. The downside is we might get trailing/leading + // spaces or empty items so we'll need to trim and filter this list. + const types = type.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s != ""); // Loop over all of the types we've discovered, and check them for being // importable @@ -113,34 +117,6 @@ class Module { } } - private extractTypes(type: string): string[] { - let types: string[] = []; - - // Before we do anything else, we need to split apart any collections, these - // always have the syntax of something[something, maybesomething], so we'll - // check for [] first. - if (type.match(/[^\[]*\[.+\]/)) { - let [, genericType, parsedTypes] = type.match(/([^\[]*)\[(.+)\]/) as any[]; - parsedTypes = parsedTypes.split(",").map((s: string) => s.trim()); - - const innerTypes: string[] = []; - for (let innerType of parsedTypes) { - if (innerType.match(/\[/)) { - innerTypes.push(...this.extractTypes(innerType)); - } else { - innerTypes.push(innerType); - } - } - - types.push(genericType.trim()); - types.push(...innerTypes); - } else { - types.push(type.trim()); - } - - return types; - } - // We're purposely replicating the API of CodeMaker here, because CodeMaker cannot // Operate on more than one file at a time, so we have to buffer our calls to // CodeMaker, otherwise we can end up in inconsistent state when we get things like: From b8b40c045918460e1a53fbdc533c9358936ef8ac Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 29 Sep 2018 00:33:53 -0400 Subject: [PATCH 29/88] Add a bit of extra documentation --- packages/jsii-pacmak/lib/targets/python.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 17fb2205b8..828df3e22d 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -83,6 +83,8 @@ class Module { } } + // Giving a type hint, extract all of the relevant types that are involved, and if + // they are defined in another module, mark that module for import. public maybeImportType(type: string) { // If we split our types by any of the "special" characters that can't appear in // identifiers (like "[],") then we will get a list of all of the identifiers, From 848ed7ac9de08889fa162fd465d6e7a5eae8a718 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 29 Sep 2018 03:20:25 -0400 Subject: [PATCH 30/88] Bump version --- .../jsii-python-runtime/src/jsii/_kernel/providers/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py b/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py index 11d7fe62c3..55b5808570 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py @@ -213,7 +213,7 @@ def handshake(self): # TODO: Replace with proper error. assert ( - resp.hello == "jsii-runtime@0.7.4" + resp.hello == "jsii-runtime@0.7.6" ), f"Invalid JSII Runtime Version: {resp.hello!r}" def send( From 855c2a8f40002a67fe9a396303c9930422595397 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 29 Sep 2018 18:16:24 -0400 Subject: [PATCH 31/88] Fix formatting to handle nested types --- packages/jsii-pacmak/lib/targets/python.ts | 52 ++++++++++++++-------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 828df3e22d..da86068cc1 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -379,7 +379,7 @@ class PythonGenerator extends Generator { let interfaceBases: string[] = []; for (let interfaceBase of (ifc.interfaces || [])) { let interfaceBaseType = this.toPythonType(interfaceBase); - interfaceBases.push(this.formatPythonType(interfaceBaseType, true)); + interfaceBases.push(this.formatPythonType(interfaceBaseType, true, currentModule.name)); currentModule.maybeImportType(interfaceBaseType); } interfaceBases.push("_Protocol"); @@ -435,10 +435,10 @@ class PythonGenerator extends Generator { module.maybeImportType(paramType); - pythonParams.push(`${paramName}: ${this.formatPythonType(paramType)}`); + pythonParams.push(`${paramName}: ${this.formatPythonType(paramType, false, module.name)}`); } - module.openBlock(`def ${name}(${pythonParams.join(", ")}) -> ${this.formatPythonType(returnType)}`); + module.openBlock(`def ${name}(${pythonParams.join(", ")}) -> ${this.formatPythonType(returnType, false, module.name)}`); module.line("..."); module.closeBlock(); } @@ -506,26 +506,40 @@ class PythonGenerator extends Generator { }).join("."); } - private formatPythonType(type: string, fowardReference: boolean = false) { - // Built in types do not need formatted in any particular way. - if(PYTHON_BUILTIN_TYPES.indexOf(type) > -1) { - return type; - } + private formatPythonType(type: string, forwardReference: boolean = false, moduleName: string) { + // If we split our types by any of the "special" characters that can't appear in + // identifiers (like "[],") then we will get a list of all of the identifiers, + // no matter how nested they are. The downside is we might get trailing/leading + // spaces or empty items so we'll need to trim and filter this list. + const types = type.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s != ""); - const [, typeModule, typeName] = type.match(/(.*)\.(.*)/) as any[]; + for (let innerType of types) { + // Built in types do not need formatted in any particular way. + if(PYTHON_BUILTIN_TYPES.indexOf(innerType) > -1) { + continue; + } - // Types whose module is different than our current module, can also just be - // formatted as they are. - if (this.currentModule().name != typeModule) { - return type; + // If we do not have a current moduleName, or the type is not within that + // module, then we don't format it any particular way. + if (!innerType.startsWith(moduleName + ".")) { + continue; + } else { + const re = new RegExp('[^\[,\s]"?' + innerType + '"?[$\],\s]'); + + // If this is our current module, then we need to correctly handle our + // forward references, by placing the type inside of quotes, unless + // we're returning real forward references. + if (!forwardReference) { + type = type.replace(re, `"${innerType}"`); + } + + // Now that we've handled (or not) our forward references, then we want + // to replace the module with just the type name. + type = type.replace(re, innerType.substring(moduleName.length + 1, innerType.length)); + } } - // Otherwise, this is a type that exists in this module, and we can jsut emit - // the name. - // TODO: We currently emit this as a string, because that's how forward - // references used to work prior to 3.7. Ideally we will move to using 3.7 - // and can just use native forward references. - return fowardReference ? typeName : `"${typeName}"`; + return type; } private currentModule(): Module { From d4b1fe82db0ad7808a7e095229a6d19c6b37e44e Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 29 Sep 2018 22:51:45 -0400 Subject: [PATCH 32/88] Refactor various toPythonXXX functions into stand alone functions --- packages/jsii-pacmak/lib/targets/python.ts | 286 +++++++++++---------- 1 file changed, 147 insertions(+), 139 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index da86068cc1..c65c166103 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -5,7 +5,7 @@ import util = require('util'); import * as spec from 'jsii-spec'; import { Generator, GeneratorOptions } from '../generator'; import { Target, TargetOptions } from '../target'; -import { CodeMaker } from 'codemaker'; +import { CodeMaker, toSnakeCase } from 'codemaker'; import { shell } from '../util'; export default class Python extends Target { @@ -46,6 +46,135 @@ const PYTHON_KEYWORDS = [ ] +function toPythonModuleName(name: string): string { + if (name.match(/^@[^/]+\/[^/]+$/)) { + name = name.replace(/^@/g, ""); + name = name.replace(/\//g, "."); + } + + name = toSnakeCase(name.replace(/-/g, "_")); + + return name; +} + + +function toPythonModuleFilename(name: string): string { + if (name.match(/^@[^/]+\/[^/]+$/)) { + name = name.replace(/^@/g, ""); + name = name.replace(/\//g, "."); + } + + name = name.replace(/\./g, "/"); + + return name; +} + + +function toPythonPackageName(name: string): string { + return toPythonModuleName(name).replace(/_/g, "-"); +} + + +function toPythonIdentifier(name: string): string { + if (PYTHON_KEYWORDS.indexOf(name) > -1) { + return name + "_"; + } + + return name; +} + + +function toPythonType(typeref: spec.TypeReference): string { + if (spec.isPrimitiveTypeReference(typeref)) { + return toPythonPrimitive(typeref.primitive); + } else if (spec.isCollectionTypeReference(typeref)) { + return toPythonCollection(typeref); + } else if (spec.isNamedTypeReference(typeref)) { + return toPythonFQN(typeref.fqn); + } else if (typeref.union) { + const types = new Array(); + for (const subtype of typeref.union.types) { + types.push(toPythonType(subtype)); + } + return `typing.Union[${types.join(", ")}]`; + } else { + throw new Error("Invalid type reference: " + JSON.stringify(typeref)); + } +} + +function toPythonCollection(ref: spec.CollectionTypeReference) { + const elementPythonType = toPythonType(ref.collection.elementtype); + switch (ref.collection.kind) { + case spec.CollectionKind.Array: return `typing.List[${elementPythonType}]`; + case spec.CollectionKind.Map: return `typing.Mapping[str,${elementPythonType}]`; + default: + throw new Error(`Unsupported collection kind: ${ref.collection.kind}`); + } +} + + +function toPythonPrimitive(primitive: spec.PrimitiveType): string { + switch (primitive) { + case spec.PrimitiveType.Boolean: return "bool"; + case spec.PrimitiveType.Date: return "dateetime.datetime"; + case spec.PrimitiveType.Json: return "typing.Mapping[typing.Any, typing.Any]"; + case spec.PrimitiveType.Number: return "numbers.Number"; + case spec.PrimitiveType.String: return "str"; + case spec.PrimitiveType.Any: return "typing.Any"; + default: + throw new Error("Unknown primitive type: " + primitive); + } +} + + +function toPythonFQN(name: string): string { + return name.split(".").map((cur, idx, arr) => { + if (idx == arr.length - 1) { + return toPythonIdentifier(cur); + } else { + return toPythonModuleName(cur); + } + }).join("."); +} + + +function formatPythonType(type: string, forwardReference: boolean = false, moduleName: string) { + // If we split our types by any of the "special" characters that can't appear in + // identifiers (like "[],") then we will get a list of all of the identifiers, + // no matter how nested they are. The downside is we might get trailing/leading + // spaces or empty items so we'll need to trim and filter this list. + const types = type.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s != ""); + + for (let innerType of types) { + // Built in types do not need formatted in any particular way. + if(PYTHON_BUILTIN_TYPES.indexOf(innerType) > -1) { + continue; + } + + // If we do not have a current moduleName, or the type is not within that + // module, then we don't format it any particular way. + if (!innerType.startsWith(moduleName + ".")) { + continue; + } else { + const re = new RegExp('[^\[,\s]"?' + innerType + '"?[$\],\s]'); + + // If this is our current module, then we need to correctly handle our + // forward references, by placing the type inside of quotes, unless + // we're returning real forward references. + if (!forwardReference) { + type = type.replace(re, `"${innerType}"`); + } + + // Now that we've handled (or not) our forward references, then we want + // to replace the module with just the type name. + type = type.replace(re, innerType.substring(moduleName.length + 1, innerType.length)); + } + } + + return type; +} + + class Module { readonly name: string; @@ -235,7 +364,7 @@ class PythonGenerator extends Generator { } protected getAssemblyOutputDir(mod: spec.Assembly) { - return path.join("src", this.toPythonModuleFilename(this.toPythonModuleName(mod.name)), "_jsii"); + return path.join("src", toPythonModuleFilename(toPythonModuleName(mod.name)), "_jsii"); } protected onBeginAssembly(assm: spec.Assembly, _fingerprint: boolean) { @@ -248,8 +377,8 @@ class PythonGenerator extends Generator { } protected onEndAssembly(assm: spec.Assembly, _fingerprint: boolean) { - const packageName = this.toPythonPackageName(assm.name); - const topLevelModuleName = this.toPythonModuleName(packageName); + const packageName = toPythonPackageName(assm.name); + const topLevelModuleName = toPythonModuleName(packageName); const moduleNames = this.modules.map(m => m.name); moduleNames.push(`${topLevelModuleName}._jsii`); @@ -292,7 +421,7 @@ class PythonGenerator extends Generator { } protected onBeginNamespace(ns: string) { - const moduleName = this.toPythonModuleName(ns); + const moduleName = toPythonModuleName(ns); const loadAssembly = this.assembly.name == ns ? true : false; let moduleArgs: any[] = []; @@ -309,7 +438,7 @@ class PythonGenerator extends Generator { protected onEndNamespace(_ns: string) { let module = this.moduleStack.pop() as Module; - let moduleFilename = path.join("src", this.toPythonModuleFilename(module.name), "__init__.py"); + let moduleFilename = path.join("src", toPythonModuleFilename(module.name), "__init__.py"); this.code.openFile(moduleFilename); module.write(this.code); @@ -317,7 +446,7 @@ class PythonGenerator extends Generator { } protected onBeginClass(cls: spec.ClassType, abstract: boolean | undefined) { - const clsName = this.toPythonIdentifier(cls.name); + const clsName = toPythonIdentifier(cls.name); // TODO: Figure out what to do with abstract here. abstract; @@ -340,7 +469,7 @@ class PythonGenerator extends Generator { protected onStaticMethod(_cls: spec.ClassType, method: spec.Method) { // TODO: Handle the case where the Python name and the JSII name differ. - const methodName = this.toPythonIdentifier(method.name!); + const methodName = toPythonIdentifier(method.name!); this.currentModule().line("@_jsii_classmethod"); this.emitPythonMethod(methodName, "cls", method.parameters, method.returns); @@ -348,7 +477,7 @@ class PythonGenerator extends Generator { protected onMethod(_cls: spec.ClassType, method: spec.Method) { // TODO: Handle the case where the Python name and the JSII name differ. - const methodName = this.toPythonIdentifier(method.name!); + const methodName = toPythonIdentifier(method.name!); this.currentModule().line("@_jsii_method"); this.emitPythonMethod(methodName, "self", method.parameters, method.returns); @@ -356,7 +485,7 @@ class PythonGenerator extends Generator { protected onStaticProperty(_cls: spec.ClassType, prop: spec.Property) { // TODO: Handle the case where the Python name and the JSII name differ. - const propertyName = this.toPythonIdentifier(prop.name!); + const propertyName = toPythonIdentifier(prop.name!); // TODO: Properties have a bunch of states, they can have getters and setters // we need to better handle all of these cases. @@ -366,7 +495,7 @@ class PythonGenerator extends Generator { protected onProperty(_cls: spec.ClassType, prop: spec.Property) { // TODO: Handle the case where the Python name and the JSII name differ. - const propertyName = this.toPythonIdentifier(prop.name!); + const propertyName = toPythonIdentifier(prop.name!); this.currentModule().line("@_jsii_property"); this.emitPythonMethod(propertyName, "self", [], prop.type); @@ -403,13 +532,13 @@ class PythonGenerator extends Generator { } protected onInterfaceMethod(_ifc: spec.InterfaceType, method: spec.Method) { - const methodName = this.toPythonIdentifier(method.name!); + const methodName = toPythonIdentifier(method.name!); this.emitPythonMethod(methodName, "self", method.parameters, method.returns); } protected onInterfaceProperty(_ifc: spec.InterfaceType, prop: spec.Property) { - const propertyName = this.toPythonIdentifier(prop.name!); + const propertyName = toPythonIdentifier(prop.name!); this.currentModule().line("@property"); this.emitPythonMethod(propertyName, "self", [], prop.type); @@ -419,7 +548,7 @@ class PythonGenerator extends Generator { let module = this.currentModule(); // TODO: Handle imports (if needed) for type. - const returnType = returns ? this.toPythonType(returns) : "None"; + const returnType = returns ? toPythonType(returns) : "None"; module.maybeImportType(returnType); @@ -430,144 +559,23 @@ class PythonGenerator extends Generator { let pythonParams: string[] = implicitParam ? [implicitParam] : []; for (let param of params) { - let paramName = this.toPythonIdentifier(param.name); - let paramType = this.toPythonType(param.type); + let paramName = toPythonIdentifier(param.name); + let paramType = toPythonType(param.type); module.maybeImportType(paramType); - pythonParams.push(`${paramName}: ${this.formatPythonType(paramType, false, module.name)}`); + pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, module.name)}`); } - module.openBlock(`def ${name}(${pythonParams.join(", ")}) -> ${this.formatPythonType(returnType, false, module.name)}`); + module.openBlock(`def ${name}(${pythonParams.join(", ")}) -> ${formatPythonType(returnType, false, module.name)}`); module.line("..."); module.closeBlock(); } - private toPythonPackageName(name: string): string { - return this.toPythonModuleName(name).replace(/_/g, "-"); - } - - private toPythonIdentifier(name: string): string { - if (PYTHON_KEYWORDS.indexOf(name) > -1) { - return name + "_"; - } - - return name; - } - - private toPythonType(typeref: spec.TypeReference): string { - if (spec.isPrimitiveTypeReference(typeref)) { - return this.toPythonPrimitive(typeref.primitive); - } else if (spec.isCollectionTypeReference(typeref)) { - return this.toPythonCollection(typeref); - } else if (spec.isNamedTypeReference(typeref)) { - return this.toPythonFQN(typeref.fqn); - } else if (typeref.union) { - const types = new Array(); - for (const subtype of typeref.union.types) { - types.push(this.toPythonType(subtype)); - } - return `typing.Union[${types.join(", ")}]`; - } else { - throw new Error("Invalid type reference: " + JSON.stringify(typeref)); - } - } - - private toPythonCollection(ref: spec.CollectionTypeReference) { - const elementPythonType = this.toPythonType(ref.collection.elementtype); - switch (ref.collection.kind) { - case spec.CollectionKind.Array: return `typing.List[${elementPythonType}]`; - case spec.CollectionKind.Map: return `typing.Mapping[str,${elementPythonType}]`; - default: - throw new Error(`Unsupported collection kind: ${ref.collection.kind}`); - } - } - - private toPythonPrimitive(primitive: spec.PrimitiveType): string { - switch (primitive) { - case spec.PrimitiveType.Boolean: return "bool"; - case spec.PrimitiveType.Date: return "dateetime.datetime"; - case spec.PrimitiveType.Json: return "typing.Mapping[typing.Any, typing.Any]"; - case spec.PrimitiveType.Number: return "numbers.Number"; - case spec.PrimitiveType.String: return "str"; - case spec.PrimitiveType.Any: return "typing.Any"; - default: - throw new Error("Unknown primitive type: " + primitive); - } - } - - private toPythonFQN(name: string): string { - return name.split(".").map((cur, idx, arr) => { - if (idx == arr.length - 1) { - return this.toPythonIdentifier(cur); - } else { - return this.toPythonModuleName(cur); - } - }).join("."); - } - - private formatPythonType(type: string, forwardReference: boolean = false, moduleName: string) { - // If we split our types by any of the "special" characters that can't appear in - // identifiers (like "[],") then we will get a list of all of the identifiers, - // no matter how nested they are. The downside is we might get trailing/leading - // spaces or empty items so we'll need to trim and filter this list. - const types = type.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s != ""); - - for (let innerType of types) { - // Built in types do not need formatted in any particular way. - if(PYTHON_BUILTIN_TYPES.indexOf(innerType) > -1) { - continue; - } - - // If we do not have a current moduleName, or the type is not within that - // module, then we don't format it any particular way. - if (!innerType.startsWith(moduleName + ".")) { - continue; - } else { - const re = new RegExp('[^\[,\s]"?' + innerType + '"?[$\],\s]'); - - // If this is our current module, then we need to correctly handle our - // forward references, by placing the type inside of quotes, unless - // we're returning real forward references. - if (!forwardReference) { - type = type.replace(re, `"${innerType}"`); - } - - // Now that we've handled (or not) our forward references, then we want - // to replace the module with just the type name. - type = type.replace(re, innerType.substring(moduleName.length + 1, innerType.length)); - } - } - - return type; - } - private currentModule(): Module { return this.moduleStack.slice(-1)[0]; } - private toPythonModuleName(name: string): string { - if (name.match(/^@[^/]+\/[^/]+$/)) { - name = name.replace(/^@/g, ""); - name = name.replace(/\//g, "."); - } - - name = this.code.toSnakeCase(name.replace(/-/g, "_")); - - return name; - } - - private toPythonModuleFilename(name: string): string { - if (name.match(/^@[^/]+\/[^/]+$/)) { - name = name.replace(/^@/g, ""); - name = name.replace(/\//g, "."); - } - - name = name.replace(/\./g, "/"); - - return name; - } - // Not Currently Used protected onInterfaceMethodOverload(_ifc: spec.InterfaceType, _overload: spec.Method, _originalMethod: spec.Method) { From 322f04f6dd2883db0610ee0e00ca488799f85e0a Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 30 Sep 2018 02:00:40 -0400 Subject: [PATCH 33/88] Drastic refactoring to delay emiting lines until the very end This is a WIP refactoring, it all works but it needs a lot of clean up. --- packages/jsii-pacmak/lib/targets/python.ts | 675 +++++++++++++++------ 1 file changed, 487 insertions(+), 188 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index c65c166103..700fa7cc47 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -156,18 +156,20 @@ function formatPythonType(type: string, forwardReference: boolean = false, modul if (!innerType.startsWith(moduleName + ".")) { continue; } else { - const re = new RegExp('[^\[,\s]"?' + innerType + '"?[$\],\s]'); + const typeName = innerType.substring(moduleName.length + 1, innerType.length); + const re = new RegExp('((?:^|[[,\\s])"?)' + innerType + '("?(?:$|[\\],\\s]))'); // If this is our current module, then we need to correctly handle our // forward references, by placing the type inside of quotes, unless // we're returning real forward references. if (!forwardReference) { - type = type.replace(re, `"${innerType}"`); + type = type.replace(re, `$1"${innerType}"$2`); } // Now that we've handled (or not) our forward references, then we want // to replace the module with just the type name. - type = type.replace(re, innerType.substring(moduleName.length + 1, innerType.length)); + // type = type.replace(re, "$1" + innerType.substring(moduleName.length + 1, innerType.length) + "$2"); + type = type.replace(re, `$1${typeName}$2`); } } @@ -175,125 +177,387 @@ function formatPythonType(type: string, forwardReference: boolean = false, modul } -class Module { +interface Writable { + write(code: CodeMaker): void; +} + + +interface WithMembers { + addMember(member: PythonItem): PythonItem; +} + + +interface PythonItem { readonly name: string; - readonly assembly?: spec.Assembly; - readonly assemblyFilename?: string; + requiredTypes(): string[]; +} - private buffer: object[]; - private exportedNames: string[]; - private importedModules: string[]; - constructor(ns: string, assembly?: [spec.Assembly, string]) { - this.name = ns; +type ModuleMember = PythonItem & WithMembers & Writable; - if (assembly != undefined) { - this.assembly = assembly[0]; - this.assemblyFilename = assembly[1]; + +class InterfaceMethod implements PythonItem, Writable { + + public readonly moduleName: string; + public readonly name: string; + + private readonly parameters: spec.Parameter[]; + private readonly returns?: spec.TypeReference; + + constructor(moduleName: string, name: string, parameters: spec.Parameter[], returns?: spec.TypeReference) { + this.moduleName = moduleName; + this.name = name; + this.parameters = parameters; + this.returns = returns; + } + + public requiredTypes(): string[] { + const types: string[] = [this.getReturnType(this.returns)]; + + for (let param of this.parameters) { + types.push(toPythonType(param.type)); } - this.buffer = []; - this.exportedNames = []; - this.importedModules = []; + return types; } - // Adds a name to the list of names that this module will export, this will control - // things like what is listed inside of __all__. - public exportName(name: string) { - this.exportedNames.push(name); + public write(code: CodeMaker) { + const returnType = this.getReturnType(this.returns); + + // We need to turn a list of JSII parameters, into Python style arguments with + // gradual typing, so we'll have to iterate over the list of parameters, and + // build the list, converting as we go. + // TODO: Handle imports (if needed) for all of these types. + let pythonParams: string[] = ["self"] + for (let param of this.parameters) { + const paramName = toPythonIdentifier(param.name); + const paramType = toPythonType(param.type); + + pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, this.moduleName)}`); + } + + code.openBlock(`def ${this.name}(${pythonParams.join(", ")}) -> ${formatPythonType(returnType, false, this.moduleName)}`); + code.line("..."); + code.closeBlock(); + } + + private getReturnType(type?: spec.TypeReference): string { + return type ? toPythonType(type) : "None"; + } +} + +class InterfaceProperty implements PythonItem, Writable { + + public readonly moduleName: string; + public readonly name: string; + private readonly type: spec.TypeReference; + + constructor(moduleName: string, name: string, type: spec.TypeReference) { + this.moduleName = moduleName; + this.name = name; + this.type = type; + } + + public requiredTypes(): string[] { + return [toPythonType(this.type)]; + } + + public write(code: CodeMaker) { + const returnType = toPythonType(this.type); + + code.line("@property"); + code.openBlock(`def ${this.name}(self) -> ${formatPythonType(returnType, false, this.moduleName)}`); + code.line("..."); + code.closeBlock(); + } +} + + +class Interface implements ModuleMember { + + public readonly moduleName: string; + public readonly name: string; + + private bases: string[]; + private members: (PythonItem & Writable)[]; + + constructor(moduleName: string, name: string, bases: string[]) { + this.moduleName = moduleName; + this.name = name; + + this.bases = bases; + this.members = []; + } + + public addMember(member: PythonItem & Writable): PythonItem { + this.members.push(member); + return member; } - // Adds a name to the list of modules that should be imported at the top of this - // file. - public importModule(name: string) { - if (!this.importedModules.includes(name)) { - this.importedModules.push(name); + public requiredTypes(): string[] { + const types = this.bases.slice(); + + for (let member of this.members) { + types.push(...member.requiredTypes()); } + + return types; } - // Giving a type hint, extract all of the relevant types that are involved, and if - // they are defined in another module, mark that module for import. - public maybeImportType(type: string) { - // If we split our types by any of the "special" characters that can't appear in - // identifiers (like "[],") then we will get a list of all of the identifiers, - // no matter how nested they are. The downside is we might get trailing/leading - // spaces or empty items so we'll need to trim and filter this list. - const types = type.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s != ""); + public write(code: CodeMaker) { + // TODO: Data Types? + + const interfaceBases = this.bases.map(baseType => formatPythonType(baseType, true, this.moduleName)); + interfaceBases.push("_Protocol"); - // Loop over all of the types we've discovered, and check them for being - // importable - for (let type of types) { - // For built in types, we don't need to do anything, and can just move on. - if (PYTHON_BUILTIN_TYPES.indexOf(type) > -1) { continue; } - - let [, typeModule] = type.match(/(.*)\.(.*)/) as any[]; - - // Given a name like foo.bar.Frob, we want to import the module that Frob exists - // in. Given that all classes exported by JSII have to be pascal cased, and all - // of our imports are snake cases, this should be safe. We're going to double - // check this though, to ensure that our importable here is safe. - if (typeModule != typeModule.toLowerCase()) { - // If we ever get to this point, we'll need to implment aliasing for our - // imports. - throw new Error(`Type module is not lower case: '${typeModule}'`); + code.openBlock(`class ${this.name}(${interfaceBases.join(",")})`); + if (this.members.length > 0) { + for (let member of this.members) { + member.write(code); } + } else { + code.line("pass"); + } + code.closeBlock(); + } +} - // We only want to actually import the type for this module, if it isn't the - // module that we're currently in, otherwise we'll jus rely on the module scope - // to make the name available to us. - if (typeModule != this.name) { - this.importModule(typeModule); - } + +class StaticMethod implements PythonItem, Writable { + + public readonly moduleName: string; + public readonly name: string; + + private readonly parameters: spec.Parameter[]; + private readonly returns?: spec.TypeReference; + + constructor(moduleName: string, name: string, parameters: spec.Parameter[], returns?: spec.TypeReference) { + this.moduleName = moduleName; + this.name = name; + this.parameters = parameters; + this.returns = returns; + } + + public requiredTypes(): string[] { + const types: string[] = [this.getReturnType(this.returns)]; + + for (let param of this.parameters) { + types.push(toPythonType(param.type)); } + + return types; + } + + public write(code: CodeMaker) { + const returnType = this.getReturnType(this.returns); + + // We need to turn a list of JSII parameters, into Python style arguments with + // gradual typing, so we'll have to iterate over the list of parameters, and + // build the list, converting as we go. + // TODO: Handle imports (if needed) for all of these types. + let pythonParams: string[] = ["cls"] + for (let param of this.parameters) { + const paramName = toPythonIdentifier(param.name); + const paramType = toPythonType(param.type); + + pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, this.moduleName)}`); + } + + code.line("@_jsii_classmethod"); + code.openBlock(`def ${this.name}(${pythonParams.join(", ")}) -> ${formatPythonType(returnType, false, this.moduleName)}`); + code.line("..."); + code.closeBlock(); + } + + private getReturnType(type?: spec.TypeReference): string { + return type ? toPythonType(type) : "None"; + } +} + + +class Method implements PythonItem, Writable { + + public readonly moduleName: string; + public readonly name: string; + + private readonly parameters: spec.Parameter[]; + private readonly returns?: spec.TypeReference; + + constructor(moduleName: string, name: string, parameters: spec.Parameter[], returns?: spec.TypeReference) { + this.moduleName = moduleName; + this.name = name; + this.parameters = parameters; + this.returns = returns; + } + + public requiredTypes(): string[] { + const types: string[] = [this.getReturnType(this.returns)]; + + for (let param of this.parameters) { + types.push(toPythonType(param.type)); + } + + return types; + } + + public write(code: CodeMaker) { + const returnType = this.getReturnType(this.returns); + + // We need to turn a list of JSII parameters, into Python style arguments with + // gradual typing, so we'll have to iterate over the list of parameters, and + // build the list, converting as we go. + // TODO: Handle imports (if needed) for all of these types. + let pythonParams: string[] = ["self"] + for (let param of this.parameters) { + const paramName = toPythonIdentifier(param.name); + const paramType = toPythonType(param.type); + + pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, this.moduleName)}`); + } + + code.line("@_jsii_method"); + code.openBlock(`def ${this.name}(${pythonParams.join(", ")}) -> ${formatPythonType(returnType, false, this.moduleName)}`); + code.line("..."); + code.closeBlock(); + } + + private getReturnType(type?: spec.TypeReference): string { + return type ? toPythonType(type) : "None"; + } +} + + +class StaticProperty implements PythonItem, Writable { + + public readonly moduleName: string; + public readonly name: string; + private readonly type: spec.TypeReference; + + constructor(moduleName: string, name: string, type: spec.TypeReference) { + this.moduleName = moduleName; + this.name = name; + this.type = type; + } + + public requiredTypes(): string[] { + return [toPythonType(this.type)]; } - // We're purposely replicating the API of CodeMaker here, because CodeMaker cannot - // Operate on more than one file at a time, so we have to buffer our calls to - // CodeMaker, otherwise we can end up in inconsistent state when we get things like: - // - onBeginNamespace(foo) - // - onBeginNamespace(foo.bar) - // - OnEndNamespace(foo.bar) - // - Inconsitent State, where we're now back in the scope of foo, but due to - // the fact that if we had opened a file in onBeginNamespace(foo), we would - // have had to close it for onBeginNamespace(foo.bar), and re-opening it - // would overwrite the contents. - // - OnEndNamespace(foo) - // To solve this, we buffer all of the things we *would* have written out via - // CodeMaker via this API, and then we will just iterate over it in the - // onEndNamespace event and write it out then. + public write(code: CodeMaker) { + const returnType = toPythonType(this.type); - public line(...args: any[]) { - this.buffer.push({method: "line", args: args}); + code.line("@_jsii_classproperty"); + code.openBlock(`def ${this.name}(cls) -> ${formatPythonType(returnType, false, this.moduleName)}`); + code.line("..."); + code.closeBlock(); } +} + - public indent(...args: any[]) { - this.buffer.push({method: "indent", args: args}); +class Property implements PythonItem, Writable { + + public readonly moduleName: string; + public readonly name: string; + private readonly type: spec.TypeReference; + + constructor(moduleName: string, name: string, type: spec.TypeReference) { + this.moduleName = moduleName; + this.name = name; + this.type = type; } - public unindent(...args: any[]) { - this.buffer.push({method: "unindent", args: args}); + public requiredTypes(): string[] { + return [toPythonType(this.type)]; } - public open(...args: any[]) { - this.buffer.push({method: "open", args: args}); + public write(code: CodeMaker) { + const returnType = toPythonType(this.type); + + code.line("@_jsii_property"); + code.openBlock(`def ${this.name}(self) -> ${formatPythonType(returnType, false, this.moduleName)}`); + code.line("..."); + code.closeBlock(); } +} + + +class Class implements ModuleMember { + public readonly moduleName: string; + public readonly name: string; - public close(...args: any[]) { - this.buffer.push({method: "close", args: args}); + private jsii_fqn: string; + private members: (PythonItem & Writable)[]; + + constructor(moduleName: string, name: string, jsii_fqn: string) { + this.moduleName = moduleName; + this.name = name; + + this.jsii_fqn = jsii_fqn; + this.members = []; } - public openBlock(...args: any[]) { - this.buffer.push({method: "openBlock", args: args}); + public addMember(member: PythonItem & Writable): PythonItem { + this.members.push(member); + return member; } - public closeBlock(...args: any[]) { - this.buffer.push({method: "closeBlock", args: args}); + public requiredTypes(): string[] { + const types: string[] = []; + + for (let member of this.members) { + types.push(...member.requiredTypes()); + } + + return types; } public write(code: CodeMaker) { - // Before we do Anything, we need to write out our module headers, this is where - // we handle stuff like imports, any required initialization, etc. + // TODO: Data Types? + // TODO: Bases + + code.openBlock(`class ${this.name}(metaclass=_JSIIMeta, jsii_type="${this.jsii_fqn}")`); + if (this.members.length > 0) { + for (let member of this.members) { + member.write(code); + } + } else { + code.line("pass"); + } + code.closeBlock(); + } +} + + + +class Module { + + readonly name: string; + readonly assembly?: spec.Assembly; + readonly assemblyFilename?: string; + + private members: ModuleMember[]; + + constructor(ns: string, assembly?: [spec.Assembly, string]) { + this.name = ns; + + if (assembly != undefined) { + this.assembly = assembly[0]; + this.assemblyFilename = assembly[1]; + } + + this.members = []; + } + + public addMember(member: ModuleMember): ModuleMember { + this.members.push(member); + + return member; + } + + public write(code: CodeMaker) { + // Before we write anything else, we need to write out our module headers, this + // is where we handle stuff like imports, any required initialization, etc. code.line(this.generateImportFrom("jsii.compat", ["Protocol"])); code.line( this.generateImportFrom( @@ -310,7 +574,8 @@ class Module { ); // Go over all of the modules that we need to import, and import them. - for (let [idx, modName] of this.importedModules.sort().entries()) { + // for (let [idx, modName] of this.importedModules.sort().entries()) { + for (let [idx, modName] of this.getRequiredTypeImports().sort().entries()) { if (idx == 0) { code.line(); } @@ -320,22 +585,83 @@ class Module { // Determine if we need to write out the kernel load line. if (this.assembly && this.assemblyFilename) { - this.exportName("__jsii_assembly__"); code.line(`__jsii_assembly__ = _JSIIAssembly.load("${this.assembly.name}", "${this.assembly.version}", __name__, "${this.assemblyFilename}")`); } // Now that we've gotten all of the module header stuff done, we need to go // through and actually write out the meat of our module. - for (let buffered of this.buffer) { - let methodName = (buffered as any)["method"] as string; - let args = (buffered as any)["args"] as any[]; + // TODO: We need to handle sorting our members prior to writing them out, so + // that we write out anything that a particular member depends on, prior + // to actually writing out that member. + for (let member of this.members) { + member.write(code); + } + + // // Whatever names we've exported, we'll write out our __all__ that lists them. + code.line(`__all__ = [${this.getExportedNames().map(s => `"${s}"`).join(", ")}]`); + } + + private getRequiredTypeImports(): string[] { + const types: string[] = [] + const imports: string[] = []; + + // Compute a list of all of of the types that + for (let member of this.members) { + types.push(...member.requiredTypes()); + } + + // Go over our types, and generate a list of imports that we need to import for + // our module. + for (let type of types) { + // If we split our types by any of the "special" characters that can't appear in + // identifiers (like "[],") then we will get a list of all of the identifiers, + // no matter how nested they are. The downside is we might get trailing/leading + // spaces or empty items so we'll need to trim and filter this list. + const subTypes = type.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s != ""); + + // Loop over all of the types we've discovered, and check them for being + // importable + for (let subType of subTypes) { + // For built in types, we don't need to do anything, and can just move on. + if (PYTHON_BUILTIN_TYPES.indexOf(subType) > -1) { continue; } + + let [, typeModule] = subType.match(/(.*)\.(.*)/) as any[]; + + // Given a name like foo.bar.Frob, we want to import the module that Frob exists + // in. Given that all classes exported by JSII have to be pascal cased, and all + // of our imports are snake cases, this should be safe. We're going to double + // check this though, to ensure that our importable here is safe. + if (typeModule != typeModule.toLowerCase()) { + // If we ever get to this point, we'll need to implment aliasing for our + // imports. + throw new Error(`Type module is not lower case: '${typeModule}'`); + } + + // We only want to actually import the type for this module, if it isn't the + // module that we're currently in, otherwise we'll jus rely on the module scope + // to make the name available to us. + if (typeModule != this.name && imports.indexOf(typeModule) == -1) { + imports.push(typeModule); + } + } + } + + return imports; + } + + private getExportedNames(): string[] { + // We assume that anything that is a member of this module, will be exported by + // this module. + const exportedNames = this.members.map(m => m.name); - (code as any)[methodName](...args); + // If this module will be outputting the Assembly, then we also want to export + // our assembly variable. + if (this.assembly && this.assemblyFilename) { + exportedNames.push("__jsii_assembly__"); } - // Whatever names we've exported, we'll write out our __all__ that lists them. - const stringifiedExportedNames = this.exportedNames.sort().map(s => `"${s}"`); - code.line(`__all__ = [${stringifiedExportedNames.join(", ")}]`); + return exportedNames.sort(); + } private generateImportFrom(from: string, names: string[]): string { @@ -350,6 +676,7 @@ class Module { class PythonGenerator extends Generator { + private currentMember?: ModuleMember; private modules: Module[]; private moduleStack: Module[]; @@ -359,6 +686,7 @@ class PythonGenerator extends Generator { this.code.openBlockFormatter = s => `${s}:`; this.code.closeBlockFormatter = _s => ""; + this.currentMember = undefined; this.modules = []; this.moduleStack = []; } @@ -446,130 +774,101 @@ class PythonGenerator extends Generator { } protected onBeginClass(cls: spec.ClassType, abstract: boolean | undefined) { - const clsName = toPythonIdentifier(cls.name); + const currentModule = this.currentModule(); // TODO: Figure out what to do with abstract here. abstract; - this.currentModule().exportName(clsName); - this.currentModule().openBlock(`class ${clsName}(metaclass=_JSIIMeta, jsii_type="${cls.fqn}")`); + this.currentMember = currentModule.addMember( + new Class( + currentModule.name, + toPythonIdentifier(cls.name), + cls.fqn + ) + ); } - protected onEndClass(cls: spec.ClassType) { - const currentModule = this.currentModule(); - - // If our class does not have any members, then we need to emit a pass statement - // to give it *some* kind of a body. - if (cls.properties == undefined && cls.methods == undefined) { - currentModule.line("pass"); - } - - currentModule.closeBlock(); + protected onEndClass(_cls: spec.ClassType) { + this.currentMember = undefined; } protected onStaticMethod(_cls: spec.ClassType, method: spec.Method) { - // TODO: Handle the case where the Python name and the JSII name differ. - const methodName = toPythonIdentifier(method.name!); - - this.currentModule().line("@_jsii_classmethod"); - this.emitPythonMethod(methodName, "cls", method.parameters, method.returns); + this.currentMember!.addMember( + new StaticMethod( + this.currentModule().name, + toPythonIdentifier(method.name!), + method.parameters || [], + method.returns + ) + ); } protected onMethod(_cls: spec.ClassType, method: spec.Method) { - // TODO: Handle the case where the Python name and the JSII name differ. - const methodName = toPythonIdentifier(method.name!); - - this.currentModule().line("@_jsii_method"); - this.emitPythonMethod(methodName, "self", method.parameters, method.returns); + this.currentMember!.addMember( + new Method( + this.currentModule().name, + toPythonIdentifier(method.name!), + method.parameters || [], + method.returns + ) + ); } protected onStaticProperty(_cls: spec.ClassType, prop: spec.Property) { - // TODO: Handle the case where the Python name and the JSII name differ. - const propertyName = toPythonIdentifier(prop.name!); - - // TODO: Properties have a bunch of states, they can have getters and setters - // we need to better handle all of these cases. - this.currentModule().line("@_jsii_classproperty"); - this.emitPythonMethod(propertyName, "self", [], prop.type); + this.currentMember!.addMember( + new StaticProperty( + this.currentModule().name, + toPythonIdentifier(prop.name!), + prop.type, + ) + ); } protected onProperty(_cls: spec.ClassType, prop: spec.Property) { - // TODO: Handle the case where the Python name and the JSII name differ. - const propertyName = toPythonIdentifier(prop.name!); - - this.currentModule().line("@_jsii_property"); - this.emitPythonMethod(propertyName, "self", [], prop.type); + this.currentMember!.addMember( + new Property( + this.currentModule().name, + toPythonIdentifier(prop.name!), + prop.type, + ) + ); } protected onBeginInterface(ifc: spec.InterfaceType) { const currentModule = this.currentModule(); - const interfaceName = this.toPythonIdentifier(ifc.name); - let interfaceBases: string[] = []; - for (let interfaceBase of (ifc.interfaces || [])) { - let interfaceBaseType = this.toPythonType(interfaceBase); - interfaceBases.push(this.formatPythonType(interfaceBaseType, true, currentModule.name)); - currentModule.maybeImportType(interfaceBaseType); - } - interfaceBases.push("_Protocol"); - - // TODO: Data Type - - currentModule.exportName(interfaceName); - currentModule.openBlock(`class ${interfaceName}(${interfaceBases.join(",")})`); + this.currentMember = currentModule.addMember( + new Interface( + currentModule.name, + toPythonIdentifier(ifc.name), + (ifc.interfaces || []).map(i => toPythonType(i)) + ) + ); } - protected onEndInterface(ifc: spec.InterfaceType) { - const currentModule = this.currentModule(); - - // If our interface does not have any members, then we need to emit a pass - // statement to give it *some* kind of a body. - if (ifc.properties == undefined && ifc.methods == undefined) { - currentModule.line("pass"); - } - - currentModule.closeBlock(); + protected onEndInterface(_ifc: spec.InterfaceType) { + this.currentMember = undefined; } protected onInterfaceMethod(_ifc: spec.InterfaceType, method: spec.Method) { - const methodName = toPythonIdentifier(method.name!); - - this.emitPythonMethod(methodName, "self", method.parameters, method.returns); + this.currentMember!.addMember( + new InterfaceMethod( + this.currentModule().name, + toPythonIdentifier(method.name!), + method.parameters || [], + method.returns + ) + ); } protected onInterfaceProperty(_ifc: spec.InterfaceType, prop: spec.Property) { - const propertyName = toPythonIdentifier(prop.name!); - - this.currentModule().line("@property"); - this.emitPythonMethod(propertyName, "self", [], prop.type); - } - - private emitPythonMethod(name?: string, implicitParam?: string, params: spec.Parameter[] = [], returns?: spec.TypeReference) { - let module = this.currentModule(); - - // TODO: Handle imports (if needed) for type. - const returnType = returns ? toPythonType(returns) : "None"; - module.maybeImportType(returnType); - - - // We need to turn a list of JSII parameters, into Python style arguments with - // gradual typing, so we'll have to iterate over the list of parameters, and - // build the list, converting as we go. - // TODO: Handle imports (if needed) for all of these types. - - let pythonParams: string[] = implicitParam ? [implicitParam] : []; - for (let param of params) { - let paramName = toPythonIdentifier(param.name); - let paramType = toPythonType(param.type); - - module.maybeImportType(paramType); - - pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, module.name)}`); - } - - module.openBlock(`def ${name}(${pythonParams.join(", ")}) -> ${formatPythonType(returnType, false, module.name)}`); - module.line("..."); - module.closeBlock(); + this.currentMember!.addMember( + new InterfaceProperty( + this.currentModule().name, + toPythonIdentifier(prop.name!), + prop.type, + ) + ); } private currentModule(): Module { From 6f636dd85756ea657baf348c6ccf7b4472d290a6 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 30 Sep 2018 02:12:48 -0400 Subject: [PATCH 34/88] Remove code duplication amongst the many Method types --- packages/jsii-pacmak/lib/targets/python.ts | 121 ++++----------------- 1 file changed, 21 insertions(+), 100 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 700fa7cc47..97c817a4c6 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -197,11 +197,14 @@ interface PythonItem { type ModuleMember = PythonItem & WithMembers & Writable; -class InterfaceMethod implements PythonItem, Writable { +class BaseMethod implements PythonItem, Writable { public readonly moduleName: string; public readonly name: string; + protected readonly decorator?: string; + protected readonly implicitParameter: string; + private readonly parameters: spec.Parameter[]; private readonly returns?: spec.TypeReference; @@ -229,7 +232,7 @@ class InterfaceMethod implements PythonItem, Writable { // gradual typing, so we'll have to iterate over the list of parameters, and // build the list, converting as we go. // TODO: Handle imports (if needed) for all of these types. - let pythonParams: string[] = ["self"] + let pythonParams: string[] = [this.implicitParameter]; for (let param of this.parameters) { const paramName = toPythonIdentifier(param.name); const paramType = toPythonType(param.type); @@ -237,6 +240,10 @@ class InterfaceMethod implements PythonItem, Writable { pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, this.moduleName)}`); } + if (this.decorator != undefined) { + code.line(`@${this.decorator}`); + } + code.openBlock(`def ${this.name}(${pythonParams.join(", ")}) -> ${formatPythonType(returnType, false, this.moduleName)}`); code.line("..."); code.closeBlock(); @@ -247,6 +254,12 @@ class InterfaceMethod implements PythonItem, Writable { } } + +class InterfaceMethod extends BaseMethod { + protected readonly implicitParameter: string = "self"; +} + + class InterfaceProperty implements PythonItem, Writable { public readonly moduleName: string; @@ -324,107 +337,15 @@ class Interface implements ModuleMember { } -class StaticMethod implements PythonItem, Writable { - - public readonly moduleName: string; - public readonly name: string; - - private readonly parameters: spec.Parameter[]; - private readonly returns?: spec.TypeReference; - - constructor(moduleName: string, name: string, parameters: spec.Parameter[], returns?: spec.TypeReference) { - this.moduleName = moduleName; - this.name = name; - this.parameters = parameters; - this.returns = returns; - } - - public requiredTypes(): string[] { - const types: string[] = [this.getReturnType(this.returns)]; - - for (let param of this.parameters) { - types.push(toPythonType(param.type)); - } - - return types; - } - - public write(code: CodeMaker) { - const returnType = this.getReturnType(this.returns); - - // We need to turn a list of JSII parameters, into Python style arguments with - // gradual typing, so we'll have to iterate over the list of parameters, and - // build the list, converting as we go. - // TODO: Handle imports (if needed) for all of these types. - let pythonParams: string[] = ["cls"] - for (let param of this.parameters) { - const paramName = toPythonIdentifier(param.name); - const paramType = toPythonType(param.type); - - pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, this.moduleName)}`); - } - - code.line("@_jsii_classmethod"); - code.openBlock(`def ${this.name}(${pythonParams.join(", ")}) -> ${formatPythonType(returnType, false, this.moduleName)}`); - code.line("..."); - code.closeBlock(); - } - - private getReturnType(type?: spec.TypeReference): string { - return type ? toPythonType(type) : "None"; - } +class StaticMethod extends BaseMethod { + protected readonly decorator?: string = "_jsii_classmethod"; + protected readonly implicitParameter: string = "cls"; } -class Method implements PythonItem, Writable { - - public readonly moduleName: string; - public readonly name: string; - - private readonly parameters: spec.Parameter[]; - private readonly returns?: spec.TypeReference; - - constructor(moduleName: string, name: string, parameters: spec.Parameter[], returns?: spec.TypeReference) { - this.moduleName = moduleName; - this.name = name; - this.parameters = parameters; - this.returns = returns; - } - - public requiredTypes(): string[] { - const types: string[] = [this.getReturnType(this.returns)]; - - for (let param of this.parameters) { - types.push(toPythonType(param.type)); - } - - return types; - } - - public write(code: CodeMaker) { - const returnType = this.getReturnType(this.returns); - - // We need to turn a list of JSII parameters, into Python style arguments with - // gradual typing, so we'll have to iterate over the list of parameters, and - // build the list, converting as we go. - // TODO: Handle imports (if needed) for all of these types. - let pythonParams: string[] = ["self"] - for (let param of this.parameters) { - const paramName = toPythonIdentifier(param.name); - const paramType = toPythonType(param.type); - - pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, this.moduleName)}`); - } - - code.line("@_jsii_method"); - code.openBlock(`def ${this.name}(${pythonParams.join(", ")}) -> ${formatPythonType(returnType, false, this.moduleName)}`); - code.line("..."); - code.closeBlock(); - } - - private getReturnType(type?: spec.TypeReference): string { - return type ? toPythonType(type) : "None"; - } +class Method extends BaseMethod { + protected readonly decorator?: string = "_jsii_method"; + protected readonly implicitParameter: string = "self"; } From 78d986dd61ac980964883797972131992e7cb174 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 30 Sep 2018 02:22:57 -0400 Subject: [PATCH 35/88] Remove duplication from property types --- packages/jsii-pacmak/lib/targets/python.ts | 80 +++++++--------------- 1 file changed, 24 insertions(+), 56 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 97c817a4c6..677b24a3d4 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -255,15 +255,14 @@ class BaseMethod implements PythonItem, Writable { } -class InterfaceMethod extends BaseMethod { - protected readonly implicitParameter: string = "self"; -} - - -class InterfaceProperty implements PythonItem, Writable { +class BaseProperty implements PythonItem, Writable { public readonly moduleName: string; public readonly name: string; + + protected readonly decorator: string; + protected readonly implicitParameter: string; + private readonly type: spec.TypeReference; constructor(moduleName: string, name: string, type: spec.TypeReference) { @@ -279,14 +278,25 @@ class InterfaceProperty implements PythonItem, Writable { public write(code: CodeMaker) { const returnType = toPythonType(this.type); - code.line("@property"); - code.openBlock(`def ${this.name}(self) -> ${formatPythonType(returnType, false, this.moduleName)}`); + code.line(`@${this.decorator}`); + code.openBlock(`def ${this.name}(${this.implicitParameter}) -> ${formatPythonType(returnType, false, this.moduleName)}`); code.line("..."); code.closeBlock(); } } +class InterfaceMethod extends BaseMethod { + protected readonly implicitParameter: string = "self"; +} + + +class InterfaceProperty extends BaseProperty { + protected readonly decorator: string = "property"; + protected readonly implicitParameter: string = "self"; +} + + class Interface implements ModuleMember { public readonly moduleName: string; @@ -349,57 +359,15 @@ class Method extends BaseMethod { } -class StaticProperty implements PythonItem, Writable { - - public readonly moduleName: string; - public readonly name: string; - private readonly type: spec.TypeReference; - - constructor(moduleName: string, name: string, type: spec.TypeReference) { - this.moduleName = moduleName; - this.name = name; - this.type = type; - } - - public requiredTypes(): string[] { - return [toPythonType(this.type)]; - } - - public write(code: CodeMaker) { - const returnType = toPythonType(this.type); - - code.line("@_jsii_classproperty"); - code.openBlock(`def ${this.name}(cls) -> ${formatPythonType(returnType, false, this.moduleName)}`); - code.line("..."); - code.closeBlock(); - } +class StaticProperty extends BaseProperty { + protected readonly decorator: string = "_jsii_classproperty"; + protected readonly implicitParameter: string = "cls"; } -class Property implements PythonItem, Writable { - - public readonly moduleName: string; - public readonly name: string; - private readonly type: spec.TypeReference; - - constructor(moduleName: string, name: string, type: spec.TypeReference) { - this.moduleName = moduleName; - this.name = name; - this.type = type; - } - - public requiredTypes(): string[] { - return [toPythonType(this.type)]; - } - - public write(code: CodeMaker) { - const returnType = toPythonType(this.type); - - code.line("@_jsii_property"); - code.openBlock(`def ${this.name}(self) -> ${formatPythonType(returnType, false, this.moduleName)}`); - code.line("..."); - code.closeBlock(); - } +class Property extends BaseProperty { + protected readonly decorator: string = "_jsii_property"; + protected readonly implicitParameter: string = "self"; } From ea7736309f02e0c36dc0941b4019a12a5f81ae8c Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 30 Sep 2018 02:53:13 -0400 Subject: [PATCH 36/88] Ensure we generate inherited from classes before we inherit from them --- packages/jsii-pacmak/lib/targets/python.ts | 69 ++++++++++++++++++++-- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 677b24a3d4..f8a01a7f08 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -176,6 +176,54 @@ function formatPythonType(type: string, forwardReference: boolean = false, modul return type; } +function sortMembers(sortable: (PythonItem & WithDependencies)[]): (PythonItem & WithDependencies)[] { + // We're going to take a copy of our sortable item, because it'll make it easier if + // this method doesn't have side effects. + sortable = sortable.slice(); + + // Actually sort our now copied array. + sortable.sort((first, second): number => { + // There are 4 possible states that the first and second members could be in: + // 1. Neither member depends on the other. + // 2. The left member depends on the right member. + // 3. The right member depends on the left member. + // 4. The two members depends on each other. + // + // In the case of (1), we can't determine ordering by using the dependencies + // so we'll move along and fall back to another ordering mechanism. + // + // In the case of (2), then we want the right member to come first, so that + // we render it prior to the left member. The opposite is true for (3). + // + // In the case of (4), we've got a cyclic dependency, and we have to error + // out because there's no way for us to represent that. + if (first.depends_on.indexOf(second.name) > -1 && second.depends_on.indexOf(first.name) > -1) { + throw new Error(`${first.name} and ${second.name} have a cyclic dependency and cannot be rendered.`); + } else if (first.depends_on.indexOf(second.name)) { + return 1; + } else if (second.depends_on.indexOf(first.name)) { + return -1; + } + + // If we've gotten here, then the two members given to us do not depend on + // each other. Perhaps in the future we could be smarter and try to do a + // non mandatory dependency so Type Hints might not need forward references + // in *every* case... however for now we'll just do the simpliest thing, and + // sort the two items alphabetically. + if (first.name < second.name) { + return -1; + } else if (first.name > second.name) { + return 1; + } + + // Finally, if we've gotten all the way here, then the two members must + // somehow be equal, and we'll just have to return a 0. + return 0; + }); + + return sortable; +} + interface Writable { write(code: CodeMaker): void; @@ -187,6 +235,10 @@ interface WithMembers { } +interface WithDependencies { + readonly depends_on: string[]; +} + interface PythonItem { readonly name: string; @@ -194,7 +246,7 @@ interface PythonItem { } -type ModuleMember = PythonItem & WithMembers & Writable; +type ModuleMember = PythonItem & WithMembers & WithDependencies & Writable; class BaseMethod implements PythonItem, Writable { @@ -308,11 +360,15 @@ class Interface implements ModuleMember { constructor(moduleName: string, name: string, bases: string[]) { this.moduleName = moduleName; this.name = name; - this.bases = bases; + this.members = []; } + get depends_on(): string[] { + return this.bases; + } + public addMember(member: PythonItem & Writable): PythonItem { this.members.push(member); return member; @@ -386,6 +442,10 @@ class Class implements ModuleMember { this.members = []; } + get depends_on(): string[] { + return []; + } + public addMember(member: PythonItem & Writable): PythonItem { this.members.push(member); return member; @@ -479,10 +539,7 @@ class Module { // Now that we've gotten all of the module header stuff done, we need to go // through and actually write out the meat of our module. - // TODO: We need to handle sorting our members prior to writing them out, so - // that we write out anything that a particular member depends on, prior - // to actually writing out that member. - for (let member of this.members) { + for (let member of (sortMembers(this.members)) as ModuleMember[]) { member.write(code); } From 4bf7672f06577e8502827c526ed42c0e219c0be8 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 30 Sep 2018 03:46:42 -0400 Subject: [PATCH 37/88] Re-enable tslint on python.ts --- packages/jsii-pacmak/lib/targets/python.ts | 193 ++++++++++----------- 1 file changed, 87 insertions(+), 106 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index f8a01a7f08..cb7224be88 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -1,11 +1,10 @@ -/* tslint:disable */ import path = require('path'); import util = require('util'); +import { CodeMaker, toSnakeCase } from 'codemaker'; import * as spec from 'jsii-spec'; import { Generator, GeneratorOptions } from '../generator'; import { Target, TargetOptions } from '../target'; -import { CodeMaker, toSnakeCase } from 'codemaker'; import { shell } from '../util'; export default class Python extends Target { @@ -31,22 +30,21 @@ export default class Python extends Target { // ################## // # CODE GENERATOR # // ################## -const debug = function(o: any) { +const debug = (o: any) => { + // tslint:disable-next-line:no-console console.log(util.inspect(o, false, null, true)); -} - +}; -const PYTHON_BUILTIN_TYPES = ["bool", "str", "None"] +const PYTHON_BUILTIN_TYPES = ["bool", "str", "None"]; const PYTHON_KEYWORDS = [ "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", "continue", "def", "del", "elif", "else", "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", "return", "try", "while", "with", "yield" -] +]; - -function toPythonModuleName(name: string): string { +const toPythonModuleName = (name: string): string => { if (name.match(/^@[^/]+\/[^/]+$/)) { name = name.replace(/^@/g, ""); name = name.replace(/\//g, "."); @@ -55,10 +53,9 @@ function toPythonModuleName(name: string): string { name = toSnakeCase(name.replace(/-/g, "_")); return name; -} - +}; -function toPythonModuleFilename(name: string): string { +const toPythonModuleFilename = (name: string): string => { if (name.match(/^@[^/]+\/[^/]+$/)) { name = name.replace(/^@/g, ""); name = name.replace(/\//g, "."); @@ -67,24 +64,21 @@ function toPythonModuleFilename(name: string): string { name = name.replace(/\./g, "/"); return name; -} - +}; -function toPythonPackageName(name: string): string { +const toPythonPackageName = (name: string): string => { return toPythonModuleName(name).replace(/_/g, "-"); -} +}; - -function toPythonIdentifier(name: string): string { +const toPythonIdentifier = (name: string): string => { if (PYTHON_KEYWORDS.indexOf(name) > -1) { return name + "_"; } return name; -} - +}; -function toPythonType(typeref: spec.TypeReference): string { +const toPythonType = (typeref: spec.TypeReference): string => { if (spec.isPrimitiveTypeReference(typeref)) { return toPythonPrimitive(typeref.primitive); } else if (spec.isCollectionTypeReference(typeref)) { @@ -100,9 +94,9 @@ function toPythonType(typeref: spec.TypeReference): string { } else { throw new Error("Invalid type reference: " + JSON.stringify(typeref)); } -} +}; -function toPythonCollection(ref: spec.CollectionTypeReference) { +const toPythonCollection = (ref: spec.CollectionTypeReference) => { const elementPythonType = toPythonType(ref.collection.elementtype); switch (ref.collection.kind) { case spec.CollectionKind.Array: return `typing.List[${elementPythonType}]`; @@ -110,44 +104,41 @@ function toPythonCollection(ref: spec.CollectionTypeReference) { default: throw new Error(`Unsupported collection kind: ${ref.collection.kind}`); } -} +}; - -function toPythonPrimitive(primitive: spec.PrimitiveType): string { +const toPythonPrimitive = (primitive: spec.PrimitiveType): string => { switch (primitive) { case spec.PrimitiveType.Boolean: return "bool"; - case spec.PrimitiveType.Date: return "dateetime.datetime"; - case spec.PrimitiveType.Json: return "typing.Mapping[typing.Any, typing.Any]"; - case spec.PrimitiveType.Number: return "numbers.Number"; - case spec.PrimitiveType.String: return "str"; - case spec.PrimitiveType.Any: return "typing.Any"; + case spec.PrimitiveType.Date: return "dateetime.datetime"; + case spec.PrimitiveType.Json: return "typing.Mapping[typing.Any, typing.Any]"; + case spec.PrimitiveType.Number: return "numbers.Number"; + case spec.PrimitiveType.String: return "str"; + case spec.PrimitiveType.Any: return "typing.Any"; default: throw new Error("Unknown primitive type: " + primitive); } -} +}; - -function toPythonFQN(name: string): string { +const toPythonFQN = (name: string): string => { return name.split(".").map((cur, idx, arr) => { - if (idx == arr.length - 1) { + if (idx === arr.length - 1) { return toPythonIdentifier(cur); } else { return toPythonModuleName(cur); } }).join("."); -} - +}; -function formatPythonType(type: string, forwardReference: boolean = false, moduleName: string) { +const formatPythonType = (type: string, forwardReference: boolean = false, moduleName: string) => { // If we split our types by any of the "special" characters that can't appear in // identifiers (like "[],") then we will get a list of all of the identifiers, // no matter how nested they are. The downside is we might get trailing/leading // spaces or empty items so we'll need to trim and filter this list. - const types = type.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s != ""); + const types = type.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s !== ""); - for (let innerType of types) { + for (const innerType of types) { // Built in types do not need formatted in any particular way. - if(PYTHON_BUILTIN_TYPES.indexOf(innerType) > -1) { + if (PYTHON_BUILTIN_TYPES.indexOf(innerType) > -1) { continue; } @@ -174,9 +165,9 @@ function formatPythonType(type: string, forwardReference: boolean = false, modul } return type; -} +}; -function sortMembers(sortable: (PythonItem & WithDependencies)[]): (PythonItem & WithDependencies)[] { +const sortMembers = (sortable: Array): Array => { // We're going to take a copy of our sortable item, because it'll make it easier if // this method doesn't have side effects. sortable = sortable.slice(); @@ -222,33 +213,27 @@ function sortMembers(sortable: (PythonItem & WithDependencies)[]): (PythonItem & }); return sortable; -} - +}; interface Writable { write(code: CodeMaker): void; } - interface WithMembers { addMember(member: PythonItem): PythonItem; } - interface WithDependencies { readonly depends_on: string[]; } - interface PythonItem { readonly name: string; requiredTypes(): string[]; } - type ModuleMember = PythonItem & WithMembers & WithDependencies & Writable; - class BaseMethod implements PythonItem, Writable { public readonly moduleName: string; @@ -270,7 +255,7 @@ class BaseMethod implements PythonItem, Writable { public requiredTypes(): string[] { const types: string[] = [this.getReturnType(this.returns)]; - for (let param of this.parameters) { + for (const param of this.parameters) { types.push(toPythonType(param.type)); } @@ -284,15 +269,15 @@ class BaseMethod implements PythonItem, Writable { // gradual typing, so we'll have to iterate over the list of parameters, and // build the list, converting as we go. // TODO: Handle imports (if needed) for all of these types. - let pythonParams: string[] = [this.implicitParameter]; - for (let param of this.parameters) { + const pythonParams: string[] = [this.implicitParameter]; + for (const param of this.parameters) { const paramName = toPythonIdentifier(param.name); const paramType = toPythonType(param.type); pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, this.moduleName)}`); } - if (this.decorator != undefined) { + if (this.decorator !== undefined) { code.line(`@${this.decorator}`); } @@ -306,7 +291,6 @@ class BaseMethod implements PythonItem, Writable { } } - class BaseProperty implements PythonItem, Writable { public readonly moduleName: string; @@ -337,25 +321,22 @@ class BaseProperty implements PythonItem, Writable { } } - class InterfaceMethod extends BaseMethod { protected readonly implicitParameter: string = "self"; } - class InterfaceProperty extends BaseProperty { protected readonly decorator: string = "property"; protected readonly implicitParameter: string = "self"; } - class Interface implements ModuleMember { public readonly moduleName: string; public readonly name: string; private bases: string[]; - private members: (PythonItem & Writable)[]; + private members: Array; constructor(moduleName: string, name: string, bases: string[]) { this.moduleName = moduleName; @@ -377,7 +358,7 @@ class Interface implements ModuleMember { public requiredTypes(): string[] { const types = this.bases.slice(); - for (let member of this.members) { + for (const member of this.members) { types.push(...member.requiredTypes()); } @@ -392,7 +373,7 @@ class Interface implements ModuleMember { code.openBlock(`class ${this.name}(${interfaceBases.join(",")})`); if (this.members.length > 0) { - for (let member of this.members) { + for (const member of this.members) { member.write(code); } } else { @@ -402,43 +383,38 @@ class Interface implements ModuleMember { } } - class StaticMethod extends BaseMethod { protected readonly decorator?: string = "_jsii_classmethod"; protected readonly implicitParameter: string = "cls"; } - class Method extends BaseMethod { protected readonly decorator?: string = "_jsii_method"; protected readonly implicitParameter: string = "self"; } - class StaticProperty extends BaseProperty { protected readonly decorator: string = "_jsii_classproperty"; protected readonly implicitParameter: string = "cls"; } - class Property extends BaseProperty { protected readonly decorator: string = "_jsii_property"; protected readonly implicitParameter: string = "self"; } - class Class implements ModuleMember { public readonly moduleName: string; public readonly name: string; - private jsii_fqn: string; - private members: (PythonItem & Writable)[]; + private jsiiFQN: string; + private members: Array; - constructor(moduleName: string, name: string, jsii_fqn: string) { + constructor(moduleName: string, name: string, jsiiFQN: string) { this.moduleName = moduleName; this.name = name; - this.jsii_fqn = jsii_fqn; + this.jsiiFQN = jsiiFQN; this.members = []; } @@ -454,7 +430,7 @@ class Class implements ModuleMember { public requiredTypes(): string[] { const types: string[] = []; - for (let member of this.members) { + for (const member of this.members) { types.push(...member.requiredTypes()); } @@ -465,9 +441,9 @@ class Class implements ModuleMember { // TODO: Data Types? // TODO: Bases - code.openBlock(`class ${this.name}(metaclass=_JSIIMeta, jsii_type="${this.jsii_fqn}")`); + code.openBlock(`class ${this.name}(metaclass=_JSIIMeta, jsii_type="${this.jsiiFQN}")`); if (this.members.length > 0) { - for (let member of this.members) { + for (const member of this.members) { member.write(code); } } else { @@ -477,20 +453,18 @@ class Class implements ModuleMember { } } - - class Module { - readonly name: string; - readonly assembly?: spec.Assembly; - readonly assemblyFilename?: string; + public readonly name: string; + public readonly assembly?: spec.Assembly; + public readonly assemblyFilename?: string; private members: ModuleMember[]; constructor(ns: string, assembly?: [spec.Assembly, string]) { this.name = ns; - if (assembly != undefined) { + if (assembly !== undefined) { this.assembly = assembly[0]; this.assemblyFilename = assembly[1]; } @@ -524,8 +498,8 @@ class Module { // Go over all of the modules that we need to import, and import them. // for (let [idx, modName] of this.importedModules.sort().entries()) { - for (let [idx, modName] of this.getRequiredTypeImports().sort().entries()) { - if (idx == 0) { + for (const [idx, modName] of this.getRequiredTypeImports().sort().entries()) { + if (idx === 0) { code.line(); } @@ -534,12 +508,18 @@ class Module { // Determine if we need to write out the kernel load line. if (this.assembly && this.assemblyFilename) { - code.line(`__jsii_assembly__ = _JSIIAssembly.load("${this.assembly.name}", "${this.assembly.version}", __name__, "${this.assemblyFilename}")`); + code.line( + `__jsii_assembly__ = _JSIIAssembly.load(` + + `"${this.assembly.name}", ` + + `"${this.assembly.version}", ` + + `__name__, ` + + `"${this.assemblyFilename}")` + ); } // Now that we've gotten all of the module header stuff done, we need to go // through and actually write out the meat of our module. - for (let member of (sortMembers(this.members)) as ModuleMember[]) { + for (const member of (sortMembers(this.members)) as ModuleMember[]) { member.write(code); } @@ -548,36 +528,36 @@ class Module { } private getRequiredTypeImports(): string[] { - const types: string[] = [] + const types: string[] = []; const imports: string[] = []; // Compute a list of all of of the types that - for (let member of this.members) { + for (const member of this.members) { types.push(...member.requiredTypes()); } // Go over our types, and generate a list of imports that we need to import for // our module. - for (let type of types) { + for (const type of types) { // If we split our types by any of the "special" characters that can't appear in // identifiers (like "[],") then we will get a list of all of the identifiers, // no matter how nested they are. The downside is we might get trailing/leading // spaces or empty items so we'll need to trim and filter this list. - const subTypes = type.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s != ""); + const subTypes = type.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s !== ""); // Loop over all of the types we've discovered, and check them for being // importable - for (let subType of subTypes) { + for (const subType of subTypes) { // For built in types, we don't need to do anything, and can just move on. if (PYTHON_BUILTIN_TYPES.indexOf(subType) > -1) { continue; } - let [, typeModule] = subType.match(/(.*)\.(.*)/) as any[]; + const [, typeModule] = subType.match(/(.*)\.(.*)/) as any[]; // Given a name like foo.bar.Frob, we want to import the module that Frob exists // in. Given that all classes exported by JSII have to be pascal cased, and all // of our imports are snake cases, this should be safe. We're going to double // check this though, to ensure that our importable here is safe. - if (typeModule != typeModule.toLowerCase()) { + if (typeModule !== typeModule.toLowerCase()) { // If we ever get to this point, we'll need to implment aliasing for our // imports. throw new Error(`Type module is not lower case: '${typeModule}'`); @@ -586,7 +566,7 @@ class Module { // We only want to actually import the type for this module, if it isn't the // module that we're currently in, otherwise we'll jus rely on the module scope // to make the name available to us. - if (typeModule != this.name && imports.indexOf(typeModule) == -1) { + if (typeModule !== this.name && imports.indexOf(typeModule) === -1) { imports.push(typeModule); } } @@ -673,7 +653,7 @@ class PythonGenerator extends Generator { this.code.line(`description="${assm.description}",`); this.code.line(`url="${assm.homepage}",`); this.code.line('package_dir={"": "src"},'); - this.code.line(`packages=[${moduleNames.map(m => `"${m}"`).join(",")}],`) + this.code.line(`packages=[${moduleNames.map(m => `"${m}"`).join(",")}],`); this.code.line(`package_data={"${topLevelModuleName}._jsii": ["*.jsii.tgz"]},`); this.code.line('python_requires=">=3.6",'); this.code.unindent(")"); @@ -689,41 +669,40 @@ class PythonGenerator extends Generator { // We also need to write out a MANIFEST.in to ensure that all of our required // files are included. - this.code.openFile("MANIFEST.in") - this.code.line("include pyproject.toml") - this.code.closeFile("MANIFEST.in") + this.code.openFile("MANIFEST.in"); + this.code.line("include pyproject.toml"); + this.code.closeFile("MANIFEST.in"); } protected onBeginNamespace(ns: string) { const moduleName = toPythonModuleName(ns); - const loadAssembly = this.assembly.name == ns ? true : false; + const loadAssembly = this.assembly.name === ns ? true : false; - let moduleArgs: any[] = []; + const moduleArgs: any[] = []; if (loadAssembly) { moduleArgs.push([this.assembly, this.getAssemblyFileName()]); } - let mod = new Module(moduleName, ...moduleArgs); + const mod = new Module(moduleName, ...moduleArgs); this.modules.push(mod); this.moduleStack.push(mod); } protected onEndNamespace(_ns: string) { - let module = this.moduleStack.pop() as Module; - let moduleFilename = path.join("src", toPythonModuleFilename(module.name), "__init__.py"); + const module = this.moduleStack.pop() as Module; + const moduleFilename = path.join("src", toPythonModuleFilename(module.name), "__init__.py"); this.code.openFile(moduleFilename); module.write(this.code); this.code.closeFile(moduleFilename); } - protected onBeginClass(cls: spec.ClassType, abstract: boolean | undefined) { + protected onBeginClass(cls: spec.ClassType, _abstract: boolean | undefined) { const currentModule = this.currentModule(); // TODO: Figure out what to do with abstract here. - abstract; this.currentMember = currentModule.addMember( new Class( @@ -817,10 +796,6 @@ class PythonGenerator extends Generator { ); } - private currentModule(): Module { - return this.moduleStack.slice(-1)[0]; - } - // Not Currently Used protected onInterfaceMethodOverload(_ifc: spec.InterfaceType, _overload: spec.Method, _originalMethod: spec.Method) { @@ -842,4 +817,10 @@ class PythonGenerator extends Generator { debug("onStaticMethodOverload"); throw new Error("Unhandled Type: StaticMethodOverload"); } + + // End Not Currently Used + + private currentModule(): Module { + return this.moduleStack.slice(-1)[0]; + } } From 99fa04dc3308a3400b99d0a4ee649c316de55679 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 30 Sep 2018 04:31:18 -0400 Subject: [PATCH 38/88] Cleanup interfaces for the nodes --- packages/jsii-pacmak/lib/targets/python.ts | 78 ++++++++++++---------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index cb7224be88..99ddeb8028 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -167,7 +167,7 @@ const formatPythonType = (type: string, forwardReference: boolean = false, modul return type; }; -const sortMembers = (sortable: Array): Array => { +const sortMembers = (sortable: PythonCollectionNode[]): PythonCollectionNode[] => { // We're going to take a copy of our sortable item, because it'll make it easier if // this method doesn't have side effects. sortable = sortable.slice(); @@ -215,26 +215,35 @@ const sortMembers = (sortable: Array): Array; + private members: PythonNode[]; constructor(moduleName: string, name: string, bases: string[]) { this.moduleName = moduleName; @@ -350,7 +359,7 @@ class Interface implements ModuleMember { return this.bases; } - public addMember(member: PythonItem & Writable): PythonItem { + public addMember(member: PythonNode): PythonNode { this.members.push(member); return member; } @@ -365,7 +374,7 @@ class Interface implements ModuleMember { return types; } - public write(code: CodeMaker) { + public emit(code: CodeMaker) { // TODO: Data Types? const interfaceBases = this.bases.map(baseType => formatPythonType(baseType, true, this.moduleName)); @@ -374,7 +383,7 @@ class Interface implements ModuleMember { code.openBlock(`class ${this.name}(${interfaceBases.join(",")})`); if (this.members.length > 0) { for (const member of this.members) { - member.write(code); + member.emit(code); } } else { code.line("pass"); @@ -403,12 +412,12 @@ class Property extends BaseProperty { protected readonly implicitParameter: string = "self"; } -class Class implements ModuleMember { +class Class implements PythonCollectionNode { public readonly moduleName: string; public readonly name: string; private jsiiFQN: string; - private members: Array; + private members: PythonNode[]; constructor(moduleName: string, name: string, jsiiFQN: string) { this.moduleName = moduleName; @@ -422,7 +431,7 @@ class Class implements ModuleMember { return []; } - public addMember(member: PythonItem & Writable): PythonItem { + public addMember(member: PythonNode): PythonNode { this.members.push(member); return member; } @@ -437,14 +446,14 @@ class Class implements ModuleMember { return types; } - public write(code: CodeMaker) { + public emit(code: CodeMaker) { // TODO: Data Types? // TODO: Bases code.openBlock(`class ${this.name}(metaclass=_JSIIMeta, jsii_type="${this.jsiiFQN}")`); if (this.members.length > 0) { for (const member of this.members) { - member.write(code); + member.emit(code); } } else { code.line("pass"); @@ -459,7 +468,7 @@ class Module { public readonly assembly?: spec.Assembly; public readonly assemblyFilename?: string; - private members: ModuleMember[]; + private members: PythonCollectionNode[]; constructor(ns: string, assembly?: [spec.Assembly, string]) { this.name = ns; @@ -472,13 +481,12 @@ class Module { this.members = []; } - public addMember(member: ModuleMember): ModuleMember { + public addMember(member: PythonCollectionNode): PythonCollectionNode { this.members.push(member); - return member; } - public write(code: CodeMaker) { + public emit(code: CodeMaker) { // Before we write anything else, we need to write out our module headers, this // is where we handle stuff like imports, any required initialization, etc. code.line(this.generateImportFrom("jsii.compat", ["Protocol"])); @@ -519,8 +527,8 @@ class Module { // Now that we've gotten all of the module header stuff done, we need to go // through and actually write out the meat of our module. - for (const member of (sortMembers(this.members)) as ModuleMember[]) { - member.write(code); + for (const member of sortMembers(this.members)) { + member.emit(code); } // // Whatever names we've exported, we'll write out our __all__ that lists them. @@ -602,7 +610,7 @@ class Module { class PythonGenerator extends Generator { - private currentMember?: ModuleMember; + private currentMember?: PythonCollectionNode; private modules: Module[]; private moduleStack: Module[]; @@ -695,7 +703,7 @@ class PythonGenerator extends Generator { const moduleFilename = path.join("src", toPythonModuleFilename(module.name), "__init__.py"); this.code.openFile(moduleFilename); - module.write(this.code); + module.emit(this.code); this.code.closeFile(moduleFilename); } From 34098504d7de6033a54fee55b2ad5236dd601673 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 30 Sep 2018 17:50:17 -0400 Subject: [PATCH 39/88] Fix sorting by dependency --- packages/jsii-pacmak/lib/targets/python.ts | 87 ++++++++++++---------- 1 file changed, 49 insertions(+), 38 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 99ddeb8028..8c6bc06266 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -167,53 +167,45 @@ const formatPythonType = (type: string, forwardReference: boolean = false, modul return type; }; +const setDifference = (setA: Set, setB: Set): Set => { + const difference = new Set(setA); + for (const elem of setB) { + difference.delete(elem); + } + return difference; +}; + const sortMembers = (sortable: PythonCollectionNode[]): PythonCollectionNode[] => { + const sorted: PythonCollectionNode[] = []; + const sortedFQNs: Set = new Set(); + // We're going to take a copy of our sortable item, because it'll make it easier if // this method doesn't have side effects. sortable = sortable.slice(); - // Actually sort our now copied array. - sortable.sort((first, second): number => { - // There are 4 possible states that the first and second members could be in: - // 1. Neither member depends on the other. - // 2. The left member depends on the right member. - // 3. The right member depends on the left member. - // 4. The two members depends on each other. - // - // In the case of (1), we can't determine ordering by using the dependencies - // so we'll move along and fall back to another ordering mechanism. - // - // In the case of (2), then we want the right member to come first, so that - // we render it prior to the left member. The opposite is true for (3). - // - // In the case of (4), we've got a cyclic dependency, and we have to error - // out because there's no way for us to represent that. - if (first.depends_on.indexOf(second.name) > -1 && second.depends_on.indexOf(first.name) > -1) { - throw new Error(`${first.name} and ${second.name} have a cyclic dependency and cannot be rendered.`); - } else if (first.depends_on.indexOf(second.name)) { - return 1; - } else if (second.depends_on.indexOf(first.name)) { - return -1; + while (sortable.length > 0) { + let idx: number | undefined; + + for (const [idx2, item] of sortable.entries()) { + if (setDifference(new Set(item.depends_on), sortedFQNs).size === 0) { + sorted.push(item); + sortedFQNs.add(item.fqn); + idx = idx2; + break; + } else { + idx = undefined; + } } - // If we've gotten here, then the two members given to us do not depend on - // each other. Perhaps in the future we could be smarter and try to do a - // non mandatory dependency so Type Hints might not need forward references - // in *every* case... however for now we'll just do the simpliest thing, and - // sort the two items alphabetically. - if (first.name < second.name) { - return -1; - } else if (first.name > second.name) { - return 1; + if (idx === undefined) { + throw new Error("Could not sort members."); + } else { + sortable.splice(idx, 1); } + } - // Finally, if we've gotten all the way here, then the two members must - // somehow be equal, and we'll just have to return a 0. - return 0; - }); - - return sortable; -}; + return sorted; + }; interface PythonNode { @@ -223,6 +215,9 @@ interface PythonNode { // The name of the given Node. readonly name: string; + // The fully qualifed name of this node. + readonly fqn: string; + // Returns a list of all of the FQN Python types that this Node requires, this // should traverse all of it's members to get the full list of all types required to // exist (i.e. be imported). @@ -261,6 +256,10 @@ class BaseMethod implements PythonNode { this.returns = returns; } + get fqn(): string { + return `${this.moduleName}.${this.name}`; + } + public requiredTypes(): string[] { const types: string[] = [this.getReturnType(this.returns)]; @@ -316,6 +315,10 @@ class BaseProperty implements PythonNode { this.type = type; } + get fqn(): string { + return `${this.moduleName}.${this.name}`; + } + public requiredTypes(): string[] { return [toPythonType(this.type)]; } @@ -355,6 +358,10 @@ class Interface implements PythonCollectionNode { this.members = []; } + get fqn(): string { + return `${this.moduleName}.${this.name}`; + } + get depends_on(): string[] { return this.bases; } @@ -427,6 +434,10 @@ class Class implements PythonCollectionNode { this.members = []; } + get fqn(): string { + return `${this.moduleName}.${this.name}`; + } + get depends_on(): string[] { return []; } From 912a593ad5d4a9997b35d097d66d8603bd4837c1 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Wed, 10 Oct 2018 13:46:43 -0400 Subject: [PATCH 40/88] Support passing arguments to the class constructor --- packages/jsii-python-runtime/src/jsii/_kernel/__init__.py | 7 +++++-- .../src/jsii/_kernel/providers/process.py | 8 +++++++- packages/jsii-python-runtime/src/jsii/runtime.py | 6 +++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py index 4bf1270923..c64e6dc08e 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py @@ -67,8 +67,11 @@ def load(self, name: str, version: str, tarball: str) -> None: self.provider.load(LoadRequest(name=name, version=version, tarball=tarball)) # TODO: Can we do protocols in typing? - def create(self, klass: Any) -> ObjRef: - return self.provider.create(CreateRequest(fqn=klass.__jsii_type__)) + def create(self, klass: Any, args: Optional[List[Any]] = None) -> ObjRef: + if args is None: + args = [] + + return self.provider.create(CreateRequest(fqn=klass.__jsii_type__, args=args)) def delete(self, ref: ObjRef) -> None: self.provider.delete(DeleteRequest(objref=ref)) diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py b/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py index 55b5808570..5087d407cd 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py @@ -92,6 +92,12 @@ def ohook(d): return d +def jdefault(obj): + if hasattr(obj, "__jsii_ref__"): + return _unstructure_ref(obj.__jsii_ref__) + raise TypeError + + class _NodeProcess: def __init__(self): self._serializer = cattr.Converter() @@ -224,7 +230,7 @@ def send( # #python-attrs/attrs#429 fixed. if "property_" in req_dict: req_dict["property"] = req_dict.pop("property_") - data = json.dumps(req_dict).encode("utf8") + data = json.dumps(req_dict, default=jdefault).encode("utf8") # Send our data, ensure that it is framed with a trailing \n self._process.stdin.write(b"%b\n" % (data,)) diff --git a/packages/jsii-python-runtime/src/jsii/runtime.py b/packages/jsii-python-runtime/src/jsii/runtime.py index f31e7e2a45..dbee37883c 100644 --- a/packages/jsii-python-runtime/src/jsii/runtime.py +++ b/packages/jsii-python-runtime/src/jsii/runtime.py @@ -84,10 +84,10 @@ def __call__(cls, *args, **kwargs): # Create our object at the JS level. # TODO: Handle args/kwargs # TODO: Handle Overrides - ref = cls.__jsii_kernel__.create(cls) + ref = cls.__jsii_kernel__.create(cls, args=args) # Create our object at the Python level. - obj = cls.__class__.from_reference(cls, ref, *args, **kwargs) + obj = cls.__class__.from_reference(cls, ref, **kwargs) # Whenever the object we're creating gets garbage collected, then we want to # delete it from the JS runtime as well. @@ -98,7 +98,7 @@ def __call__(cls, *args, **kwargs): # Instatiate our object at the Python level. if isinstance(obj, cls): - obj.__init__(*args, **kwargs) + obj.__init__(**kwargs) return obj From be62f7cd46c0cf0f933ff34d2ede6ba3a693eda2 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Wed, 10 Oct 2018 13:47:17 -0400 Subject: [PATCH 41/88] Handle FQN that contain a @ sign --- packages/jsii-python-runtime/src/jsii/_reference_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-python-runtime/src/jsii/_reference_map.py b/packages/jsii-python-runtime/src/jsii/_reference_map.py index c04d74eeaa..47bfdd49f5 100644 --- a/packages/jsii-python-runtime/src/jsii/_reference_map.py +++ b/packages/jsii-python-runtime/src/jsii/_reference_map.py @@ -28,7 +28,7 @@ def resolve(self, ref): # If we got to this point, then we didn't have a referene for this, in that case # we want to create a new instance, but we need to create it in such a way that # we don't try to recreate the type inside of the JSII interface. - class_ = _types[ref.ref.split("@", 1)[0]] + class_ = _types[ref.ref.rsplit("@", 1)[0]] return class_.__class__.from_reference(class_, ref) From 44c43f4059647a83309fb8d26dde29438eaa9063 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Wed, 10 Oct 2018 16:56:39 -0400 Subject: [PATCH 42/88] Update the JSII Provider Version Number --- .../jsii-python-runtime/src/jsii/_kernel/providers/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py b/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py index 5087d407cd..718de28e7c 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/providers/process.py @@ -219,7 +219,7 @@ def handshake(self): # TODO: Replace with proper error. assert ( - resp.hello == "jsii-runtime@0.7.6" + resp.hello == "jsii-runtime@0.7.7" ), f"Invalid JSII Runtime Version: {resp.hello!r}" def send( From bac9dc706a949c1979200f0ba220d08a33221d00 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Tue, 16 Oct 2018 20:33:23 -0400 Subject: [PATCH 43/88] Interfaces only need to declare dependencies in the same module --- packages/jsii-pacmak/lib/targets/python.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 8c6bc06266..934404246a 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -363,7 +363,7 @@ class Interface implements PythonCollectionNode { } get depends_on(): string[] { - return this.bases; + return this.bases.filter(base => base.startsWith(this.moduleName + ".")); } public addMember(member: PythonNode): PythonNode { From 379d83157d1b7d84f502da233957a004a4f29069 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Tue, 16 Oct 2018 23:43:48 -0400 Subject: [PATCH 44/88] Implement Enums in the Python generator --- packages/jsii-pacmak/lib/targets/python.ts | 92 ++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 934404246a..ee3360ea09 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -473,6 +473,73 @@ class Class implements PythonCollectionNode { } } +class Enum implements PythonCollectionNode { + public readonly moduleName: string; + public readonly name: string; + + private members: PythonNode[]; + + constructor(moduleName: string, name: string) { + this.moduleName = moduleName; + this.name = name; + this.members = []; + } + + get fqn(): string { + return `${this.moduleName}.${this.name}`; + } + + get depends_on(): string[] { + return []; + } + + public addMember(member: PythonNode): PythonNode { + this.members.push(member); + return member; + } + + public requiredTypes(): string[] { + return ["enum.Enum"]; + } + + public emit(code: CodeMaker) { + code.openBlock(`class ${this.name}(enum.Enum)`); + if (this.members.length > 0) { + for (const member of this.members) { + member.emit(code); + } + } else { + code.line("pass"); + } + code.closeBlock(); + } +} + +class EnumMember implements PythonNode { + public readonly moduleName: string; + public readonly name: string; + + private readonly value: string; + + constructor(moduleName: string, name: string, value: string) { + this.moduleName = moduleName; + this.name = name; + this.value = value; + } + + get fqn(): string { + return `${this.moduleName}.${this.name}`; + } + + public requiredTypes(): string[] { + return []; + } + + public emit(code: CodeMaker) { + code.line(`${this.name} = "${this.value}"`); + } +} + class Module { public readonly name: string; @@ -815,6 +882,31 @@ class PythonGenerator extends Generator { ); } + protected onBeginEnum(enm: spec.EnumType) { + const currentModule = this.currentModule(); + + this.currentMember = currentModule.addMember( + new Enum( + currentModule.name, + toPythonIdentifier(enm.name), + ) + ); + } + + protected onEndEnum(_enm: spec.EnumType) { + this.currentMember = undefined; + } + + protected onEnumMember(_enm: spec.EnumType, member: spec.EnumMember) { + this.currentMember!.addMember( + new EnumMember( + this.currentModule().name, + toPythonIdentifier(member.name), + member.name + ) + ); + } + // Not Currently Used protected onInterfaceMethodOverload(_ifc: spec.InterfaceType, _overload: spec.Method, _originalMethod: spec.Method) { From 4fdda680d9bd6a0ce1799ec6792930e5602e01bd Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Tue, 16 Oct 2018 23:44:59 -0400 Subject: [PATCH 45/88] Default to None if there is no result from an Invoke --- packages/jsii-python-runtime/src/jsii/_kernel/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/types.py b/packages/jsii-python-runtime/src/jsii/_kernel/types.py index 095153ac8c..961e2bdf7c 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/types.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/types.py @@ -122,7 +122,7 @@ class InvokeRequest: @attr.s(auto_attribs=True, frozen=True, slots=True) class InvokeResponse: - result: Any + result: Any = None @attr.s(auto_attribs=True, frozen=True, slots=True) From 8f93d692a6ec34b259cc231dee3a1f23fa640a59 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Wed, 17 Oct 2018 00:25:00 -0400 Subject: [PATCH 46/88] Fix a spelling mistake --- packages/jsii-pacmak/lib/targets/python.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index ee3360ea09..0275807cfb 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -109,7 +109,7 @@ const toPythonCollection = (ref: spec.CollectionTypeReference) => { const toPythonPrimitive = (primitive: spec.PrimitiveType): string => { switch (primitive) { case spec.PrimitiveType.Boolean: return "bool"; - case spec.PrimitiveType.Date: return "dateetime.datetime"; + case spec.PrimitiveType.Date: return "datetime.datetime"; case spec.PrimitiveType.Json: return "typing.Mapping[typing.Any, typing.Any]"; case spec.PrimitiveType.Number: return "numbers.Number"; case spec.PrimitiveType.String: return "str"; From 18ba6e6e41bf49121b3788df14746be54440b8d8 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 18 Oct 2018 01:27:55 -0400 Subject: [PATCH 47/88] Make sure all dependencies have been imported --- packages/jsii-pacmak/lib/targets/python.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 0275807cfb..e2b911e497 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -658,6 +658,18 @@ class Module { } } + // Add the additional dependencies that we require in order for imports to + // work correctly. These are dependencies at the JS level, isntead of at the + // Python level. + if (this.assembly !== undefined && this.assembly!.dependencies !== undefined) { + for (const depName of Object.keys(this.assembly!.dependencies!)) { + const moduleName = toPythonModuleName(depName); + if (imports.indexOf(moduleName) === -1) { + imports.push(moduleName); + } + } + } + return imports; } From 83929ef9210cbd541cc04fe0a8c9324d006d273a Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 20 Oct 2018 18:42:07 -0400 Subject: [PATCH 48/88] Magical Removal Greatly simplify the implementation by removing the magic decorators and pushing more of the work into the generated Python code. This makes the generated Python code have more noise to seeing the structure of the API but it means that there is less magic happening behind the scenes. This also makes it better for tools like MyPy since this code more more like "normal" Python code. --- packages/jsii-pacmak/lib/targets/python.ts | 94 +++++-- .../jsii-python-runtime/src/jsii/__init__.py | 31 +++ .../src/jsii/_kernel/__init__.py | 30 ++- .../src/jsii/_kernel/types.py | 20 ++ .../jsii-python-runtime/src/jsii/_runtime.py | 93 +++++++ .../jsii-python-runtime/src/jsii/python.py | 38 +++ .../jsii-python-runtime/src/jsii/runtime.py | 244 ------------------ 7 files changed, 271 insertions(+), 279 deletions(-) create mode 100644 packages/jsii-python-runtime/src/jsii/_runtime.py create mode 100644 packages/jsii-python-runtime/src/jsii/python.py delete mode 100644 packages/jsii-python-runtime/src/jsii/runtime.py diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index e2b911e497..2245940cd3 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -246,8 +246,8 @@ class BaseMethod implements PythonNode { protected readonly decorator?: string; protected readonly implicitParameter: string; - private readonly parameters: spec.Parameter[]; - private readonly returns?: spec.TypeReference; + protected readonly parameters: spec.Parameter[]; + protected readonly returns?: spec.TypeReference; constructor(moduleName: string, name: string, parameters: spec.Parameter[], returns?: spec.TypeReference) { this.moduleName = moduleName; @@ -290,10 +290,14 @@ class BaseMethod implements PythonNode { } code.openBlock(`def ${this.name}(${pythonParams.join(", ")}) -> ${formatPythonType(returnType, false, this.moduleName)}`); - code.line("..."); + this.emitBody(code); code.closeBlock(); } + protected emitBody(code: CodeMaker) { + code.line("..."); + } + private getReturnType(type?: spec.TypeReference): string { return type ? toPythonType(type) : "None"; } @@ -308,11 +312,13 @@ class BaseProperty implements PythonNode { protected readonly implicitParameter: string; private readonly type: spec.TypeReference; + private readonly immutable: boolean; - constructor(moduleName: string, name: string, type: spec.TypeReference) { + constructor(moduleName: string, name: string, type: spec.TypeReference, immutable: boolean) { this.moduleName = moduleName; this.name = name; this.type = type; + this.immutable = immutable; } get fqn(): string { @@ -328,8 +334,23 @@ class BaseProperty implements PythonNode { code.line(`@${this.decorator}`); code.openBlock(`def ${this.name}(${this.implicitParameter}) -> ${formatPythonType(returnType, false, this.moduleName)}`); - code.line("..."); + this.emitGetterBody(code); code.closeBlock(); + + if (!this.immutable) { + code.line(`@${this.name}.setter`); + code.openBlock(`def ${this.name}(${this.implicitParameter}, value: ${formatPythonType(returnType, false, this.moduleName)})`); + this.emitSetterBody(code); + code.closeBlock(); + } + } + + protected emitGetterBody(code: CodeMaker) { + code.line("..."); + } + + protected emitSetterBody(code: CodeMaker) { + code.line("..."); } } @@ -400,23 +421,56 @@ class Interface implements PythonCollectionNode { } class StaticMethod extends BaseMethod { - protected readonly decorator?: string = "_jsii_classmethod"; + protected readonly decorator?: string = "classmethod"; protected readonly implicitParameter: string = "cls"; + + protected emitBody(code: CodeMaker) { + const paramNames: string[] = []; + for (const param of this.parameters) { + paramNames.push(toPythonIdentifier(param.name)); + } + + code.line(`return jsii.sinvoke(${this.implicitParameter}, "${this.name}", [${paramNames.join(", ")}])`); + } } class Method extends BaseMethod { - protected readonly decorator?: string = "_jsii_method"; protected readonly implicitParameter: string = "self"; + + protected emitBody(code: CodeMaker) { + const paramNames: string[] = []; + for (const param of this.parameters) { + paramNames.push(toPythonIdentifier(param.name)); + } + + code.line(`return jsii.invoke(${this.implicitParameter}, "${this.name}", [${paramNames.join(", ")}])`); + } } class StaticProperty extends BaseProperty { - protected readonly decorator: string = "_jsii_classproperty"; + protected readonly decorator: string = "classproperty"; protected readonly implicitParameter: string = "cls"; + + protected emitGetterBody(code: CodeMaker) { + code.line(`return jsii.sget(${this.implicitParameter}, "${this.name}")`); + } + + protected emitSetterBody(code: CodeMaker) { + code.line(`return jsii.sset(${this.implicitParameter}, "${this.name}", value)`); + } } class Property extends BaseProperty { - protected readonly decorator: string = "_jsii_property"; + protected readonly decorator: string = "property"; protected readonly implicitParameter: string = "self"; + + protected emitGetterBody(code: CodeMaker) { + code.line(`return jsii.get(${this.implicitParameter}, "${this.name}")`); + } + + protected emitSetterBody(code: CodeMaker) { + code.line(`return jsii.set(${this.implicitParameter}, "${this.name}", value)`); + } } class Class implements PythonCollectionNode { @@ -461,7 +515,7 @@ class Class implements PythonCollectionNode { // TODO: Data Types? // TODO: Bases - code.openBlock(`class ${this.name}(metaclass=_JSIIMeta, jsii_type="${this.jsiiFQN}")`); + code.openBlock(`class ${this.name}(metaclass=jsii.JSIIMeta, jsii_type="${this.jsiiFQN}")`); if (this.members.length > 0) { for (const member of this.members) { member.emit(code); @@ -567,20 +621,9 @@ class Module { public emit(code: CodeMaker) { // Before we write anything else, we need to write out our module headers, this // is where we handle stuff like imports, any required initialization, etc. + code.line("import jsii"); code.line(this.generateImportFrom("jsii.compat", ["Protocol"])); - code.line( - this.generateImportFrom( - "jsii.runtime", - [ - "JSIIAssembly", - "JSIIMeta", - "jsii_method", - "jsii_property", - "jsii_classmethod", - "jsii_classproperty", - ] - ) - ); + code.line("from jsii.python import classproperty"); // Go over all of the modules that we need to import, and import them. // for (let [idx, modName] of this.importedModules.sort().entries()) { @@ -595,7 +638,7 @@ class Module { // Determine if we need to write out the kernel load line. if (this.assembly && this.assemblyFilename) { code.line( - `__jsii_assembly__ = _JSIIAssembly.load(` + + `__jsii_assembly__ = jsii.JSIIAssembly.load(` + `"${this.assembly.name}", ` + `"${this.assembly.version}", ` + `__name__, ` + @@ -843,6 +886,7 @@ class PythonGenerator extends Generator { this.currentModule().name, toPythonIdentifier(prop.name!), prop.type, + prop.immutable || false ) ); } @@ -853,6 +897,7 @@ class PythonGenerator extends Generator { this.currentModule().name, toPythonIdentifier(prop.name!), prop.type, + prop.immutable || false, ) ); } @@ -890,6 +935,7 @@ class PythonGenerator extends Generator { this.currentModule().name, toPythonIdentifier(prop.name!), prop.type, + true, ) ); } diff --git a/packages/jsii-python-runtime/src/jsii/__init__.py b/packages/jsii-python-runtime/src/jsii/__init__.py index e69de29bb2..39e6dd8f21 100644 --- a/packages/jsii-python-runtime/src/jsii/__init__.py +++ b/packages/jsii-python-runtime/src/jsii/__init__.py @@ -0,0 +1,31 @@ +from ._runtime import JSIIAssembly, JSIIMeta, kernel + + +# Alias our Kernel methods here, so that jsii. works. This will hide the fact +# that there is a kernel at all from our callers. +load = kernel.load +create = kernel.create +delete = kernel.delete +get = kernel.get +set = kernel.set +sget = kernel.sget +sset = kernel.sset +invoke = kernel.invoke +sinvoke = kernel.sinvoke +stats = kernel.stats + + +__all__ = [ + "JSIIAssembly", + "JSIIMeta", + "load", + "create", + "delete", + "get", + "set", + "sget", + "sset", + "invoke", + "sinvoke", + "stats", +] diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py index c64e6dc08e..9ba82e3504 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py @@ -8,6 +8,7 @@ from jsii import _reference_map from jsii._utils import Singleton from jsii._kernel.providers import BaseKernel, ProcessKernel +from jsii._kernel.types import JSClass, Referenceable from jsii._kernel.types import ( LoadRequest, CreateRequest, @@ -66,8 +67,7 @@ def __init__(self, provider_class: Type[BaseKernel] = ProcessKernel) -> None: def load(self, name: str, version: str, tarball: str) -> None: self.provider.load(LoadRequest(name=name, version=version, tarball=tarball)) - # TODO: Can we do protocols in typing? - def create(self, klass: Any, args: Optional[List[Any]] = None) -> ObjRef: + def create(self, klass: JSClass, args: Optional[List[Any]] = None) -> ObjRef: if args is None: args = [] @@ -77,34 +77,42 @@ def delete(self, ref: ObjRef) -> None: self.provider.delete(DeleteRequest(objref=ref)) @_dereferenced - def get(self, ref: ObjRef, property: str) -> Any: - return self.provider.get(GetRequest(objref=ref, property_=property)).value + def get(self, obj: Referenceable, property: str) -> Any: + return self.provider.get( + GetRequest(objref=obj.__jsii_ref__, property_=property) + ).value - def set(self, ref: ObjRef, property: str, value: Any) -> None: - self.provider.set(SetRequest(objref=ref, property_=property, value=value)) + def set(self, obj: Referenceable, property: str, value: Any) -> None: + self.provider.set( + SetRequest(objref=obj.__jsii_ref__, property_=property, value=value) + ) @_dereferenced - def sget(self, klass: Any, property: str) -> Any: + def sget(self, klass: JSClass, property: str) -> Any: return self.provider.sget( StaticGetRequest(fqn=klass.__jsii_type__, property_=property) ).value - def sset(self, klass: Any, property: str, value: Any) -> None: + def sset(self, klass: JSClass, property: str, value: Any) -> None: self.provider.sset( StaticSetRequest(fqn=klass.__jsii_type__, property_=property, value=value) ) @_dereferenced - def invoke(self, ref: ObjRef, method: str, args: Optional[List[Any]] = None) -> Any: + def invoke( + self, obj: Referenceable, method: str, args: Optional[List[Any]] = None + ) -> Any: if args is None: args = [] return self.provider.invoke( - InvokeRequest(objref=ref, method=method, args=args) + InvokeRequest(objref=obj.__jsii_ref__, method=method, args=args) ).result @_dereferenced - def sinvoke(self, klass: Any, method: str, args: Optional[List[Any]] = None) -> Any: + def sinvoke( + self, klass: JSClass, method: str, args: Optional[List[Any]] = None + ) -> Any: if args is None: args = [] diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/types.py b/packages/jsii-python-runtime/src/jsii/_kernel/types.py index 961e2bdf7c..84fe8e8638 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/types.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/types.py @@ -2,6 +2,8 @@ import attr +from jsii.compat import Protocol + # TODO: # - HelloResponse @@ -228,3 +230,21 @@ class StatsResponse: InvokeResponse, StatsResponse, ] + + +class JSClass(Protocol): + + @property + def __jsii_type__(self) -> str: + """ + Returns a str that points to this class inside of the Javascript runtime. + """ + + +class Referenceable(Protocol): + + @property + def __jsii_ref__(self) -> ObjRef: + """ + Returns an ObjRef that points to this object on the JS side. + """ diff --git a/packages/jsii-python-runtime/src/jsii/_runtime.py b/packages/jsii-python-runtime/src/jsii/_runtime.py new file mode 100644 index 0000000000..2c51cd2939 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/_runtime.py @@ -0,0 +1,93 @@ +import weakref +import os + +import attr + +from jsii import _reference_map +from jsii._compat import importlib_resources +from jsii._kernel import Kernel +from jsii.python import _ClassPropertyMeta + + +# Yea, a global here is kind of gross, however, there's not really a better way of +# handling this. Fundamentally this is a global value, since we can only reasonably +# have a single kernel active at any one time in a real program. +kernel = Kernel() + + +@attr.s(auto_attribs=True, frozen=True, slots=True) +class JSIIAssembly: + + name: str + version: str + module: str + filename: str + + @classmethod + def load(cls, *args, _kernel=kernel, **kwargs): + # Our object here really just acts as a record for our JSIIAssembly, it doesn't + # offer any functionality itself, besides this class method that will trigger + # the loading of the given assembly in the JSII Kernel. + assembly = cls(*args, **kwargs) + + # Actually load the assembly into the kernel, we're using the + # importlib.resources API here isntead of manually constructing the path, in + # the hopes that this will make JSII modules able to be used with zipimport + # instead of only on the FS. + with importlib_resources.path( + f"{assembly.module}._jsii", assembly.filename + ) as assembly_path: + _kernel.load(assembly.name, assembly.version, os.fspath(assembly_path)) + + # Give our record of the assembly back to the caller. + return assembly + + +class JSIIMeta(_ClassPropertyMeta, type): + def __new__(cls, name, bases, attrs, *, jsii_type): + # TODO: We need to figure out exactly what we're going to do this the kernel + # here. Right now we're "creating" a new one per type, and relying on + # the fact it's a singleton so that everything is in the same kernel. + # That might not make complete sense though, particularly if we ever + # want to have multiple kernels in a single process (does that even + # make sense?). Perhaps we should pass it in like we pass the type, and + # have the autogenerated code either create it once per library or once + # per process. + attrs["__jsii_kernel__"] = kernel + attrs["__jsii_type__"] = jsii_type + + obj = super().__new__(name, bases, attrs) + + _reference_map.register_type(obj.__jsii_type__, obj) + + return obj + + def __call__(cls, *args, **kwargs): + # Create our object at the JS level. + # TODO: Handle args/kwargs + # TODO: Handle Overrides + ref = cls.__jsii_kernel__.create(cls, args=args) + + # Create our object at the Python level. + obj = cls.__class__.from_reference(cls, ref, **kwargs) + + # Whenever the object we're creating gets garbage collected, then we want to + # delete it from the JS runtime as well. + # TODO: Figure out if this is *really* true, what happens if something goes + # out of scope at the Python level, but something is holding onto it + # at the JS level? What mechanics are in place for this if any? + weakref.finalize(obj, cls.__jsii_kernel__.delete, obj.__jsii_ref__) + + # Instatiate our object at the Python level. + if isinstance(obj, cls): + obj.__init__(**kwargs) + + return obj + + def from_reference(cls, ref, *args, **kwargs): + obj = cls.__new__(cls, *args, **kwargs) + obj.__jsii_ref__ = ref + + _reference_map.register_reference(obj.__jsii_ref__.ref, obj) + + return obj diff --git a/packages/jsii-python-runtime/src/jsii/python.py b/packages/jsii-python-runtime/src/jsii/python.py new file mode 100644 index 0000000000..406e738ca6 --- /dev/null +++ b/packages/jsii-python-runtime/src/jsii/python.py @@ -0,0 +1,38 @@ +class _ClassProperty: + def __init__(self, fget, fset=None): + self.fget = fget + self.fset = fset + + def __get__(self, obj, klass=None): + if klass is None: + klass = type(obj) + return self.fget.__get__(obj, klass)() + + def __set__(self, obj, value): + if not self.fset: + raise AttributeError("Can't set attribute.") + + klass = type(obj) + return self.fset.__get__(obj, klass)(value) + + def setter(self, func): + if not isinstance(func, (classmethod, staticmethod)): + func = classmethod(func) + + self.fset = func + + return self + + +def classproperty(func): + return _ClassProperty(func) + + +class _ClassPropertyMeta(type): + + def __setattr__(self, key, value): + obj = getattr(self, key, None) + if isinstance(obj, _ClassProperty): + return obj.__set__(self, value) + + return super().__setattr__(key, value) diff --git a/packages/jsii-python-runtime/src/jsii/runtime.py b/packages/jsii-python-runtime/src/jsii/runtime.py deleted file mode 100644 index dbee37883c..0000000000 --- a/packages/jsii-python-runtime/src/jsii/runtime.py +++ /dev/null @@ -1,244 +0,0 @@ -from typing import Any, Callable, TypeVar - -import typing -import weakref -import os - -import attr - -from jsii import _reference_map -from jsii._compat import importlib_resources -from jsii._kernel import Kernel - - -# Yea, a global here is kind of gross, however, there's not really a better way of -# handling this. Fundamentally this is a global value, since we can only reasonably -# have a single kernel active at any one time in a real program. -kernel = Kernel() - - -@attr.s(auto_attribs=True, frozen=True, slots=True) -class JSIIAssembly: - - name: str - version: str - module: str - filename: str - - @classmethod - def load(cls, *args, _kernel=kernel, **kwargs): - # Our object here really just acts as a record for our JSIIAssembly, it doesn't - # offer any functionality itself, besides this class method that will trigger - # the loading of the given assembly in the JSII Kernel. - assembly = cls(*args, **kwargs) - - # Actually load the assembly into the kernel, we're using the - # importlib.resources API here isntead of manually constructing the path, in - # the hopes that this will make JSII modules able to be used with zipimport - # instead of only on the FS. - with importlib_resources.path( - f"{assembly.module}._jsii", assembly.filename - ) as assembly_path: - _kernel.load(assembly.name, assembly.version, os.fspath(assembly_path)) - - # Give our record of the assembly back to the caller. - return assembly - - -class JSIIMeta(type): - def __new__(cls, name, bases, attrs, *, jsii_type): - # TODO: We need to figure out exactly what we're going to do this the kernel - # here. Right now we're "creating" a new one per type, and relying on - # the fact it's a singleton so that everything is in the same kernel. - # That might not make complete sense though, particularly if we ever - # want to have multiple kernels in a single process (does that even - # make sense?). Perhaps we should pass it in like we pass the type, and - # have the autogenerated code either create it once per library or once - # per process. - attrs["__jsii_kernel__"] = kernel - attrs["__jsii_type__"] = jsii_type - - # We need to lift all of the classproperty instances out of the class, because - # they need to actually be set *on* the metaclass.. which is us. However we - # don't want to have to derive a separate metaclass for each class, so instead - # we're going to dynamically handle these. - class_properties, new_attrs = {}, {} - for key, value in attrs.items(): - if isinstance(value, jsii_classproperty): - class_properties[key] = value - else: - new_attrs[key] = value - new_attrs["__jsii_class_properties__"] = class_properties - - obj = type.__new__(cls, name, bases, new_attrs) - - # We need to go through an implement the __set_name__ portion of our API. - for key, value in obj.__jsii_class_properties__.items(): - value.__set_name__(obj, key) - - _reference_map.register_type(obj.__jsii_type__, obj) - - return obj - - def __call__(cls, *args, **kwargs): - # Create our object at the JS level. - # TODO: Handle args/kwargs - # TODO: Handle Overrides - ref = cls.__jsii_kernel__.create(cls, args=args) - - # Create our object at the Python level. - obj = cls.__class__.from_reference(cls, ref, **kwargs) - - # Whenever the object we're creating gets garbage collected, then we want to - # delete it from the JS runtime as well. - # TODO: Figure out if this is *really* true, what happens if something goes - # out of scope at the Python level, but something is holding onto it - # at the JS level? What mechanics are in place for this if any? - weakref.finalize(obj, cls.__jsii_kernel__.delete, obj.__jsii_ref__) - - # Instatiate our object at the Python level. - if isinstance(obj, cls): - obj.__init__(**kwargs) - - return obj - - def from_reference(cls, ref, *args, **kwargs): - obj = cls.__new__(cls, *args, **kwargs) - obj.__jsii_ref__ = ref - - _reference_map.register_reference(obj.__jsii_ref__.ref, obj) - - return obj - - def __getattr__(obj, name): - if name in obj.__jsii_class_properties__: - return obj.__jsii_class_properties__[name].__get__(obj, None) - - raise AttributeError(f"type object {obj.__name__!r} has no attribute {name!r}") - - def __setattr__(obj, name, value): - if name in obj.__jsii_class_properties__: - return obj.__jsii_class_properties__[name].__set__(obj, value) - - return super().__setattr__(name, value) - - -# NOTE: We do this goofy thing so that typing works on our generated stub classes, -# because MyPy does not currently understand the decorator protocol very well. -# Something like https://github.com/python/peps/pull/242 should make this no -# longer required. -if typing.TYPE_CHECKING: - jsii_property = property -else: - # The naming is a little funky on this, since it's a class but named like a - # function. This is done to better mimic other decorators like @proeprty. - class jsii_property: - - # We throw away the getter function here, because it doesn't really matter or - # provide us with anything useful. It exists only to provide a way to pass - # naming into us, and so that consumers of this library can "look at the - # source" and at least see something that resembles the structure of the - # library they're using, even though it won't have any of the body of the code. - def __init__(self, getter): - pass - - # TODO: Figure out a way to let the caller of this override the name. This might - # be useful in cases where the name that the JS level code is using isn't - # a valid python identifier, but we still want to be able to bind it, and - # doing so would require giving it a different name at the Python level. - def __set_name__(self, owner, name): - self.name = name - - def __get__(self, instance, owner): - return instance.__jsii_kernel__.get(instance.__jsii_ref__, self.name) - - def __set__(self, instance, value): - instance.__jsii_kernel__.set(instance.__jsii_ref__, self.name, value) - - -class _JSIIMethod: - def __init__(self, obj, method_name): - self.obj = obj - self.method_name = method_name - - def __call__(self, *args): - kernel = self.obj.__jsii_kernel__ - return kernel.invoke(self.obj.__jsii_ref__, self.method_name, args) - - -# NOTE: We do this goofy thing so that typing works on our generated stub classes, -# because MyPy does not currently understand the decorator protocol very well. -# Something like https://github.com/python/peps/pull/242 should make this no -# longer required. -if typing.TYPE_CHECKING: - F = TypeVar("F", bound=Callable[..., Any]) - - def jsii_method(func: F) -> F: - ... - - -else: - # Again, the naming is a little funky on this, since it's a class but named like a - # function. This is done to better mimic other decorators like @classmethod. - class jsii_method: - - # Again, we're throwing away the actual function body, because it exists only - # to provide the structure of the library for people who read the code, and a - # way to pass the name/typing signatures through. - def __init__(self, meth): - pass - - # TODO: Figure out a way to let the caller of this override the name. This might - # be useful in cases where the name that the JS level code is using isn't - # a valid python identifier, but we still want to be able to bind it, and - # doing so would require giving it a different name at the Python level. - def __set_name__(self, owner, name): - self.name = name - - def __get__(self, instance, owner): - return _JSIIMethod(instance, self.name) - - -class _JSIIClassMethod: - def __init__(self, klass, method_name): - self.klass = klass - self.method_name = method_name - - def __call__(self, *args): - kernel = self.klass.__jsii_kernel__ - return kernel.sinvoke(self.klass, self.method_name, args) - - -if typing.TYPE_CHECKING: - jsii_classmethod = classmethod -else: - - class jsii_classmethod: - def __init__(self, meth): - pass - - def __set_name__(self, owner, name): - self.name = name - - def __get__(self, instance, owner): - return _JSIIClassMethod(owner, self.name) - - -if typing.TYPE_CHECKING: - # TODO: Figure out if this type checks correctly, if not how do we make it type - # check correctly... or is it even possible at all? - jsii_classproperty = property -else: - - class jsii_classproperty: - def __init__(self, meth): - pass - - def __set_name__(self, owner, name): - self.name = name - - def __get__(self, instance, owner): - return instance.__jsii_kernel__.sget(instance, self.name) - - def __set__(self, instance, value): - instance.__jsii_kernel__.sset(instance, self.name, value) From 86b92fba20e410fa749f903776a41feb4029d746 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 20 Oct 2018 19:15:04 -0400 Subject: [PATCH 49/88] Enable Typechecking installed JSII packages --- packages/jsii-pacmak/lib/targets/python.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 2245940cd3..fd08b3d9b2 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -775,6 +775,7 @@ class PythonGenerator extends Generator { const packageName = toPythonPackageName(assm.name); const topLevelModuleName = toPythonModuleName(packageName); const moduleNames = this.modules.map(m => m.name); + const pyTypedFilename = path.join("src", toPythonModuleFilename(topLevelModuleName), "py.typed"); moduleNames.push(`${topLevelModuleName}._jsii`); moduleNames.sort(); @@ -795,7 +796,7 @@ class PythonGenerator extends Generator { this.code.line(`url="${assm.homepage}",`); this.code.line('package_dir={"": "src"},'); this.code.line(`packages=[${moduleNames.map(m => `"${m}"`).join(",")}],`); - this.code.line(`package_data={"${topLevelModuleName}._jsii": ["*.jsii.tgz"]},`); + this.code.line(`package_data={"${topLevelModuleName}": ["py.typed"], "${topLevelModuleName}._jsii": ["*.jsii.tgz"]},`); this.code.line('python_requires=">=3.6",'); this.code.unindent(")"); this.code.closeFile("setup.py"); @@ -813,6 +814,11 @@ class PythonGenerator extends Generator { this.code.openFile("MANIFEST.in"); this.code.line("include pyproject.toml"); this.code.closeFile("MANIFEST.in"); + + // We also need to write out a py.typed file, to Signal to MyPy that these files + // are safe to use for typechecking. + this.code.openFile(pyTypedFilename); + this.code.closeFile(pyTypedFilename); } protected onBeginNamespace(ns: string) { From 98875baebe7f9606147af6609521b8c79827e6d8 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 20 Oct 2018 19:39:50 -0400 Subject: [PATCH 50/88] Implement Python name mangling --- packages/jsii-pacmak/lib/targets/python.ts | 46 +++++++++++++++------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index fd08b3d9b2..f4f846bbf7 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -78,6 +78,14 @@ const toPythonIdentifier = (name: string): string => { return name; }; +const toPythonMethodName = (name: string): string => { + return toPythonIdentifier(toSnakeCase(name)); +}; + +const toPythonPropertyName = (name: string): string => { + return toPythonIdentifier(toSnakeCase(name)); +}; + const toPythonType = (typeref: spec.TypeReference): string => { if (spec.isPrimitiveTypeReference(typeref)) { return toPythonPrimitive(typeref.primitive); @@ -246,12 +254,14 @@ class BaseMethod implements PythonNode { protected readonly decorator?: string; protected readonly implicitParameter: string; + protected readonly jsName: string; protected readonly parameters: spec.Parameter[]; protected readonly returns?: spec.TypeReference; - constructor(moduleName: string, name: string, parameters: spec.Parameter[], returns?: spec.TypeReference) { + constructor(moduleName: string, name: string, jsName: string, parameters: spec.Parameter[], returns?: spec.TypeReference) { this.moduleName = moduleName; this.name = name; + this.jsName = jsName; this.parameters = parameters; this.returns = returns; } @@ -311,12 +321,14 @@ class BaseProperty implements PythonNode { protected readonly decorator: string; protected readonly implicitParameter: string; + protected readonly jsName: string; private readonly type: spec.TypeReference; private readonly immutable: boolean; - constructor(moduleName: string, name: string, type: spec.TypeReference, immutable: boolean) { + constructor(moduleName: string, name: string, jsName: string, type: spec.TypeReference, immutable: boolean) { this.moduleName = moduleName; this.name = name; + this.jsName = jsName; this.type = type; this.immutable = immutable; } @@ -430,7 +442,7 @@ class StaticMethod extends BaseMethod { paramNames.push(toPythonIdentifier(param.name)); } - code.line(`return jsii.sinvoke(${this.implicitParameter}, "${this.name}", [${paramNames.join(", ")}])`); + code.line(`return jsii.sinvoke(${this.implicitParameter}, "${this.jsName}", [${paramNames.join(", ")}])`); } } @@ -443,7 +455,7 @@ class Method extends BaseMethod { paramNames.push(toPythonIdentifier(param.name)); } - code.line(`return jsii.invoke(${this.implicitParameter}, "${this.name}", [${paramNames.join(", ")}])`); + code.line(`return jsii.invoke(${this.implicitParameter}, "${this.jsName}", [${paramNames.join(", ")}])`); } } @@ -452,11 +464,11 @@ class StaticProperty extends BaseProperty { protected readonly implicitParameter: string = "cls"; protected emitGetterBody(code: CodeMaker) { - code.line(`return jsii.sget(${this.implicitParameter}, "${this.name}")`); + code.line(`return jsii.sget(${this.implicitParameter}, "${this.jsName}")`); } protected emitSetterBody(code: CodeMaker) { - code.line(`return jsii.sset(${this.implicitParameter}, "${this.name}", value)`); + code.line(`return jsii.sset(${this.implicitParameter}, "${this.jsName}", value)`); } } @@ -465,11 +477,11 @@ class Property extends BaseProperty { protected readonly implicitParameter: string = "self"; protected emitGetterBody(code: CodeMaker) { - code.line(`return jsii.get(${this.implicitParameter}, "${this.name}")`); + code.line(`return jsii.get(${this.implicitParameter}, "${this.jsName}")`); } protected emitSetterBody(code: CodeMaker) { - code.line(`return jsii.set(${this.implicitParameter}, "${this.name}", value)`); + code.line(`return jsii.set(${this.implicitParameter}, "${this.jsName}", value)`); } } @@ -868,7 +880,8 @@ class PythonGenerator extends Generator { this.currentMember!.addMember( new StaticMethod( this.currentModule().name, - toPythonIdentifier(method.name!), + toPythonMethodName(method.name!), + method.name!, method.parameters || [], method.returns ) @@ -879,7 +892,8 @@ class PythonGenerator extends Generator { this.currentMember!.addMember( new Method( this.currentModule().name, - toPythonIdentifier(method.name!), + toPythonMethodName(method.name!), + method.name!, method.parameters || [], method.returns ) @@ -890,7 +904,8 @@ class PythonGenerator extends Generator { this.currentMember!.addMember( new StaticProperty( this.currentModule().name, - toPythonIdentifier(prop.name!), + toPythonPropertyName(prop.name!), + prop.name!, prop.type, prop.immutable || false ) @@ -901,7 +916,8 @@ class PythonGenerator extends Generator { this.currentMember!.addMember( new Property( this.currentModule().name, - toPythonIdentifier(prop.name!), + toPythonPropertyName(prop.name!), + prop.name!, prop.type, prop.immutable || false, ) @@ -928,7 +944,8 @@ class PythonGenerator extends Generator { this.currentMember!.addMember( new InterfaceMethod( this.currentModule().name, - toPythonIdentifier(method.name!), + toPythonMethodName(method.name!), + method.name!, method.parameters || [], method.returns ) @@ -939,7 +956,8 @@ class PythonGenerator extends Generator { this.currentMember!.addMember( new InterfaceProperty( this.currentModule().name, - toPythonIdentifier(prop.name!), + toPythonPropertyName(prop.name!), + prop.name!, prop.type, true, ) From dd3e31d796e2b3a521c658885dfe5532ba5f7a14 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 20 Oct 2018 19:50:13 -0400 Subject: [PATCH 51/88] Refactor to remove duplication --- packages/jsii-pacmak/lib/targets/python.ts | 72 ++++++++++------------ 1 file changed, 32 insertions(+), 40 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index f4f846bbf7..c914d4b7ca 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -253,6 +253,7 @@ class BaseMethod implements PythonNode { protected readonly decorator?: string; protected readonly implicitParameter: string; + protected readonly jsiiMethod?: string; protected readonly jsName: string; protected readonly parameters: spec.Parameter[]; @@ -304,8 +305,17 @@ class BaseMethod implements PythonNode { code.closeBlock(); } - protected emitBody(code: CodeMaker) { - code.line("..."); + private emitBody(code: CodeMaker) { + if (this.jsiiMethod === undefined) { + code.line("..."); + } else { + const paramNames: string[] = []; + for (const param of this.parameters) { + paramNames.push(toPythonIdentifier(param.name)); + } + + code.line(`return jsii.${this.jsiiMethod}(${this.implicitParameter}, "${this.jsName}", [${paramNames.join(", ")}])`); + } } private getReturnType(type?: spec.TypeReference): string { @@ -320,6 +330,8 @@ class BaseProperty implements PythonNode { protected readonly decorator: string; protected readonly implicitParameter: string; + protected readonly jsiiGetMethod?: string; + protected readonly jsiiSetMethod?: string; protected readonly jsName: string; private readonly type: spec.TypeReference; @@ -357,12 +369,20 @@ class BaseProperty implements PythonNode { } } - protected emitGetterBody(code: CodeMaker) { - code.line("..."); + private emitGetterBody(code: CodeMaker) { + if (this.jsiiGetMethod === undefined) { + code.line("..."); + } else { + code.line(`return jsii.${this.jsiiGetMethod}(${this.implicitParameter}, "${this.jsName}")`); + } } - protected emitSetterBody(code: CodeMaker) { - code.line("..."); + private emitSetterBody(code: CodeMaker) { + if (this.jsiiSetMethod === undefined) { + code.line("..."); + } else { + code.line(`return jsii.${this.jsiiSetMethod}(${this.implicitParameter}, "${this.jsName}", value)`); + } } } @@ -435,54 +455,26 @@ class Interface implements PythonCollectionNode { class StaticMethod extends BaseMethod { protected readonly decorator?: string = "classmethod"; protected readonly implicitParameter: string = "cls"; - - protected emitBody(code: CodeMaker) { - const paramNames: string[] = []; - for (const param of this.parameters) { - paramNames.push(toPythonIdentifier(param.name)); - } - - code.line(`return jsii.sinvoke(${this.implicitParameter}, "${this.jsName}", [${paramNames.join(", ")}])`); - } + protected readonly jsiiMethod: string = "sinvoke"; } class Method extends BaseMethod { protected readonly implicitParameter: string = "self"; - - protected emitBody(code: CodeMaker) { - const paramNames: string[] = []; - for (const param of this.parameters) { - paramNames.push(toPythonIdentifier(param.name)); - } - - code.line(`return jsii.invoke(${this.implicitParameter}, "${this.jsName}", [${paramNames.join(", ")}])`); - } + protected readonly jsiiMethod: string = "invoke"; } class StaticProperty extends BaseProperty { protected readonly decorator: string = "classproperty"; protected readonly implicitParameter: string = "cls"; - - protected emitGetterBody(code: CodeMaker) { - code.line(`return jsii.sget(${this.implicitParameter}, "${this.jsName}")`); - } - - protected emitSetterBody(code: CodeMaker) { - code.line(`return jsii.sset(${this.implicitParameter}, "${this.jsName}", value)`); - } + protected readonly jsiiGetMethod: string = "sget"; + protected readonly jsiiSetMethod: string = "sset"; } class Property extends BaseProperty { protected readonly decorator: string = "property"; protected readonly implicitParameter: string = "self"; - - protected emitGetterBody(code: CodeMaker) { - code.line(`return jsii.get(${this.implicitParameter}, "${this.jsName}")`); - } - - protected emitSetterBody(code: CodeMaker) { - code.line(`return jsii.set(${this.implicitParameter}, "${this.jsName}", value)`); - } + protected readonly jsiiGetMethod: string = "get"; + protected readonly jsiiSetMethod: string = "set"; } class Class implements PythonCollectionNode { From 4e9fc0851b258764fa55510491176bfc70d21802 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 20 Oct 2018 20:10:34 -0400 Subject: [PATCH 52/88] Always use the global kernel instead of the __jsii_kernel__ indrection --- packages/jsii-python-runtime/src/jsii/_runtime.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/packages/jsii-python-runtime/src/jsii/_runtime.py b/packages/jsii-python-runtime/src/jsii/_runtime.py index 2c51cd2939..8c39d5a530 100644 --- a/packages/jsii-python-runtime/src/jsii/_runtime.py +++ b/packages/jsii-python-runtime/src/jsii/_runtime.py @@ -45,15 +45,6 @@ def load(cls, *args, _kernel=kernel, **kwargs): class JSIIMeta(_ClassPropertyMeta, type): def __new__(cls, name, bases, attrs, *, jsii_type): - # TODO: We need to figure out exactly what we're going to do this the kernel - # here. Right now we're "creating" a new one per type, and relying on - # the fact it's a singleton so that everything is in the same kernel. - # That might not make complete sense though, particularly if we ever - # want to have multiple kernels in a single process (does that even - # make sense?). Perhaps we should pass it in like we pass the type, and - # have the autogenerated code either create it once per library or once - # per process. - attrs["__jsii_kernel__"] = kernel attrs["__jsii_type__"] = jsii_type obj = super().__new__(name, bases, attrs) @@ -66,7 +57,7 @@ def __call__(cls, *args, **kwargs): # Create our object at the JS level. # TODO: Handle args/kwargs # TODO: Handle Overrides - ref = cls.__jsii_kernel__.create(cls, args=args) + ref = kernel.create(cls, args=args) # Create our object at the Python level. obj = cls.__class__.from_reference(cls, ref, **kwargs) @@ -76,7 +67,7 @@ def __call__(cls, *args, **kwargs): # TODO: Figure out if this is *really* true, what happens if something goes # out of scope at the Python level, but something is holding onto it # at the JS level? What mechanics are in place for this if any? - weakref.finalize(obj, cls.__jsii_kernel__.delete, obj.__jsii_ref__) + weakref.finalize(obj, kernel.delete, obj.__jsii_ref__) # Instatiate our object at the Python level. if isinstance(obj, cls): From f1c6986aadaa2caa3473a3fc6ed2f91f38534263 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 20 Oct 2018 20:12:11 -0400 Subject: [PATCH 53/88] Enable subclassing of JSII Classes --- packages/jsii-python-runtime/src/jsii/_runtime.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/jsii-python-runtime/src/jsii/_runtime.py b/packages/jsii-python-runtime/src/jsii/_runtime.py index 8c39d5a530..3abb43f5c9 100644 --- a/packages/jsii-python-runtime/src/jsii/_runtime.py +++ b/packages/jsii-python-runtime/src/jsii/_runtime.py @@ -44,8 +44,13 @@ def load(cls, *args, _kernel=kernel, **kwargs): class JSIIMeta(_ClassPropertyMeta, type): - def __new__(cls, name, bases, attrs, *, jsii_type): - attrs["__jsii_type__"] = jsii_type + def __new__(cls, name, bases, attrs, *, jsii_type=None): + # We want to ensure that subclasses of a JSII class do not require setting the + # jsii_type keyword argument. They should be able to subclass it as normal. + # Since their parent class will have the __jsii_type__ variable defined, they + # will as well anyways. + if jsii_type is not None: + attrs["__jsii_type__"] = jsii_type obj = super().__new__(name, bases, attrs) From 721f677b7b780b75e3382a504ae1a7806334c581 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 20 Oct 2018 20:13:13 -0400 Subject: [PATCH 54/88] Correct the call to the super class --- packages/jsii-python-runtime/src/jsii/_runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-python-runtime/src/jsii/_runtime.py b/packages/jsii-python-runtime/src/jsii/_runtime.py index 3abb43f5c9..5dfaeeb9a6 100644 --- a/packages/jsii-python-runtime/src/jsii/_runtime.py +++ b/packages/jsii-python-runtime/src/jsii/_runtime.py @@ -52,7 +52,7 @@ def __new__(cls, name, bases, attrs, *, jsii_type=None): if jsii_type is not None: attrs["__jsii_type__"] = jsii_type - obj = super().__new__(name, bases, attrs) + obj = super().__new__(cls, name, bases, attrs) _reference_map.register_type(obj.__jsii_type__, obj) From 9e038519a85c1f7a9e03c0cb7d935c2dd15ab4cc Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 20 Oct 2018 22:19:32 -0400 Subject: [PATCH 55/88] Use real __init__ methods to enable sublcassing of JSII classes --- packages/jsii-pacmak/lib/targets/python.ts | 51 +++++++++++++++++-- .../src/jsii/_kernel/__init__.py | 10 +++- .../src/jsii/_reference_map.py | 20 +++++--- .../jsii-python-runtime/src/jsii/_runtime.py | 31 ++++------- 4 files changed, 79 insertions(+), 33 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index c914d4b7ca..cce8f708f6 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -213,7 +213,7 @@ const sortMembers = (sortable: PythonCollectionNode[]): PythonCollectionNode[] = } return sorted; - }; +}; interface PythonNode { @@ -249,18 +249,27 @@ interface PythonCollectionNode extends PythonNode { class BaseMethod implements PythonNode { public readonly moduleName: string; + public readonly parent: PythonCollectionNode; public readonly name: string; protected readonly decorator?: string; protected readonly implicitParameter: string; protected readonly jsiiMethod?: string; + protected readonly classAsFirstParameter: boolean = false; + protected readonly returnFromJSIIMethod: boolean = true; - protected readonly jsName: string; + protected readonly jsName?: string; protected readonly parameters: spec.Parameter[]; protected readonly returns?: spec.TypeReference; - constructor(moduleName: string, name: string, jsName: string, parameters: spec.Parameter[], returns?: spec.TypeReference) { + constructor(moduleName: string, + parent: PythonCollectionNode, + name: string, + jsName: string | undefined, + parameters: spec.Parameter[], + returns?: spec.TypeReference) { this.moduleName = moduleName; + this.parent = parent; this.name = name; this.jsName = jsName; this.parameters = parameters; @@ -309,12 +318,23 @@ class BaseMethod implements PythonNode { if (this.jsiiMethod === undefined) { code.line("..."); } else { + const methodPrefix: string = this.returnFromJSIIMethod ? "return " : ""; + + const jsiiMethodParams: string[] = []; + if (this.classAsFirstParameter) { + jsiiMethodParams.push(this.parent.name); + } + jsiiMethodParams.push(this.implicitParameter); + if (this.jsName !== undefined) { + jsiiMethodParams.push(`"${this.jsName}"`); + } + const paramNames: string[] = []; for (const param of this.parameters) { paramNames.push(toPythonIdentifier(param.name)); } - code.line(`return jsii.${this.jsiiMethod}(${this.implicitParameter}, "${this.jsName}", [${paramNames.join(", ")}])`); + code.line(`${methodPrefix}jsii.${this.jsiiMethod}(${jsiiMethodParams.join(", ")}, [${paramNames.join(", ")}])`); } } @@ -458,6 +478,13 @@ class StaticMethod extends BaseMethod { protected readonly jsiiMethod: string = "sinvoke"; } +class Initializer extends BaseMethod { + protected readonly implicitParameter: string = "self"; + protected readonly jsiiMethod: string = "create"; + protected readonly classAsFirstParameter: boolean = true; + protected readonly returnFromJSIIMethod: boolean = false; +} + class Method extends BaseMethod { protected readonly implicitParameter: string = "self"; protected readonly jsiiMethod: string = "invoke"; @@ -862,6 +889,19 @@ class PythonGenerator extends Generator { cls.fqn ) ); + + if (cls.initializer !== undefined) { + this.currentMember.addMember( + new Initializer( + currentModule.name, + this.currentMember, + "__init__", + undefined, + cls.initializer.parameters || [], + cls.initializer.returns + ) + ); + } } protected onEndClass(_cls: spec.ClassType) { @@ -872,6 +912,7 @@ class PythonGenerator extends Generator { this.currentMember!.addMember( new StaticMethod( this.currentModule().name, + this.currentMember!, toPythonMethodName(method.name!), method.name!, method.parameters || [], @@ -884,6 +925,7 @@ class PythonGenerator extends Generator { this.currentMember!.addMember( new Method( this.currentModule().name, + this.currentMember!, toPythonMethodName(method.name!), method.name!, method.parameters || [], @@ -936,6 +978,7 @@ class PythonGenerator extends Generator { this.currentMember!.addMember( new InterfaceMethod( this.currentModule().name, + this.currentMember!, toPythonMethodName(method.name!), method.name!, method.parameters || [], diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py index 9ba82e3504..28e1b30b52 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py @@ -67,11 +67,17 @@ def __init__(self, provider_class: Type[BaseKernel] = ProcessKernel) -> None: def load(self, name: str, version: str, tarball: str) -> None: self.provider.load(LoadRequest(name=name, version=version, tarball=tarball)) - def create(self, klass: JSClass, args: Optional[List[Any]] = None) -> ObjRef: + # TODO: Is there a way to say that obj has to be an instance of klass? + def create( + self, klass: JSClass, obj: Any, args: Optional[List[Any]] = None + ) -> ObjRef: if args is None: args = [] - return self.provider.create(CreateRequest(fqn=klass.__jsii_type__, args=args)) + # TODO: Handle Overrides + obj.__jsii_ref__ = self.provider.create( + CreateRequest(fqn=klass.__jsii_type__, args=args) + ) def delete(self, ref: ObjRef) -> None: self.provider.delete(DeleteRequest(objref=ref)) diff --git a/packages/jsii-python-runtime/src/jsii/_reference_map.py b/packages/jsii-python-runtime/src/jsii/_reference_map.py index 47bfdd49f5..baed667da7 100644 --- a/packages/jsii-python-runtime/src/jsii/_reference_map.py +++ b/packages/jsii-python-runtime/src/jsii/_reference_map.py @@ -1,12 +1,14 @@ # This module exists to break an import cycle between jsii.runtime and jsii.kernel import weakref +from ._kernel.types import JSClass, Referenceable + _types = {} -def register_type(type_, obj): - _types[type_] = obj +def register_type(klass: JSClass): + _types[klass.__jsii_type__] = klass class _ReferenceMap: @@ -14,8 +16,8 @@ def __init__(self, types): self._refs = weakref.WeakValueDictionary() self._types = types - def register(self, ref, inst): - self._refs[ref] = inst + def register(self, inst: Referenceable): + self._refs[inst.__jsii_ref__.ref] = inst def resolve(self, ref): # First we need to check our reference map to see if we have any instance that @@ -28,8 +30,14 @@ def resolve(self, ref): # If we got to this point, then we didn't have a referene for this, in that case # we want to create a new instance, but we need to create it in such a way that # we don't try to recreate the type inside of the JSII interface. - class_ = _types[ref.ref.rsplit("@", 1)[0]] - return class_.__class__.from_reference(class_, ref) + klass = _types[ref.ref.rsplit("@", 1)[0]] + + # Create our instance, bypassing __init__ by directly calling __new__, and then + # assign our reference to __jsii_ref__ + inst = klass.__new__(klass) + inst.__jsii_ref__ = ref + + return inst _refs = _ReferenceMap(_types) diff --git a/packages/jsii-python-runtime/src/jsii/_runtime.py b/packages/jsii-python-runtime/src/jsii/_runtime.py index 5dfaeeb9a6..aa044144e2 100644 --- a/packages/jsii-python-runtime/src/jsii/_runtime.py +++ b/packages/jsii-python-runtime/src/jsii/_runtime.py @@ -54,36 +54,25 @@ def __new__(cls, name, bases, attrs, *, jsii_type=None): obj = super().__new__(cls, name, bases, attrs) - _reference_map.register_type(obj.__jsii_type__, obj) + # Now that we've created the class, we'll need to register it with our reference + # mapper. We only do this for types that are actually jsii types, and not any + # subclasses of them. + if jsii_type is not None: + _reference_map.register_type(obj) return obj def __call__(cls, *args, **kwargs): - # Create our object at the JS level. - # TODO: Handle args/kwargs - # TODO: Handle Overrides - ref = kernel.create(cls, args=args) + inst = super().__call__(*args, **kwargs) - # Create our object at the Python level. - obj = cls.__class__.from_reference(cls, ref, **kwargs) + # Register this instance with our reference map. + _reference_map.register_reference(inst) # Whenever the object we're creating gets garbage collected, then we want to # delete it from the JS runtime as well. # TODO: Figure out if this is *really* true, what happens if something goes # out of scope at the Python level, but something is holding onto it # at the JS level? What mechanics are in place for this if any? - weakref.finalize(obj, kernel.delete, obj.__jsii_ref__) - - # Instatiate our object at the Python level. - if isinstance(obj, cls): - obj.__init__(**kwargs) - - return obj + weakref.finalize(inst, kernel.delete, inst.__jsii_ref__) - def from_reference(cls, ref, *args, **kwargs): - obj = cls.__new__(cls, *args, **kwargs) - obj.__jsii_ref__ = ref - - _reference_map.register_reference(obj.__jsii_ref__.ref, obj) - - return obj + return inst From 79b4d4d63320d48b27d8bb1ae94355f526866b56 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sat, 20 Oct 2018 22:57:03 -0400 Subject: [PATCH 56/88] Make sure that classes inherent from their bases and interfaces --- packages/jsii-pacmak/lib/targets/python.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index cce8f708f6..78fb5714ee 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -509,13 +509,15 @@ class Class implements PythonCollectionNode { public readonly name: string; private jsiiFQN: string; + private bases: string[]; private members: PythonNode[]; - constructor(moduleName: string, name: string, jsiiFQN: string) { + constructor(moduleName: string, name: string, jsiiFQN: string, bases: string[]) { this.moduleName = moduleName; this.name = name; this.jsiiFQN = jsiiFQN; + this.bases = bases; this.members = []; } @@ -524,7 +526,7 @@ class Class implements PythonCollectionNode { } get depends_on(): string[] { - return []; + return this.bases.filter(base => base.startsWith(this.moduleName + ".")); } public addMember(member: PythonNode): PythonNode { @@ -544,9 +546,13 @@ class Class implements PythonCollectionNode { public emit(code: CodeMaker) { // TODO: Data Types? - // TODO: Bases - code.openBlock(`class ${this.name}(metaclass=jsii.JSIIMeta, jsii_type="${this.jsiiFQN}")`); + let basesString: string = ""; + if (this.bases.length >= 1) { + basesString = this.bases.join(", ") + ", "; + } + + code.openBlock(`class ${this.name}(${basesString}metaclass=jsii.JSIIMeta, jsii_type="${this.jsiiFQN}")`); if (this.members.length > 0) { for (const member of this.members) { member.emit(code); @@ -886,7 +892,8 @@ class PythonGenerator extends Generator { new Class( currentModule.name, toPythonIdentifier(cls.name), - cls.fqn + cls.fqn, + ((cls.base !== undefined ? [cls.base] : []).concat(cls.interfaces || [])).map(b => toPythonType(b)), ) ); From 67058b9dfb525db32465fbd108247ab0f13cc4b0 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 21 Oct 2018 00:52:55 -0400 Subject: [PATCH 57/88] Correctly limit dependencies to this module, even when faced with submodules --- packages/jsii-pacmak/lib/targets/python.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 78fb5714ee..5793dc91a4 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -2,6 +2,7 @@ import path = require('path'); import util = require('util'); import { CodeMaker, toSnakeCase } from 'codemaker'; +import * as escapeStringRegexp from 'escape-string-regexp'; import * as spec from 'jsii-spec'; import { Generator, GeneratorOptions } from '../generator'; import { Target, TargetOptions } from '../target'; @@ -215,6 +216,10 @@ const sortMembers = (sortable: PythonCollectionNode[]): PythonCollectionNode[] = return sorted; }; +const isInModule = (modName: string, fqn: string): boolean => { + return new RegExp(`^${escapeStringRegexp(modName)}\.[^\.]+$`).test(fqn); +}; + interface PythonNode { // The name of the module that this Node exists in. @@ -436,7 +441,7 @@ class Interface implements PythonCollectionNode { } get depends_on(): string[] { - return this.bases.filter(base => base.startsWith(this.moduleName + ".")); + return this.bases.filter(base => isInModule(this.moduleName, base)); } public addMember(member: PythonNode): PythonNode { @@ -526,7 +531,7 @@ class Class implements PythonCollectionNode { } get depends_on(): string[] { - return this.bases.filter(base => base.startsWith(this.moduleName + ".")); + return this.bases.filter(base => isInModule(this.moduleName, base)); } public addMember(member: PythonNode): PythonNode { From f5f0ec5f0f3b52b1f571849b0b1571f23f59a676 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 21 Oct 2018 01:33:06 -0400 Subject: [PATCH 58/88] Implement Dataclasses as TypedDicts --- packages/jsii-pacmak/lib/targets/python.ts | 139 +++++++++++++++--- packages/jsii-python-runtime/setup.py | 1 + .../jsii-python-runtime/src/jsii/compat.py | 3 +- 3 files changed, 121 insertions(+), 22 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 5793dc91a4..b4031aae6a 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -460,8 +460,6 @@ class Interface implements PythonCollectionNode { } public emit(code: CodeMaker) { - // TODO: Data Types? - const interfaceBases = this.bases.map(baseType => formatPythonType(baseType, true, this.moduleName)); interfaceBases.push("_Protocol"); @@ -477,6 +475,82 @@ class Interface implements PythonCollectionNode { } } +class TypedDictProperty implements PythonNode { + + public readonly moduleName: string; + public readonly name: string; + + private readonly type: spec.TypeReference; + + constructor(moduleName: string, name: string, type: spec.TypeReference) { + this.moduleName = moduleName; + this.name = name; + this.type = type; + } + + get fqn(): string { + return `${this.moduleName}.${this.name}`; + } + + public requiredTypes(): string[] { + return [toPythonType(this.type)]; + } + + public emit(code: CodeMaker) { + const propType: string = formatPythonType(toPythonType(this.type), undefined, this.moduleName); + code.line(`${this.name}: ${propType}`); + } +} + +class TypedDict implements PythonCollectionNode { + public readonly moduleName: string; + public readonly name: string; + + private members: PythonNode[]; + + constructor(moduleName: string, name: string) { + this.moduleName = moduleName; + this.name = name; + + this.members = []; + } + + get fqn(): string { + return `${this.moduleName}.${this.name}`; + } + + get depends_on(): string[] { + return []; + } + + public addMember(member: TypedDictProperty): TypedDictProperty { + this.members.push(member); + return member; + } + + public requiredTypes(): string[] { + const types: string[] = []; + + for (const member of this.members) { + types.push(...member.requiredTypes()); + } + + return types; + } + + public emit(code: CodeMaker) { + code.openBlock(`class ${this.name}(_TypedDict, total=False)`); + if (this.members.length > 0) { + for (const member of this.members) { + member.emit(code); + } + } else { + code.line("pass"); + } + code.closeBlock(); + } +} + class StaticMethod extends BaseMethod { protected readonly decorator?: string = "classmethod"; protected readonly implicitParameter: string = "cls"; @@ -664,7 +738,7 @@ class Module { // Before we write anything else, we need to write out our module headers, this // is where we handle stuff like imports, any required initialization, etc. code.line("import jsii"); - code.line(this.generateImportFrom("jsii.compat", ["Protocol"])); + code.line(this.generateImportFrom("jsii.compat", ["Protocol", "TypedDict"])); code.line("from jsii.python import classproperty"); // Go over all of the modules that we need to import, and import them. @@ -973,20 +1047,33 @@ class PythonGenerator extends Generator { protected onBeginInterface(ifc: spec.InterfaceType) { const currentModule = this.currentModule(); - this.currentMember = currentModule.addMember( - new Interface( - currentModule.name, - toPythonIdentifier(ifc.name), - (ifc.interfaces || []).map(i => toPythonType(i)) - ) - ); + if (ifc.datatype) { + this.currentMember = currentModule.addMember( + new TypedDict( + currentModule.name, + toPythonIdentifier(ifc.name) + ) + ); + } else { + this.currentMember = currentModule.addMember( + new Interface( + currentModule.name, + toPythonIdentifier(ifc.name), + (ifc.interfaces || []).map(i => toPythonType(i)) + ) + ); + } } protected onEndInterface(_ifc: spec.InterfaceType) { this.currentMember = undefined; } - protected onInterfaceMethod(_ifc: spec.InterfaceType, method: spec.Method) { + protected onInterfaceMethod(ifc: spec.InterfaceType, method: spec.Method) { + if (ifc.datatype) { + throw new Error("Cannot have a method on a data type."); + } + this.currentMember!.addMember( new InterfaceMethod( this.currentModule().name, @@ -999,16 +1086,26 @@ class PythonGenerator extends Generator { ); } - protected onInterfaceProperty(_ifc: spec.InterfaceType, prop: spec.Property) { - this.currentMember!.addMember( - new InterfaceProperty( - this.currentModule().name, - toPythonPropertyName(prop.name!), - prop.name!, - prop.type, - true, - ) - ); + protected onInterfaceProperty(ifc: spec.InterfaceType, prop: spec.Property) { + if (ifc.datatype) { + this.currentMember!.addMember( + new TypedDictProperty( + this.currentModule().name, + toPythonIdentifier(prop.name!), + prop.type, + ) + ); + } else { + this.currentMember!.addMember( + new InterfaceProperty( + this.currentModule().name, + toPythonPropertyName(prop.name!), + prop.name!, + prop.type, + true, + ) + ); + } } protected onBeginEnum(enm: spec.EnumType) { diff --git a/packages/jsii-python-runtime/setup.py b/packages/jsii-python-runtime/setup.py index 03c115f664..86440270d2 100644 --- a/packages/jsii-python-runtime/setup.py +++ b/packages/jsii-python-runtime/setup.py @@ -20,6 +20,7 @@ "cattrs", "importlib_resources ; python_version < '3.7'", "typing_extensions>=3.6.4", + "mypy_extensions>=0.4.0", ], python_requires=">=3.6", ) diff --git a/packages/jsii-python-runtime/src/jsii/compat.py b/packages/jsii-python-runtime/src/jsii/compat.py index 1c1e037c51..ea8b6c8d87 100644 --- a/packages/jsii-python-runtime/src/jsii/compat.py +++ b/packages/jsii-python-runtime/src/jsii/compat.py @@ -1,6 +1,7 @@ # External Compatability Shims +from mypy_extensions import TypedDict from typing_extensions import Protocol -__all__ = ["Protocol"] +__all__ = ["Protocol", "TypedDict"] From a2d6c50d7a6fb8f4588f96341dce686bfd3adb81 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 21 Oct 2018 02:50:40 -0400 Subject: [PATCH 59/88] Support typing the JSII package --- packages/jsii-python-runtime/setup.py | 2 +- packages/jsii-python-runtime/src/jsii/py.typed | 0 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 packages/jsii-python-runtime/src/jsii/py.typed diff --git a/packages/jsii-python-runtime/setup.py b/packages/jsii-python-runtime/setup.py index 86440270d2..d3d7396128 100644 --- a/packages/jsii-python-runtime/setup.py +++ b/packages/jsii-python-runtime/setup.py @@ -12,7 +12,7 @@ package_dir={"": "src"}, packages=setuptools.find_packages(where="src"), package_data={ - "jsii": ["_metadata.json"], + "jsii": ["_metadata.json", "py.typed"], "jsii._embedded.jsii": ["*.js", "*.js.map", "*.wasm"], }, install_requires=[ diff --git a/packages/jsii-python-runtime/src/jsii/py.typed b/packages/jsii-python-runtime/src/jsii/py.typed new file mode 100644 index 0000000000..e69de29bb2 From 99278c9b084bceec652df6eb8653478216bf53cc Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 21 Oct 2018 03:23:14 -0400 Subject: [PATCH 60/88] Better handle numbers --- packages/jsii-pacmak/lib/targets/python.ts | 2 +- packages/jsii-python-runtime/src/jsii/__init__.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index b4031aae6a..a47d204906 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -120,7 +120,7 @@ const toPythonPrimitive = (primitive: spec.PrimitiveType): string => { case spec.PrimitiveType.Boolean: return "bool"; case spec.PrimitiveType.Date: return "datetime.datetime"; case spec.PrimitiveType.Json: return "typing.Mapping[typing.Any, typing.Any]"; - case spec.PrimitiveType.Number: return "numbers.Number"; + case spec.PrimitiveType.Number: return "jsii.Number"; case spec.PrimitiveType.String: return "str"; case spec.PrimitiveType.Any: return "typing.Any"; default: diff --git a/packages/jsii-python-runtime/src/jsii/__init__.py b/packages/jsii-python-runtime/src/jsii/__init__.py index 39e6dd8f21..db2c4a9a37 100644 --- a/packages/jsii-python-runtime/src/jsii/__init__.py +++ b/packages/jsii-python-runtime/src/jsii/__init__.py @@ -1,6 +1,13 @@ +from typing import Union + from ._runtime import JSIIAssembly, JSIIMeta, kernel +# JS doesn't have distinct float or integer types, but we do. So we'll define our own +# type alias that will allow either. +Number = Union[int, float] + + # Alias our Kernel methods here, so that jsii. works. This will hide the fact # that there is a kernel at all from our callers. load = kernel.load @@ -18,6 +25,7 @@ __all__ = [ "JSIIAssembly", "JSIIMeta", + "Number", "load", "create", "delete", From e75585cfc731044be3f154d0e0b9d0d3a99ff17b Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 21 Oct 2018 03:33:05 -0400 Subject: [PATCH 61/88] Correctly emit class bases --- packages/jsii-pacmak/lib/targets/python.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index a47d204906..ed17a71018 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -624,14 +624,12 @@ class Class implements PythonCollectionNode { } public emit(code: CodeMaker) { - // TODO: Data Types? + const classParams: string[] = this.bases.map(baseType => formatPythonType(baseType, true, this.moduleName)); - let basesString: string = ""; - if (this.bases.length >= 1) { - basesString = this.bases.join(", ") + ", "; - } + classParams.push("metaclass=jsii.JSIIMeta"); + classParams.push(`jsii_type="${this.jsiiFQN}"`); - code.openBlock(`class ${this.name}(${basesString}metaclass=jsii.JSIIMeta, jsii_type="${this.jsiiFQN}")`); + code.openBlock(`class ${this.name}(${classParams.join(", ")})`); if (this.members.length > 0) { for (const member of this.members) { member.emit(code); From 530f6c715fd223b12be3b6e46f118688b56f2c46 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 21 Oct 2018 03:37:20 -0400 Subject: [PATCH 62/88] Don't inhereit from Interfaces MyPy Protocols do not require inheritence to work, however we were using that as documentation for what interfaces a particular class implements. However, as it turns out, the Protocol class uses a metaclass, which conflicts with the JSIIMeta in use by our JSII classes. So we'll have to rely on the duck typing support of MyPy instead. --- packages/jsii-pacmak/lib/targets/python.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index ed17a71018..442dbbe464 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -970,7 +970,7 @@ class PythonGenerator extends Generator { currentModule.name, toPythonIdentifier(cls.name), cls.fqn, - ((cls.base !== undefined ? [cls.base] : []).concat(cls.interfaces || [])).map(b => toPythonType(b)), + (cls.base !== undefined ? [cls.base] : []).map(b => toPythonType(b)), ) ); From e9b025042135ce76cc9d0470bc6d283e28f93eb9 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 21 Oct 2018 14:56:41 -0400 Subject: [PATCH 63/88] These can be private --- packages/jsii-pacmak/lib/targets/python.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 442dbbe464..cd80e170a9 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -263,9 +263,9 @@ class BaseMethod implements PythonNode { protected readonly classAsFirstParameter: boolean = false; protected readonly returnFromJSIIMethod: boolean = true; - protected readonly jsName?: string; - protected readonly parameters: spec.Parameter[]; - protected readonly returns?: spec.TypeReference; + private readonly jsName?: string; + private readonly parameters: spec.Parameter[]; + private readonly returns?: spec.TypeReference; constructor(moduleName: string, parent: PythonCollectionNode, From 7edc04eb8320c54b2f0f0ba6ce385a6f238a9b5c Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 21 Oct 2018 16:17:16 -0400 Subject: [PATCH 64/88] Handle lifting a data type into keyword arguments --- packages/jsii-pacmak/lib/targets/python.ts | 124 +++++++++++++++++---- 1 file changed, 101 insertions(+), 23 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index cd80e170a9..96bed91a38 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -88,21 +88,31 @@ const toPythonPropertyName = (name: string): string => { }; const toPythonType = (typeref: spec.TypeReference): string => { + let pythonType: string; + + // Get the underlying python type. if (spec.isPrimitiveTypeReference(typeref)) { - return toPythonPrimitive(typeref.primitive); + pythonType = toPythonPrimitive(typeref.primitive); } else if (spec.isCollectionTypeReference(typeref)) { - return toPythonCollection(typeref); + pythonType = toPythonCollection(typeref); } else if (spec.isNamedTypeReference(typeref)) { - return toPythonFQN(typeref.fqn); + pythonType = toPythonFQN(typeref.fqn); } else if (typeref.union) { const types = new Array(); for (const subtype of typeref.union.types) { types.push(toPythonType(subtype)); } - return `typing.Union[${types.join(", ")}]`; + pythonType = `typing.Union[${types.join(", ")}]`; } else { throw new Error("Invalid type reference: " + JSON.stringify(typeref)); } + + // If our type is Optional, then we'll wrap our underlying type with typing.Optional + if (typeref.optional) { + pythonType = `typing.Optional[${pythonType}]`; + } + + return pythonType; }; const toPythonCollection = (ref: spec.CollectionTypeReference) => { @@ -266,19 +276,22 @@ class BaseMethod implements PythonNode { private readonly jsName?: string; private readonly parameters: spec.Parameter[]; private readonly returns?: spec.TypeReference; + private readonly liftedProp?: spec.InterfaceType; constructor(moduleName: string, parent: PythonCollectionNode, name: string, jsName: string | undefined, parameters: spec.Parameter[], - returns?: spec.TypeReference) { + returns?: spec.TypeReference, + liftedProp?: spec.InterfaceType) { this.moduleName = moduleName; this.parent = parent; this.name = name; this.jsName = jsName; this.parameters = parameters; this.returns = returns; + this.liftedProp = liftedProp; } get fqn(): string { @@ -310,6 +323,29 @@ class BaseMethod implements PythonNode { pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, this.moduleName)}`); } + // If we have a lifted parameter, then we'll drop the last argument to our params + // and then we'll lift all of the params of the lifted type as keyword arguments + // to the function. + if (this.liftedProp !== undefined) { + // Remove our last item. + pythonParams.pop(); + + if (this.liftedProp.properties !== undefined && this.liftedProp.properties.length >= 1) { + // All of these parameters are keyword only arguments, so we'll mark them + // as such. + pythonParams.push("*"); + + // Iterate over all of our props, and reflect them into our params. + for (const prop of this.liftedProp.properties) { + const paramName = toPythonIdentifier(prop.name); + const paramType = toPythonType(prop.type); + const paramDefault = prop.type.optional ? "=None" : ""; + + pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, this.moduleName)}${paramDefault}`); + } + } + } + if (this.decorator !== undefined) { code.line(`@${this.decorator}`); } @@ -323,24 +359,45 @@ class BaseMethod implements PythonNode { if (this.jsiiMethod === undefined) { code.line("..."); } else { - const methodPrefix: string = this.returnFromJSIIMethod ? "return " : ""; - - const jsiiMethodParams: string[] = []; - if (this.classAsFirstParameter) { - jsiiMethodParams.push(this.parent.name); - } - jsiiMethodParams.push(this.implicitParameter); - if (this.jsName !== undefined) { - jsiiMethodParams.push(`"${this.jsName}"`); + if (this.liftedProp !== undefined) { + this.emitAutoProps(code); } - const paramNames: string[] = []; - for (const param of this.parameters) { - paramNames.push(toPythonIdentifier(param.name)); - } + this.emitJsiiMethodCall(code); + } + } + + private emitAutoProps(code: CodeMaker) { + const lastParameter = this.parameters.slice(-1)[0]; + const argName: string = toPythonIdentifier(lastParameter.name); + const typeName: string = formatPythonType(toPythonType(lastParameter.type), true, this.moduleName); + + const propMembers: string[] = []; + for (const prop of this.liftedProp!.properties || []) { + propMembers.push(`"${toPythonIdentifier(prop.name)}": ${toPythonIdentifier(prop.name)}`); + } + + code.line(`${argName}: ${typeName} = {${propMembers.join(", ")}}`); + } + + private emitJsiiMethodCall(code: CodeMaker) { + const methodPrefix: string = this.returnFromJSIIMethod ? "return " : ""; - code.line(`${methodPrefix}jsii.${this.jsiiMethod}(${jsiiMethodParams.join(", ")}, [${paramNames.join(", ")}])`); + const jsiiMethodParams: string[] = []; + if (this.classAsFirstParameter) { + jsiiMethodParams.push(this.parent.name); + } + jsiiMethodParams.push(this.implicitParameter); + if (this.jsName !== undefined) { + jsiiMethodParams.push(`"${this.jsName}"`); + } + + const paramNames: string[] = []; + for (const param of this.parameters) { + paramNames.push(toPythonIdentifier(param.name)); } + + code.line(`${methodPrefix}jsii.${this.jsiiMethod}(${jsiiMethodParams.join(", ")}, [${paramNames.join(", ")}])`); } private getReturnType(type?: spec.TypeReference): string { @@ -982,7 +1039,8 @@ class PythonGenerator extends Generator { "__init__", undefined, cls.initializer.parameters || [], - cls.initializer.returns + cls.initializer.returns, + this.getliftedProp(cls.initializer), ) ); } @@ -1000,7 +1058,8 @@ class PythonGenerator extends Generator { toPythonMethodName(method.name!), method.name!, method.parameters || [], - method.returns + method.returns, + this.getliftedProp(method), ) ); } @@ -1013,7 +1072,8 @@ class PythonGenerator extends Generator { toPythonMethodName(method.name!), method.name!, method.parameters || [], - method.returns + method.returns, + this.getliftedProp(method), ) ); } @@ -1079,7 +1139,8 @@ class PythonGenerator extends Generator { toPythonMethodName(method.name!), method.name!, method.parameters || [], - method.returns + method.returns, + this.getliftedProp(method), ) ); } @@ -1155,6 +1216,23 @@ class PythonGenerator extends Generator { // End Not Currently Used + private getliftedProp(method: spec.Method): spec.InterfaceType | undefined { + // If there are parameters to this method, and if the last parameter's type is + // a datatype interface, then we want to lift the members of that last paramter + // as keyword arguments to this function. + if (method.parameters !== undefined && method.parameters.length >= 1) { + const lastParameter = method.parameters.slice(-1)[0]; + if (spec.isNamedTypeReference(lastParameter.type)) { + const lastParameterType = this.findType(lastParameter.type.fqn); + if (spec.isInterfaceType(lastParameterType) && lastParameterType.datatype) { + return lastParameterType; + } + } + } + + return undefined; + } + private currentModule(): Module { return this.moduleStack.slice(-1)[0]; } From 771e9232015414cc31f91d6d0ca7d705155e3fd0 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 21 Oct 2018 19:52:00 -0400 Subject: [PATCH 65/88] Handle Mandatory/Optional data type members --- packages/jsii-pacmak/lib/targets/python.ts | 86 +++++++++++++++++++--- 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 96bed91a38..182c465c2c 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -87,7 +87,7 @@ const toPythonPropertyName = (name: string): string => { return toPythonIdentifier(toSnakeCase(name)); }; -const toPythonType = (typeref: spec.TypeReference): string => { +const toPythonType = (typeref: spec.TypeReference, respectOptional: boolean = true): string => { let pythonType: string; // Get the underlying python type. @@ -108,7 +108,8 @@ const toPythonType = (typeref: spec.TypeReference): string => { } // If our type is Optional, then we'll wrap our underlying type with typing.Optional - if (typeref.optional) { + // However, if we're not respecting optionals, then we'll just skip over this. + if (respectOptional && typeref.optional) { pythonType = `typing.Optional[${pythonType}]`; } @@ -372,12 +373,30 @@ class BaseMethod implements PythonNode { const argName: string = toPythonIdentifier(lastParameter.name); const typeName: string = formatPythonType(toPythonType(lastParameter.type), true, this.moduleName); - const propMembers: string[] = []; + // We need to build up a list of properties, which are mandatory, these are the + // ones we will specifiy to start with in our dictionary literal. + const mandatoryPropMembers: string[] = []; for (const prop of this.liftedProp!.properties || []) { - propMembers.push(`"${toPythonIdentifier(prop.name)}": ${toPythonIdentifier(prop.name)}`); + if (prop.type.optional) { + continue; + } + + mandatoryPropMembers.push(`"${toPythonIdentifier(prop.name)}": ${toPythonIdentifier(prop.name)}`); } + code.line(`${argName}: ${typeName} = {${mandatoryPropMembers.join(", ")}}`); + code.line(); - code.line(`${argName}: ${typeName} = {${propMembers.join(", ")}}`); + // Now we'll go through our optional properties, and if they haven't been set + // we'll add them to our dictionary. + for (const prop of this.liftedProp!.properties || []) { + if (!prop.type.optional) { + continue; + } + + code.openBlock(`if ${toPythonIdentifier(prop.name)} is not None`); + code.line(`${argName}["${toPythonIdentifier(prop.name)}"] = ${toPythonIdentifier(prop.name)}`); + code.closeBlock(); + } } private emitJsiiMethodCall(code: CodeMaker) { @@ -549,12 +568,16 @@ class TypedDictProperty implements PythonNode { return `${this.moduleName}.${this.name}`; } + get optional(): boolean { + return this.type.optional || false; + } + public requiredTypes(): string[] { return [toPythonType(this.type)]; } public emit(code: CodeMaker) { - const propType: string = formatPythonType(toPythonType(this.type), undefined, this.moduleName); + const propType: string = formatPythonType(toPythonType(this.type, false), undefined, this.moduleName); code.line(`${this.name}: ${propType}`); } } @@ -563,7 +586,7 @@ class TypedDict implements PythonCollectionNode { public readonly moduleName: string; public readonly name: string; - private members: PythonNode[]; + private members: TypedDictProperty[]; constructor(moduleName: string, name: string) { this.moduleName = moduleName; @@ -596,15 +619,54 @@ class TypedDict implements PythonCollectionNode { } public emit(code: CodeMaker) { - code.openBlock(`class ${this.name}(_TypedDict, total=False)`); - if (this.members.length > 0) { - for (const member of this.members) { + // MyPy doesn't let us mark some keys as optional, and some keys as mandatory, + // we can either mark either the entire class as mandatory or the entire class + // as optional. However, we can make two classes, one with all mandatory keys + // and one with all optional keys in order to emulate this. So we'll go ahead + // and implement this "split" class logic. + + const mandatoryMembers = this.members.filter(item => !item.optional); + const optionalMembers = this.members.filter(item => item.optional); + + if (mandatoryMembers.length >= 1 && optionalMembers.length >= 1) { + // In this case, we have both mandatory *and* optional members, so we'll + // do our split class logic. + + // We'll emit the optional members first, just because it's a little nicer + // for the final class in the chain to have the mandatory members. + code.openBlock(`class _${this.name}(_TypedDict, total=False)`); + for (const member of optionalMembers) { member.emit(code); } + code.closeBlock(); + + // Now we'll emit the mandatory members. + code.openBlock(`class ${this.name}(_${this.name})`); + for (const member of mandatoryMembers) { + member.emit(code); + } + code.closeBlock(); } else { - code.line("pass"); + // In this case we either have no members, or we have all of one type, so + // we'll see if we have any optional members, if we don't then we'll use + // total=True instead of total=False for the class. + if (optionalMembers.length >= 1) { + code.openBlock(`class ${this.name}(_TypedDict, total=False)`); + } else { + code.openBlock(`class ${this.name}(_TypedDict)`); + } + + // Finally we'll just iterate over and emit all of our members. + if (this.members.length > 0) { + for (const member of this.members) { + member.emit(code); + } + } else { + code.line("pass"); + } + + code.closeBlock(); } - code.closeBlock(); } } From 25def601478a0927f8d320f84169b65d991c8545 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Sun, 21 Oct 2018 20:02:08 -0400 Subject: [PATCH 66/88] Use publication to hide implementation details from end users --- packages/jsii-pacmak/lib/targets/python.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 182c465c2c..372afc2f46 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -855,6 +855,8 @@ class Module { // Before we write anything else, we need to write out our module headers, this // is where we handle stuff like imports, any required initialization, etc. code.line("import jsii"); + code.line("import publication"); + code.line(); code.line(this.generateImportFrom("jsii.compat", ["Protocol", "TypedDict"])); code.line("from jsii.python import classproperty"); @@ -885,8 +887,13 @@ class Module { member.emit(code); } - // // Whatever names we've exported, we'll write out our __all__ that lists them. + // Whatever names we've exported, we'll write out our __all__ that lists them. code.line(`__all__ = [${this.getExportedNames().map(s => `"${s}"`).join(", ")}]`); + + // Finally, we'll use publication to ensure that all of the non-public names + // get hidden from dir(), tab-complete, etc. + code.line(); + code.line("publication.publish()"); } private getRequiredTypeImports(): string[] { @@ -1031,6 +1038,7 @@ class PythonGenerator extends Generator { this.code.line(`packages=[${moduleNames.map(m => `"${m}"`).join(",")}],`); this.code.line(`package_data={"${topLevelModuleName}": ["py.typed"], "${topLevelModuleName}._jsii": ["*.jsii.tgz"]},`); this.code.line('python_requires=">=3.6",'); + this.code.line(`install_requires=["publication"],`); this.code.unindent(")"); this.code.closeFile("setup.py"); From a000fb7e15993c62512874c9eda57dbe112b6483 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 19 Oct 2018 17:13:47 -0400 Subject: [PATCH 67/88] Refactor module dependency to use Assembly metadata --- packages/jsii-pacmak/lib/targets/python.ts | 208 ++++++--------------- packages/jsii-pacmak/package.json | 2 + 2 files changed, 57 insertions(+), 153 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 372afc2f46..f73b4dbaf1 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -140,13 +140,13 @@ const toPythonPrimitive = (primitive: spec.PrimitiveType): string => { }; const toPythonFQN = (name: string): string => { - return name.split(".").map((cur, idx, arr) => { - if (idx === arr.length - 1) { - return toPythonIdentifier(cur); - } else { - return toPythonModuleName(cur); - } - }).join("."); + const [, modulePart, typePart] = name.match(/^((?:[^A-Z\.][^\.]+\.?)+)\.([A-Z].+)$/) as string[]; + const fqnParts = [ + toPythonModuleName(modulePart), + typePart.split(".").map(cur => toPythonIdentifier(cur)).join(".") + ]; + + return fqnParts.join("."); }; const formatPythonType = (type: string, forwardReference: boolean = false, moduleName: string) => { @@ -155,6 +155,7 @@ const formatPythonType = (type: string, forwardReference: boolean = false, modul // no matter how nested they are. The downside is we might get trailing/leading // spaces or empty items so we'll need to trim and filter this list. const types = type.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s !== ""); + // const moduleRe = new RegExp(`^${escapeStringRegexp(moduleName)}\.([A-Z].+)$`); for (const innerType of types) { // Built in types do not need formatted in any particular way. @@ -164,16 +165,18 @@ const formatPythonType = (type: string, forwardReference: boolean = false, modul // If we do not have a current moduleName, or the type is not within that // module, then we don't format it any particular way. + // if (!moduleRe.test(innerType)) { if (!innerType.startsWith(moduleName + ".")) { continue; } else { const typeName = innerType.substring(moduleName.length + 1, innerType.length); + // const [, typeName] = innerType.match(moduleRe) as string[]; const re = new RegExp('((?:^|[[,\\s])"?)' + innerType + '("?(?:$|[\\],\\s]))'); // If this is our current module, then we need to correctly handle our // forward references, by placing the type inside of quotes, unless // we're returning real forward references. - if (!forwardReference) { + if (!forwardReference && !typeName.match(/^[a-z]/)) { type = type.replace(re, `$1"${innerType}"$2`); } @@ -242,11 +245,6 @@ interface PythonNode { // The fully qualifed name of this node. readonly fqn: string; - // Returns a list of all of the FQN Python types that this Node requires, this - // should traverse all of it's members to get the full list of all types required to - // exist (i.e. be imported). - requiredTypes(): string[]; - // Emits the entire tree of objects represented by this object into the given // CodeMaker object. emit(code: CodeMaker): void; @@ -299,16 +297,6 @@ class BaseMethod implements PythonNode { return `${this.moduleName}.${this.name}`; } - public requiredTypes(): string[] { - const types: string[] = [this.getReturnType(this.returns)]; - - for (const param of this.parameters) { - types.push(toPythonType(param.type)); - } - - return types; - } - public emit(code: CodeMaker) { const returnType = this.getReturnType(this.returns); @@ -450,10 +438,6 @@ class BaseProperty implements PythonNode { return `${this.moduleName}.${this.name}`; } - public requiredTypes(): string[] { - return [toPythonType(this.type)]; - } - public emit(code: CodeMaker) { const returnType = toPythonType(this.type); @@ -525,16 +509,6 @@ class Interface implements PythonCollectionNode { return member; } - public requiredTypes(): string[] { - const types = this.bases.slice(); - - for (const member of this.members) { - types.push(...member.requiredTypes()); - } - - return types; - } - public emit(code: CodeMaker) { const interfaceBases = this.bases.map(baseType => formatPythonType(baseType, true, this.moduleName)); interfaceBases.push("_Protocol"); @@ -572,10 +546,6 @@ class TypedDictProperty implements PythonNode { return this.type.optional || false; } - public requiredTypes(): string[] { - return [toPythonType(this.type)]; - } - public emit(code: CodeMaker) { const propType: string = formatPythonType(toPythonType(this.type, false), undefined, this.moduleName); code.line(`${this.name}: ${propType}`); @@ -608,16 +578,6 @@ class TypedDict implements PythonCollectionNode { return member; } - public requiredTypes(): string[] { - const types: string[] = []; - - for (const member of this.members) { - types.push(...member.requiredTypes()); - } - - return types; - } - public emit(code: CodeMaker) { // MyPy doesn't let us mark some keys as optional, and some keys as mandatory, // we can either mark either the entire class as mandatory or the entire class @@ -732,16 +692,6 @@ class Class implements PythonCollectionNode { return member; } - public requiredTypes(): string[] { - const types: string[] = []; - - for (const member of this.members) { - types.push(...member.requiredTypes()); - } - - return types; - } - public emit(code: CodeMaker) { const classParams: string[] = this.bases.map(baseType => formatPythonType(baseType, true, this.moduleName)); @@ -785,10 +735,6 @@ class Enum implements PythonCollectionNode { return member; } - public requiredTypes(): string[] { - return ["enum.Enum"]; - } - public emit(code: CodeMaker) { code.openBlock(`class ${this.name}(enum.Enum)`); if (this.members.length > 0) { @@ -818,10 +764,6 @@ class EnumMember implements PythonNode { return `${this.moduleName}.${this.name}`; } - public requiredTypes(): string[] { - return []; - } - public emit(code: CodeMaker) { code.line(`${this.name} = "${this.value}"`); } @@ -830,20 +772,21 @@ class EnumMember implements PythonNode { class Module { public readonly name: string; - public readonly assembly?: spec.Assembly; - public readonly assemblyFilename?: string; + public readonly assembly: spec.Assembly; + public readonly assemblyFilename: string; + public readonly loadAssembly: boolean; private members: PythonCollectionNode[]; + private subModules: string[]; - constructor(ns: string, assembly?: [spec.Assembly, string]) { + constructor(ns: string, assembly: spec.Assembly, assemblyFilename: string, loadAssembly: boolean = false) { this.name = ns; - - if (assembly !== undefined) { - this.assembly = assembly[0]; - this.assemblyFilename = assembly[1]; - } + this.assembly = assembly; + this.assemblyFilename = assemblyFilename; + this.loadAssembly = loadAssembly; this.members = []; + this.subModules = []; } public addMember(member: PythonCollectionNode): PythonCollectionNode { @@ -851,9 +794,17 @@ class Module { return member; } + public addSubmodule(module: string) { + this.subModules.push(module); + } + public emit(code: CodeMaker) { // Before we write anything else, we need to write out our module headers, this // is where we handle stuff like imports, any required initialization, etc. + code.line("import datetime"); + code.line("import enum"); + code.line("import typing"); + code.line(); code.line("import jsii"); code.line("import publication"); code.line(); @@ -862,16 +813,30 @@ class Module { // Go over all of the modules that we need to import, and import them. // for (let [idx, modName] of this.importedModules.sort().entries()) { - for (const [idx, modName] of this.getRequiredTypeImports().sort().entries()) { + const dependencies = Object.keys(this.assembly.dependencies || {}); + for (const [idx, depName] of dependencies.sort().entries()) { + // If this our first dependency, add a blank line to format our imports + // slightly nicer. if (idx === 0) { code.line(); } - code.line(`import ${modName}`); + code.line(`import ${toPythonModuleName(depName)}`); + } + + const moduleRe = new RegExp(`^${escapeStringRegexp(this.name)}\.`); + for (const [idx, subModule] of this.subModules.sort().entries()) { + // If this our first subModule, add a blank line to format our imports + // slightly nicer. + if (idx === 0) { + code.line(); + } + + code.line(`from . import ${subModule.replace(moduleRe, "")}`); } // Determine if we need to write out the kernel load line. - if (this.assembly && this.assemblyFilename) { + if (this.loadAssembly) { code.line( `__jsii_assembly__ = jsii.JSIIAssembly.load(` + `"${this.assembly.name}", ` + @@ -896,66 +861,6 @@ class Module { code.line("publication.publish()"); } - private getRequiredTypeImports(): string[] { - const types: string[] = []; - const imports: string[] = []; - - // Compute a list of all of of the types that - for (const member of this.members) { - types.push(...member.requiredTypes()); - } - - // Go over our types, and generate a list of imports that we need to import for - // our module. - for (const type of types) { - // If we split our types by any of the "special" characters that can't appear in - // identifiers (like "[],") then we will get a list of all of the identifiers, - // no matter how nested they are. The downside is we might get trailing/leading - // spaces or empty items so we'll need to trim and filter this list. - const subTypes = type.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s !== ""); - - // Loop over all of the types we've discovered, and check them for being - // importable - for (const subType of subTypes) { - // For built in types, we don't need to do anything, and can just move on. - if (PYTHON_BUILTIN_TYPES.indexOf(subType) > -1) { continue; } - - const [, typeModule] = subType.match(/(.*)\.(.*)/) as any[]; - - // Given a name like foo.bar.Frob, we want to import the module that Frob exists - // in. Given that all classes exported by JSII have to be pascal cased, and all - // of our imports are snake cases, this should be safe. We're going to double - // check this though, to ensure that our importable here is safe. - if (typeModule !== typeModule.toLowerCase()) { - // If we ever get to this point, we'll need to implment aliasing for our - // imports. - throw new Error(`Type module is not lower case: '${typeModule}'`); - } - - // We only want to actually import the type for this module, if it isn't the - // module that we're currently in, otherwise we'll jus rely on the module scope - // to make the name available to us. - if (typeModule !== this.name && imports.indexOf(typeModule) === -1) { - imports.push(typeModule); - } - } - } - - // Add the additional dependencies that we require in order for imports to - // work correctly. These are dependencies at the JS level, isntead of at the - // Python level. - if (this.assembly !== undefined && this.assembly!.dependencies !== undefined) { - for (const depName of Object.keys(this.assembly!.dependencies!)) { - const moduleName = toPythonModuleName(depName); - if (imports.indexOf(moduleName) === -1) { - imports.push(moduleName); - } - } - } - - return imports; - } - private getExportedNames(): string[] { // We assume that anything that is a member of this module, will be exported by // this module. @@ -963,10 +868,14 @@ class Module { // If this module will be outputting the Assembly, then we also want to export // our assembly variable. - if (this.assembly && this.assemblyFilename) { + if (this.loadAssembly) { exportedNames.push("__jsii_assembly__"); } + // We also need to export all of our submodules. + const moduleRe = new RegExp(`^${escapeStringRegexp(this.name)}\.`); + exportedNames.push(...this.subModules.map(item => item.replace(moduleRe, ""))); + return exportedNames.sort(); } @@ -1065,15 +974,12 @@ class PythonGenerator extends Generator { protected onBeginNamespace(ns: string) { const moduleName = toPythonModuleName(ns); const loadAssembly = this.assembly.name === ns ? true : false; + const mod = new Module(moduleName, this.assembly, this.getAssemblyFileName(), loadAssembly); - const moduleArgs: any[] = []; - - if (loadAssembly) { - moduleArgs.push([this.assembly, this.getAssemblyFileName()]); + for (const parentMod of this.moduleStack) { + parentMod.addSubmodule(moduleName); } - const mod = new Module(moduleName, ...moduleArgs); - this.modules.push(mod); this.moduleStack.push(mod); } @@ -1239,13 +1145,9 @@ class PythonGenerator extends Generator { protected onBeginEnum(enm: spec.EnumType) { const currentModule = this.currentModule(); + const newMember = new Enum(currentModule.name, toPythonIdentifier(enm.name)); - this.currentMember = currentModule.addMember( - new Enum( - currentModule.name, - toPythonIdentifier(enm.name), - ) - ); + this.currentMember = currentModule.addMember(newMember); } protected onEndEnum(_enm: spec.EnumType) { diff --git a/packages/jsii-pacmak/package.json b/packages/jsii-pacmak/package.json index fc956f1009..405130a07c 100644 --- a/packages/jsii-pacmak/package.json +++ b/packages/jsii-pacmak/package.json @@ -20,8 +20,10 @@ "aws" ], "dependencies": { + "@types/escape-string-regexp": "^1.0.0", "clone": "^2.1.1", "codemaker": "^0.7.7", + "escape-string-regexp": "^1.0.5", "fs-extra": "^4.0.3", "jsii-spec": "^0.7.7", "spdx-license-list": "^4.1.0", From a964267b9338fb0feb2926aa9eb7f0eb8f63d4f9 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Mon, 22 Oct 2018 15:31:37 -0400 Subject: [PATCH 68/88] Handle variadic parameters --- packages/jsii-pacmak/lib/targets/python.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index f73b4dbaf1..d84273f3fd 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -333,6 +333,17 @@ class BaseMethod implements PythonNode { pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, this.moduleName)}${paramDefault}`); } } + } else if (this.parameters.length >= 1 && this.parameters.slice(-1)[0].variadic) { + // Another situation we could be in, is that instead of having a plain parameter + // we have a variadic parameter where we need to expand the last parameter as a + // *args. + pythonParams.pop(); + + const lastParameter = this.parameters.slice(-1)[0]; + const paramName = toPythonIdentifier(lastParameter.name); + const paramType = toPythonType(lastParameter.type, false); + + pythonParams.push(`*${paramName}: ${formatPythonType(paramType, false, this.moduleName)}`); } if (this.decorator !== undefined) { From 4575e855d0551510bbbd1616da09d83005cb9236 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Wed, 24 Oct 2018 09:35:05 -0400 Subject: [PATCH 69/88] Massive refactor to greatly simplify Python generator --- packages/jsii-pacmak/lib/targets/python.ts | 1320 ++++++++++---------- packages/jsii-pacmak/package.json | 2 - 2 files changed, 669 insertions(+), 653 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index d84273f3fd..8175f7908f 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -2,7 +2,6 @@ import path = require('path'); import util = require('util'); import { CodeMaker, toSnakeCase } from 'codemaker'; -import * as escapeStringRegexp from 'escape-string-regexp'; import * as spec from 'jsii-spec'; import { Generator, GeneratorOptions } from '../generator'; import { Target, TargetOptions } from '../target'; @@ -31,7 +30,7 @@ export default class Python extends Target { // ################## // # CODE GENERATOR # // ################## -const debug = (o: any) => { +export const debug = (o: any) => { // tslint:disable-next-line:no-console console.log(util.inspect(o, false, null, true)); }; @@ -56,15 +55,8 @@ const toPythonModuleName = (name: string): string => { return name; }; -const toPythonModuleFilename = (name: string): string => { - if (name.match(/^@[^/]+\/[^/]+$/)) { - name = name.replace(/^@/g, ""); - name = name.replace(/\//g, "."); - } - - name = name.replace(/\./g, "/"); - - return name; +const pythonModuleNameToFilename = (name: string): string => { + return name.replace(/\./g, "/"); }; const toPythonPackageName = (name: string): string => { @@ -87,109 +79,6 @@ const toPythonPropertyName = (name: string): string => { return toPythonIdentifier(toSnakeCase(name)); }; -const toPythonType = (typeref: spec.TypeReference, respectOptional: boolean = true): string => { - let pythonType: string; - - // Get the underlying python type. - if (spec.isPrimitiveTypeReference(typeref)) { - pythonType = toPythonPrimitive(typeref.primitive); - } else if (spec.isCollectionTypeReference(typeref)) { - pythonType = toPythonCollection(typeref); - } else if (spec.isNamedTypeReference(typeref)) { - pythonType = toPythonFQN(typeref.fqn); - } else if (typeref.union) { - const types = new Array(); - for (const subtype of typeref.union.types) { - types.push(toPythonType(subtype)); - } - pythonType = `typing.Union[${types.join(", ")}]`; - } else { - throw new Error("Invalid type reference: " + JSON.stringify(typeref)); - } - - // If our type is Optional, then we'll wrap our underlying type with typing.Optional - // However, if we're not respecting optionals, then we'll just skip over this. - if (respectOptional && typeref.optional) { - pythonType = `typing.Optional[${pythonType}]`; - } - - return pythonType; -}; - -const toPythonCollection = (ref: spec.CollectionTypeReference) => { - const elementPythonType = toPythonType(ref.collection.elementtype); - switch (ref.collection.kind) { - case spec.CollectionKind.Array: return `typing.List[${elementPythonType}]`; - case spec.CollectionKind.Map: return `typing.Mapping[str,${elementPythonType}]`; - default: - throw new Error(`Unsupported collection kind: ${ref.collection.kind}`); - } -}; - -const toPythonPrimitive = (primitive: spec.PrimitiveType): string => { - switch (primitive) { - case spec.PrimitiveType.Boolean: return "bool"; - case spec.PrimitiveType.Date: return "datetime.datetime"; - case spec.PrimitiveType.Json: return "typing.Mapping[typing.Any, typing.Any]"; - case spec.PrimitiveType.Number: return "jsii.Number"; - case spec.PrimitiveType.String: return "str"; - case spec.PrimitiveType.Any: return "typing.Any"; - default: - throw new Error("Unknown primitive type: " + primitive); - } -}; - -const toPythonFQN = (name: string): string => { - const [, modulePart, typePart] = name.match(/^((?:[^A-Z\.][^\.]+\.?)+)\.([A-Z].+)$/) as string[]; - const fqnParts = [ - toPythonModuleName(modulePart), - typePart.split(".").map(cur => toPythonIdentifier(cur)).join(".") - ]; - - return fqnParts.join("."); -}; - -const formatPythonType = (type: string, forwardReference: boolean = false, moduleName: string) => { - // If we split our types by any of the "special" characters that can't appear in - // identifiers (like "[],") then we will get a list of all of the identifiers, - // no matter how nested they are. The downside is we might get trailing/leading - // spaces or empty items so we'll need to trim and filter this list. - const types = type.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s !== ""); - // const moduleRe = new RegExp(`^${escapeStringRegexp(moduleName)}\.([A-Z].+)$`); - - for (const innerType of types) { - // Built in types do not need formatted in any particular way. - if (PYTHON_BUILTIN_TYPES.indexOf(innerType) > -1) { - continue; - } - - // If we do not have a current moduleName, or the type is not within that - // module, then we don't format it any particular way. - // if (!moduleRe.test(innerType)) { - if (!innerType.startsWith(moduleName + ".")) { - continue; - } else { - const typeName = innerType.substring(moduleName.length + 1, innerType.length); - // const [, typeName] = innerType.match(moduleRe) as string[]; - const re = new RegExp('((?:^|[[,\\s])"?)' + innerType + '("?(?:$|[\\],\\s]))'); - - // If this is our current module, then we need to correctly handle our - // forward references, by placing the type inside of quotes, unless - // we're returning real forward references. - if (!forwardReference && !typeName.match(/^[a-z]/)) { - type = type.replace(re, `$1"${innerType}"$2`); - } - - // Now that we've handled (or not) our forward references, then we want - // to replace the module with just the type name. - // type = type.replace(re, "$1" + innerType.substring(moduleName.length + 1, innerType.length) + "$2"); - type = type.replace(re, `$1${typeName}$2`); - } - } - - return type; -}; - const setDifference = (setA: Set, setB: Set): Set => { const difference = new Set(setA); for (const elem of setB) { @@ -198,77 +87,142 @@ const setDifference = (setA: Set, setB: Set): Set => { return difference; }; -const sortMembers = (sortable: PythonCollectionNode[]): PythonCollectionNode[] => { - const sorted: PythonCollectionNode[] = []; - const sortedFQNs: Set = new Set(); +const sortMembers = (sortable: PythonBase[], resolver: TypeResolver): PythonBase[] => { + const sorted: PythonBase[] = []; + const seen: Set = new Set(); // We're going to take a copy of our sortable item, because it'll make it easier if // this method doesn't have side effects. sortable = sortable.slice(); - while (sortable.length > 0) { - let idx: number | undefined; + // The first thing we want to do, is push any item which is not sortable to the very + // front of the list. This will be things like methods, properties, etc. + for (const item of sortable) { + if (!isSortableType(item)) { + sorted.push(item); + seen.add(item); + } + } + sortable = sortable.filter(i => !seen.has(i)); - for (const [idx2, item] of sortable.entries()) { - if (setDifference(new Set(item.depends_on), sortedFQNs).size === 0) { + // Now that we've pulled out everything that couldn't possibly have dependencies, + // we will go through the remaining items, and pull off any items which have no + // dependencies that we haven't already sorted. + while (sortable.length > 0) { + for (const item of (sortable as Array)) { + const itemDeps: Set = new Set(item.dependsOn(resolver).map(i => resolver.getType(i))); + if (setDifference(itemDeps, seen).size === 0) { sorted.push(item); - sortedFQNs.add(item.fqn); - idx = idx2; + seen.add(item); + break; - } else { - idx = undefined; } } - if (idx === undefined) { + const leftover = sortable.filter(i => !seen.has(i)); + if (leftover === sortable) { throw new Error("Could not sort members."); } else { - sortable.splice(idx, 1); + sortable = leftover; } } return sorted; }; -const isInModule = (modName: string, fqn: string): boolean => { - return new RegExp(`^${escapeStringRegexp(modName)}\.[^\.]+$`).test(fqn); -}; +interface PythonBase { + readonly name: string; -interface PythonNode { + emit(code: CodeMaker, resolver: TypeResolver): void; +} - // The name of the module that this Node exists in. - readonly moduleName: string; +interface PythonType extends PythonBase { + // The JSII FQN for this item, if this item doesn't exist as a JSII type, then it + // doesn't have a FQN and it should be null; + readonly fqn: string | null; - // The name of the given Node. - readonly name: string; + addMember(member: PythonBase): void; +} - // The fully qualifed name of this node. - readonly fqn: string; +interface ISortableType { + dependsOn(resolver: TypeResolver): spec.NamedTypeReference[]; +} - // Emits the entire tree of objects represented by this object into the given - // CodeMaker object. - emit(code: CodeMaker): void; +function isSortableType(arg: any): arg is ISortableType { + return arg.dependsOn !== undefined; } -interface PythonCollectionNode extends PythonNode { - // A list of other nodes that this node depends on, can be used to sort a list of - // nodes so that nodes get emited *after* the nodes it depends on. - readonly depends_on: string[]; +interface PythonTypeOpts { + bases?: spec.TypeReference[]; +} - // Given a particular item, add it as a member of this collection of nodes, returns - // the original member back. - addMember(member: PythonNode): PythonNode; +abstract class BasePythonClassType implements PythonType, ISortableType { + + public readonly name: string; + public readonly fqn: string | null; + + protected bases: spec.TypeReference[]; + protected members: PythonBase[]; + + constructor(name: string, fqn: string, opts: PythonTypeOpts) { + const { + bases = [], + } = opts; + + this.name = name; + this.fqn = fqn; + this.bases = bases; + this.members = []; + } + + public dependsOn(resolver: TypeResolver): spec.NamedTypeReference[] { + const dependencies: spec.NamedTypeReference[] = []; + + // We need to return any bases that are in the same module. + for (const base of this.bases) { + if (spec.isNamedTypeReference(base)) { + if (resolver.isInModule(base)) { + dependencies.push(base); + } + } + } + + return dependencies; + } + + public addMember(member: PythonBase) { + this.members.push(member); + } + + public emit(code: CodeMaker, resolver: TypeResolver) { + code.openBlock(`class ${this.name}(${this.getClassParams(resolver).join(", ")})`); + + if (this.members.length > 0) { + for (const member of sortMembers(this.members, resolver)) { + member.emit(code, resolver); + } + } else { + code.line("pass"); + } + + code.closeBlock(); + } + + protected abstract getClassParams(resolver: TypeResolver): string[]; +} + +interface BaseMethodOpts { + liftedProp?: spec.InterfaceType, + parent?: spec.NamedTypeReference, } -class BaseMethod implements PythonNode { +abstract class BaseMethod implements PythonBase { - public readonly moduleName: string; - public readonly parent: PythonCollectionNode; public readonly name: string; - protected readonly decorator?: string; - protected readonly implicitParameter: string; + protected readonly abstract implicitParameter: string; protected readonly jsiiMethod?: string; + protected readonly decorator?: string; protected readonly classAsFirstParameter: boolean = false; protected readonly returnFromJSIIMethod: boolean = true; @@ -276,40 +230,38 @@ class BaseMethod implements PythonNode { private readonly parameters: spec.Parameter[]; private readonly returns?: spec.TypeReference; private readonly liftedProp?: spec.InterfaceType; + private readonly parent?: spec.NamedTypeReference; - constructor(moduleName: string, - parent: PythonCollectionNode, - name: string, + constructor(name: string, jsName: string | undefined, parameters: spec.Parameter[], returns?: spec.TypeReference, - liftedProp?: spec.InterfaceType) { - this.moduleName = moduleName; - this.parent = parent; + opts: BaseMethodOpts = {}) { this.name = name; this.jsName = jsName; this.parameters = parameters; this.returns = returns; - this.liftedProp = liftedProp; - } - - get fqn(): string { - return `${this.moduleName}.${this.name}`; + this.liftedProp = opts.liftedProp; + this.parent = opts.parent; } - public emit(code: CodeMaker) { - const returnType = this.getReturnType(this.returns); + public emit(code: CodeMaker, resolver: TypeResolver) { + let returnType: string; + if (this.returns !== undefined) { + returnType = resolver.resolve(this.returns, { forwardReferences: false }); + } else { + returnType = "None"; + } // We need to turn a list of JSII parameters, into Python style arguments with // gradual typing, so we'll have to iterate over the list of parameters, and // build the list, converting as we go. - // TODO: Handle imports (if needed) for all of these types. const pythonParams: string[] = [this.implicitParameter]; for (const param of this.parameters) { const paramName = toPythonIdentifier(param.name); - const paramType = toPythonType(param.type); + const paramType = resolver.resolve(param.type, { forwardReferences: false}); - pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, this.moduleName)}`); + pythonParams.push(`${paramName}: ${paramType}`); } // If we have a lifted parameter, then we'll drop the last argument to our params @@ -327,10 +279,10 @@ class BaseMethod implements PythonNode { // Iterate over all of our props, and reflect them into our params. for (const prop of this.liftedProp.properties) { const paramName = toPythonIdentifier(prop.name); - const paramType = toPythonType(prop.type); + const paramType = resolver.resolve(prop.type, { forwardReferences: false }); const paramDefault = prop.type.optional ? "=None" : ""; - pythonParams.push(`${paramName}: ${formatPythonType(paramType, false, this.moduleName)}${paramDefault}`); + pythonParams.push(`${paramName}: ${paramType}${paramDefault}`); } } } else if (this.parameters.length >= 1 && this.parameters.slice(-1)[0].variadic) { @@ -341,36 +293,39 @@ class BaseMethod implements PythonNode { const lastParameter = this.parameters.slice(-1)[0]; const paramName = toPythonIdentifier(lastParameter.name); - const paramType = toPythonType(lastParameter.type, false); + const paramType = resolver.resolve( + lastParameter.type, + { forwardReferences: false, ignoreOptional: true }, + ); - pythonParams.push(`*${paramName}: ${formatPythonType(paramType, false, this.moduleName)}`); + pythonParams.push(`*${paramName}: ${paramType}`); } if (this.decorator !== undefined) { code.line(`@${this.decorator}`); } - code.openBlock(`def ${this.name}(${pythonParams.join(", ")}) -> ${formatPythonType(returnType, false, this.moduleName)}`); - this.emitBody(code); + code.openBlock(`def ${this.name}(${pythonParams.join(", ")}) -> ${returnType}`); + this.emitBody(code, resolver); code.closeBlock(); } - private emitBody(code: CodeMaker) { + private emitBody(code: CodeMaker, resolver: TypeResolver) { if (this.jsiiMethod === undefined) { code.line("..."); } else { if (this.liftedProp !== undefined) { - this.emitAutoProps(code); + this.emitAutoProps(code, resolver); } - this.emitJsiiMethodCall(code); + this.emitJsiiMethodCall(code, resolver); } } - private emitAutoProps(code: CodeMaker) { + private emitAutoProps(code: CodeMaker, resolver: TypeResolver) { const lastParameter = this.parameters.slice(-1)[0]; - const argName: string = toPythonIdentifier(lastParameter.name); - const typeName: string = formatPythonType(toPythonType(lastParameter.type), true, this.moduleName); + const argName = toPythonIdentifier(lastParameter.name); + const typeName = resolver.resolve(lastParameter.type); // We need to build up a list of properties, which are mandatory, these are the // ones we will specifiy to start with in our dictionary literal. @@ -398,12 +353,15 @@ class BaseMethod implements PythonNode { } } - private emitJsiiMethodCall(code: CodeMaker) { + private emitJsiiMethodCall(code: CodeMaker, resolver: TypeResolver) { const methodPrefix: string = this.returnFromJSIIMethod ? "return " : ""; const jsiiMethodParams: string[] = []; if (this.classAsFirstParameter) { - jsiiMethodParams.push(this.parent.name); + if (this.parent === undefined) { + throw new Error("Parent not known."); + } + jsiiMethodParams.push(resolver.resolve(this.parent)); } jsiiMethodParams.push(this.implicitParameter); if (this.jsName !== undefined) { @@ -417,69 +375,71 @@ class BaseMethod implements PythonNode { code.line(`${methodPrefix}jsii.${this.jsiiMethod}(${jsiiMethodParams.join(", ")}, [${paramNames.join(", ")}])`); } +} - private getReturnType(type?: spec.TypeReference): string { - return type ? toPythonType(type) : "None"; - } +interface BasePropertyOpts { + immutable?: boolean; } -class BaseProperty implements PythonNode { +abstract class BaseProperty implements PythonBase { - public readonly moduleName: string; public readonly name: string; - protected readonly decorator: string; - protected readonly implicitParameter: string; + protected readonly abstract decorator: string; + protected readonly abstract implicitParameter: string; protected readonly jsiiGetMethod?: string; protected readonly jsiiSetMethod?: string; - protected readonly jsName: string; + private readonly jsName: string; private readonly type: spec.TypeReference; private readonly immutable: boolean; - constructor(moduleName: string, name: string, jsName: string, type: spec.TypeReference, immutable: boolean) { - this.moduleName = moduleName; + constructor(name: string, jsName: string, type: spec.TypeReference, opts: BasePropertyOpts = {}) { + const { + immutable = false, + } = opts; + this.name = name; this.jsName = jsName; this.type = type; this.immutable = immutable; } - get fqn(): string { - return `${this.moduleName}.${this.name}`; - } - - public emit(code: CodeMaker) { - const returnType = toPythonType(this.type); + public emit(code: CodeMaker, resolver: TypeResolver) { + const pythonType = resolver.resolve(this.type, { forwardReferences: false }); code.line(`@${this.decorator}`); - code.openBlock(`def ${this.name}(${this.implicitParameter}) -> ${formatPythonType(returnType, false, this.moduleName)}`); - this.emitGetterBody(code); + code.openBlock(`def ${this.name}(${this.implicitParameter}) -> ${pythonType}`); + if (this.jsiiGetMethod !== undefined) { + code.line(`return jsii.${this.jsiiGetMethod}(${this.implicitParameter}, "${this.jsName}")`); + } else { + code.line("..."); + } code.closeBlock(); if (!this.immutable) { code.line(`@${this.name}.setter`); - code.openBlock(`def ${this.name}(${this.implicitParameter}, value: ${formatPythonType(returnType, false, this.moduleName)})`); - this.emitSetterBody(code); + code.openBlock(`def ${this.name}(${this.implicitParameter}, value: ${pythonType})`); + if (this.jsiiSetMethod !== undefined) { + code.line(`return jsii.${this.jsiiSetMethod}(${this.implicitParameter}, "${this.jsName}", value)`); + } else { + code.line("..."); + } code.closeBlock(); } } +} - private emitGetterBody(code: CodeMaker) { - if (this.jsiiGetMethod === undefined) { - code.line("..."); - } else { - code.line(`return jsii.${this.jsiiGetMethod}(${this.implicitParameter}, "${this.jsName}")`); - } - } +class Interface extends BasePythonClassType { - private emitSetterBody(code: CodeMaker) { - if (this.jsiiSetMethod === undefined) { - code.line("..."); - } else { - code.line(`return jsii.${this.jsiiSetMethod}(${this.implicitParameter}, "${this.jsName}", value)`); - } + protected getClassParams(resolver: TypeResolver): string[] { + const params: string[] = this.bases.map(b => resolver.resolve(b)); + + params.push("jsii.compat.Protocol"); + + return params; } + } class InterfaceMethod extends BaseMethod { @@ -491,113 +451,23 @@ class InterfaceProperty extends BaseProperty { protected readonly implicitParameter: string = "self"; } -class Interface implements PythonCollectionNode { - - public readonly moduleName: string; - public readonly name: string; - - private bases: string[]; - private members: PythonNode[]; - - constructor(moduleName: string, name: string, bases: string[]) { - this.moduleName = moduleName; - this.name = name; - this.bases = bases; - - this.members = []; - } - - get fqn(): string { - return `${this.moduleName}.${this.name}`; - } - - get depends_on(): string[] { - return this.bases.filter(base => isInModule(this.moduleName, base)); - } - - public addMember(member: PythonNode): PythonNode { - this.members.push(member); - return member; - } - - public emit(code: CodeMaker) { - const interfaceBases = this.bases.map(baseType => formatPythonType(baseType, true, this.moduleName)); - interfaceBases.push("_Protocol"); - - code.openBlock(`class ${this.name}(${interfaceBases.join(",")})`); - if (this.members.length > 0) { - for (const member of this.members) { - member.emit(code); - } - } else { - code.line("pass"); - } - code.closeBlock(); - } -} - -class TypedDictProperty implements PythonNode { - - public readonly moduleName: string; - public readonly name: string; - - private readonly type: spec.TypeReference; - - constructor(moduleName: string, name: string, type: spec.TypeReference) { - this.moduleName = moduleName; - this.name = name; - this.type = type; - } - - get fqn(): string { - return `${this.moduleName}.${this.name}`; - } - - get optional(): boolean { - return this.type.optional || false; - } - - public emit(code: CodeMaker) { - const propType: string = formatPythonType(toPythonType(this.type, false), undefined, this.moduleName); - code.line(`${this.name}: ${propType}`); - } -} - -class TypedDict implements PythonCollectionNode { - public readonly moduleName: string; - public readonly name: string; - - private members: TypedDictProperty[]; - - constructor(moduleName: string, name: string) { - this.moduleName = moduleName; - this.name = name; - - this.members = []; - } - - get fqn(): string { - return `${this.moduleName}.${this.name}`; - } - - get depends_on(): string[] { - return []; - } - - public addMember(member: TypedDictProperty): TypedDictProperty { - this.members.push(member); - return member; - } +class TypedDict extends BasePythonClassType { - public emit(code: CodeMaker) { + public emit(code: CodeMaker, resolver: TypeResolver) { // MyPy doesn't let us mark some keys as optional, and some keys as mandatory, // we can either mark either the entire class as mandatory or the entire class // as optional. However, we can make two classes, one with all mandatory keys // and one with all optional keys in order to emulate this. So we'll go ahead // and implement this "split" class logic. - const mandatoryMembers = this.members.filter(item => !item.optional); - const optionalMembers = this.members.filter(item => item.optional); + const classParams = this.getClassParams(resolver); + + const mandatoryMembers = this.members.filter( + item => item instanceof TypedDictProperty ? !item.optional : true + ); + const optionalMembers = this.members.filter( + item => item instanceof TypedDictProperty ? item.optional : false + ); if (mandatoryMembers.length >= 1 && optionalMembers.length >= 1) { // In this case, we have both mandatory *and* optional members, so we'll @@ -605,16 +475,16 @@ class TypedDict implements PythonCollectionNode { // We'll emit the optional members first, just because it's a little nicer // for the final class in the chain to have the mandatory members. - code.openBlock(`class _${this.name}(_TypedDict, total=False)`); + code.openBlock(`class _${this.name}(${classParams.concat(["total=False"]).join(", ")})`); for (const member of optionalMembers) { - member.emit(code); + member.emit(code, resolver); } code.closeBlock(); // Now we'll emit the mandatory members. code.openBlock(`class ${this.name}(_${this.name})`); - for (const member of mandatoryMembers) { - member.emit(code); + for (const member of sortMembers(mandatoryMembers, resolver)) { + member.emit(code, resolver); } code.closeBlock(); } else { @@ -622,15 +492,15 @@ class TypedDict implements PythonCollectionNode { // we'll see if we have any optional members, if we don't then we'll use // total=True instead of total=False for the class. if (optionalMembers.length >= 1) { - code.openBlock(`class ${this.name}(_TypedDict, total=False)`); + code.openBlock(`class ${this.name}(${classParams.concat(["total=False"]).join(", ")})`); } else { - code.openBlock(`class ${this.name}(_TypedDict)`); + code.openBlock(`class ${this.name}(${classParams.join(", ")})`); } // Finally we'll just iterate over and emit all of our members. if (this.members.length > 0) { - for (const member of this.members) { - member.emit(code); + for (const member of sortMembers(this.members, resolver)) { + member.emit(code, resolver); } } else { code.line("pass"); @@ -639,6 +509,52 @@ class TypedDict implements PythonCollectionNode { code.closeBlock(); } } + + protected getClassParams(resolver: TypeResolver): string[] { + const params: string[] = this.bases.map(b => resolver.resolve(b)); + + params.push("jsii.compat.TypedDict"); + + return params; + } + +} + +class TypedDictProperty implements PythonBase { + + public readonly name: string; + + private readonly type: spec.TypeReference; + + constructor(name: string, type: spec.TypeReference) { + this.name = name; + this.type = type; + } + + get optional(): boolean { + return this.type.optional !== undefined ? this.type.optional : false; + } + + public emit(code: CodeMaker, resolver: TypeResolver) { + const resolvedType = resolver.resolve( + this.type, + { forwardReferences: false, ignoreOptional: true } + ); + code.line(`${this.name}: ${resolvedType}`); + } +} + +class Class extends BasePythonClassType { + + protected getClassParams(resolver: TypeResolver): string[] { + const params: string[] = this.bases.map(b => resolver.resolve(b)); + + params.push("metaclass=jsii.JSIIMeta"); + params.push(`jsii_type="${this.fqn}"`); + + return params; + } + } class StaticMethod extends BaseMethod { @@ -673,143 +589,63 @@ class Property extends BaseProperty { protected readonly jsiiSetMethod: string = "set"; } -class Class implements PythonCollectionNode { - public readonly moduleName: string; - public readonly name: string; - - private jsiiFQN: string; - private bases: string[]; - private members: PythonNode[]; - - constructor(moduleName: string, name: string, jsiiFQN: string, bases: string[]) { - this.moduleName = moduleName; - this.name = name; - - this.jsiiFQN = jsiiFQN; - this.bases = bases; - this.members = []; - } - - get fqn(): string { - return `${this.moduleName}.${this.name}`; - } +class Enum extends BasePythonClassType { - get depends_on(): string[] { - return this.bases.filter(base => isInModule(this.moduleName, base)); + protected getClassParams(_resolver: TypeResolver): string[] { + return ["enum.Enum"]; } - public addMember(member: PythonNode): PythonNode { - this.members.push(member); - return member; - } - - public emit(code: CodeMaker) { - const classParams: string[] = this.bases.map(baseType => formatPythonType(baseType, true, this.moduleName)); - - classParams.push("metaclass=jsii.JSIIMeta"); - classParams.push(`jsii_type="${this.jsiiFQN}"`); - - code.openBlock(`class ${this.name}(${classParams.join(", ")})`); - if (this.members.length > 0) { - for (const member of this.members) { - member.emit(code); - } - } else { - code.line("pass"); - } - code.closeBlock(); - } } -class Enum implements PythonCollectionNode { - public readonly moduleName: string; - public readonly name: string; - - private members: PythonNode[]; - - constructor(moduleName: string, name: string) { - this.moduleName = moduleName; - this.name = name; - this.members = []; - } - - get fqn(): string { - return `${this.moduleName}.${this.name}`; - } - - get depends_on(): string[] { - return []; - } - - public addMember(member: PythonNode): PythonNode { - this.members.push(member); - return member; - } +class EnumMember implements PythonBase { - public emit(code: CodeMaker) { - code.openBlock(`class ${this.name}(enum.Enum)`); - if (this.members.length > 0) { - for (const member of this.members) { - member.emit(code); - } - } else { - code.line("pass"); - } - code.closeBlock(); - } -} - -class EnumMember implements PythonNode { - public readonly moduleName: string; public readonly name: string; private readonly value: string; - constructor(moduleName: string, name: string, value: string) { - this.moduleName = moduleName; + constructor(name: string, value: string) { this.name = name; this.value = value; } - get fqn(): string { - return `${this.moduleName}.${this.name}`; - } - - public emit(code: CodeMaker) { + public emit(code: CodeMaker, _resolver: TypeResolver) { code.line(`${this.name} = "${this.value}"`); } } -class Module { +interface ModuleOpts { + assembly: spec.Assembly, + assemblyFilename: string; + loadAssembly: boolean; +} + +class Module implements PythonType { public readonly name: string; - public readonly assembly: spec.Assembly; - public readonly assemblyFilename: string; - public readonly loadAssembly: boolean; + public readonly fqn: string | null; - private members: PythonCollectionNode[]; - private subModules: string[]; + private assembly: spec.Assembly; + private assemblyFilename: string; + private loadAssembly: boolean; + private members: PythonBase[]; - constructor(ns: string, assembly: spec.Assembly, assemblyFilename: string, loadAssembly: boolean = false) { - this.name = ns; - this.assembly = assembly; - this.assemblyFilename = assemblyFilename; - this.loadAssembly = loadAssembly; + constructor(name: string, fqn: string | null, opts: ModuleOpts) { + this.name = name; + this.fqn = fqn; + this.assembly = opts.assembly; + this.assemblyFilename = opts.assemblyFilename; + this.loadAssembly = opts.loadAssembly; this.members = []; - this.subModules = []; } - public addMember(member: PythonCollectionNode): PythonCollectionNode { + public addMember(member: PythonBase) { this.members.push(member); - return member; } - public addSubmodule(module: string) { - this.subModules.push(module); - } + public emit(code: CodeMaker, resolver: TypeResolver) { + resolver = this.fqn ? resolver.bind(this.fqn) : resolver; - public emit(code: CodeMaker) { // Before we write anything else, we need to write out our module headers, this // is where we handle stuff like imports, any required initialization, etc. code.line("import datetime"); @@ -817,9 +653,9 @@ class Module { code.line("import typing"); code.line(); code.line("import jsii"); + code.line("import jsii.compat"); code.line("import publication"); code.line(); - code.line(this.generateImportFrom("jsii.compat", ["Protocol", "TypedDict"])); code.line("from jsii.python import classproperty"); // Go over all of the modules that we need to import, and import them. @@ -835,17 +671,6 @@ class Module { code.line(`import ${toPythonModuleName(depName)}`); } - const moduleRe = new RegExp(`^${escapeStringRegexp(this.name)}\.`); - for (const [idx, subModule] of this.subModules.sort().entries()) { - // If this our first subModule, add a blank line to format our imports - // slightly nicer. - if (idx === 0) { - code.line(); - } - - code.line(`from . import ${subModule.replace(moduleRe, "")}`); - } - // Determine if we need to write out the kernel load line. if (this.loadAssembly) { code.line( @@ -857,347 +682,544 @@ class Module { ); } - // Now that we've gotten all of the module header stuff done, we need to go - // through and actually write out the meat of our module. - for (const member of sortMembers(this.members)) { - member.emit(code); + // Emit all of our members. + for (const member of sortMembers(this.members, resolver)) { + member.emit(code, resolver); } // Whatever names we've exported, we'll write out our __all__ that lists them. - code.line(`__all__ = [${this.getExportedNames().map(s => `"${s}"`).join(", ")}]`); + code.line(`__all__ = [${this.members.map(m => `"${m.name}"`).sort().join(", ")}]`); // Finally, we'll use publication to ensure that all of the non-public names // get hidden from dir(), tab-complete, etc. code.line(); code.line("publication.publish()"); } +} - private getExportedNames(): string[] { - // We assume that anything that is a member of this module, will be exported by - // this module. - const exportedNames = this.members.map(m => m.name); - - // If this module will be outputting the Assembly, then we also want to export - // our assembly variable. - if (this.loadAssembly) { - exportedNames.push("__jsii_assembly__"); - } - - // We also need to export all of our submodules. - const moduleRe = new RegExp(`^${escapeStringRegexp(this.name)}\.`); - exportedNames.push(...this.subModules.map(item => item.replace(moduleRe, ""))); - - return exportedNames.sort(); - - } +interface PackageMetadata { + summary?: string; + readme?: string; + url?: string; +} - private generateImportFrom(from: string, names: string[]): string { - // Whenever we import something, we want to prefix all of the names we're - // importing with an underscore to indicate that these names are private. We - // do this, because otherwise we could get clashes in the names we use, and the - // names of exported classes. - const importNames = names.map(n => `${n} as _${n}`); - return `from ${from} import ${importNames.join(", ")}`; - } +interface PackageData { + filename: string; + data: string | null; } -class PythonGenerator extends Generator { +class Package { - private currentMember?: PythonCollectionNode; - private modules: Module[]; - private moduleStack: Module[]; + public readonly name: string; + public readonly version: string; + public readonly metadata: PackageMetadata; - constructor(options = new GeneratorOptions()) { - super(options); + private modules: Map; + private data: Map; - this.code.openBlockFormatter = s => `${s}:`; - this.code.closeBlockFormatter = _s => ""; + constructor(name: string, version: string, metadata: PackageMetadata) { + this.name = name; + this.version = version; + this.metadata = metadata; - this.currentMember = undefined; - this.modules = []; - this.moduleStack = []; + this.modules = new Map(); + this.data = new Map(); } - protected getAssemblyOutputDir(mod: spec.Assembly) { - return path.join("src", toPythonModuleFilename(toPythonModuleName(mod.name)), "_jsii"); + public addModule(module: Module) { + this.modules.set(module.name, module); } - protected onBeginAssembly(assm: spec.Assembly, _fingerprint: boolean) { - // We need to write out an __init__.py for our _jsii package so that - // importlib.resources will be able to load our assembly from it. - const assemblyInitFilename = path.join(this.getAssemblyOutputDir(assm), "__init__.py"); + public addData(module: Module, filename: string, data: string | null) { + if (!this.data.has(module.name)) { + this.data.set(module.name, new Array()); + } - this.code.openFile(assemblyInitFilename); - this.code.closeFile(assemblyInitFilename); + this.data.get(module.name)!.push({filename, data}); } - protected onEndAssembly(assm: spec.Assembly, _fingerprint: boolean) { - const packageName = toPythonPackageName(assm.name); - const topLevelModuleName = toPythonModuleName(packageName); - const moduleNames = this.modules.map(m => m.name); - const pyTypedFilename = path.join("src", toPythonModuleFilename(topLevelModuleName), "py.typed"); + public write(code: CodeMaker, resolver: TypeResolver) { + const modules = [...this.modules.values()].sort((a, b) => a.name.localeCompare(b.name)); + + // Iterate over all of our modules, and write them out to disk. + for (const mod of modules) { + const filename = path.join("src", pythonModuleNameToFilename(mod.name), "__init__.py"); + + code.openFile(filename); + mod.emit(code, resolver); + code.closeFile(filename); + } - moduleNames.push(`${topLevelModuleName}._jsii`); - moduleNames.sort(); + // Handle our package data. + const packageData: {[key: string]: string[]} = {}; + for (const [mod, pdata] of this.data) { + for (const data of pdata) { + if (data.data != null) { + const filepath = path.join("src", pythonModuleNameToFilename(mod), data.filename); - // We need to write out our packaging for the Python ecosystem here. + code.openFile(filepath); + code.line(data.data); + code.closeFile(filepath); + } + } + + packageData[mod] = pdata.map(pd => pd.filename); + } + + const setupKwargs = { + name: this.name, + version: this.version, + description: this.metadata.summary, + url: this.metadata.url, + package_dir: {"": "src"}, + packages: modules.map(m => m.name), + package_data: packageData, + python_requires: ">=3.6", + install_requires: ["publication"], + }; + + // We Need a setup.py to make this Package, actually a Package. // TODO: // - Author - // - README // - License // - Classifiers - // - install_requires - this.code.openFile("setup.py"); - this.code.line("import setuptools"); - this.code.indent("setuptools.setup("); - this.code.line(`name="${packageName}",`); - this.code.line(`version="${assm.version}",`); - this.code.line(`description="${assm.description}",`); - this.code.line(`url="${assm.homepage}",`); - this.code.line('package_dir={"": "src"},'); - this.code.line(`packages=[${moduleNames.map(m => `"${m}"`).join(",")}],`); - this.code.line(`package_data={"${topLevelModuleName}": ["py.typed"], "${topLevelModuleName}._jsii": ["*.jsii.tgz"]},`); - this.code.line('python_requires=">=3.6",'); - this.code.line(`install_requires=["publication"],`); - this.code.unindent(")"); - this.code.closeFile("setup.py"); + code.openFile("setup.py"); + code.line("import json"); + code.line("import setuptools"); + code.line(); + code.line('kwargs = json.loads("""'); + code.line(JSON.stringify(setupKwargs, null, 4)); + code.line('""")'); + code.line(); + code.line("setuptools.setup(**kwargs)"); + code.closeFile("setup.py"); // Because we're good citizens, we're going to go ahead and support pyproject.toml // as well. // TODO: Might be easier to just use a TOML library to write this out. - this.code.openFile("pyproject.toml"); - this.code.line("[build-system]"); - this.code.line('requires = ["setuptools", "wheel"]'); - this.code.closeFile("pyproject.toml"); + code.openFile("pyproject.toml"); + code.line("[build-system]"); + code.line('requires = ["setuptools", "wheel"]'); + code.closeFile("pyproject.toml"); // We also need to write out a MANIFEST.in to ensure that all of our required // files are included. - this.code.openFile("MANIFEST.in"); - this.code.line("include pyproject.toml"); - this.code.closeFile("MANIFEST.in"); + code.openFile("MANIFEST.in"); + code.line("include pyproject.toml"); + code.closeFile("MANIFEST.in"); + } +} + +interface TypeResolverOpts { + forwardReferences?: boolean; + ignoreOptional?: boolean; +} + +class TypeResolver { + + private readonly types: Map; + private boundTo?: string; + private readonly moduleRe = new RegExp("^((?:[^A-Z\.][^\.]+\.?)+)\.([A-Z].+)$"); - // We also need to write out a py.typed file, to Signal to MyPy that these files - // are safe to use for typechecking. - this.code.openFile(pyTypedFilename); - this.code.closeFile(pyTypedFilename); + constructor(types: Map, boundTo?: string) { + this.types = types; + this.boundTo = boundTo !== undefined ? this.toPythonFQN(boundTo) : boundTo; } - protected onBeginNamespace(ns: string) { - const moduleName = toPythonModuleName(ns); - const loadAssembly = this.assembly.name === ns ? true : false; - const mod = new Module(moduleName, this.assembly, this.getAssemblyFileName(), loadAssembly); + public bind(fqn: string): TypeResolver { + return new TypeResolver(this.types, fqn); + } + + public isInModule(typeRef: spec.NamedTypeReference): boolean { + const pythonType = this.toPythonFQN(typeRef.fqn); + const [, moduleName] = pythonType.match(this.moduleRe) as string[]; + + return this.boundTo !== undefined && this.boundTo === moduleName; + } + + public getType(typeRef: spec.NamedTypeReference): PythonType { + const type = this.types.get(typeRef.fqn); - for (const parentMod of this.moduleStack) { - parentMod.addSubmodule(moduleName); + if (type === undefined) { + throw new Error(`Could not locate type: "${typeRef.fqn}"`); } - this.modules.push(mod); - this.moduleStack.push(mod); + return type; } - protected onEndNamespace(_ns: string) { - const module = this.moduleStack.pop() as Module; - const moduleFilename = path.join("src", toPythonModuleFilename(module.name), "__init__.py"); + public resolve( + typeRef: spec.TypeReference, + opts: TypeResolverOpts = { forwardReferences: true, ignoreOptional: false }): string { + // First, we need to resolve our given type reference into the Python type. + let pythonType = this.toPythonType(typeRef, opts.ignoreOptional); + + // If we split our types by any of the "special" characters that can't appear in + // identifiers (like "[],") then we will get a list of all of the identifiers, + // no matter how nested they are. The downside is we might get trailing/leading + // spaces or empty items so we'll need to trim and filter this list. + const types = pythonType.split(/[\[\],]/).map((s: string) => s.trim()).filter(s => s !== ""); + + for (const innerType of types) { + // Built in types do not need formatted in any particular way. + if (PYTHON_BUILTIN_TYPES.indexOf(innerType) > -1) { + continue; + } + + const [, moduleName, typeName] = innerType.match(this.moduleRe) as string[]; + + // If our resolver is not bound to the same module as the type we're trying to + // resolve, then we'll just skip ahead to the next one, no further changes are + // reqired. + if (this.boundTo === undefined || moduleName !== this.boundTo) { + continue; + } else { + // Otherwise, we have to implement some particular logic in order to deal + // with forward references and trimming our module name out of the type. + // This re will look for the entire type, boxed by either the start/end of + // a string, a comma, a space, a quote, or open/closing brackets. This will + // ensure that we only match whole type names, and not partial ones. + const re = new RegExp('((?:^|[[,\\s])"?)' + innerType + '("?(?:$|[\\],\\s]))'); + + // We need to handle forward references, our caller knows if we're able to + // use them in the current context or not, so if not, we'll wrap our forward + // reference in quotes. + if (opts.forwardReferences !== undefined && !opts.forwardReferences) { + pythonType = pythonType.replace(re, `$1"${innerType}"$2`); + } + + // Now that we've gotten forward references out of the way, we will want + // to replace the entire type string, with just the type portion. + pythonType = pythonType.replace(re, `$1${typeName}$2`); + } + } - this.code.openFile(moduleFilename); - module.emit(this.code); - this.code.closeFile(moduleFilename); + return pythonType; } - protected onBeginClass(cls: spec.ClassType, _abstract: boolean | undefined) { - const currentModule = this.currentModule(); + private toPythonType(typeRef: spec.TypeReference, ignoreOptional?: boolean): string { + let pythonType: string; - // TODO: Figure out what to do with abstract here. + // Get the underlying python type. + if (spec.isPrimitiveTypeReference(typeRef)) { + pythonType = this.toPythonPrimitive(typeRef.primitive); + } else if (spec.isCollectionTypeReference(typeRef)) { + pythonType = this.toPythonCollection(typeRef); + } else if (spec.isNamedTypeReference(typeRef)) { + pythonType = this.toPythonFQN(typeRef.fqn); + } else if (typeRef.union) { + const types = new Array(); + for (const subtype of typeRef.union.types) { + types.push(this.toPythonType(subtype)); + } + pythonType = `typing.Union[${types.join(", ")}]`; + } else { + throw new Error("Invalid type reference: " + JSON.stringify(typeRef)); + } - this.currentMember = currentModule.addMember( - new Class( - currentModule.name, - toPythonIdentifier(cls.name), - cls.fqn, - (cls.base !== undefined ? [cls.base] : []).map(b => toPythonType(b)), - ) + // If our type is Optional, then we'll wrap our underlying type with typing.Optional + // However, if we're not respecting optionals, then we'll just skip over this. + if (!ignoreOptional && typeRef.optional) { + pythonType = `typing.Optional[${pythonType}]`; + } + + return pythonType; + } + + private toPythonPrimitive(primitive: spec.PrimitiveType): string { + switch (primitive) { + case spec.PrimitiveType.Boolean: return "bool"; + case spec.PrimitiveType.Date: return "datetime.datetime"; + case spec.PrimitiveType.Json: return "typing.Mapping[typing.Any, typing.Any]"; + case spec.PrimitiveType.Number: return "jsii.Number"; + case spec.PrimitiveType.String: return "str"; + case spec.PrimitiveType.Any: return "typing.Any"; + default: + throw new Error("Unknown primitive type: " + primitive); + } + } + + private toPythonCollection(ref: spec.CollectionTypeReference): string { + const elementPythonType = this.toPythonType(ref.collection.elementtype); + switch (ref.collection.kind) { + case spec.CollectionKind.Array: return `typing.List[${elementPythonType}]`; + case spec.CollectionKind.Map: return `typing.Mapping[str,${elementPythonType}]`; + default: + throw new Error(`Unsupported collection kind: ${ref.collection.kind}`); + } + } + + private toPythonFQN(fqn: string): string { + const [, modulePart, typePart] = fqn.match(/^((?:[^A-Z\.][^\.]+\.?)+)(?:\.([A-Z].+))?$/) as string[]; + const fqnParts: string[] = [toPythonModuleName(modulePart)]; + + if (typePart) { + fqnParts.push(typePart.split(".").map(cur => toPythonIdentifier(cur)).join(".")); + } + + return fqnParts.join("."); + } +} + +class PythonGenerator extends Generator { + + private package: Package; + private types: Map; + + constructor(options = new GeneratorOptions()) { + super(options); + + this.code.openBlockFormatter = s => `${s}:`; + this.code.closeBlockFormatter = _s => ""; + + this.types = new Map(); + } + + protected getAssemblyOutputDir(assm: spec.Assembly) { + return path.join("src", pythonModuleNameToFilename(this.getAssemblyModuleName(assm))); + } + + protected onBeginAssembly(assm: spec.Assembly, _fingerprint: boolean) { + this.package = new Package( + toPythonPackageName(assm.name), + assm.version, + { + summary: assm.description, + readme: assm.readme !== undefined ? assm.readme.markdown : "", + url: assm.homepage, + }, + ); + + const assemblyModule = new Module( + this.getAssemblyModuleName(assm), + null, + { assembly: assm, + assemblyFilename: this.getAssemblyFileName(), + loadAssembly: false }, + ); + + this.package.addModule(assemblyModule); + this.package.addData(assemblyModule, this.getAssemblyFileName(), null); + } + + protected onEndAssembly(_assm: spec.Assembly, _fingerprint: boolean) { + this.package.write(this.code, new TypeResolver(this.types)); + } + + protected onBeginNamespace(ns: string) { + const module = new Module( + toPythonModuleName(ns), + ns, + { assembly: this.assembly, + assemblyFilename: this.getAssemblyFileName(), + loadAssembly: ns === this.assembly.name }, + ); + + this.package.addModule(module); + this.types.set(ns, module); + + // If this is our top level namespace, then we'll want to add a py.typed marker + // so that all of our typing works. + if (ns === this.assembly.name) { + this.package.addData(module, "py.typed", ""); + } + } + + protected onBeginClass(cls: spec.ClassType, _abstract: boolean | undefined) { + // TODO: Figure out what to do with abstract here. + const klass = new Class( + toPythonIdentifier(cls.name), + cls.fqn, + { bases: cls.base !== undefined ? [cls.base] : [] } ); if (cls.initializer !== undefined) { - this.currentMember.addMember( + const { parameters = [] } = cls.initializer; + + klass.addMember( new Initializer( - currentModule.name, - this.currentMember, "__init__", undefined, - cls.initializer.parameters || [], + parameters, cls.initializer.returns, - this.getliftedProp(cls.initializer), + { liftedProp: this.getliftedProp(cls.initializer), parent: cls }, ) ); } - } - protected onEndClass(_cls: spec.ClassType) { - this.currentMember = undefined; + this.addPythonType(klass); } - protected onStaticMethod(_cls: spec.ClassType, method: spec.Method) { - this.currentMember!.addMember( + protected onStaticMethod(cls: spec.ClassType, method: spec.Method) { + const { parameters = [] } = method; + + this.getPythonType(cls.fqn).addMember( new StaticMethod( - this.currentModule().name, - this.currentMember!, toPythonMethodName(method.name!), - method.name!, - method.parameters || [], + method.name, + parameters, method.returns, - this.getliftedProp(method), + { liftedProp: this.getliftedProp(method) }, ) ); } - protected onMethod(_cls: spec.ClassType, method: spec.Method) { - this.currentMember!.addMember( - new Method( - this.currentModule().name, - this.currentMember!, - toPythonMethodName(method.name!), - method.name!, - method.parameters || [], - method.returns, - this.getliftedProp(method), + protected onStaticProperty(cls: spec.ClassType, prop: spec.Property) { + this.getPythonType(cls.fqn).addMember( + new StaticProperty( + toPythonPropertyName(prop.name), + prop.name, + prop.type, + { immutable: prop.immutable }, ) ); } - protected onStaticProperty(_cls: spec.ClassType, prop: spec.Property) { - this.currentMember!.addMember( - new StaticProperty( - this.currentModule().name, - toPythonPropertyName(prop.name!), - prop.name!, - prop.type, - prop.immutable || false + protected onMethod(cls: spec.ClassType, method: spec.Method) { + const { parameters = [] } = method; + + this.getPythonType(cls.fqn).addMember( + new Method( + toPythonMethodName(method.name!), + method.name, + parameters, + method.returns, + { liftedProp: this.getliftedProp(method) }, ) ); } - protected onProperty(_cls: spec.ClassType, prop: spec.Property) { - this.currentMember!.addMember( + protected onProperty(cls: spec.ClassType, prop: spec.Property) { + this.getPythonType(cls.fqn).addMember( new Property( - this.currentModule().name, - toPythonPropertyName(prop.name!), - prop.name!, + toPythonPropertyName(prop.name), + prop.name, prop.type, - prop.immutable || false, + { immutable: prop.immutable }, ) ); } protected onBeginInterface(ifc: spec.InterfaceType) { - const currentModule = this.currentModule(); + let iface: Interface | TypedDict; if (ifc.datatype) { - this.currentMember = currentModule.addMember( - new TypedDict( - currentModule.name, - toPythonIdentifier(ifc.name) - ) + iface = new TypedDict( + toPythonIdentifier(ifc.name), + ifc.fqn, + { bases: ifc.interfaces }, ); } else { - this.currentMember = currentModule.addMember( - new Interface( - currentModule.name, - toPythonIdentifier(ifc.name), - (ifc.interfaces || []).map(i => toPythonType(i)) - ) + iface = new Interface( + toPythonIdentifier(ifc.name), + ifc.fqn, + { bases: ifc.interfaces }, ); } - } - protected onEndInterface(_ifc: spec.InterfaceType) { - this.currentMember = undefined; + this.addPythonType(iface); } + protected onEndInterface(_ifc: spec.InterfaceType) { return; } + protected onInterfaceMethod(ifc: spec.InterfaceType, method: spec.Method) { - if (ifc.datatype) { - throw new Error("Cannot have a method on a data type."); - } + const { parameters = [] } = method; - this.currentMember!.addMember( + this.getPythonType(ifc.fqn).addMember( new InterfaceMethod( - this.currentModule().name, - this.currentMember!, toPythonMethodName(method.name!), - method.name!, - method.parameters || [], + method.name, + parameters, method.returns, - this.getliftedProp(method), + { liftedProp: this.getliftedProp(method) }, ) ); } protected onInterfaceProperty(ifc: spec.InterfaceType, prop: spec.Property) { + let ifaceProperty: InterfaceProperty | TypedDictProperty; + if (ifc.datatype) { - this.currentMember!.addMember( - new TypedDictProperty( - this.currentModule().name, - toPythonIdentifier(prop.name!), - prop.type, - ) + ifaceProperty = new TypedDictProperty( + toPythonIdentifier(prop.name), + prop.type, ); } else { - this.currentMember!.addMember( - new InterfaceProperty( - this.currentModule().name, - toPythonPropertyName(prop.name!), - prop.name!, - prop.type, - true, - ) + ifaceProperty = new InterfaceProperty( + toPythonPropertyName(prop.name), + prop.name, + prop.type, + { immutable: prop.immutable }, ); } - } - - protected onBeginEnum(enm: spec.EnumType) { - const currentModule = this.currentModule(); - const newMember = new Enum(currentModule.name, toPythonIdentifier(enm.name)); - this.currentMember = currentModule.addMember(newMember); + this.getPythonType(ifc.fqn).addMember(ifaceProperty); } - protected onEndEnum(_enm: spec.EnumType) { - this.currentMember = undefined; + protected onBeginEnum(enm: spec.EnumType) { + this.addPythonType(new Enum(toPythonIdentifier(enm.name), enm.fqn, {})); } - protected onEnumMember(_enm: spec.EnumType, member: spec.EnumMember) { - this.currentMember!.addMember( + protected onEnumMember(enm: spec.EnumType, member: spec.EnumMember) { + this.getPythonType(enm.fqn).addMember( new EnumMember( - this.currentModule().name, toPythonIdentifier(member.name), - member.name + member.name, ) ); } - // Not Currently Used - protected onInterfaceMethodOverload(_ifc: spec.InterfaceType, _overload: spec.Method, _originalMethod: spec.Method) { - debug("onInterfaceMethodOverload"); throw new Error("Unhandled Type: InterfaceMethodOverload"); } protected onUnionProperty(_cls: spec.ClassType, _prop: spec.Property, _union: spec.UnionTypeReference) { - debug("onUnionProperty"); throw new Error("Unhandled Type: UnionProperty"); } protected onMethodOverload(_cls: spec.ClassType, _overload: spec.Method, _originalMethod: spec.Method) { - debug("onMethodOverload"); throw new Error("Unhandled Type: MethodOverload"); } protected onStaticMethodOverload(_cls: spec.ClassType, _overload: spec.Method, _originalMethod: spec.Method) { - debug("onStaticMethodOverload"); throw new Error("Unhandled Type: StaticMethodOverload"); } - // End Not Currently Used + private getAssemblyModuleName(assm: spec.Assembly): string { + return `${toPythonModuleName(assm.name)}._jsii`; + } + + private getParentFQN(fqn: string): string { + const m = fqn.match(/^(.+)\.[^\.]+$/); + + if (m === null) { + throw new Error(`Could not determine parent FQN of: ${fqn}`); + } + + return m[1]; + } + + private getParent(fqn: string): PythonType { + return this.getPythonType(this.getParentFQN(fqn)); + } + + private getPythonType(fqn: string): PythonType { + const type = this.types.get(fqn); + + if (type === undefined) { + throw new Error(`Could not locate type: "${fqn}"`); + } + + return type; + } + + private addPythonType(type: PythonType) { + if (type.fqn === null) { + throw new Error("Cannot add a Python type without a FQN."); + } + + this.getParent(type.fqn).addMember(type); + this.types.set(type.fqn, type); + } private getliftedProp(method: spec.Method): spec.InterfaceType | undefined { // If there are parameters to this method, and if the last parameter's type is @@ -1215,8 +1237,4 @@ class PythonGenerator extends Generator { return undefined; } - - private currentModule(): Module { - return this.moduleStack.slice(-1)[0]; - } } diff --git a/packages/jsii-pacmak/package.json b/packages/jsii-pacmak/package.json index 405130a07c..fc956f1009 100644 --- a/packages/jsii-pacmak/package.json +++ b/packages/jsii-pacmak/package.json @@ -20,10 +20,8 @@ "aws" ], "dependencies": { - "@types/escape-string-regexp": "^1.0.0", "clone": "^2.1.1", "codemaker": "^0.7.7", - "escape-string-regexp": "^1.0.5", "fs-extra": "^4.0.3", "jsii-spec": "^0.7.7", "spdx-license-list": "^4.1.0", From 366c44b2eb9eacaa5dd9fdcf8137622b01e3cba3 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Wed, 24 Oct 2018 16:47:10 -0400 Subject: [PATCH 70/88] Fix a bug with non-builtin but standard types --- packages/jsii-pacmak/lib/targets/python.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 8175f7908f..42041684e5 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -819,6 +819,7 @@ class TypeResolver { private readonly types: Map; private boundTo?: string; + private readonly stdTypesRe = new RegExp("^(datetime\.datetime|typing\.[A-Z][a-z]+|jsii\.Number)$"); private readonly moduleRe = new RegExp("^((?:[^A-Z\.][^\.]+\.?)+)\.([A-Z].+)$"); constructor(types: Map, boundTo?: string) { @@ -865,6 +866,12 @@ class TypeResolver { continue; } + // These are not exactly built in types, but they're also not types that + // this resolver has to worry about. + if (this.stdTypesRe.test(innerType)) { + continue; + } + const [, moduleName, typeName] = innerType.match(this.moduleRe) as string[]; // If our resolver is not bound to the same module as the type we're trying to From 7505b32d61143d3c0edfd213b7ee323f9ac3ca83 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Wed, 24 Oct 2018 16:51:14 -0400 Subject: [PATCH 71/88] Make sure that __jsii_assembly__ is in __all__ --- packages/jsii-pacmak/lib/targets/python.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 42041684e5..3c87870343 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -688,7 +688,11 @@ class Module implements PythonType { } // Whatever names we've exported, we'll write out our __all__ that lists them. - code.line(`__all__ = [${this.members.map(m => `"${m.name}"`).sort().join(", ")}]`); + const exportedMembers = this.members.map(m => `"${m.name}"`); + if (this.loadAssembly) { + exportedMembers.push(`"__jsii_assembly__"`); + } + code.line(`__all__ = [${exportedMembers.sort().join(", ")}]`); // Finally, we'll use publication to ensure that all of the non-public names // get hidden from dir(), tab-complete, etc. From 70d15d0746926b3fc252719b1f797d66e54c63bf Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Wed, 24 Oct 2018 20:51:47 -0400 Subject: [PATCH 72/88] Deterministic regex --- packages/jsii-pacmak/lib/targets/python.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 3c87870343..055b6bd9f6 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -824,7 +824,7 @@ class TypeResolver { private readonly types: Map; private boundTo?: string; private readonly stdTypesRe = new RegExp("^(datetime\.datetime|typing\.[A-Z][a-z]+|jsii\.Number)$"); - private readonly moduleRe = new RegExp("^((?:[^A-Z\.][^\.]+\.?)+)\.([A-Z].+)$"); + private readonly moduleRe = new RegExp("^((?:[^A-Z\.][^\.]+\.)*(?:[^A-Z\.][^\.]+))\.([A-Z].+)$"); constructor(types: Map, boundTo?: string) { this.types = types; From e76d4aab1ba205dee5878d9d2f9e339ef0fcdad0 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 25 Oct 2018 10:30:07 -0400 Subject: [PATCH 73/88] Fix support for imports from submodules --- packages/jsii-pacmak/lib/targets/python.ts | 155 +++++++++++++++++---- packages/jsii-pacmak/package.json | 2 + 2 files changed, 133 insertions(+), 24 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 055b6bd9f6..92081243b9 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -2,6 +2,7 @@ import path = require('path'); import util = require('util'); import { CodeMaker, toSnakeCase } from 'codemaker'; +import * as escapeStringRegexp from 'escape-string-regexp'; import * as spec from 'jsii-spec'; import { Generator, GeneratorOptions } from '../generator'; import { Target, TargetOptions } from '../target'; @@ -130,9 +131,29 @@ const sortMembers = (sortable: PythonBase[], resolver: TypeResolver): PythonBase return sorted; }; +const recurseForNamedTypeReferences = (typeRef: spec.TypeReference): spec.NamedTypeReference[] => { + if (spec.isPrimitiveTypeReference(typeRef)) { + return []; + } else if (spec.isCollectionTypeReference(typeRef)) { + return recurseForNamedTypeReferences(typeRef.collection.elementtype); + } else if (spec.isNamedTypeReference(typeRef)) { + return [typeRef]; + } else if (typeRef.union) { + const types: spec.NamedTypeReference[] = []; + for (const type of typeRef.union.types) { + types.push(...recurseForNamedTypeReferences(type)); + } + return types; + } else { + throw new Error("Invalid type reference: " + JSON.stringify(typeRef)); + } +}; + interface PythonBase { readonly name: string; + getTypes(): spec.NamedTypeReference[]; + emit(code: CodeMaker, resolver: TypeResolver): void; } @@ -190,6 +211,14 @@ abstract class BasePythonClassType implements PythonType, ISortableType { return dependencies; } + public getTypes(): spec.NamedTypeReference[] { + const types: spec.NamedTypeReference[] = []; + for (const member of this.members) { + types.push(...member.getTypes()); + } + return types; + } + public addMember(member: PythonBase) { this.members.push(member); } @@ -245,6 +274,22 @@ abstract class BaseMethod implements PythonBase { this.parent = opts.parent; } + public getTypes(): spec.NamedTypeReference[] { + const types: spec.NamedTypeReference[] = []; + + // Look into our parameters and see what we need from there. + for (const parameter of this.parameters) { + types.push(...recurseForNamedTypeReferences(parameter.type)); + } + + // If we return anything, also check it. + if (this.returns !== undefined) { + types.push(...recurseForNamedTypeReferences(this.returns)); + } + + return types; + } + public emit(code: CodeMaker, resolver: TypeResolver) { let returnType: string; if (this.returns !== undefined) { @@ -405,6 +450,10 @@ abstract class BaseProperty implements PythonBase { this.immutable = immutable; } + public getTypes(): spec.NamedTypeReference[] { + return recurseForNamedTypeReferences(this.type); + } + public emit(code: CodeMaker, resolver: TypeResolver) { const pythonType = resolver.resolve(this.type, { forwardReferences: false }); @@ -535,6 +584,10 @@ class TypedDictProperty implements PythonBase { return this.type.optional !== undefined ? this.type.optional : false; } + public getTypes(): spec.NamedTypeReference[] { + return recurseForNamedTypeReferences(this.type); + } + public emit(code: CodeMaker, resolver: TypeResolver) { const resolvedType = resolver.resolve( this.type, @@ -608,6 +661,10 @@ class EnumMember implements PythonBase { this.value = value; } + public getTypes(): spec.NamedTypeReference[] { + return []; + } + public emit(code: CodeMaker, _resolver: TypeResolver) { code.line(`${this.name} = "${this.value}"`); } @@ -643,6 +700,14 @@ class Module implements PythonType { this.members.push(member); } + public getTypes(): spec.NamedTypeReference[] { + const types: spec.NamedTypeReference[] = []; + for (const member of this.members) { + types.push(...member.getTypes()); + } + return types; + } + public emit(code: CodeMaker, resolver: TypeResolver) { resolver = this.fqn ? resolver.bind(this.fqn) : resolver; @@ -659,17 +724,7 @@ class Module implements PythonType { code.line("from jsii.python import classproperty"); // Go over all of the modules that we need to import, and import them. - // for (let [idx, modName] of this.importedModules.sort().entries()) { - const dependencies = Object.keys(this.assembly.dependencies || {}); - for (const [idx, depName] of dependencies.sort().entries()) { - // If this our first dependency, add a blank line to format our imports - // slightly nicer. - if (idx === 0) { - code.line(); - } - - code.line(`import ${toPythonModuleName(depName)}`); - } + this.emitDependencyImports(code, resolver); // Determine if we need to write out the kernel load line. if (this.loadAssembly) { @@ -699,6 +754,40 @@ class Module implements PythonType { code.line(); code.line("publication.publish()"); } + + private emitDependencyImports(code: CodeMaker, resolver: TypeResolver) { + const moduleRe = new RegExp(`^${escapeStringRegexp(this.name)}\.(.+)$`); + const deps = Array.from( + new Set([ + ...Object.keys(this.assembly.dependencies || {}).map(d => toPythonModuleName(d)), + ...resolver.requiredModules(this.getTypes()), + ]) + ); + + // Only emit dependencies that are *not* submodules to our current module. + for (const [idx, moduleName] of deps.filter(d => !moduleRe.test(d)).sort().entries()) { + // If this our first dependency, add a blank line to format our imports + // slightly nicer. + if (idx === 0) { + code.line(); + } + + code.line(`import ${moduleName}`); + } + + // Only emit dependencies that *are* submodules to our current module. + for (const [idx, moduleName] of deps.filter(d => moduleRe.test(d)).sort().entries()) { + // If this our first dependency, add a blank line to format our imports + // slightly nicer. + if (idx === 0) { + code.line(); + } + + const [, submoduleName] = moduleName.match(moduleRe) as string[]; + + code.line(`from . import ${submoduleName}`); + } + } } interface PackageMetadata { @@ -824,19 +913,24 @@ class TypeResolver { private readonly types: Map; private boundTo?: string; private readonly stdTypesRe = new RegExp("^(datetime\.datetime|typing\.[A-Z][a-z]+|jsii\.Number)$"); + private readonly boundRe: RegExp; private readonly moduleRe = new RegExp("^((?:[^A-Z\.][^\.]+\.)*(?:[^A-Z\.][^\.]+))\.([A-Z].+)$"); constructor(types: Map, boundTo?: string) { this.types = types; this.boundTo = boundTo !== undefined ? this.toPythonFQN(boundTo) : boundTo; + + if (this.boundTo !== undefined) { + this.boundRe = new RegExp(`^(${escapeStringRegexp(this.boundTo)})\.(.+)$`); + } } public bind(fqn: string): TypeResolver { return new TypeResolver(this.types, fqn); } - public isInModule(typeRef: spec.NamedTypeReference): boolean { - const pythonType = this.toPythonFQN(typeRef.fqn); + public isInModule(typeRef: spec.NamedTypeReference | string): boolean { + const pythonType = typeof typeRef !== "string" ? this.toPythonFQN(typeRef.fqn) : typeRef; const [, moduleName] = pythonType.match(this.moduleRe) as string[]; return this.boundTo !== undefined && this.boundTo === moduleName; @@ -852,9 +946,24 @@ class TypeResolver { return type; } + public requiredModules(types: spec.NamedTypeReference[]): Set { + const modules = new Set(); + for (const type of types.map(t => this.toPythonType(t, true))) { + if (!this.isInModule(type)) { + const [, moduleName] = type.match(this.moduleRe) as string[]; + modules.add(moduleName); + } + } + return modules; + } + public resolve( typeRef: spec.TypeReference, opts: TypeResolverOpts = { forwardReferences: true, ignoreOptional: false }): string { + const { + forwardReferences = true, + } = opts; + // First, we need to resolve our given type reference into the Python type. let pythonType = this.toPythonType(typeRef, opts.ignoreOptional); @@ -876,25 +985,23 @@ class TypeResolver { continue; } - const [, moduleName, typeName] = innerType.match(this.moduleRe) as string[]; - - // If our resolver is not bound to the same module as the type we're trying to - // resolve, then we'll just skip ahead to the next one, no further changes are - // reqired. - if (this.boundTo === undefined || moduleName !== this.boundTo) { - continue; - } else { - // Otherwise, we have to implement some particular logic in order to deal - // with forward references and trimming our module name out of the type. + // If our resolver is bound to the same module as the type we're trying to + // resolve, then we'll implement the needed logic to use module relative naming + // and to handle forward references (if needed). + if (this.boundRe !== undefined && this.boundRe.test(innerType)) { // This re will look for the entire type, boxed by either the start/end of // a string, a comma, a space, a quote, or open/closing brackets. This will // ensure that we only match whole type names, and not partial ones. const re = new RegExp('((?:^|[[,\\s])"?)' + innerType + '("?(?:$|[\\],\\s]))'); + const [, , typeName] = innerType.match(this.boundRe) as string[]; // We need to handle forward references, our caller knows if we're able to // use them in the current context or not, so if not, we'll wrap our forward // reference in quotes. - if (opts.forwardReferences !== undefined && !opts.forwardReferences) { + // We have special logic here for checking if our thing is actually *in* + // our module, behond what we've already done, because our other logic will + // work for submodules, but this can't. + if (!forwardReferences && this.isInModule(innerType)) { pythonType = pythonType.replace(re, `$1"${innerType}"$2`); } diff --git a/packages/jsii-pacmak/package.json b/packages/jsii-pacmak/package.json index fc956f1009..6c7a127402 100644 --- a/packages/jsii-pacmak/package.json +++ b/packages/jsii-pacmak/package.json @@ -22,6 +22,7 @@ "dependencies": { "clone": "^2.1.1", "codemaker": "^0.7.7", + "escape-string-regexp": "^1.0.5", "fs-extra": "^4.0.3", "jsii-spec": "^0.7.7", "spdx-license-list": "^4.1.0", @@ -31,6 +32,7 @@ "devDependencies": { "@scope/jsii-calc-lib": "^0.7.7", "@types/clone": "^0.1.30", + "@types/escape-string-regexp": "^1.0.0", "@types/fs-extra": "^4.0.8", "@types/node": "^9.6.18", "@types/nodeunit": "0.0.30", From 1288b032842e19cb8bd5db5cb7596c098842598e Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 25 Oct 2018 10:31:01 -0400 Subject: [PATCH 74/88] Correctly pass in the klass argument to @classproperty properties --- packages/jsii-python-runtime/src/jsii/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-python-runtime/src/jsii/python.py b/packages/jsii-python-runtime/src/jsii/python.py index 406e738ca6..d8d2d5396f 100644 --- a/packages/jsii-python-runtime/src/jsii/python.py +++ b/packages/jsii-python-runtime/src/jsii/python.py @@ -6,7 +6,7 @@ def __init__(self, fget, fset=None): def __get__(self, obj, klass=None): if klass is None: klass = type(obj) - return self.fget.__get__(obj, klass)() + return self.fget.__get__(obj, klass)(klass) def __set__(self, obj, value): if not self.fset: From efcded0ab24159d7ed5f77b28272465a28b4072c Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 25 Oct 2018 10:45:01 -0400 Subject: [PATCH 75/88] Emit a None default for optional positional params --- packages/jsii-pacmak/lib/targets/python.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 92081243b9..6561897480 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -305,8 +305,9 @@ abstract class BaseMethod implements PythonBase { for (const param of this.parameters) { const paramName = toPythonIdentifier(param.name); const paramType = resolver.resolve(param.type, { forwardReferences: false}); + const paramDefault = param.type.optional ? "=None" : ""; - pythonParams.push(`${paramName}: ${paramType}`); + pythonParams.push(`${paramName}: ${paramType}${paramDefault}`); } // If we have a lifted parameter, then we'll drop the last argument to our params From 7e7bbc712a8c60ebc6b7a8f6f4e3a3cfa406dec8 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 25 Oct 2018 10:51:40 -0400 Subject: [PATCH 76/88] Don't add typing.Optional to the auto props expansion --- packages/jsii-pacmak/lib/targets/python.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 6561897480..c5352ac07d 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -371,7 +371,7 @@ abstract class BaseMethod implements PythonBase { private emitAutoProps(code: CodeMaker, resolver: TypeResolver) { const lastParameter = this.parameters.slice(-1)[0]; const argName = toPythonIdentifier(lastParameter.name); - const typeName = resolver.resolve(lastParameter.type); + const typeName = resolver.resolve(lastParameter.type, {ignoreOptional: true }); // We need to build up a list of properties, which are mandatory, these are the // ones we will specifiy to start with in our dictionary literal. From 6a2ec914b23acb6ef87f5bbbf1e457ef0a96ec01 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 25 Oct 2018 11:14:37 -0400 Subject: [PATCH 77/88] Use "real" forward refs if possible, even with forwardReferences = false --- packages/jsii-pacmak/lib/targets/python.ts | 34 +++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index c5352ac07d..86db155ff3 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -165,6 +165,10 @@ interface PythonType extends PythonBase { addMember(member: PythonBase): void; } +function isPythonType(arg: PythonBase): arg is PythonType { + return (arg as any).fqn !== undefined; +} + interface ISortableType { dependsOn(resolver: TypeResolver): spec.NamedTypeReference[]; } @@ -229,6 +233,10 @@ abstract class BasePythonClassType implements PythonType, ISortableType { if (this.members.length > 0) { for (const member of sortMembers(this.members, resolver)) { member.emit(code, resolver); + + if (isPythonType(member)) { + resolver.markTypeEmitted(member); + } } } else { code.line("pass"); @@ -528,6 +536,10 @@ class TypedDict extends BasePythonClassType { code.openBlock(`class _${this.name}(${classParams.concat(["total=False"]).join(", ")})`); for (const member of optionalMembers) { member.emit(code, resolver); + + if (isPythonType(member)) { + resolver.markTypeEmitted(member); + } } code.closeBlock(); @@ -535,6 +547,10 @@ class TypedDict extends BasePythonClassType { code.openBlock(`class ${this.name}(_${this.name})`); for (const member of sortMembers(mandatoryMembers, resolver)) { member.emit(code, resolver); + + if (isPythonType(member)) { + resolver.markTypeEmitted(member); + } } code.closeBlock(); } else { @@ -551,6 +567,10 @@ class TypedDict extends BasePythonClassType { if (this.members.length > 0) { for (const member of sortMembers(this.members, resolver)) { member.emit(code, resolver); + + if (isPythonType(member)) { + resolver.markTypeEmitted(member); + } } } else { code.line("pass"); @@ -741,6 +761,10 @@ class Module implements PythonType { // Emit all of our members. for (const member of sortMembers(this.members, resolver)) { member.emit(code, resolver); + + if (isPythonType(member)) { + resolver.markTypeEmitted(member); + } } // Whatever names we've exported, we'll write out our __all__ that lists them. @@ -916,10 +940,12 @@ class TypeResolver { private readonly stdTypesRe = new RegExp("^(datetime\.datetime|typing\.[A-Z][a-z]+|jsii\.Number)$"); private readonly boundRe: RegExp; private readonly moduleRe = new RegExp("^((?:[^A-Z\.][^\.]+\.)*(?:[^A-Z\.][^\.]+))\.([A-Z].+)$"); + private readonly emitted: Set; constructor(types: Map, boundTo?: string) { this.types = types; this.boundTo = boundTo !== undefined ? this.toPythonFQN(boundTo) : boundTo; + this.emitted = new Set(); if (this.boundTo !== undefined) { this.boundRe = new RegExp(`^(${escapeStringRegexp(this.boundTo)})\.(.+)$`); @@ -958,6 +984,12 @@ class TypeResolver { return modules; } + public markTypeEmitted(type: PythonType) { + if (type.fqn) { + this.emitted.add(this.toPythonFQN(type.fqn)); + } + } + public resolve( typeRef: spec.TypeReference, opts: TypeResolverOpts = { forwardReferences: true, ignoreOptional: false }): string { @@ -1002,7 +1034,7 @@ class TypeResolver { // We have special logic here for checking if our thing is actually *in* // our module, behond what we've already done, because our other logic will // work for submodules, but this can't. - if (!forwardReferences && this.isInModule(innerType)) { + if (!forwardReferences && this.isInModule(innerType) && !this.emitted.has(innerType)) { pythonType = pythonType.replace(re, `$1"${innerType}"$2`); } From 22d2a34420a9a97b5c7d03ed5658776e5ae47afa Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 25 Oct 2018 12:13:14 -0400 Subject: [PATCH 78/88] Implement Abstract classes --- packages/jsii-pacmak/lib/targets/python.ts | 56 +++++++++++++++---- .../jsii-python-runtime/src/jsii/__init__.py | 3 +- .../jsii-python-runtime/src/jsii/_runtime.py | 5 ++ 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 86db155ff3..2e347b18c3 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -249,6 +249,7 @@ abstract class BasePythonClassType implements PythonType, ISortableType { } interface BaseMethodOpts { + abstract?: boolean; liftedProp?: spec.InterfaceType, parent?: spec.NamedTypeReference, } @@ -268,18 +269,22 @@ abstract class BaseMethod implements PythonBase { private readonly returns?: spec.TypeReference; private readonly liftedProp?: spec.InterfaceType; private readonly parent?: spec.NamedTypeReference; + private readonly abstract: boolean; constructor(name: string, jsName: string | undefined, parameters: spec.Parameter[], returns?: spec.TypeReference, opts: BaseMethodOpts = {}) { + const { abstract = false } = opts; + this.name = name; this.jsName = jsName; this.parameters = parameters; this.returns = returns; this.liftedProp = opts.liftedProp; this.parent = opts.parent; + this.abstract = abstract; } public getTypes(): spec.NamedTypeReference[] { @@ -359,13 +364,17 @@ abstract class BaseMethod implements PythonBase { code.line(`@${this.decorator}`); } + if (this.abstract) { + code.line("@abc.abstractmethod"); + } + code.openBlock(`def ${this.name}(${pythonParams.join(", ")}) -> ${returnType}`); this.emitBody(code, resolver); code.closeBlock(); } private emitBody(code: CodeMaker, resolver: TypeResolver) { - if (this.jsiiMethod === undefined) { + if (this.jsiiMethod === undefined || this.abstract) { code.line("..."); } else { if (this.liftedProp !== undefined) { @@ -432,6 +441,7 @@ abstract class BaseMethod implements PythonBase { } interface BasePropertyOpts { + abstract?: boolean; immutable?: boolean; } @@ -446,16 +456,19 @@ abstract class BaseProperty implements PythonBase { private readonly jsName: string; private readonly type: spec.TypeReference; + private readonly abstract: boolean; private readonly immutable: boolean; constructor(name: string, jsName: string, type: spec.TypeReference, opts: BasePropertyOpts = {}) { const { + abstract = false, immutable = false, } = opts; this.name = name; this.jsName = jsName; this.type = type; + this.abstract = abstract; this.immutable = immutable; } @@ -467,8 +480,11 @@ abstract class BaseProperty implements PythonBase { const pythonType = resolver.resolve(this.type, { forwardReferences: false }); code.line(`@${this.decorator}`); + if (this.abstract) { + code.line("@abc.abstractmethod"); + } code.openBlock(`def ${this.name}(${this.implicitParameter}) -> ${pythonType}`); - if (this.jsiiGetMethod !== undefined) { + if (this.jsiiGetMethod !== undefined && !this.abstract) { code.line(`return jsii.${this.jsiiGetMethod}(${this.implicitParameter}, "${this.jsName}")`); } else { code.line("..."); @@ -477,8 +493,11 @@ abstract class BaseProperty implements PythonBase { if (!this.immutable) { code.line(`@${this.name}.setter`); + if (this.abstract) { + code.line("@abc.abstractmethod"); + } code.openBlock(`def ${this.name}(${this.implicitParameter}, value: ${pythonType})`); - if (this.jsiiSetMethod !== undefined) { + if (this.jsiiSetMethod !== undefined && !this.abstract) { code.line(`return jsii.${this.jsiiSetMethod}(${this.implicitParameter}, "${this.jsName}", value)`); } else { code.line("..."); @@ -618,12 +637,27 @@ class TypedDictProperty implements PythonBase { } } +interface ClassOpts extends PythonTypeOpts { + abstract?: boolean; +} + class Class extends BasePythonClassType { + private abstract: boolean; + + constructor(name: string, fqn: string, opts: ClassOpts) { + super(name, fqn, opts); + + const { abstract = false } = opts; + + this.abstract = abstract; + } + protected getClassParams(resolver: TypeResolver): string[] { const params: string[] = this.bases.map(b => resolver.resolve(b)); + const metaclass: string = this.abstract ? "JSIIAbstractClass" : "JSIIMeta"; - params.push("metaclass=jsii.JSIIMeta"); + params.push(`metaclass=jsii.${metaclass}`); params.push(`jsii_type="${this.fqn}"`); return params; @@ -734,6 +768,7 @@ class Module implements PythonType { // Before we write anything else, we need to write out our module headers, this // is where we handle stuff like imports, any required initialization, etc. + code.line("import abc"); code.line("import datetime"); code.line("import enum"); code.line("import typing"); @@ -1175,12 +1210,11 @@ class PythonGenerator extends Generator { } } - protected onBeginClass(cls: spec.ClassType, _abstract: boolean | undefined) { - // TODO: Figure out what to do with abstract here. + protected onBeginClass(cls: spec.ClassType, abstract: boolean | undefined) { const klass = new Class( toPythonIdentifier(cls.name), cls.fqn, - { bases: cls.base !== undefined ? [cls.base] : [] } + { abstract, bases: cls.base !== undefined ? [cls.base] : [] } ); if (cls.initializer !== undefined) { @@ -1209,7 +1243,7 @@ class PythonGenerator extends Generator { method.name, parameters, method.returns, - { liftedProp: this.getliftedProp(method) }, + { abstract: method.abstract, liftedProp: this.getliftedProp(method) }, ) ); } @@ -1220,7 +1254,7 @@ class PythonGenerator extends Generator { toPythonPropertyName(prop.name), prop.name, prop.type, - { immutable: prop.immutable }, + { abstract: prop.abstract, immutable: prop.immutable }, ) ); } @@ -1234,7 +1268,7 @@ class PythonGenerator extends Generator { method.name, parameters, method.returns, - { liftedProp: this.getliftedProp(method) }, + { abstract: method.abstract, liftedProp: this.getliftedProp(method) }, ) ); } @@ -1245,7 +1279,7 @@ class PythonGenerator extends Generator { toPythonPropertyName(prop.name), prop.name, prop.type, - { immutable: prop.immutable }, + { abstract: prop.abstract, immutable: prop.immutable }, ) ); } diff --git a/packages/jsii-python-runtime/src/jsii/__init__.py b/packages/jsii-python-runtime/src/jsii/__init__.py index db2c4a9a37..7a976e4fb3 100644 --- a/packages/jsii-python-runtime/src/jsii/__init__.py +++ b/packages/jsii-python-runtime/src/jsii/__init__.py @@ -1,6 +1,6 @@ from typing import Union -from ._runtime import JSIIAssembly, JSIIMeta, kernel +from ._runtime import JSIIAssembly, JSIIMeta, JSIIAbstractClass, kernel # JS doesn't have distinct float or integer types, but we do. So we'll define our own @@ -25,6 +25,7 @@ __all__ = [ "JSIIAssembly", "JSIIMeta", + "JSIIAbstractClass", "Number", "load", "create", diff --git a/packages/jsii-python-runtime/src/jsii/_runtime.py b/packages/jsii-python-runtime/src/jsii/_runtime.py index aa044144e2..9b3a4cecd2 100644 --- a/packages/jsii-python-runtime/src/jsii/_runtime.py +++ b/packages/jsii-python-runtime/src/jsii/_runtime.py @@ -1,3 +1,4 @@ +import abc import weakref import os @@ -76,3 +77,7 @@ def __call__(cls, *args, **kwargs): weakref.finalize(inst, kernel.delete, inst.__jsii_ref__) return inst + + +class JSIIAbstractClass(abc.ABCMeta, JSIIMeta): + pass From 2987f3633180bb8bef803e1158443613212747f6 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 26 Oct 2018 11:55:37 -0400 Subject: [PATCH 79/88] Don't trust whether the JSII tells us a parameter is optional or not --- packages/jsii-pacmak/lib/targets/python.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 2e347b18c3..4805003008 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -311,14 +311,28 @@ abstract class BaseMethod implements PythonBase { returnType = "None"; } + // We cannot (currently?) trust the JSII to accurately tell us whether a + // parameter is truly optional or not. Because of that, we have to selectively + // choose when we're going to respect the optional flag and emit a default value + // to only be at the tail end of the method signature. + // See: https://github.com/awslabs/jsii/issues/284 + let optionalStartsAt: number | undefined; + for (const [idx, param] of this.parameters.entries()) { + if (param.type.optional && optionalStartsAt === undefined) { + optionalStartsAt = idx; + } else if (!param.type.optional) { + optionalStartsAt = undefined; + } + } + // We need to turn a list of JSII parameters, into Python style arguments with // gradual typing, so we'll have to iterate over the list of parameters, and // build the list, converting as we go. const pythonParams: string[] = [this.implicitParameter]; - for (const param of this.parameters) { + for (const [idx, param] of this.parameters.entries()) { const paramName = toPythonIdentifier(param.name); const paramType = resolver.resolve(param.type, { forwardReferences: false}); - const paramDefault = param.type.optional ? "=None" : ""; + const paramDefault = optionalStartsAt !== undefined && idx >= optionalStartsAt ? "=None" : ""; pythonParams.push(`${paramName}: ${paramType}${paramDefault}`); } From 72fb08dca6631fa3ccd9fe7978aa10d5144bae6b Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 26 Oct 2018 12:09:30 -0400 Subject: [PATCH 80/88] Don't include Optional when emitting Any --- packages/jsii-pacmak/lib/targets/python.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 4805003008..7a91bccc28 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -1118,7 +1118,10 @@ class TypeResolver { // If our type is Optional, then we'll wrap our underlying type with typing.Optional // However, if we're not respecting optionals, then we'll just skip over this. - if (!ignoreOptional && typeRef.optional) { + // We explicitly don't emit this when our type is typing.Any, because typing.Any + // already implied that None is an accepted type. + // See: https://github.com/awslabs/jsii/issues/284 + if (!ignoreOptional && typeRef.optional && pythonType !== "typing.Any") { pythonType = `typing.Optional[${pythonType}]`; } From 785ce86e4dd14e11a82801815c820d738cdea6bb Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 26 Oct 2018 14:37:33 -0400 Subject: [PATCH 81/88] Revert "Use "real" forward refs if possible, even with forwardReferences = false" This reverts commit 6a2ec914b23acb6ef87f5bbbf1e457ef0a96ec01. --- packages/jsii-pacmak/lib/targets/python.ts | 34 +--------------------- 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 7a91bccc28..6ee62f073d 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -165,10 +165,6 @@ interface PythonType extends PythonBase { addMember(member: PythonBase): void; } -function isPythonType(arg: PythonBase): arg is PythonType { - return (arg as any).fqn !== undefined; -} - interface ISortableType { dependsOn(resolver: TypeResolver): spec.NamedTypeReference[]; } @@ -233,10 +229,6 @@ abstract class BasePythonClassType implements PythonType, ISortableType { if (this.members.length > 0) { for (const member of sortMembers(this.members, resolver)) { member.emit(code, resolver); - - if (isPythonType(member)) { - resolver.markTypeEmitted(member); - } } } else { code.line("pass"); @@ -569,10 +561,6 @@ class TypedDict extends BasePythonClassType { code.openBlock(`class _${this.name}(${classParams.concat(["total=False"]).join(", ")})`); for (const member of optionalMembers) { member.emit(code, resolver); - - if (isPythonType(member)) { - resolver.markTypeEmitted(member); - } } code.closeBlock(); @@ -580,10 +568,6 @@ class TypedDict extends BasePythonClassType { code.openBlock(`class ${this.name}(_${this.name})`); for (const member of sortMembers(mandatoryMembers, resolver)) { member.emit(code, resolver); - - if (isPythonType(member)) { - resolver.markTypeEmitted(member); - } } code.closeBlock(); } else { @@ -600,10 +584,6 @@ class TypedDict extends BasePythonClassType { if (this.members.length > 0) { for (const member of sortMembers(this.members, resolver)) { member.emit(code, resolver); - - if (isPythonType(member)) { - resolver.markTypeEmitted(member); - } } } else { code.line("pass"); @@ -810,10 +790,6 @@ class Module implements PythonType { // Emit all of our members. for (const member of sortMembers(this.members, resolver)) { member.emit(code, resolver); - - if (isPythonType(member)) { - resolver.markTypeEmitted(member); - } } // Whatever names we've exported, we'll write out our __all__ that lists them. @@ -989,12 +965,10 @@ class TypeResolver { private readonly stdTypesRe = new RegExp("^(datetime\.datetime|typing\.[A-Z][a-z]+|jsii\.Number)$"); private readonly boundRe: RegExp; private readonly moduleRe = new RegExp("^((?:[^A-Z\.][^\.]+\.)*(?:[^A-Z\.][^\.]+))\.([A-Z].+)$"); - private readonly emitted: Set; constructor(types: Map, boundTo?: string) { this.types = types; this.boundTo = boundTo !== undefined ? this.toPythonFQN(boundTo) : boundTo; - this.emitted = new Set(); if (this.boundTo !== undefined) { this.boundRe = new RegExp(`^(${escapeStringRegexp(this.boundTo)})\.(.+)$`); @@ -1033,12 +1007,6 @@ class TypeResolver { return modules; } - public markTypeEmitted(type: PythonType) { - if (type.fqn) { - this.emitted.add(this.toPythonFQN(type.fqn)); - } - } - public resolve( typeRef: spec.TypeReference, opts: TypeResolverOpts = { forwardReferences: true, ignoreOptional: false }): string { @@ -1083,7 +1051,7 @@ class TypeResolver { // We have special logic here for checking if our thing is actually *in* // our module, behond what we've already done, because our other logic will // work for submodules, but this can't. - if (!forwardReferences && this.isInModule(innerType) && !this.emitted.has(innerType)) { + if (!forwardReferences && this.isInModule(innerType)) { pythonType = pythonType.replace(re, `$1"${innerType}"$2`); } From 413ec40eeb40c1f4b58d2b1c13b22a947559d093 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 26 Oct 2018 16:55:21 -0400 Subject: [PATCH 82/88] Ensure all of regexp strings are correctly escaped --- packages/jsii-pacmak/lib/targets/python.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 6ee62f073d..841474ab9c 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -806,7 +806,7 @@ class Module implements PythonType { } private emitDependencyImports(code: CodeMaker, resolver: TypeResolver) { - const moduleRe = new RegExp(`^${escapeStringRegexp(this.name)}\.(.+)$`); + const moduleRe = new RegExp(`^${escapeStringRegexp(this.name)}\\.(.+)$`); const deps = Array.from( new Set([ ...Object.keys(this.assembly.dependencies || {}).map(d => toPythonModuleName(d)), @@ -962,16 +962,16 @@ class TypeResolver { private readonly types: Map; private boundTo?: string; - private readonly stdTypesRe = new RegExp("^(datetime\.datetime|typing\.[A-Z][a-z]+|jsii\.Number)$"); + private readonly stdTypesRe = new RegExp("^(datetime\\.datetime|typing\\.[A-Z][a-z]+|jsii\\.Number)$"); private readonly boundRe: RegExp; - private readonly moduleRe = new RegExp("^((?:[^A-Z\.][^\.]+\.)*(?:[^A-Z\.][^\.]+))\.([A-Z].+)$"); + private readonly moduleRe = new RegExp("^((?:[^A-Z\.][^\\.]+\\.)*(?:[^A-Z\\.][^\\.]+))\\.([A-Z].+)$"); constructor(types: Map, boundTo?: string) { this.types = types; this.boundTo = boundTo !== undefined ? this.toPythonFQN(boundTo) : boundTo; if (this.boundTo !== undefined) { - this.boundRe = new RegExp(`^(${escapeStringRegexp(this.boundTo)})\.(.+)$`); + this.boundRe = new RegExp(`^(${escapeStringRegexp(this.boundTo)})\\.(.+)$`); } } From db2e266d90ee6d6c103f8c76b02dd46f79ecc1c8 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 1 Nov 2018 18:26:38 -0400 Subject: [PATCH 83/88] Move Namespaces as classes instead of modules --- packages/jsii-pacmak/lib/targets/python.ts | 217 +++++++++++---------- 1 file changed, 111 insertions(+), 106 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 841474ab9c..55d63e0b76 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -111,7 +111,7 @@ const sortMembers = (sortable: PythonBase[], resolver: TypeResolver): PythonBase // dependencies that we haven't already sorted. while (sortable.length > 0) { for (const item of (sortable as Array)) { - const itemDeps: Set = new Set(item.dependsOn(resolver).map(i => resolver.getType(i))); + const itemDeps: Set = new Set(item.dependsOn(resolver)); if (setDifference(itemDeps, seen).size === 0) { sorted.push(item); seen.add(item); @@ -121,8 +121,8 @@ const sortMembers = (sortable: PythonBase[], resolver: TypeResolver): PythonBase } const leftover = sortable.filter(i => !seen.has(i)); - if (leftover === sortable) { - throw new Error("Could not sort members."); + if (leftover.length === sortable.length) { + throw new Error("Could not sort members (circular dependency?)."); } else { sortable = leftover; } @@ -152,8 +152,6 @@ const recurseForNamedTypeReferences = (typeRef: spec.TypeReference): spec.NamedT interface PythonBase { readonly name: string; - getTypes(): spec.NamedTypeReference[]; - emit(code: CodeMaker, resolver: TypeResolver): void; } @@ -166,7 +164,7 @@ interface PythonType extends PythonBase { } interface ISortableType { - dependsOn(resolver: TypeResolver): spec.NamedTypeReference[]; + dependsOn(resolver: TypeResolver): PythonType[]; } function isSortableType(arg: any): arg is ISortableType { @@ -196,14 +194,32 @@ abstract class BasePythonClassType implements PythonType, ISortableType { this.members = []; } - public dependsOn(resolver: TypeResolver): spec.NamedTypeReference[] { - const dependencies: spec.NamedTypeReference[] = []; + public dependsOn(resolver: TypeResolver): PythonType[] { + const dependencies: PythonType[] = []; + const parent = resolver.getParent(this.fqn!); - // We need to return any bases that are in the same module. + // We need to return any bases that are in the same module at the same level of + // nesting. + const seen: Set = new Set(); for (const base of this.bases) { if (spec.isNamedTypeReference(base)) { if (resolver.isInModule(base)) { - dependencies.push(base); + // Given a base, we need to locate the base's parent that is the same as + // our parent, because we only care about dependencies that are at the + // same level of our own. + // TODO: We might need to recurse into our members to also find their + // dependencies. + let baseItem = resolver.getType(base); + let baseParent = resolver.getParent(base); + while (baseParent !== parent) { + baseItem = baseParent; + baseParent = resolver.getParent(baseItem.fqn!); + } + + if (!seen.has(baseItem.fqn!)) { + dependencies.push(baseItem); + seen.add(baseItem.fqn!); + } } } } @@ -211,20 +227,17 @@ abstract class BasePythonClassType implements PythonType, ISortableType { return dependencies; } - public getTypes(): spec.NamedTypeReference[] { - const types: spec.NamedTypeReference[] = []; - for (const member of this.members) { - types.push(...member.getTypes()); - } - return types; - } - public addMember(member: PythonBase) { this.members.push(member); } public emit(code: CodeMaker, resolver: TypeResolver) { - code.openBlock(`class ${this.name}(${this.getClassParams(resolver).join(", ")})`); + resolver = this.fqn ? resolver.bind(this.fqn) : resolver; + + const classParams = this.getClassParams(resolver); + const bases = classParams.length > 0 ? `(${classParams.join(", ")})` : ""; + + code.openBlock(`class ${this.name}${bases}`); if (this.members.length > 0) { for (const member of sortMembers(this.members, resolver)) { @@ -279,22 +292,6 @@ abstract class BaseMethod implements PythonBase { this.abstract = abstract; } - public getTypes(): spec.NamedTypeReference[] { - const types: spec.NamedTypeReference[] = []; - - // Look into our parameters and see what we need from there. - for (const parameter of this.parameters) { - types.push(...recurseForNamedTypeReferences(parameter.type)); - } - - // If we return anything, also check it. - if (this.returns !== undefined) { - types.push(...recurseForNamedTypeReferences(this.returns)); - } - - return types; - } - public emit(code: CodeMaker, resolver: TypeResolver) { let returnType: string; if (this.returns !== undefined) { @@ -478,10 +475,6 @@ abstract class BaseProperty implements PythonBase { this.immutable = immutable; } - public getTypes(): spec.NamedTypeReference[] { - return recurseForNamedTypeReferences(this.type); - } - public emit(code: CodeMaker, resolver: TypeResolver) { const pythonType = resolver.resolve(this.type, { forwardReferences: false }); @@ -537,6 +530,8 @@ class InterfaceProperty extends BaseProperty { class TypedDict extends BasePythonClassType { public emit(code: CodeMaker, resolver: TypeResolver) { + resolver = this.fqn ? resolver.bind(this.fqn) : resolver; + // MyPy doesn't let us mark some keys as optional, and some keys as mandatory, // we can either mark either the entire class as mandatory or the entire class // as optional. However, we can make two classes, one with all mandatory keys @@ -618,10 +613,6 @@ class TypedDictProperty implements PythonBase { return this.type.optional !== undefined ? this.type.optional : false; } - public getTypes(): spec.NamedTypeReference[] { - return recurseForNamedTypeReferences(this.type); - } - public emit(code: CodeMaker, resolver: TypeResolver) { const resolvedType = resolver.resolve( this.type, @@ -710,15 +701,17 @@ class EnumMember implements PythonBase { this.value = value; } - public getTypes(): spec.NamedTypeReference[] { - return []; - } - public emit(code: CodeMaker, _resolver: TypeResolver) { code.line(`${this.name} = "${this.value}"`); } } +class Namespace extends BasePythonClassType { + protected getClassParams(_resolver: TypeResolver): string[] { + return []; + } +} + interface ModuleOpts { assembly: spec.Assembly, assemblyFilename: string; @@ -749,16 +742,8 @@ class Module implements PythonType { this.members.push(member); } - public getTypes(): spec.NamedTypeReference[] { - const types: spec.NamedTypeReference[] = []; - for (const member of this.members) { - types.push(...member.getTypes()); - } - return types; - } - public emit(code: CodeMaker, resolver: TypeResolver) { - resolver = this.fqn ? resolver.bind(this.fqn) : resolver; + resolver = this.fqn ? resolver.bind(this.fqn, this.name) : resolver; // Before we write anything else, we need to write out our module headers, this // is where we handle stuff like imports, any required initialization, etc. @@ -805,17 +790,14 @@ class Module implements PythonType { code.line("publication.publish()"); } - private emitDependencyImports(code: CodeMaker, resolver: TypeResolver) { - const moduleRe = new RegExp(`^${escapeStringRegexp(this.name)}\\.(.+)$`); + private emitDependencyImports(code: CodeMaker, _resolver: TypeResolver) { const deps = Array.from( new Set([ ...Object.keys(this.assembly.dependencies || {}).map(d => toPythonModuleName(d)), - ...resolver.requiredModules(this.getTypes()), ]) ); - // Only emit dependencies that are *not* submodules to our current module. - for (const [idx, moduleName] of deps.filter(d => !moduleRe.test(d)).sort().entries()) { + for (const [idx, moduleName] of deps.sort().entries()) { // If this our first dependency, add a blank line to format our imports // slightly nicer. if (idx === 0) { @@ -824,19 +806,6 @@ class Module implements PythonType { code.line(`import ${moduleName}`); } - - // Only emit dependencies that *are* submodules to our current module. - for (const [idx, moduleName] of deps.filter(d => moduleRe.test(d)).sort().entries()) { - // If this our first dependency, add a blank line to format our imports - // slightly nicer. - if (idx === 0) { - code.line(); - } - - const [, submoduleName] = moduleName.match(moduleRe) as string[]; - - code.line(`from . import ${submoduleName}`); - } } } @@ -964,26 +933,51 @@ class TypeResolver { private boundTo?: string; private readonly stdTypesRe = new RegExp("^(datetime\\.datetime|typing\\.[A-Z][a-z]+|jsii\\.Number)$"); private readonly boundRe: RegExp; - private readonly moduleRe = new RegExp("^((?:[^A-Z\.][^\\.]+\\.)*(?:[^A-Z\\.][^\\.]+))\\.([A-Z].+)$"); + private readonly moduleName?: string; + private readonly moduleRe: RegExp; - constructor(types: Map, boundTo?: string) { + constructor(types: Map, boundTo?: string, moduleName?: string) { this.types = types; + this.moduleName = moduleName; this.boundTo = boundTo !== undefined ? this.toPythonFQN(boundTo) : boundTo; + if (this.moduleName !== undefined) { + this.moduleRe = new RegExp(`^(${escapeStringRegexp(this.moduleName)})\\.(.+)$`); + } + if (this.boundTo !== undefined) { this.boundRe = new RegExp(`^(${escapeStringRegexp(this.boundTo)})\\.(.+)$`); } } - public bind(fqn: string): TypeResolver { - return new TypeResolver(this.types, fqn); + public bind(fqn: string, moduleName?: string): TypeResolver { + return new TypeResolver( + this.types, + fqn, + moduleName !== undefined ? moduleName : this.moduleName, + ); } public isInModule(typeRef: spec.NamedTypeReference | string): boolean { const pythonType = typeof typeRef !== "string" ? this.toPythonFQN(typeRef.fqn) : typeRef; - const [, moduleName] = pythonType.match(this.moduleRe) as string[]; + return this.moduleRe.test(pythonType); + } - return this.boundTo !== undefined && this.boundTo === moduleName; + public isInNamespace(typeRef: spec.NamedTypeReference | string): boolean { + const pythonType = typeof typeRef !== "string" ? this.toPythonFQN(typeRef.fqn) : typeRef; + return this.boundRe.test(pythonType); + } + + public getParent(typeRef: spec.NamedTypeReference | string): PythonType { + const fqn = typeof typeRef !== "string" ? typeRef.fqn : typeRef; + const [, parentFQN] = fqn.match(/^(.+)\.[^\.]+$/) as string[]; + const parent = this.types.get(parentFQN); + + if (parent === undefined) { + throw new Error(`Could not find parent: ${parentFQN}`); + } + + return parent; } public getType(typeRef: spec.NamedTypeReference): PythonType { @@ -996,17 +990,6 @@ class TypeResolver { return type; } - public requiredModules(types: spec.NamedTypeReference[]): Set { - const modules = new Set(); - for (const type of types.map(t => this.toPythonType(t, true))) { - if (!this.isInModule(type)) { - const [, moduleName] = type.match(this.moduleRe) as string[]; - modules.add(moduleName); - } - } - return modules; - } - public resolve( typeRef: spec.TypeReference, opts: TypeResolverOpts = { forwardReferences: true, ignoreOptional: false }): string { @@ -1038,12 +1021,20 @@ class TypeResolver { // If our resolver is bound to the same module as the type we're trying to // resolve, then we'll implement the needed logic to use module relative naming // and to handle forward references (if needed). - if (this.boundRe !== undefined && this.boundRe.test(innerType)) { + if (this.isInModule(innerType)) { + // If our type is part of the same namespace, then we'll return a namespace + // relative name, otherwise a module relative name. + let typeName: string; + if (this.isInNamespace(innerType)) { + [, , typeName] = innerType.match(this.boundRe) as string[]; + } else { + [, , typeName] = innerType.match(this.moduleRe) as string[]; + } + // This re will look for the entire type, boxed by either the start/end of // a string, a comma, a space, a quote, or open/closing brackets. This will // ensure that we only match whole type names, and not partial ones. const re = new RegExp('((?:^|[[,\\s])"?)' + innerType + '("?(?:$|[\\],\\s]))'); - const [, , typeName] = innerType.match(this.boundRe) as string[]; // We need to handle forward references, our caller knows if we're able to // use them in the current context or not, so if not, we'll wrap our forward @@ -1177,21 +1168,35 @@ class PythonGenerator extends Generator { } protected onBeginNamespace(ns: string) { - const module = new Module( - toPythonModuleName(ns), - ns, - { assembly: this.assembly, - assemblyFilename: this.getAssemblyFileName(), - loadAssembly: ns === this.assembly.name }, - ); - - this.package.addModule(module); - this.types.set(ns, module); - - // If this is our top level namespace, then we'll want to add a py.typed marker - // so that all of our typing works. + // If we're generating the Namespace that matches our assembly, then we'll + // actually be generating a module, otherwise we'll generate a class within + // that module. if (ns === this.assembly.name) { + const module = new Module( + toPythonModuleName(ns), + ns, + { assembly: this.assembly, + assemblyFilename: this.getAssemblyFileName(), + loadAssembly: ns === this.assembly.name }, + ); + + this.package.addModule(module); + // Add our py.typed marker to ensure that gradual typing works for this + // package. this.package.addData(module, "py.typed", ""); + + this.types.set(ns, module); + } else { + // This should be temporary code, which can be removed and turned into an + // error case once https://github.com/awslabs/jsii/issues/270 and + // https://github.com/awslabs/jsii/issues/283 are solved. + this.addPythonType( + new Namespace( + toPythonIdentifier(ns.replace(/^.+\.([^\.]+)$/, "$1")), + ns, + {}, + ), + ); } } From 4754feda6466cf23d0e84ce77bcac5f11549f8e6 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 1 Nov 2018 18:26:58 -0400 Subject: [PATCH 84/88] Explicitly use Python3 --- packages/jsii-pacmak/lib/targets/python.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 55d63e0b76..2b0e8b3635 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -23,8 +23,8 @@ export default class Python extends Target { await shell("black", ["--py36", sourceDir], {}); // Actually package up our code, both as a sdist and a wheel for publishing. - await shell("python", ["setup.py", "sdist", "--dist-dir", outDir], { cwd: sourceDir }); - await shell("python", ["setup.py", "bdist_wheel", "--dist-dir", outDir], { cwd: sourceDir }); + await shell("python3", ["setup.py", "sdist", "--dist-dir", outDir], { cwd: sourceDir }); + await shell("python3", ["setup.py", "bdist_wheel", "--dist-dir", outDir], { cwd: sourceDir }); } } From 4cdec2008a4572227ffdfb0d7331cbe74e9d8c4c Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 1 Nov 2018 19:18:21 -0400 Subject: [PATCH 85/88] Handle return values that are data types --- packages/jsii-pacmak/lib/targets/python.ts | 3 ++ .../jsii-python-runtime/src/jsii/__init__.py | 4 +- .../src/jsii/_kernel/__init__.py | 10 ++--- .../src/jsii/_kernel/types.py | 2 +- .../src/jsii/_reference_map.py | 41 +++++++++++++++---- .../jsii-python-runtime/src/jsii/_runtime.py | 9 ++++ 6 files changed, 55 insertions(+), 14 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 2b0e8b3635..096426162c 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -560,12 +560,15 @@ class TypedDict extends BasePythonClassType { code.closeBlock(); // Now we'll emit the mandatory members. + code.line(`@jsii.data_type(jsii_type="${this.fqn}")`); code.openBlock(`class ${this.name}(_${this.name})`); for (const member of sortMembers(mandatoryMembers, resolver)) { member.emit(code, resolver); } code.closeBlock(); } else { + code.line(`@jsii.data_type(jsii_type="${this.fqn}")`); + // In this case we either have no members, or we have all of one type, so // we'll see if we have any optional members, if we don't then we'll use // total=True instead of total=False for the class. diff --git a/packages/jsii-python-runtime/src/jsii/__init__.py b/packages/jsii-python-runtime/src/jsii/__init__.py index 7a976e4fb3..c8c2d3908c 100644 --- a/packages/jsii-python-runtime/src/jsii/__init__.py +++ b/packages/jsii-python-runtime/src/jsii/__init__.py @@ -1,6 +1,6 @@ from typing import Union -from ._runtime import JSIIAssembly, JSIIMeta, JSIIAbstractClass, kernel +from ._runtime import JSIIAssembly, JSIIMeta, JSIIAbstractClass, data_type, kernel # JS doesn't have distinct float or integer types, but we do. So we'll define our own @@ -27,6 +27,8 @@ "JSIIMeta", "JSIIAbstractClass", "Number", + "data_type", + "kernel", "load", "create", "delete", diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py index 28e1b30b52..42875d6e83 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py @@ -24,19 +24,19 @@ ) -def _recursize_dereference(d): +def _recursize_dereference(kernel, d): if isinstance(d, collections.abc.Mapping): - return {k: _recursize_dereference(v) for k, v in d.items()} + return {k: _recursize_dereference(kernel, v) for k, v in d.items()} elif isinstance(d, ObjRef): - return _reference_map.resolve_reference(d) + return _reference_map.resolve_reference(kernel, d) else: return d def _dereferenced(fn): @functools.wraps(fn) - def wrapped(*args, **kwargs): - return _recursize_dereference(fn(*args, **kwargs)) + def wrapped(kernel, *args, **kwargs): + return _recursize_dereference(kernel, fn(kernel, *args, **kwargs)) return wrapped diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/types.py b/packages/jsii-python-runtime/src/jsii/_kernel/types.py index 84fe8e8638..46b100f861 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/types.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/types.py @@ -81,7 +81,7 @@ class StaticGetRequest: @attr.s(auto_attribs=True, frozen=True, slots=True) class GetResponse: - value: Any + value: Any = None @attr.s(auto_attribs=True, frozen=True, slots=True) diff --git a/packages/jsii-python-runtime/src/jsii/_reference_map.py b/packages/jsii-python-runtime/src/jsii/_reference_map.py index baed667da7..c1223fe281 100644 --- a/packages/jsii-python-runtime/src/jsii/_reference_map.py +++ b/packages/jsii-python-runtime/src/jsii/_reference_map.py @@ -1,16 +1,27 @@ # This module exists to break an import cycle between jsii.runtime and jsii.kernel import weakref +from .compat import TypedDict from ._kernel.types import JSClass, Referenceable _types = {} +_data_types = {} def register_type(klass: JSClass): _types[klass.__jsii_type__] = klass +def register_data_type(data_type: TypedDict): + _data_types[data_type.__jsii_type__] = data_type + + +class _FakeReference: + def __init__(self, ref: str) -> None: + self.__jsii_ref__ = ref + + class _ReferenceMap: def __init__(self, types): self._refs = weakref.WeakValueDictionary() @@ -19,7 +30,7 @@ def __init__(self, types): def register(self, inst: Referenceable): self._refs[inst.__jsii_ref__.ref] = inst - def resolve(self, ref): + def resolve(self, kernel, ref): # First we need to check our reference map to see if we have any instance that # already matches this reference. try: @@ -30,12 +41,28 @@ def resolve(self, ref): # If we got to this point, then we didn't have a referene for this, in that case # we want to create a new instance, but we need to create it in such a way that # we don't try to recreate the type inside of the JSII interface. - klass = _types[ref.ref.rsplit("@", 1)[0]] - - # Create our instance, bypassing __init__ by directly calling __new__, and then - # assign our reference to __jsii_ref__ - inst = klass.__new__(klass) - inst.__jsii_ref__ = ref + class_fqn = ref.ref.rsplit("@", 1)[0] + if class_fqn in _types: + klass = _types[class_fqn] + + # Create our instance, bypassing __init__ by directly calling __new__, and then + # assign our reference to __jsii_ref__ + inst = klass.__new__(klass) + inst.__jsii_ref__ = ref + elif class_fqn in _data_types: + data_type = _data_types[class_fqn] + + # A Data type is nothing more than a dictionary, however we need to iterate + # over all of it's properties, and ask the kernel for the values of each of + # then in order to constitute our dict + inst = {} + + for name in data_type.__annotations__.keys(): + # This is a hack, because our kernel expects an object that has a + # __jsii_ref__ attached to it, and we don't have one of those. + inst[name] = kernel.get(_FakeReference(ref), name) + else: + raise ValueError(f"Unknown type: {class_fqn}") return inst diff --git a/packages/jsii-python-runtime/src/jsii/_runtime.py b/packages/jsii-python-runtime/src/jsii/_runtime.py index 9b3a4cecd2..313f2260f1 100644 --- a/packages/jsii-python-runtime/src/jsii/_runtime.py +++ b/packages/jsii-python-runtime/src/jsii/_runtime.py @@ -81,3 +81,12 @@ def __call__(cls, *args, **kwargs): class JSIIAbstractClass(abc.ABCMeta, JSIIMeta): pass + + +def data_type(*, jsii_type): + def deco(cls): + cls.__jsii_type__ = jsii_type + _reference_map.register_data_type(cls) + return cls + + return deco From 620d8502e663690123ecde4a83ae1ea398bc6800 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 1 Nov 2018 21:11:40 -0400 Subject: [PATCH 86/88] Generate Proxy class to use when JSII runtime returns an abstract class --- packages/jsii-pacmak/lib/targets/python.ts | 85 +++++++++++++++---- .../src/jsii/_reference_map.py | 10 ++- 2 files changed, 78 insertions(+), 17 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index 096426162c..a8a4cf2e96 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -152,7 +152,7 @@ const recurseForNamedTypeReferences = (typeRef: spec.TypeReference): spec.NamedT interface PythonBase { readonly name: string; - emit(code: CodeMaker, resolver: TypeResolver): void; + emit(code: CodeMaker, resolver: TypeResolver, opts?: any): void; } interface PythonType extends PythonBase { @@ -239,6 +239,8 @@ abstract class BasePythonClassType implements PythonType, ISortableType { code.openBlock(`class ${this.name}${bases}`); + this.emitPreamble(code, resolver); + if (this.members.length > 0) { for (const member of sortMembers(this.members, resolver)) { member.emit(code, resolver); @@ -251,6 +253,8 @@ abstract class BasePythonClassType implements PythonType, ISortableType { } protected abstract getClassParams(resolver: TypeResolver): string[]; + + protected emitPreamble(_code: CodeMaker, _resolver: TypeResolver) { return; } } interface BaseMethodOpts { @@ -259,9 +263,14 @@ interface BaseMethodOpts { parent?: spec.NamedTypeReference, } +interface BaseMethodEmitOpts { + renderAbstract?: boolean; +} + abstract class BaseMethod implements PythonBase { public readonly name: string; + public readonly abstract: boolean; protected readonly abstract implicitParameter: string; protected readonly jsiiMethod?: string; @@ -274,7 +283,6 @@ abstract class BaseMethod implements PythonBase { private readonly returns?: spec.TypeReference; private readonly liftedProp?: spec.InterfaceType; private readonly parent?: spec.NamedTypeReference; - private readonly abstract: boolean; constructor(name: string, jsName: string | undefined, @@ -284,15 +292,17 @@ abstract class BaseMethod implements PythonBase { const { abstract = false } = opts; this.name = name; + this.abstract = abstract; this.jsName = jsName; this.parameters = parameters; this.returns = returns; this.liftedProp = opts.liftedProp; this.parent = opts.parent; - this.abstract = abstract; } - public emit(code: CodeMaker, resolver: TypeResolver) { + public emit(code: CodeMaker, resolver: TypeResolver, opts?: BaseMethodEmitOpts) { + const { renderAbstract = true } = opts || {}; + let returnType: string; if (this.returns !== undefined) { returnType = resolver.resolve(this.returns, { forwardReferences: false }); @@ -367,17 +377,17 @@ abstract class BaseMethod implements PythonBase { code.line(`@${this.decorator}`); } - if (this.abstract) { + if (renderAbstract && this.abstract) { code.line("@abc.abstractmethod"); } code.openBlock(`def ${this.name}(${pythonParams.join(", ")}) -> ${returnType}`); - this.emitBody(code, resolver); + this.emitBody(code, resolver, renderAbstract); code.closeBlock(); } - private emitBody(code: CodeMaker, resolver: TypeResolver) { - if (this.jsiiMethod === undefined || this.abstract) { + private emitBody(code: CodeMaker, resolver: TypeResolver, renderAbstract: boolean) { + if (this.jsiiMethod === undefined || (renderAbstract && this.abstract)) { code.line("..."); } else { if (this.liftedProp !== undefined) { @@ -448,9 +458,14 @@ interface BasePropertyOpts { immutable?: boolean; } +interface BasePropertyEmitOpts { + renderAbstract?: boolean; +} + abstract class BaseProperty implements PythonBase { public readonly name: string; + public readonly abstract: boolean; protected readonly abstract decorator: string; protected readonly abstract implicitParameter: string; @@ -459,7 +474,6 @@ abstract class BaseProperty implements PythonBase { private readonly jsName: string; private readonly type: spec.TypeReference; - private readonly abstract: boolean; private readonly immutable: boolean; constructor(name: string, jsName: string, type: spec.TypeReference, opts: BasePropertyOpts = {}) { @@ -469,21 +483,22 @@ abstract class BaseProperty implements PythonBase { } = opts; this.name = name; + this.abstract = abstract; this.jsName = jsName; this.type = type; - this.abstract = abstract; this.immutable = immutable; } - public emit(code: CodeMaker, resolver: TypeResolver) { + public emit(code: CodeMaker, resolver: TypeResolver, opts?: BasePropertyEmitOpts) { + const { renderAbstract = true } = opts || {}; const pythonType = resolver.resolve(this.type, { forwardReferences: false }); code.line(`@${this.decorator}`); - if (this.abstract) { + if (renderAbstract && this.abstract) { code.line("@abc.abstractmethod"); } code.openBlock(`def ${this.name}(${this.implicitParameter}) -> ${pythonType}`); - if (this.jsiiGetMethod !== undefined && !this.abstract) { + if (this.jsiiGetMethod !== undefined && (!renderAbstract || !this.abstract)) { code.line(`return jsii.${this.jsiiGetMethod}(${this.implicitParameter}, "${this.jsName}")`); } else { code.line("..."); @@ -492,11 +507,11 @@ abstract class BaseProperty implements PythonBase { if (!this.immutable) { code.line(`@${this.name}.setter`); - if (this.abstract) { + if (renderAbstract && this.abstract) { code.line("@abc.abstractmethod"); } code.openBlock(`def ${this.name}(${this.implicitParameter}, value: ${pythonType})`); - if (this.jsiiSetMethod !== undefined && !this.abstract) { + if (this.jsiiSetMethod !== undefined && (!renderAbstract || !this.abstract)) { code.line(`return jsii.${this.jsiiSetMethod}(${this.implicitParameter}, "${this.jsName}", value)`); } else { code.line("..."); @@ -641,6 +656,43 @@ class Class extends BasePythonClassType { this.abstract = abstract; } + public emit(code: CodeMaker, resolver: TypeResolver) { + // First we do our normal class logic for emitting our members. + super.emit(code, resolver); + + // Then, if our class is Abstract, we have to go through and redo all of + // this logic, except only emiting abstract methods and properties as non + // abstract, and subclassing our initial class. + if (this.abstract) { + resolver = this.fqn ? resolver.bind(this.fqn) : resolver; + code.openBlock(`class ${this.getProxyClassName()}(${this.name})`); + + // Filter our list of members to *only* be abstract members, and not any + // other types. + const abstractMembers = this.members.filter( + m => (m instanceof BaseMethod || m instanceof BaseProperty) && m.abstract + ); + if (abstractMembers.length > 0) { + for (const member of abstractMembers) { + member.emit(code, resolver, { renderAbstract: false }); + } + } else { + code.line("pass"); + } + + code.closeBlock(); + } + } + + protected emitPreamble(code: CodeMaker, _resolver: TypeResolver) { + if (this.abstract) { + code.line("@staticmethod"); + code.openBlock("def __jsii_proxy_class__()"); + code.line(`return ${this.getProxyClassName()}`); + code.closeBlock(); + } + } + protected getClassParams(resolver: TypeResolver): string[] { const params: string[] = this.bases.map(b => resolver.resolve(b)); const metaclass: string = this.abstract ? "JSIIAbstractClass" : "JSIIMeta"; @@ -651,6 +703,9 @@ class Class extends BasePythonClassType { return params; } + private getProxyClassName(): string { + return `_${this.name}Proxy`; + } } class StaticMethod extends BaseMethod { diff --git a/packages/jsii-python-runtime/src/jsii/_reference_map.py b/packages/jsii-python-runtime/src/jsii/_reference_map.py index c1223fe281..960bc860f6 100644 --- a/packages/jsii-python-runtime/src/jsii/_reference_map.py +++ b/packages/jsii-python-runtime/src/jsii/_reference_map.py @@ -1,4 +1,5 @@ # This module exists to break an import cycle between jsii.runtime and jsii.kernel +import inspect import weakref from .compat import TypedDict @@ -45,8 +46,13 @@ def resolve(self, kernel, ref): if class_fqn in _types: klass = _types[class_fqn] - # Create our instance, bypassing __init__ by directly calling __new__, and then - # assign our reference to __jsii_ref__ + # If this class is an abstract class, then we'll use the generated proxy + # class instead of the abstract class to handle return values for this type. + if inspect.isabstract(klass): + klass = klass.__jsii_proxy_class__() + + # Create our instance, bypassing __init__ by directly calling __new__, and + # then assign our reference to __jsii_ref__ inst = klass.__new__(klass) inst.__jsii_ref__ = ref elif class_fqn in _data_types: From 55f297ead99fb215afb79d016994d00e524ab115 Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Fri, 2 Nov 2018 15:48:24 -0400 Subject: [PATCH 87/88] Pass overriden methods/properties into the JSII runtime --- packages/jsii-pacmak/lib/targets/python.ts | 5 +++ .../jsii-python-runtime/src/jsii/__init__.py | 10 ++++- .../src/jsii/_kernel/__init__.py | 40 ++++++++++++++++++- .../jsii-python-runtime/src/jsii/_runtime.py | 8 ++++ 4 files changed, 60 insertions(+), 3 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index a8a4cf2e96..af1a262bcc 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -373,6 +373,10 @@ abstract class BaseMethod implements PythonBase { pythonParams.push(`*${paramName}: ${paramType}`); } + if (this.jsName !== undefined) { + code.line(`@jsii.member(jsii_name="${this.jsName}")`); + } + if (this.decorator !== undefined) { code.line(`@${this.decorator}`); } @@ -494,6 +498,7 @@ abstract class BaseProperty implements PythonBase { const pythonType = resolver.resolve(this.type, { forwardReferences: false }); code.line(`@${this.decorator}`); + code.line(`@jsii.member(jsii_name="${this.jsName}")`); if (renderAbstract && this.abstract) { code.line("@abc.abstractmethod"); } diff --git a/packages/jsii-python-runtime/src/jsii/__init__.py b/packages/jsii-python-runtime/src/jsii/__init__.py index c8c2d3908c..cddd95e6fa 100644 --- a/packages/jsii-python-runtime/src/jsii/__init__.py +++ b/packages/jsii-python-runtime/src/jsii/__init__.py @@ -1,6 +1,13 @@ from typing import Union -from ._runtime import JSIIAssembly, JSIIMeta, JSIIAbstractClass, data_type, kernel +from ._runtime import ( + JSIIAssembly, + JSIIMeta, + JSIIAbstractClass, + data_type, + member, + kernel, +) # JS doesn't have distinct float or integer types, but we do. So we'll define our own @@ -28,6 +35,7 @@ "JSIIAbstractClass", "Number", "data_type", + "member", "kernel", "load", "create", diff --git a/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py index 42875d6e83..59d20604e6 100644 --- a/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py +++ b/packages/jsii-python-runtime/src/jsii/_kernel/__init__.py @@ -1,3 +1,5 @@ +import inspect + from typing import Any, List, Optional, Type import collections.abc @@ -21,9 +23,42 @@ StaticSetRequest, StatsRequest, ObjRef, + Override, ) +_nothing = object() + + +def _get_overides(klass: JSClass, obj: Any) -> List[Override]: + overrides = [] + + # We need to inspect each item in the MRO, until we get to our JSClass, at that + # point we'll bail, because those methods are not the overriden methods, but the + # "real" methods. + for mro_klass in type(obj).mro(): + if mro_klass is klass: + break + + for name, item in mro_klass.__dict__.items(): + # We're only interested in things that also exist on the JSII class, and + # which are themselves, jsii members. + original = getattr(klass, name, _nothing) + if original is not _nothing: + if inspect.isfunction(item) and hasattr(original, "__jsii_name__"): + overrides.append( + Override(method=original.__jsii_name__, cookie=name) + ) + elif inspect.isdatadescriptor(item) and hasattr( + original.fget, "__jsii_name__" + ): + overrides.append( + Override(property=original.fget.__jsii_name__, cookie=name) + ) + + return overrides + + def _recursize_dereference(kernel, d): if isinstance(d, collections.abc.Mapping): return {k: _recursize_dereference(kernel, v) for k, v in d.items()} @@ -74,9 +109,10 @@ def create( if args is None: args = [] - # TODO: Handle Overrides + overrides = _get_overides(klass, obj) + obj.__jsii_ref__ = self.provider.create( - CreateRequest(fqn=klass.__jsii_type__, args=args) + CreateRequest(fqn=klass.__jsii_type__, args=args, overrides=overrides) ) def delete(self, ref: ObjRef) -> None: diff --git a/packages/jsii-python-runtime/src/jsii/_runtime.py b/packages/jsii-python-runtime/src/jsii/_runtime.py index 313f2260f1..96ec90a698 100644 --- a/packages/jsii-python-runtime/src/jsii/_runtime.py +++ b/packages/jsii-python-runtime/src/jsii/_runtime.py @@ -90,3 +90,11 @@ def deco(cls): return cls return deco + + +def member(*, jsii_name): + def deco(fn): + fn.__jsii_name__ = jsii_name + return fn + + return deco From b01bf3da7c6df125efff3981e4b56e7f00e5444e Mon Sep 17 00:00:00 2001 From: Donald Stufft Date: Thu, 15 Nov 2018 15:53:44 -0500 Subject: [PATCH 88/88] Deal with name collisions --- packages/jsii-pacmak/lib/targets/python.ts | 24 +++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/packages/jsii-pacmak/lib/targets/python.ts b/packages/jsii-pacmak/lib/targets/python.ts index af1a262bcc..95631979c6 100644 --- a/packages/jsii-pacmak/lib/targets/python.ts +++ b/packages/jsii-pacmak/lib/targets/python.ts @@ -324,12 +324,34 @@ abstract class BaseMethod implements PythonBase { } } + // We cannot (currently?) blindly use the names given to us by the JSII for + // initializers, because our keyword lifting will allow two names to clash. + // This can hopefully be removed once we get https://github.com/awslabs/jsii/issues/288 + // resolved, so build up a list of all of the prop names so we can check against + // them later. + const liftedPropNames: Set = new Set(); + if (this.liftedProp !== undefined + && this.liftedProp.properties !== undefined + && this.liftedProp.properties.length >= 1) { + for (const prop of this.liftedProp.properties) { + liftedPropNames.add(toPythonIdentifier(prop.name)); + } + } + // We need to turn a list of JSII parameters, into Python style arguments with // gradual typing, so we'll have to iterate over the list of parameters, and // build the list, converting as we go. const pythonParams: string[] = [this.implicitParameter]; for (const [idx, param] of this.parameters.entries()) { - const paramName = toPythonIdentifier(param.name); + // We cannot (currently?) blindly use the names given to us by the JSII for + // initializers, because our keyword lifting will allow two names to clash. + // This can hopefully be removed once we get https://github.com/awslabs/jsii/issues/288 + // resolved. + let paramName: string = toPythonIdentifier(param.name); + while (liftedPropNames.has(paramName)) { + paramName = `${paramName}_`; + } + const paramType = resolver.resolve(param.type, { forwardReferences: false}); const paramDefault = optionalStartsAt !== undefined && idx >= optionalStartsAt ? "=None" : "";