From 9c55608be15c7ea6040f93fba1d1f1ade32b8f44 Mon Sep 17 00:00:00 2001 From: henribru <6639509+henribru@users.noreply.github.com> Date: Fri, 24 Jan 2025 21:02:21 +0100 Subject: [PATCH] Add stub generation scripts (#51) --- .gitignore | 7 ++ README.md | 8 ++ create_enums.py | 38 ++++++++++ create_service_overloads.py | 40 ++++++++++ create_type_stubs.py | 141 ++++++++++++++++++++++++++++++++++++ create_types_pyi.py | 18 +++++ gen.sh | 34 +++++++++ stubdefaulter.sh | 13 ++++ stubgen.py | 27 +++++++ uv.lock | 2 +- 10 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 create_enums.py create mode 100644 create_service_overloads.py create mode 100644 create_type_stubs.py create mode 100644 create_types_pyi.py create mode 100644 gen.sh create mode 100644 stubdefaulter.sh create mode 100644 stubgen.py diff --git a/.gitignore b/.gitignore index 9810906f..7b322be5 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,11 @@ !/uv.lock !/README.md !/LICENSE +!/gen.sh +!/create_enums.py +!/create_service_overloads.py +!/create_type_stubs.py +!/create_types_pyi.py +!/stubdefaulter.sh +!/stubgen.py __pycache__ diff --git a/README.md b/README.md index 1de20081..45d40be8 100644 --- a/README.md +++ b/README.md @@ -57,3 +57,11 @@ AdGroupAd( ad=Ad(type=AdTypeEnum.AdType.TEXT_AD), ) ``` + + +## Development +You can run `gen.sh` to generate new stubs, but note that there are a few manual steps before and after mentioned in the script. + +Feel free to use this to open a PR when new versions come out. + +Contributions to automate the remaining manual steps are also welcome. \ No newline at end of file diff --git a/create_enums.py b/create_enums.py new file mode 100644 index 00000000..f7b38f75 --- /dev/null +++ b/create_enums.py @@ -0,0 +1,38 @@ +import importlib +import re + +from proto.enums import ProtoEnumMeta + +from google.ads.googleads import client + +enums_module = importlib.import_module( + f"google.ads.googleads.{client._DEFAULT_VERSION}.enums" +) +version_package = importlib.import_module( + f"google.ads.googleads.{client._DEFAULT_VERSION}" +) + + +lines = [] +for enum in enums_module.__all__: + enum_class = getattr(version_package, enum) + for attr in dir(enum_class): + attr_val = getattr(enum_class, attr) + if isinstance(attr_val, ProtoEnumMeta): + lines.append( + f" {enum}: type[{client._DEFAULT_VERSION}.{enum}.{attr_val.__name__}]" + ) + break + +with open("google-stubs/ads/googleads/client.pyi") as f: + client_pyi = f.read() +lines.insert(0, "# Autogenerated enums") +lines.append(" # End of autogenerated enums") +client_pyi = re.sub( + "# Autogenerated enums.+# End of autogenerated enums", + "\n".join(lines), + client_pyi, + flags=re.DOTALL, +) +with open("google-stubs/ads/googleads/client.pyi", mode="w") as f: + f.write(client_pyi) diff --git a/create_service_overloads.py b/create_service_overloads.py new file mode 100644 index 00000000..3909fbae --- /dev/null +++ b/create_service_overloads.py @@ -0,0 +1,40 @@ +import importlib +import re + +from google.ads.googleads import client + +lines = [] +with open("service_overloads.txt", "w") as f: + for i, version in enumerate(sorted(client._VALID_API_VERSIONS)): + versions_package = importlib.import_module(f"google.ads.googleads.{version}") + + for type in versions_package.__all__: + if type.endswith("Client"): + name = type[: -len("Client")] + lines.append( + f' @overload\n def get_service(self, name: Literal["{name}"], version: _{version.upper()}) -> {version}.{type}: ...' + ) + if version == client._DEFAULT_VERSION: + lines.append( + f' @overload\n def get_service(self, name: Literal["{name}"]) -> {version}.{type}: ...' + ) + + if i < len(client._VALID_API_VERSIONS) - 1: + lines.append("\n") + + lines.append( + f' @overload\n def get_service(self, name: str, version: _V = "{client._DEFAULT_VERSION}") -> Any: ...' + ) + +with open("google-stubs/ads/googleads/client.pyi") as f: + client_pyi = f.read() +lines.insert(0, "# Autogenerated service overloads") +lines.append(" # End of autogenerated service overloads") +client_pyi = re.sub( + "# Autogenerated service overloads.+# End of autogenerated service overloads", + "\n".join(lines), + client_pyi, + flags=re.DOTALL, +) +with open("google-stubs/ads/googleads/client.pyi", mode="w") as f: + f.write(client_pyi) diff --git a/create_type_stubs.py b/create_type_stubs.py new file mode 100644 index 00000000..46e029aa --- /dev/null +++ b/create_type_stubs.py @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +import importlib +import inspect +from contextlib import contextmanager +from pathlib import Path + +import proto + + +class Writer: + def __init__(self, file): + self.file = file + self.lines = [] + self._indent_level = 0 + + @contextmanager + def indent(self): + self._indent_level += 1 + yield + self._indent_level -= 1 + + def write(self, s: str = "", end="\n", indent=True, prepend=False): + if not isinstance(s, str): + s = str(s) + if indent: + s = " " * self._indent_level * 4 + s + if prepend: + self.lines.insert(0, s + end) + else: + self.lines.append(s + end) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, exc_tb): + if exc_type is None: + self.file.writelines(self.lines) + + +def print_message(cls_name, cls, writer, imports): + writer.write(f"class {cls_name}(proto.Message):") + with writer.indent(): + for inner_cls_name, inner_cls in inspect.getmembers( + cls, + lambda value: inspect.isclass(value) and issubclass(value, proto.Message), + ): + if inner_cls_name != "__base__": + print_message(inner_cls_name, inner_cls, writer, imports) + + for inner_enum_name, inner_enum in inspect.getmembers( + cls, lambda value: inspect.isclass(value) and issubclass(value, proto.Enum) + ): + writer.write(f"class {inner_enum_name}(proto.Enum):") + with writer.indent(): + for member_name, member in inner_enum.__members__.items(): + writer.write(f"{member_name} = {member.value}") + + fields = {} + for field_name, field in cls.meta.fields.items(): + if field.proto_type in [proto.DOUBLE, proto.FLOAT]: + type = "float" + elif field.proto_type in [ + proto.INT64, + proto.UINT64, + proto.INT32, + proto.FIXED64, + proto.FIXED32, + proto.UINT32, + proto.SFIXED32, + proto.SFIXED64, + proto.SINT32, + proto.SINT64, + ]: + type = "int" + elif field.proto_type == proto.BOOL: + type = "bool" + elif field.proto_type == proto.STRING: + type = "str" + elif field.proto_type == proto.BYTES: + type = "bytes" + elif field.proto_type == proto.MESSAGE: + if isinstance(field.message, str): + type = field.message + else: + type = field.message.__qualname__ + if field.message.__module__ != cls.__module__: + imports.append( + f"from {field.message.__module__} import {field.message.__qualname__.split('.')[0]}" + ) + elif field.proto_type == proto.ENUM: + if isinstance(field.message, str): + type = field.enum + else: + type = field.enum.__qualname__ + if field.enum.__module__ != cls.__module__: + imports.append( + f"from {field.enum.__module__} import {field.enum.__qualname__.split('.')[0]}" + ) + else: + raise Exception(field.proto_type) + if field.repeated: + type = f"MutableSequence[{type}]" + imports.append("from collections.abc import MutableSequence") + fields[field_name] = type + writer.write(f"{field_name}: {type}") + + writer.write( + f"def __init__(self: _M, mapping: _M | Mapping | google.protobuf.message.Message | None = ..., *, ignore_unknown_fields: bool = ..., {', '.join(f'{field_name}: {type} = ...' for field_name, type in fields.items())}) -> None: ..." + ) + quoted_fields = [f'"{field}"' for field in fields] + writer.write(f"def __contains__( # type: ignore[override]") + if quoted_fields: + writer.write( + f"self, key: Literal[{', '.join(quoted_fields)}]) -> bool: ..." + ) + else: + writer.write(f"self, key: NoReturn) -> bool: ...") + + +for path in Path("./google-ads-python/google/ads/googleads/").glob("**/types/*.py"): + if path.stem == "__init__": + continue + module = importlib.import_module(".".join([*path.parent.parts[1:], path.stem])) + with open( + Path("google-stubs", *path.parent.parts[2:], f"{path.stem}.pyi"), "w" + ) as file, Writer(file) as writer: + writer.write("import proto") + writer.write("import google.protobuf.message") + writer.write("from typing import Any, TypeVar, NoReturn") + writer.write("from typing_extensions import Literal") + writer.write("from collections.abc import Mapping") + writer.write('_M = TypeVar("_M")') + imports = [] + for cls_name, cls in inspect.getmembers( + module, + lambda value: inspect.isclass(value) and issubclass(value, proto.Message), + ): + print_message(cls_name, cls, writer, imports) + for import_ in imports: + writer.write(import_, prepend=True) diff --git a/create_types_pyi.py b/create_types_pyi.py new file mode 100644 index 00000000..b042f1a3 --- /dev/null +++ b/create_types_pyi.py @@ -0,0 +1,18 @@ +import importlib +from pathlib import Path + +from google.ads.googleads import client + +for version in client._VALID_API_VERSIONS: + version_package = importlib.import_module(f"google.ads.googleads.{version}") + with open(Path("google-stubs/ads/googleads/", version, "__init__.pyi"), "w") as f: + for ( + type_name, + package_path, + ) in version_package._lazy_type_to_package_map.items(): + f.write(f"from {package_path} import {type_name} as {type_name}\n") + + # f.write("__all__ = [\n") + # for type_name in module._lazy_type_to_package_map: + # f.write(f' "{type_name}",\n') + # f.write("]\n") diff --git a/gen.sh b/gen.sh new file mode 100644 index 00000000..1132bd58 --- /dev/null +++ b/gen.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -eou pipefail + +# Manual: Clone https://github.com/googleads/google-ads-python +# Manual: Update google-ads-python dependency +# Manual: Update the following to match the API versions: +# - GoogleAdsFailure in errors.pyi +# - _V, imports and get_type default in client.pyi +# - _Request in exception_interceptor.pyi, logging_interceptor.pyi and metadata_inteceptor.pyi +cd google-ads-python +git restore . +git pull +rm __init__.py +shopt -s globstar +sed -i 's/: OptionalRetry/: Union\[retries\.Retry, gapic_v1\.method\._MethodDefault\]/' google/ads/googleads/**/*.py +cd .. +rm -rf google-stubs/ads/googleads/v* +uv run python stubgen.py +uv run python create_type_stubs.py +uv run python create_types_pyi.py +uv run create_enums.py +uv run create_service_overloads.py +./stubdefaulter.sh +mv .gitignore gitignore +uv run ruff check google-stubs --fix --unsafe-fixes +uv run ruff format google-stubs +mv gitignore .gitignore +sed -i 's/from typing/import types\nfrom typing/' google-stubs/ads/googleads/v*/**/client.pyi +sed -i 's/def operations_client(self) -> operations_v1\.OperationsClient: \.\.\./def operations_client(self) -> operations_v1\.OperationsClient: \.\.\. # type: ignore\[override\]/' google-stubs/ads/googleads/v*/**/*.pyi +mv google-stubs google +uv run mypy --namespace-packages --explicit-package-bases google || true # || true so we can move the folder back on failure. +mv google google-stubs +# Manual: Update version compatibility in README diff --git a/stubdefaulter.sh b/stubdefaulter.sh new file mode 100644 index 00000000..08f0129e --- /dev/null +++ b/stubdefaulter.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -eou pipefail + +mkdir google +mv google-stubs google/google +touch google/google/__init__.pyi +touch google/google/ads/__init__.pyi +uv run stubdefaulter -p google +mv google/google google-stubs +rm -rf google +rm google-stubs/__init__.pyi +rm google-stubs/ads/__init__.pyi diff --git a/stubgen.py b/stubgen.py new file mode 100644 index 00000000..9b4626ae --- /dev/null +++ b/stubgen.py @@ -0,0 +1,27 @@ +import os +import shutil +import subprocess +from pathlib import Path + + +def copytree(src, dst, symlinks=False, ignore=None, overwrite=True): + if not os.path.exists(dst): + os.makedirs(dst) + for item in os.listdir(src): + s = os.path.join(src, item) + d = os.path.join(dst, item) + if os.path.isdir(s): + copytree(s, d, symlinks, ignore, overwrite) + else: + if not os.path.exists(d) or ( + overwrite and os.stat(s).st_mtime - os.stat(d).st_mtime > 1 + ): + shutil.copy2(s, d) + + +# TODO: Explore options? E.g. --include-docstrings +subprocess.run(["stubgen", "google", "-o", ".."], cwd="google-ads-python", check=True) +copytree("googleads", "google-stubs/ads/googleads/", overwrite=False) +shutil.rmtree("googleads") +for dir, _, _ in os.walk("google-stubs/ads/googleads"): + Path(dir, "__init__.pyi").touch() diff --git a/uv.lock b/uv.lock index e1b9df27..d5f67625 100644 --- a/uv.lock +++ b/uv.lock @@ -124,7 +124,7 @@ wheels = [ [[package]] name = "google-ads-stubs" -version = "19.0.0" +version = "20.0.0" source = { editable = "." } dependencies = [ { name = "google-ads" },