Add stub generation scripts (#51)

.gitignore

@@ -8,4 +8,11 @@
 !/uv.lock
 !/README.md
 !/LICENSE
+!/gen.sh
+!/create_enums.py
+!/create_service_overloads.py
+!/create_type_stubs.py
+!/create_types_pyi.py
+!/stubdefaulter.sh
+!/stubgen.py
 __pycache__

README.md

@@ -57,3 +57,11 @@ AdGroupAd(
     ad=Ad(type=AdTypeEnum.AdType.TEXT_AD),
 )
 ```
+
+
+## Development
+You can run `gen.sh` to generate new stubs, but note that there are a few manual steps before and after mentioned in the script.
+
+Feel free to use this to open a PR when new versions come out.
+
+Contributions to automate the remaining manual steps are also welcome.

create_enums.py

@@ -0,0 +1,38 @@
+import importlib
+import re
+
+from proto.enums import ProtoEnumMeta
+
+from google.ads.googleads import client
+
+enums_module = importlib.import_module(
+    f"google.ads.googleads.{client._DEFAULT_VERSION}.enums"
+)
+version_package = importlib.import_module(
+    f"google.ads.googleads.{client._DEFAULT_VERSION}"
+)
+
+
+lines = []
+for enum in enums_module.__all__:
+    enum_class = getattr(version_package, enum)
+    for attr in dir(enum_class):
+        attr_val = getattr(enum_class, attr)
+        if isinstance(attr_val, ProtoEnumMeta):
+            lines.append(
+                f"    {enum}: type[{client._DEFAULT_VERSION}.{enum}.{attr_val.__name__}]"
+            )
+            break
+
+with open("google-stubs/ads/googleads/client.pyi") as f:
+    client_pyi = f.read()
+lines.insert(0, "# Autogenerated enums")
+lines.append("    # End of autogenerated enums")
+client_pyi = re.sub(
+    "# Autogenerated enums.+# End of autogenerated enums",
+    "\n".join(lines),
+    client_pyi,
+    flags=re.DOTALL,
+)
+with open("google-stubs/ads/googleads/client.pyi", mode="w") as f:
+    f.write(client_pyi)

create_service_overloads.py

@@ -0,0 +1,40 @@
+import importlib
+import re
+
+from google.ads.googleads import client
+
+lines = []
+with open("service_overloads.txt", "w") as f:
+    for i, version in enumerate(sorted(client._VALID_API_VERSIONS)):
+        versions_package = importlib.import_module(f"google.ads.googleads.{version}")
+
+        for type in versions_package.__all__:
+            if type.endswith("Client"):
+                name = type[: -len("Client")]
+                lines.append(
+                    f'    @overload\n    def get_service(self, name: Literal["{name}"], version: _{version.upper()}) -> {version}.{type}: ...'
+                )
+                if version == client._DEFAULT_VERSION:
+                    lines.append(
+                        f'    @overload\n    def get_service(self, name: Literal["{name}"]) -> {version}.{type}: ...'
+                    )
+
+        if i < len(client._VALID_API_VERSIONS) - 1:
+            lines.append("\n")
+
+    lines.append(
+        f'    @overload\n    def get_service(self, name: str, version: _V = "{client._DEFAULT_VERSION}") -> Any: ...'
+    )
+
+with open("google-stubs/ads/googleads/client.pyi") as f:
+    client_pyi = f.read()
+lines.insert(0, "# Autogenerated service overloads")
+lines.append("    # End of autogenerated service overloads")
+client_pyi = re.sub(
+    "# Autogenerated service overloads.+# End of autogenerated service overloads",
+    "\n".join(lines),
+    client_pyi,
+    flags=re.DOTALL,
+)
+with open("google-stubs/ads/googleads/client.pyi", mode="w") as f:
+    f.write(client_pyi)

create_type_stubs.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+import importlib
+import inspect
+from contextlib import contextmanager
+from pathlib import Path
+
+import proto
+
+
+class Writer:
+    def __init__(self, file):
+        self.file = file
+        self.lines = []
+        self._indent_level = 0
+
+    @contextmanager
+    def indent(self):
+        self._indent_level += 1
+        yield
+        self._indent_level -= 1
+
+    def write(self, s: str = "", end="\n", indent=True, prepend=False):
+        if not isinstance(s, str):
+            s = str(s)
+        if indent:
+            s = " " * self._indent_level * 4 + s
+        if prepend:
+            self.lines.insert(0, s + end)
+        else:
+            self.lines.append(s + end)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, exc_tb):
+        if exc_type is None:
+            self.file.writelines(self.lines)
+
+
+def print_message(cls_name, cls, writer, imports):
+    writer.write(f"class {cls_name}(proto.Message):")
+    with writer.indent():
+        for inner_cls_name, inner_cls in inspect.getmembers(
+            cls,
+            lambda value: inspect.isclass(value) and issubclass(value, proto.Message),
+        ):
+            if inner_cls_name != "__base__":
+                print_message(inner_cls_name, inner_cls, writer, imports)
+
+        for inner_enum_name, inner_enum in inspect.getmembers(
+            cls, lambda value: inspect.isclass(value) and issubclass(value, proto.Enum)
+        ):
+            writer.write(f"class {inner_enum_name}(proto.Enum):")
+            with writer.indent():
+                for member_name, member in inner_enum.__members__.items():
+                    writer.write(f"{member_name} = {member.value}")
+
+        fields = {}
+        for field_name, field in cls.meta.fields.items():
+            if field.proto_type in [proto.DOUBLE, proto.FLOAT]:
+                type = "float"
+            elif field.proto_type in [
+                proto.INT64,
+                proto.UINT64,
+                proto.INT32,
+                proto.FIXED64,
+                proto.FIXED32,
+                proto.UINT32,
+                proto.SFIXED32,
+                proto.SFIXED64,
+                proto.SINT32,
+                proto.SINT64,
+            ]:
+                type = "int"
+            elif field.proto_type == proto.BOOL:
+                type = "bool"
+            elif field.proto_type == proto.STRING:
+                type = "str"
+            elif field.proto_type == proto.BYTES:
+                type = "bytes"
+            elif field.proto_type == proto.MESSAGE:
+                if isinstance(field.message, str):
+                    type = field.message
+                else:
+                    type = field.message.__qualname__
+                    if field.message.__module__ != cls.__module__:
+                        imports.append(
+                            f"from {field.message.__module__} import {field.message.__qualname__.split('.')[0]}"
+                        )
+            elif field.proto_type == proto.ENUM:
+                if isinstance(field.message, str):
+                    type = field.enum
+                else:
+                    type = field.enum.__qualname__
+                    if field.enum.__module__ != cls.__module__:
+                        imports.append(
+                            f"from {field.enum.__module__} import {field.enum.__qualname__.split('.')[0]}"
+                        )
+            else:
+                raise Exception(field.proto_type)
+            if field.repeated:
+                type = f"MutableSequence[{type}]"
+                imports.append("from collections.abc import MutableSequence")
+            fields[field_name] = type
+            writer.write(f"{field_name}: {type}")
+
+        writer.write(
+            f"def __init__(self: _M, mapping: _M | Mapping | google.protobuf.message.Message | None = ..., *, ignore_unknown_fields: bool = ..., {', '.join(f'{field_name}: {type} = ...' for field_name, type in fields.items())}) -> None: ..."
+        )
+        quoted_fields = [f'"{field}"' for field in fields]
+        writer.write(f"def __contains__(  # type: ignore[override]")
+        if quoted_fields:
+            writer.write(
+                f"self, key: Literal[{', '.join(quoted_fields)}]) -> bool: ..."
+            )
+        else:
+            writer.write(f"self, key: NoReturn) -> bool: ...")
+
+
+for path in Path("./google-ads-python/google/ads/googleads/").glob("**/types/*.py"):
+    if path.stem == "__init__":
+        continue
+    module = importlib.import_module(".".join([*path.parent.parts[1:], path.stem]))
+    with open(
+        Path("google-stubs", *path.parent.parts[2:], f"{path.stem}.pyi"), "w"
+    ) as file, Writer(file) as writer:
+        writer.write("import proto")
+        writer.write("import google.protobuf.message")
+        writer.write("from typing import Any, TypeVar, NoReturn")
+        writer.write("from typing_extensions import Literal")
+        writer.write("from collections.abc import Mapping")
+        writer.write('_M = TypeVar("_M")')
+        imports = []
+        for cls_name, cls in inspect.getmembers(
+            module,
+            lambda value: inspect.isclass(value) and issubclass(value, proto.Message),
+        ):
+            print_message(cls_name, cls, writer, imports)
+        for import_ in imports:
+            writer.write(import_, prepend=True)

create_types_pyi.py

@@ -0,0 +1,18 @@
+import importlib
+from pathlib import Path
+
+from google.ads.googleads import client
+
+for version in client._VALID_API_VERSIONS:
+    version_package = importlib.import_module(f"google.ads.googleads.{version}")
+    with open(Path("google-stubs/ads/googleads/", version, "__init__.pyi"), "w") as f:
+        for (
+            type_name,
+            package_path,
+        ) in version_package._lazy_type_to_package_map.items():
+            f.write(f"from {package_path} import {type_name} as {type_name}\n")
+
+        # f.write("__all__ = [\n")
+        # for type_name in module._lazy_type_to_package_map:
+        #     f.write(f'    "{type_name}",\n')
+        # f.write("]\n")

gen.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+# Manual: Clone https://github.com/googleads/google-ads-python
+# Manual: Update google-ads-python dependency
+# Manual: Update the following to match the API versions:
+# - GoogleAdsFailure in errors.pyi
+# - _V, imports and get_type default in client.pyi
+# - _Request in exception_interceptor.pyi, logging_interceptor.pyi and metadata_inteceptor.pyi
+cd google-ads-python
+git restore .
+git pull
+rm __init__.py
+shopt -s globstar
+sed -i 's/: OptionalRetry/: Union\[retries\.Retry, gapic_v1\.method\._MethodDefault\]/' google/ads/googleads/**/*.py
+cd ..
+rm -rf google-stubs/ads/googleads/v*
+uv run python stubgen.py
+uv run python create_type_stubs.py
+uv run python create_types_pyi.py
+uv run create_enums.py
+uv run create_service_overloads.py
+./stubdefaulter.sh
+mv .gitignore gitignore
+uv run ruff check google-stubs --fix --unsafe-fixes
+uv run ruff format google-stubs
+mv gitignore .gitignore
+sed -i 's/from typing/import types\nfrom typing/' google-stubs/ads/googleads/v*/**/client.pyi
+sed -i 's/def operations_client(self) -> operations_v1\.OperationsClient: \.\.\./def operations_client(self) -> operations_v1\.OperationsClient: \.\.\.  # type: ignore\[override\]/' google-stubs/ads/googleads/v*/**/*.pyi
+mv google-stubs google
+uv run mypy --namespace-packages --explicit-package-bases google || true  # || true so we can move the folder back on failure.
+mv google google-stubs
+# Manual: Update version compatibility in README

stubdefaulter.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+mkdir google
+mv google-stubs google/google
+touch google/google/__init__.pyi
+touch google/google/ads/__init__.pyi
+uv run stubdefaulter -p google
+mv google/google google-stubs
+rm -rf google
+rm google-stubs/__init__.pyi
+rm google-stubs/ads/__init__.pyi

stubgen.py

@@ -0,0 +1,27 @@
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+
+def copytree(src, dst, symlinks=False, ignore=None, overwrite=True):
+    if not os.path.exists(dst):
+        os.makedirs(dst)
+    for item in os.listdir(src):
+        s = os.path.join(src, item)
+        d = os.path.join(dst, item)
+        if os.path.isdir(s):
+            copytree(s, d, symlinks, ignore, overwrite)
+        else:
+            if not os.path.exists(d) or (
+                overwrite and os.stat(s).st_mtime - os.stat(d).st_mtime > 1
+            ):
+                shutil.copy2(s, d)
+
+
+# TODO: Explore options? E.g. --include-docstrings
+subprocess.run(["stubgen", "google", "-o", ".."], cwd="google-ads-python", check=True)
+copytree("googleads", "google-stubs/ads/googleads/", overwrite=False)
+shutil.rmtree("googleads")
+for dir, _, _ in os.walk("google-stubs/ads/googleads"):
+    Path(dir, "__init__.pyi").touch()