Return Promise for lazy functions. (#689)

* Type the return value of lazy translation functions as Promise.

The return value of the lazy translation functions is a proxied
`Promise` object.
https://github.com/django/django/blob/3.2.6/django/utils/translation/__init__.py#L135-L221.

Signed-off-by: Zixuan James Li <[email protected]>

* Mark unicode translation functions for deprecation.

https://docs.djangoproject.com/en/4.0/releases/4.0/#features-removed-in-4-0.

Signed-off-by: Zixuan James Li <[email protected]>

* Add proxied functions for Promise.

Although there is nothing defined in `Promise` itself, the only
instances of `Promise` are created by the `lazy` function, with magic
methods defined on it.

https://github.com/django/django/blob/3.2.6/django/utils/functional.py#L84-L191.

Signed-off-by: Zixuan James Li <[email protected]>

* Add _StrPromise as a special type for Promise objects for str.

This allows the user to access methods defined on lazy strings while
still letting mypy be aware of that they are not instances of `str`.

The definitions for some of the magic methods are pulled from typeshed. We need
those definitions in the stubs so that `_StrPromise` objects will work properly
with operators, as refining operator types is tricky with the mypy
plugins API.

The rest of the methods will be covered by an attribute hook.

Signed-off-by: Zixuan James Li <[email protected]>

* Implement _StrPromise attribute hook.

This implements an attribute hook that provides type information for
methods that are available on `builtins.str` for `_StrPromise` except
the supported operators. This allows us to avoid copying stubs from the
builtins for all supported methods on `str`.

Signed-off-by: Zixuan James Li <[email protected]>

* Allow message being a _StrPromise object for RegexValidator.

One intended usage of lazystr is to postpone the translation of the
error message of a validation error. It is possible that we pass a
Promise (specifically _StrPromise) and only evaluate it when a
ValidationError is raised.

Signed-off-by: Zixuan James Li <[email protected]>

* Refactor _StrPromise attribtue hook with analyze_member_access.

Signed-off-by: Zixuan James Li <[email protected]>

Signed-off-by: Zixuan James Li <[email protected]>

django-stubs/core/validators.pyi

@@ -3,6 +3,7 @@ from re import RegexFlag
 from typing import Any, Callable, Collection, Dict, List, Optional, Pattern, Sequence, Sized, Tuple, Union
 
 from django.core.files.base import File
+from django.utils.functional import _StrPromise
 
 EMPTY_VALUES: Any
 
@@ -11,14 +12,14 @@ _ValidatorCallable = Callable[[Any], None]
 
 class RegexValidator:
     regex: _Regex = ...  # Pattern[str] on instance, but may be str on class definition
-    message: str = ...
+    message: Union[str, _StrPromise] = ...
     code: str = ...
     inverse_match: bool = ...
     flags: int = ...
     def __init__(
         self,
         regex: Optional[_Regex] = ...,
-        message: Optional[str] = ...,
+        message: Union[str, _StrPromise, None] = ...,
         code: Optional[str] = ...,
         inverse_match: Optional[bool] = ...,
         flags: Optional[RegexFlag] = ...,

django-stubs/utils/functional.pyi

@@ -1,8 +1,8 @@
 from functools import wraps as wraps  # noqa: F401
-from typing import Any, Callable, Generic, List, Optional, Tuple, Type, TypeVar, Union, overload
+from typing import Any, Callable, Generic, List, Optional, Sequence, Tuple, Type, TypeVar, Union, overload
 
 from django.db.models.base import Model
-from typing_extensions import Protocol
+from typing_extensions import Protocol, SupportsIndex
 
 _T = TypeVar("_T")
 
@@ -15,12 +15,38 @@ class cached_property(Generic[_T]):
     @overload
     def __get__(self, instance: object, cls: Type[Any] = ...) -> _T: ...
 
-class Promise: ...
+# Promise is only subclassed by a proxy class defined in the lazy function
+# so it makes sense for it to have all the methods available in that proxy class
+class Promise:
+    def __init__(self, args: Any, kw: Any) -> None: ...
+    def __reduce__(self) -> Tuple[Any, Tuple[Any]]: ...
+    def __lt__(self, other: Any) -> bool: ...
+    def __mod__(self, rhs: Any) -> Any: ...
+    def __add__(self, other: Any) -> Any: ...
+    def __radd__(self, other: Any) -> Any: ...
+    def __deepcopy__(self, memo: Any): ...
+
+class _StrPromise(Promise, Sequence[str]):
+    def __add__(self, __s: str) -> str: ...
+    # Incompatible with Sequence.__contains__
+    def __contains__(self, __o: str) -> bool: ...  # type: ignore[override]
+    def __ge__(self, __x: str) -> bool: ...
+    def __getitem__(self, __i: SupportsIndex | slice) -> str: ...
+    def __gt__(self, __x: str) -> bool: ...
+    def __le__(self, __x: str) -> bool: ...
+    # __len__ needed here because it defined abstract in Sequence[str]
+    def __len__(self) -> int: ...
+    def __lt__(self, __x: str) -> bool: ...
+    def __mod__(self, __x: Any) -> str: ...
+    def __mul__(self, __n: SupportsIndex) -> str: ...
+    def __rmul__(self, __n: SupportsIndex) -> str: ...
+    # Mypy requires this for the attribute hook to take effect
+    def __getattribute__(self, __name: str) -> Any: ...
 
 _C = TypeVar("_C", bound=Callable)
 
 def lazy(func: _C, *resultclasses: Any) -> _C: ...
-def lazystr(text: Any) -> str: ...
+def lazystr(text: Any) -> _StrPromise: ...
 def keep_lazy(*resultclasses: Any) -> Callable: ...
 def keep_lazy_text(func: Callable) -> Callable: ...
 

django-stubs/utils/translation/__init__.pyi

@@ -4,6 +4,7 @@ from contextlib import ContextDecorator
 from typing import Any, Callable, Optional, Type, Union
 
 from django.http.request import HttpRequest
+from django.utils.functional import _StrPromise
 
 LANGUAGE_SESSION_KEY: str
 
@@ -26,21 +27,26 @@ class Trans:
     def __getattr__(self, real_name: Any): ...
 
 def gettext_noop(message: str) -> str: ...
-def ugettext_noop(message: str) -> str: ...
 def gettext(message: str) -> str: ...
-def ugettext(message: str) -> str: ...
 def ngettext(singular: str, plural: str, number: float) -> str: ...
-def ungettext(singular: str, plural: str, number: float) -> str: ...
 def pgettext(context: str, message: str) -> str: ...
 def npgettext(context: str, singular: str, plural: str, number: int) -> str: ...
 
-gettext_lazy = gettext
-pgettext_lazy = pgettext
+# lazy evaluated translation functions
+def gettext_lazy(message: str) -> _StrPromise: ...
+def pgettext_lazy(context: str, message: str) -> _StrPromise: ...
+def ngettext_lazy(singular: str, plural: str, number: Union[int, str, None] = ...) -> _StrPromise: ...
+def npgettext_lazy(context: str, singular: str, plural: str, number: Union[int, str, None] = ...) -> _StrPromise: ...
+
+# NOTE: These translation functions are deprecated and removed in Django 4.0. We should remove them when we drop
+# support for 3.2
+def ugettext_noop(message: str) -> str: ...
+def ugettext(message: str) -> str: ...
+def ungettext(singular: str, plural: str, number: float) -> str: ...
+
+ugettext_lazy = gettext_lazy
+ungettext_lazy = ngettext_lazy
 
-def ugettext_lazy(message: str) -> str: ...
-def ngettext_lazy(singular: str, plural: str, number: Union[int, str, None] = ...) -> str: ...
-def ungettext_lazy(singular: str, plural: str, number: Union[int, str, None] = ...) -> str: ...
-def npgettext_lazy(context: str, singular: str, plural: str, number: Union[int, str, None] = ...) -> str: ...
 def activate(language: str) -> None: ...
 def deactivate() -> None: ...
 

mypy_django_plugin/lib/fullnames.py

@@ -41,3 +41,5 @@
 F_EXPRESSION_FULLNAME = "django.db.models.expressions.F"
 
 ANY_ATTR_ALLOWED_CLASS_FULLNAME = "django_stubs_ext.AnyAttrAllowed"
+
+STR_PROMISE_FULLNAME = "django.utils.functional._StrPromise"

mypy_django_plugin/main.py

@@ -22,6 +22,7 @@
 from mypy_django_plugin.django.context import DjangoContext
 from mypy_django_plugin.lib import fullnames, helpers
 from mypy_django_plugin.transformers import fields, forms, init_create, meta, querysets, request, settings
+from mypy_django_plugin.transformers.functional import resolve_str_promise_attribute
 from mypy_django_plugin.transformers.managers import (
     create_new_manager_class_from_from_queryset_method,
     resolve_manager_method,
@@ -285,6 +286,9 @@ def get_attribute_hook(self, fullname: str) -> Optional[Callable[[AttributeConte
         ):
             return resolve_manager_method
 
+        if info and info.has_base(fullnames.STR_PROMISE_FULLNAME):
+            return resolve_str_promise_attribute
+
         return None
 
     def get_type_analyze_hook(self, fullname: str) -> Optional[Callable[[AnalyzeTypeContext], MypyType]]:

mypy_django_plugin/transformers/functional.py

@@ -0,0 +1,35 @@
+from mypy.checkmember import analyze_member_access
+from mypy.errorcodes import ATTR_DEFINED
+from mypy.nodes import CallExpr, MemberExpr
+from mypy.plugin import AttributeContext
+from mypy.types import AnyType, Instance
+from mypy.types import Type as MypyType
+from mypy.types import TypeOfAny
+
+from mypy_django_plugin.lib import helpers
+
+
+def resolve_str_promise_attribute(ctx: AttributeContext) -> MypyType:
+    if isinstance(ctx.context, MemberExpr):
+        method_name = ctx.context.name
+    elif isinstance(ctx.context, CallExpr) and isinstance(ctx.context.callee, MemberExpr):
+        method_name = ctx.context.callee.name
+    else:
+        ctx.api.fail(f'Cannot resolve the attribute of "{ctx.type}"', ctx.context, code=ATTR_DEFINED)
+        return AnyType(TypeOfAny.from_error)
+
+    str_info = helpers.lookup_fully_qualified_typeinfo(helpers.get_typechecker_api(ctx), f"builtins.str")
+    assert str_info is not None
+    str_type = Instance(str_info, [])
+    return analyze_member_access(
+        method_name,
+        str_type,
+        ctx.context,
+        is_lvalue=False,
+        is_super=False,
+        # operators are already handled with magic methods defined in the stubs for _StrPromise
+        is_operator=False,
+        msg=ctx.api.msg,
+        original_type=ctx.type,
+        chk=helpers.get_typechecker_api(ctx),
+    )

tests/typecheck/utils/test_functional.yml

@@ -16,3 +16,36 @@
       f = Foo()
       reveal_type(f.attr)        # N: Revealed type is "builtins.list[builtins.str]"
       f.attr.name                # E: "List[str]" has no attribute "name"
+
+-   case: str_promise_proxy
+    main:  |
+      from typing import Union
+
+      from django.utils.functional import Promise, lazystr, _StrPromise
+
+      s = lazystr("asd")
+
+      reveal_type(s)                 # N: Revealed type is "django.utils.functional._StrPromise"
+
+      reveal_type(s.format("asd"))   # N: Revealed type is "builtins.str"
+      reveal_type(s.capitalize())    # N: Revealed type is "builtins.str"
+      reveal_type(s.swapcase)        # N: Revealed type is "def () -> builtins.str"
+      reveal_type(s.__getnewargs__)  # N: Revealed type is "def () -> Tuple[builtins.str]"
+      s.nonsense                     # E: "_StrPromise" has no attribute "nonsense"
+      f: Union[_StrPromise, str]
+      reveal_type(f.format("asd"))   # N: Revealed type is "builtins.str"
+      reveal_type(f + "asd")         # N: Revealed type is "builtins.str"
+      reveal_type("asd" + f)         # N: Revealed type is "Union[Any, builtins.str]"
+
+      reveal_type(s + "bar")         # N: Revealed type is "builtins.str"
+      reveal_type("foo" + s)         # N: Revealed type is "Any"
+      reveal_type(s % "asd")         # N: Revealed type is "builtins.str"
+
+      def foo(content: str) -> None:
+        ...
+
+      def bar(content: Promise) -> None:
+        ...
+
+      foo(s)                         # E: Argument 1 to "foo" has incompatible type "_StrPromise"; expected "str"
+      bar(s)