Misc type hint fixes

A collection of typing-related changes built while typing oslo.cache.

Closes: #278
Pull-request: #278
Pull-request-sha: aa04d4d
Signed-off-by: Stephen Finucane <stephen@that.guru>

Change-Id: I5574404aba72eeb0726418a40e036a4891ce5f33

Author: stephenfin

docs/build/unreleased/278.rst

@@ -0,0 +1,5 @@
+.. change::
+    :tags: bug, typing
+
+    A wide range of typing improvements and modernizations within the codebase.
+    Pull request courtesy Stephen Finucane.

dogpile/__init__.py

@@ -1,4 +1,10 @@
 __version__ = "1.4.2"
 
-from .lock import Lock  # noqa
-from .lock import NeedRegenerationException  # noqa
+from .lock import Lock
+from .lock import NeedRegenerationException
+
+__all__ = [
+    "Lock",
+    "NeedRegenerationException",
+    "__version__",
+]

dogpile/cache/__init__.py

@@ -1,6 +1,13 @@
-from .region import CacheRegion  # noqa
-from .region import make_region  # noqa
-from .region import register_backend  # noqa
-from .. import __version__  # noqa
+from .backends import register_backend
+from .region import CacheRegion
+from .region import make_region
+from .. import __version__
 
 # backwards compat
+
+__all__ = [
+    "CacheRegion",
+    "make_region",
+    "register_backend",
+    "__version__",
+]

dogpile/cache/api.py

@@ -1,17 +1,18 @@
 from __future__ import annotations
 
 import abc
+from collections.abc import Callable
+from collections.abc import Iterable
+from collections.abc import Mapping
+from collections.abc import Sequence
 import enum
 import pickle
 import time
 from typing import Any
-from typing import Callable
 from typing import cast
 from typing import Literal
-from typing import Mapping
 from typing import NamedTuple
 from typing import Optional
-from typing import Sequence
 from typing import Union
 
 from ..util.typing import Self
@@ -297,7 +298,7 @@ def get(self, key: KeyType) -> BackendFormatted:  # pragma NO COVERAGE
         raise NotImplementedError()
 
     def get_multi(
-        self, keys: Sequence[KeyType]
+        self, keys: Iterable[KeyType]
     ) -> Sequence[BackendFormatted]:  # pragma NO COVERAGE
         """Retrieve multiple optionally serialized values from the cache.
 
@@ -340,7 +341,7 @@ def get_serialized(self, key: KeyType) -> SerializedReturnType:
         return cast(SerializedReturnType, self.get(key))
 
     def get_serialized_multi(
-        self, keys: Sequence[KeyType]
+        self, keys: Iterable[KeyType]
     ) -> Sequence[SerializedReturnType]:  # pragma NO COVERAGE
         """Retrieve multiple serialized values from the cache.
 
@@ -478,7 +479,7 @@ def delete(self, key: KeyType) -> None:  # pragma NO COVERAGE
         raise NotImplementedError()
 
     def delete_multi(
-        self, keys: Sequence[KeyType]
+        self, keys: Iterable[KeyType]
     ) -> None:  # pragma NO COVERAGE
         """Delete multiple values from the cache.
 
@@ -498,12 +499,12 @@ def delete_multi(
 
 
 class DefaultSerialization:
-    serializer: Union[None, Serializer] = staticmethod(  # type: ignore
+    serializer: Union[None, Serializer] = staticmethod(
         pickle.dumps
-    )
-    deserializer: Union[None, Deserializer] = staticmethod(  # type: ignore
+    )  # type: ignore
+    deserializer: Union[None, Deserializer] = staticmethod(
         pickle.loads
-    )
+    )  # type: ignore
 
 
 class BytesBackend(DefaultSerialization, CacheBackend):
@@ -535,7 +536,7 @@ def get_serialized(self, key: KeyType) -> SerializedReturnType:
         raise NotImplementedError()
 
     def get_serialized_multi(
-        self, keys: Sequence[KeyType]
+        self, keys: Iterable[KeyType]
     ) -> Sequence[SerializedReturnType]:  # pragma NO COVERAGE
         """Retrieve multiple serialized values from the cache.
 

dogpile/cache/backends/memcached.py

@@ -6,12 +6,12 @@
 
 """
 
+from collections.abc import Mapping
 import random
 import threading
 import time
 import typing
 from typing import Any
-from typing import Mapping
 import warnings
 
 from ..api import CacheBackend

dogpile/cache/plugins/mako_cache.py

@@ -46,7 +46,7 @@
 from mako.cache import CacheImpl
 
 
-class MakoPlugin(CacheImpl):
+class MakoPlugin(CacheImpl):  # type: ignore[misc]
     """A Mako ``CacheImpl`` which talks to dogpile.cache."""
 
     def __init__(self, cache):

dogpile/cache/proxy.py

@@ -12,9 +12,10 @@
 
 from __future__ import annotations
 
-from typing import Mapping
+from collections.abc import Iterable
+from collections.abc import Mapping
+from collections.abc import Sequence
 from typing import Optional
-from typing import Sequence
 
 from .api import BackendFormatted
 from .api import BackendSetType
@@ -102,13 +103,13 @@ def set(self, key: KeyType, value: BackendSetType) -> None:
     def delete(self, key: KeyType) -> None:
         self.proxied.delete(key)
 
-    def get_multi(self, keys: Sequence[KeyType]) -> Sequence[BackendFormatted]:
+    def get_multi(self, keys: Iterable[KeyType]) -> Sequence[BackendFormatted]:
         return self.proxied.get_multi(keys)
 
     def set_multi(self, mapping: Mapping[KeyType, BackendSetType]) -> None:
         self.proxied.set_multi(mapping)
 
-    def delete_multi(self, keys: Sequence[KeyType]) -> None:
+    def delete_multi(self, keys: Iterable[KeyType]) -> None:
         self.proxied.delete_multi(keys)
 
     def get_mutex(self, key: KeyType) -> Optional[CacheMutex]:
@@ -118,7 +119,7 @@ def get_serialized(self, key: KeyType) -> SerializedReturnType:
         return self.proxied.get_serialized(key)
 
     def get_serialized_multi(
-        self, keys: Sequence[KeyType]
+        self, keys: Iterable[KeyType]
     ) -> Sequence[SerializedReturnType]:
         return self.proxied.get_serialized_multi(keys)
 

dogpile/cache/region.py

@@ -1,5 +1,10 @@
 from __future__ import annotations
 
+from collections.abc import Callable
+from collections.abc import Generator
+from collections.abc import Iterable
+from collections.abc import Mapping
+from collections.abc import Sequence
 import contextlib
 import datetime
 from functools import partial
@@ -10,12 +15,8 @@
 import threading
 import time
 from typing import Any
-from typing import Callable
 from typing import cast
-from typing import Mapping
 from typing import Optional
-from typing import Sequence
-from typing import Tuple
 from typing import Type
 from typing import TYPE_CHECKING
 from typing import Union
@@ -64,7 +65,7 @@
     ["CacheRegion", KeyType, Callable[[], ValuePayload], CacheMutex], None
 ]
 
-ExpirationTimeCallable = Callable[[], float]
+ExpirationTimeCallable = Callable[[], Optional[float]]
 
 ToStr = Callable[[Any], str]
 
@@ -648,7 +649,9 @@ def invalidate(self, hard=True):
         """
         self.region_invalidator.invalidate(hard)
 
-    def configure_from_config(self, config_dict, prefix):
+    def configure_from_config(
+        self, config_dict: dict[str, Any], prefix: str
+    ) -> Self:
         """Configure from a configuration dictionary
         and a prefix.
 
@@ -680,7 +683,7 @@ def configure_from_config(self, config_dict, prefix):
             ),
             _config_argument_dict=config_dict,
             _config_prefix="%sarguments." % prefix,
-            wrap=config_dict.get("%swrap" % prefix, None),
+            wrap=config_dict.get("%swrap" % prefix, None),  # type: ignore
             replace_existing_backend=config_dict.get(
                 "%sreplace_existing_backend" % prefix, False
             ),
@@ -693,7 +696,7 @@ def backend(self):
         )
 
     @property
-    def is_configured(self):
+    def is_configured(self) -> bool:
         """Return True if the backend has been configured via the
         :meth:`.CacheRegion.configure` method already.
 
@@ -902,7 +905,7 @@ def get_multi(self, keys, expiration_time=None, ignore_expiration=False):
         ]
 
     @contextlib.contextmanager
-    def _log_time(self, keys):
+    def _log_time(self, keys: Iterable[str]) -> Generator[None, None, None]:
         start_time = time.time()
         yield
         seconds = time.time() - start_time
@@ -941,7 +944,7 @@ def get_or_create(
         creator: Callable[..., ValuePayload],
         expiration_time: Optional[float] = None,
         should_cache_fn: Optional[Callable[[ValuePayload], bool]] = None,
-        creator_args: Optional[Tuple[Any, Mapping[str, Any]]] = None,
+        creator_args: Optional[tuple[Any, Mapping[str, Any]]] = None,
     ) -> ValuePayload:
         """Return a cached value based on the given key.
 
@@ -1104,7 +1107,7 @@ def go():
 
     def get_or_create_multi(
         self,
-        keys: Sequence[KeyType],
+        keys: Iterable[KeyType],
         creator: Callable[[], ValuePayload],
         expiration_time: Optional[float] = None,
         should_cache_fn: Optional[Callable[[ValuePayload], bool]] = None,
@@ -1327,7 +1330,7 @@ def _get_from_backend(self, key: KeyType) -> CacheReturnType:
             return cast(CacheReturnType, self.backend.get(key))
 
     def _get_multi_from_backend(
-        self, keys: Sequence[KeyType]
+        self, keys: Iterable[KeyType]
     ) -> Sequence[CacheReturnType]:
         if self.deserializer:
             return [
@@ -1425,7 +1428,7 @@ def delete(self, key: KeyType) -> None:
 
         self.backend.delete(key)
 
-    def delete_multi(self, keys: Sequence[KeyType]) -> None:
+    def delete_multi(self, keys: Iterable[KeyType]) -> None:
         """Remove multiple values from the cache.
 
         This operation is idempotent (can be called multiple times, or on a
@@ -1640,9 +1643,7 @@ def get_or_create_for_user_func(key_generator, user_func, *arg, **kw):
         def cache_decorator(user_func):
             if to_str is cast(Callable[[Any], str], str):
                 # backwards compatible
-                key_generator = _function_key_generator(
-                    namespace, user_func
-                )  # type: ignore
+                key_generator = _function_key_generator(namespace, user_func)
             else:
                 key_generator = _function_key_generator(
                     namespace, user_func, to_str

dogpile/cache/util.py

@@ -1,10 +1,18 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from collections.abc import Iterable
 from hashlib import sha1
+from typing import Any
+from typing import Union
 
 from ..util import compat
 from ..util import langhelpers
 
 
-def function_key_generator(namespace, fn, to_str=str):
+def function_key_generator(
+    namespace: str, fn: Callable[..., Any], to_str: Callable[[Any], str] = str
+) -> Callable[..., str]:
     """Return a function that generates a string
     key, based on a given function as well as
     arguments to the returned function itself.
@@ -45,7 +53,9 @@ def generate_key(*args, **kw):
     return generate_key
 
 
-def function_multi_key_generator(namespace, fn, to_str=str):
+def function_multi_key_generator(
+    namespace: str, fn: Callable[..., Any], to_str: Callable[[Any], str] = str
+) -> Callable[..., list[str]]:
     if namespace is None:
         namespace = "%s:%s" % (fn.__module__, fn.__name__)
     else:
@@ -67,7 +77,9 @@ def generate_keys(*args, **kw):
     return generate_keys
 
 
-def kwarg_function_key_generator(namespace, fn, to_str=str):
+def kwarg_function_key_generator(
+    namespace: str, fn: Callable[..., Any], to_str: Callable[[Any], str] = str
+) -> Callable[..., str]:
     """Return a function that generates a string
     key, based on a given function as well as
     arguments to the returned function itself.
@@ -127,22 +139,23 @@ def generate_key(*args, **kwargs):
     return generate_key
 
 
-def sha1_mangle_key(key):
+def sha1_mangle_key(key: Union[str, bytes]) -> str:
     """a SHA1 key mangler."""
 
-    if isinstance(key, str):
-        key = key.encode("utf-8")
-
-    return sha1(key).hexdigest()
+    return sha1(
+        key.encode("utf-8") if isinstance(key, str) else key
+    ).hexdigest()
 
 
-def length_conditional_mangler(length, mangler):
+def length_conditional_mangler(
+    length: int, mangler: Callable[[str], str]
+) -> Callable[[str], str]:
     """a key mangler that mangles if the length of the key is
     past a certain threshold.
 
     """
 
-    def mangle(key):
+    def mangle(key: str) -> str:
         if len(key) >= length:
             return mangler(key)
         else:
@@ -164,14 +177,16 @@ def mangle(key):
 class repr_obj:
     __slots__ = ("value", "max_chars")
 
-    def __init__(self, value, max_chars=300):
+    def __init__(self, value: Iterable[str], max_chars: int = 300) -> None:
         self.value = value
         self.max_chars = max_chars
 
-    def __eq__(self, other):
-        return other.value == self.value
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, repr_obj):
+            return NotImplemented
+        return bool(other.value == self.value)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         rep = repr(self.value)
         lenrep = len(rep)
         if lenrep > self.max_chars: