Optimize linecaching (#464)

* Optimize linecaching

* Fix annotation

Author: Tinche

HISTORY.md

@@ -4,6 +4,8 @@
 
 - Fix a regression when unstructuring dictionary values typed as `Any`.
   ([#453](https://github.com/python-attrs/cattrs/issues/453) [#462](https://github.com/python-attrs/cattrs/pull/462))
+- Optimize function source code caching.
+  ([#445](https://github.com/python-attrs/cattrs/issues/445))
 - Generate unique files only in case of linecache enabled.
   ([#445](https://github.com/python-attrs/cattrs/issues/445) [#441](https://github.com/python-attrs/cattrs/pull/461))
 

src/cattrs/gen/__init__.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import linecache
 import re
 from typing import TYPE_CHECKING, Any, Callable, Iterable, Mapping, Tuple, TypeVar
 
@@ -212,12 +211,9 @@ def make_dict_unstructure_fn(
             + ["  return res"]
         )
         script = "\n".join(total_lines)
-        fname = ""
-        if _cattrs_use_linecache:
-            fname = generate_unique_filename(
-                cl, "unstructure", reserve=_cattrs_use_linecache
-            )
-            linecache.cache[fname] = len(script), None, total_lines, fname
+        fname = generate_unique_filename(
+            cl, "unstructure", lines=total_lines if _cattrs_use_linecache else []
+        )
 
         eval(compile(script, fname, "exec"), globs)
     finally:
@@ -627,10 +623,9 @@ def make_dict_structure_fn(
     ]
 
     script = "\n".join(total_lines)
-    fname = ""
-    if _cattrs_use_linecache:
-        fname = generate_unique_filename(cl, "structure", reserve=_cattrs_use_linecache)
-        linecache.cache[fname] = len(script), None, total_lines, fname
+    fname = generate_unique_filename(
+        cl, "structure", lines=total_lines if _cattrs_use_linecache else []
+    )
 
     eval(compile(script, fname, "exec"), globs)
 

src/cattrs/gen/_lc.py

@@ -1,27 +1,25 @@
 """Line-cache functionality."""
 import linecache
-import uuid
-from typing import Any
+from typing import Any, List
 
 
-def generate_unique_filename(cls: Any, func_name: str, reserve: bool = True) -> str:
+def generate_unique_filename(cls: Any, func_name: str, lines: List[str] = []) -> str:
     """
     Create a "filename" suitable for a function being generated.
+
+    If *lines* are provided, insert them in the first free spot or stop
+    if a duplicate is found.
     """
-    unique_id = uuid.uuid4()
     extra = ""
     count = 1
 
     while True:
         unique_filename = "<cattrs generated {} {}.{}{}>".format(
             func_name, cls.__module__, getattr(cls, "__qualname__", cls.__name__), extra
         )
-        if not reserve:
+        if not lines:
             return unique_filename
-        # To handle concurrency we essentially "reserve" our spot in
-        # the linecache with a dummy line.  The caller can then
-        # set this value correctly.
-        cache_line = (1, None, (str(unique_id),), unique_filename)
+        cache_line = (len("\n".join(lines)), None, lines, unique_filename)
         if linecache.cache.setdefault(unique_filename, cache_line) == cache_line:
             return unique_filename
 

src/cattrs/gen/typeddicts.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import linecache
 import re
 import sys
 from typing import TYPE_CHECKING, Any, Callable, TypeVar
@@ -225,12 +224,9 @@ def make_dict_unstructure_fn(
         ]
         script = "\n".join(total_lines)
 
-        fname = ""
-        if _cattrs_use_linecache:
-            fname = generate_unique_filename(
-                cl, "unstructure", reserve=_cattrs_use_linecache
-            )
-            linecache.cache[fname] = len(script), None, total_lines, fname
+        fname = generate_unique_filename(
+            cl, "unstructure", lines=total_lines if _cattrs_use_linecache else []
+        )
 
         eval(compile(script, fname, "exec"), globs)
     finally:
@@ -523,10 +519,9 @@ def make_dict_structure_fn(
     ]
 
     script = "\n".join(total_lines)
-    fname = ""
-    if _cattrs_use_linecache:
-        fname = generate_unique_filename(cl, "structure", reserve=_cattrs_use_linecache)
-        linecache.cache[fname] = len(script), None, total_lines, fname
+    fname = generate_unique_filename(
+        cl, "structure", lines=total_lines if _cattrs_use_linecache else []
+    )
 
     eval(compile(script, fname, "exec"), globs)
     return globs[fn_name]

tests/test_gen.py

@@ -70,3 +70,24 @@ class B:
     c.structure(c.unstructure(B(1)), B)
 
     assert len(linecache.cache) == before
+
+
+def test_linecache_dedup():
+    """Linecaching avoids duplicates."""
+
+    @define
+    class LinecacheA:
+        a: int
+
+    c = Converter()
+    before = len(linecache.cache)
+    c.structure(c.unstructure(LinecacheA(1)), LinecacheA)
+    after = len(linecache.cache)
+
+    assert after == before + 2
+
+    c = Converter()
+
+    c.structure(c.unstructure(LinecacheA(1)), LinecacheA)
+
+    assert len(linecache.cache) == after