Use WAL with SQLite cache, fix close (#21154)

This is the more modern way to manage concurrency with SQLite. Relevant
to current discussion, it means concurrent mypy runs using the cache
will wait for each other, rather than fail

SQLite also claims this is significantly faster, but I haven't yet done
a good profile (If you are profiling this, note that WAL is a persistent
setting, so you will want to delete the cache). This might also allow
removing the `PRAGMA synchronous=OFF`

Finally, I also explicitly close the connection in main. This is
relevant to this change, because it forces checkpointing of the WAL,
which keeps reads fast, reduces disk space and means the cache.db
remains a single self-contained file under regular use

Fixes #21136, see also discussion in #13916

For what it's worth, I feel there are many legitimate uses of concurrent
mypy. At work, we often share cache between multiple projects. At home,
I often end up having parallel runs with a debugger while working on
mypy (although this PR just makes those ones hang waiting for the lock
lol)

---------

Co-authored-by: Ivan Levkivskyi <levkivskyi@gmail.com>

Author: hauntsaninja

mypy/main.py

@@ -189,14 +189,20 @@ def main(
     list([res])  # noqa: C410
 
 
+class BuildResultThunk:
+    # We pass this around so that we avoid freeing memory, which is slow
+    def __init__(self, build_result: build.BuildResult | None) -> None:
+        self._result = build_result
+
+
 def run_build(
     sources: list[BuildSource],
     options: Options,
     fscache: FileSystemCache,
     t0: float,
     stdout: TextIO,
     stderr: TextIO,
-) -> tuple[build.BuildResult | None, list[str], bool]:
+) -> tuple[BuildResultThunk | None, list[str], bool]:
     formatter = util.FancyFormatter(
         stdout, stderr, options.hide_error_codes, hide_success=bool(options.output)
     )
@@ -227,8 +233,12 @@ def flush_errors(filename: str | None, new_messages: list[str], serious: bool) -
         blockers = True
         if not e.use_stdout:
             serious = True
+
+    if res:
+        res.manager.metastore.close()
+
     maybe_write_junit_xml(time.time() - t0, serious, messages, messages_by_file, options)
-    return res, messages, blockers
+    return BuildResultThunk(res), messages, blockers
 
 
 def show_messages(

mypy/metastore.py

@@ -162,6 +162,7 @@ def connect_db(db_file: str) -> sqlite3.Connection:
     # but without this flag, commits are *very* slow, especially when using HDDs,
     # see https://www.sqlite.org/faq.html#q19 for details.
     db.execute("PRAGMA synchronous=OFF")
+    db.execute("PRAGMA journal_mode=WAL")
     db.executescript(SCHEMA)
     return db
 
@@ -171,8 +172,8 @@ def __init__(self, cache_dir_prefix: str) -> None:
         # We check startswith instead of equality because the version
         # will have already been appended by the time the cache dir is
         # passed here.
+        self.db = None
         if cache_dir_prefix.startswith(os.devnull):
-            self.db = None
             return
 
         os.makedirs(cache_dir_prefix, exist_ok=True)