Always disable sync in SQLite cache (#21184)

Fixes #21176

This seems to resolve all the HDD performance issues. Although this
makes cache less durable, this is fine, as FS is not perfect in this
sense either (and it doesn't really need to be that durable).

Author: ilevkivskyi

mypy/build.py

@@ -877,7 +877,7 @@ def __init__(
                 ]
             )
 
-        self.metastore = create_metastore(options, parallel_worker)
+        self.metastore = create_metastore(options)
 
         # a mapping from source files to their corresponding shadow files
         # for efficient lookup
@@ -1616,13 +1616,10 @@ def exclude_from_backups(target_dir: str) -> None:
         pass
 
 
-def create_metastore(options: Options, parallel_worker: bool = False) -> MetadataStore:
+def create_metastore(options: Options) -> MetadataStore:
     """Create the appropriate metadata store."""
     if options.sqlite_cache:
-        # We use this flag in both coordinator and workers to speed up commits,
-        # see mypy.metastore.connect_db() for details.
-        sync_off = options.num_workers > 0 or parallel_worker
-        mds: MetadataStore = SqliteMetadataStore(_cache_dir_prefix(options), sync_off=sync_off)
+        mds: MetadataStore = SqliteMetadataStore(_cache_dir_prefix(options))
     else:
         mds = FilesystemMetadataStore(_cache_dir_prefix(options))
     return mds

mypy/metastore.py

@@ -154,21 +154,20 @@ def close(self) -> None:
 """
 
 
-def connect_db(db_file: str, sync_off: bool = False) -> sqlite3.Connection:
+def connect_db(db_file: str) -> sqlite3.Connection:
     import sqlite3.dbapi2
 
     db = sqlite3.dbapi2.connect(db_file)
-    if sync_off:
-        # This is a bit unfortunate (as we may get corrupt cache after e.g. Ctrl + C),
-        # but without this flag, commits are *very* slow, especially when using HDDs,
-        # see https://www.sqlite.org/faq.html#q19 for details.
-        db.execute("PRAGMA synchronous=OFF")
+    # This is a bit unfortunate (as we may get corrupt cache after e.g. Ctrl + C),
+    # but without this flag, commits are *very* slow, especially when using HDDs,
+    # see https://www.sqlite.org/faq.html#q19 for details.
+    db.execute("PRAGMA synchronous=OFF")
     db.executescript(SCHEMA)
     return db
 
 
 class SqliteMetadataStore(MetadataStore):
-    def __init__(self, cache_dir_prefix: str, sync_off: bool = False) -> None:
+    def __init__(self, cache_dir_prefix: str) -> None:
         # We check startswith instead of equality because the version
         # will have already been appended by the time the cache dir is
         # passed here.
@@ -177,7 +176,7 @@ def __init__(self, cache_dir_prefix: str, sync_off: bool = False) -> None:
             return
 
         os.makedirs(cache_dir_prefix, exist_ok=True)
-        self.db = connect_db(os_path_join(cache_dir_prefix, "cache.db"), sync_off=sync_off)
+        self.db = connect_db(os_path_join(cache_dir_prefix, "cache.db"))
 
     def _query(self, name: str, field: str) -> Any:
         # Raises FileNotFound for consistency with the file system version