feat(mypyc): Enable incremental compilation

sqlglot/__init__.py

@@ -15,16 +15,25 @@
 from pathlib import Path
 from builtins import type as Type
 
+import importlib.util
+
+# Legacy single-shared-lib builds put a hash-named `*__mypyc.so` at the
+# package root; Python's import machinery can't discover that by its bare
+# name, so pre-load it explicitly. Under `separate=True`, per-module shared
+# libs (e.g. `errors__mypyc.so`) sit next to their .py sources and are
+# resolved lazily via their shims -- skip those.
 for path in Path(__file__).parent.glob("*__mypyc*.so"):
     name = path.stem.split(".")[0]
-    if name not in sys.modules:
-        import importlib.util
-
-        spec = importlib.util.spec_from_file_location(name, path)
-        if spec and spec.loader:
-            mod = importlib.util.module_from_spec(spec)
-            sys.modules[name] = mod
-            spec.loader.exec_module(mod)
+    module_base = name.removesuffix("__mypyc")
+    if (path.parent / f"{module_base}.py").exists():
+        continue
+    if name in sys.modules:
+        continue
+    spec = importlib.util.spec_from_file_location(name, path)
+    if spec and spec.loader:
+        mod = importlib.util.module_from_spec(spec)
+        sys.modules[name] = mod
+        spec.loader.exec_module(mod)
 
 import logging
 import typing as t

sqlglot/optimizer/__init__.py

@@ -1,11 +1,61 @@
 # ruff: noqa: F401
+"""Re-exports from optimizer submodules, deferred to first use.
 
-from sqlglot.optimizer.optimizer import RULES as RULES, optimize as optimize
-from sqlglot.optimizer.scope import (
-    Scope as Scope,
-    build_scope as build_scope,
-    find_all_in_scope as find_all_in_scope,
-    find_in_scope as find_in_scope,
-    traverse_scope as traverse_scope,
-    walk_in_scope as walk_in_scope,
-)
+Under mypyc's ``separate=True``, importing ``sqlglot`` can transitively
+trigger this package's ``__init__.py`` before ``sqlglot`` has finished
+binding names like ``exp`` / ``Schema``. Eager ``from sqlglot.optimizer.optimizer
+import ...`` at module top used to kick off a circular-import cascade.
+PEP 562 ``__getattr__`` lets the names resolve only when they're first
+actually accessed, by which point ``sqlglot``'s namespace is settled.
+"""
+
+from __future__ import annotations
+
+import typing as _t
+
+if _t.TYPE_CHECKING:
+    from sqlglot.optimizer.optimizer import RULES as RULES, optimize as optimize
+    from sqlglot.optimizer.scope import (
+        Scope as Scope,
+        build_scope as build_scope,
+        find_all_in_scope as find_all_in_scope,
+        find_in_scope as find_in_scope,
+        traverse_scope as traverse_scope,
+        walk_in_scope as walk_in_scope,
+    )
+
+# Explicit re-export map. An explicit mapping avoids the ambiguity of
+# fuzzy "search the first module that has the name" — several submodules
+# share names with functions inside `optimizer.py` (e.g. both a
+# `qualify` submodule and a `qualify` function re-exported in the
+# `optimizer` module), so callers doing `optimizer.qualify.qualify(...)`
+# need the submodule, not the function.
+_LAZY_ATTRS: dict[str, tuple[str, str]] = {
+    "RULES": ("sqlglot.optimizer.optimizer", "RULES"),
+    "optimize": ("sqlglot.optimizer.optimizer", "optimize"),
+    "Scope": ("sqlglot.optimizer.scope", "Scope"),
+    "build_scope": ("sqlglot.optimizer.scope", "build_scope"),
+    "find_all_in_scope": ("sqlglot.optimizer.scope", "find_all_in_scope"),
+    "find_in_scope": ("sqlglot.optimizer.scope", "find_in_scope"),
+    "traverse_scope": ("sqlglot.optimizer.scope", "traverse_scope"),
+    "walk_in_scope": ("sqlglot.optimizer.scope", "walk_in_scope"),
+}
+
+
+def __getattr__(name: str) -> _t.Any:
+    import importlib
+
+    target = _LAZY_ATTRS.get(name)
+    if target is not None:
+        module_name, attr = target
+        value = getattr(importlib.import_module(module_name), attr)
+    else:
+        # Fall back to loading `sqlglot.optimizer.<name>` as a submodule.
+        # The old eager __init__ used to populate these as a side effect
+        # of its imports; under lazy resolution we have to do it explicitly.
+        try:
+            value = importlib.import_module(f"{__name__}.{name}")
+        except ModuleNotFoundError:
+            raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None
+    globals()[name] = value
+    return value

sqlglotc/setup.py

@@ -129,6 +129,8 @@ def run(self):
 setup(
     name="sqlglotc",
     packages=[],
-    ext_modules=mypycify(_source_paths(), opt_level=os.environ.get("MYPYC_OPT", "2")),
+    ext_modules=mypycify(
+        _source_paths(), opt_level=os.environ.get("MYPYC_OPT", "2"), separate=True, verbose=True
+    ),
     cmdclass={"build_ext": build_ext, "sdist": sdist},
 )