Add separate _api.c file for each librt module

Author: p-sawicki

mypyc/build.py

@@ -53,22 +53,11 @@ class ModDesc(NamedTuple):
 
 
 LIBRT_MODULES = [
-    ModDesc(
-        "librt.internal",
-        ["internal/librt_internal_static.c", "internal/librt_internal.c"],
-        [],
-        ["internal"],
-    ),
-    ModDesc(
-        "librt.strings",
-        ["strings/librt_strings_static.c", "strings/librt_strings.c"],
-        [],
-        ["strings"],
-    ),
+    ModDesc("librt.internal", ["internal/librt_internal.c"], [], ["internal"]),
+    ModDesc("librt.strings", ["strings/librt_strings.c"], [], ["strings"]),
     ModDesc(
         "librt.base64",
         [
-            "base64/librt_base64_static.c",
             "base64/librt_base64.c",
             "base64/lib.c",
             "base64/codec_choose.c",
@@ -118,7 +107,6 @@ class ModDesc(NamedTuple):
     ModDesc(
         "librt.vecs",
         [
-            "vecs/librt_vecs_static.c",
             "vecs/librt_vecs.c",
             "vecs/vec_i64.c",
             "vecs/vec_i32.c",
@@ -132,9 +120,7 @@ class ModDesc(NamedTuple):
         ["vecs/librt_vecs.h", "vecs/vec_template.c"],
         ["vecs"],
     ),
-    ModDesc(
-        "librt.time", ["time/librt_time_static.c", "time/librt_time.c"], ["time/librt_time.h"], []
-    ),
+    ModDesc("librt.time", ["time/librt_time.c"], ["time/librt_time.h"], []),
 ]
 
 try:
@@ -531,9 +517,7 @@ def mypyc_build(
     *,
     separate: bool | list[tuple[list[str], str | None]] = False,
     only_compile_paths: Iterable[str] | None = None,
-    skip_cgen_input: (
-        tuple[list[list[tuple[str, str]]], list[tuple[str, bool, list[str]]]] | None
-    ) = None,
+    skip_cgen_input: tuple[list[list[tuple[str, str]]], list[tuple[str, list[str]]]] | None = None,
     always_use_shared_lib: bool = False,
 ) -> tuple[emitmodule.Groups, list[tuple[list[str], list[str]]], list[SourceDep]]:
     """Do the front and middle end of mypyc building, producing and writing out C source."""
@@ -567,10 +551,7 @@ def mypyc_build(
         write_file(os.path.join(compiler_options.target_dir, "ops.txt"), ops_text)
     else:
         group_cfiles = skip_cgen_input[0]
-        source_deps = [
-            SourceDep(path, has_header=hdr, include_dirs=dirs)
-            for (path, hdr, dirs) in skip_cgen_input[1]
-        ]
+        source_deps = [SourceDep(path, include_dirs=dirs) for (path, dirs) in skip_cgen_input[1]]
 
     # Write out the generated C and collect the files for each group
     # Should this be here??
@@ -687,9 +668,7 @@ def mypycify(
     strip_asserts: bool = False,
     multi_file: bool = False,
     separate: bool | list[tuple[list[str], str | None]] = False,
-    skip_cgen_input: (
-        tuple[list[list[tuple[str, str]]], list[tuple[str, bool, list[str]]]] | None
-    ) = None,
+    skip_cgen_input: tuple[list[list[tuple[str, str]]], list[tuple[str, list[str]]]] | None = None,
     target_dir: str | None = None,
     include_runtime_files: bool | None = None,
     strict_dunder_typing: bool = False,
@@ -812,8 +791,7 @@ def mypycify(
         files_to_copy = list(RUNTIME_C_FILES)
         for source_dep in source_deps:
             files_to_copy.append(source_dep.path)
-            if header := source_dep.get_header():
-                files_to_copy.append(header)
+            files_to_copy.append(source_dep.get_header())
             include_dirs.update(source_dep.include_dirs)
 
         # Copy all files

mypyc/codegen/emitmodule.py

@@ -444,7 +444,7 @@ def collect_source_dependencies(modules: dict[str, ModuleIR]) -> set[SourceDep]:
             if isinstance(dep, SourceDep):
                 source_deps.add(dep)
             else:
-                source_deps.add(dep.static_data_dep())
+                source_deps.add(dep.api_dep())
     return source_deps
 
 
@@ -588,7 +588,7 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]:
             for source_dep in sorted(source_deps, key=lambda d: d.path):
                 base_emitter.emit_line(f'#include "{source_dep.path}"')
             if self.compiler_options.depends_on_librt_internal:
-                base_emitter.emit_line("#include <internal/librt_internal_static.c>")
+                base_emitter.emit_line("#include <internal/librt_internal_api.c>")
         base_emitter.emit_line(f'#include "__native{self.short_group_suffix}.h"')
         base_emitter.emit_line(f'#include "__native_internal{self.short_group_suffix}.h"')
         emitter = base_emitter
@@ -658,8 +658,7 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]:
         # Include headers for conditional source files
         source_deps = collect_source_dependencies(self.modules)
         for source_dep in sorted(source_deps, key=lambda d: d.path):
-            if header := source_dep.get_header():
-                declarations.emit_line(f'#include "{header}"')
+            declarations.emit_line(f'#include "{source_dep.get_header()}"')
 
         declarations.emit_line(f'#include "__native{self.short_group_suffix}.h"')
         declarations.emit_line()

mypyc/ir/deps.py

@@ -19,28 +19,23 @@ def __eq__(self, other: object) -> bool:
     def __hash__(self) -> int:
         return hash(("Capsule", self.name))
 
-    def static_data_dep(self) -> SourceDep:
+    def api_dep(self) -> SourceDep:
         module = self.name.split(".")[-1]
-        return SourceDep(
-            f"{module}/librt_{module}_static.c", has_header=False, include_dirs=[module]
-        )
+        return SourceDep(f"{module}/librt_{module}_api.c", include_dirs=[module])
 
 
 class SourceDep:
     """Defines a C source file that a primitive may require.
 
-    Each source file may also have a corresponding .h file (replace .c with .h)
+    Each source file must also have a corresponding .h file (replace .c with .h)
     that gets implicitly #included if the source is used.
     include_dirs are passed to the C compiler when the file is compiled as a
     shared library separate from the C extension.
     """
 
-    def __init__(
-        self, path: str, *, has_header: bool = True, include_dirs: list[str] | None = None
-    ) -> None:
+    def __init__(self, path: str, *, include_dirs: list[str] | None = None) -> None:
         # Relative path from mypyc/lib-rt, e.g. 'bytes_extra_ops.c'
         self.path: Final = path
-        self.has_header: Final = has_header
         self.include_dirs: Final = include_dirs or []
 
     def __repr__(self) -> str:
@@ -52,9 +47,9 @@ def __eq__(self, other: object) -> bool:
     def __hash__(self) -> int:
         return hash(("SourceDep", self.path))
 
-    def get_header(self) -> str | None:
+    def get_header(self) -> str:
         """Get the header file path by replacing .c with .h"""
-        return self.path.replace(".c", ".h") if self.has_header else None
+        return self.path.replace(".c", ".h")
 
 
 Dependency = Capsule | SourceDep

mypyc/lib-rt/base64/lib_base64_api.h

@@ -0,0 +1,15 @@
+#ifndef LIBRT_BASE64_API_H
+#define LIBRT_BASE64_API_H
+
+#include "librt_base64.h"
+
+extern void *LibRTBase64_API[LIBRT_BASE64_API_LEN];
+
+#define LibRTBase64_ABIVersion (*(int (*)(void)) LibRTBase64_API[0])
+#define LibRTBase64_APIVersion (*(int (*)(void)) LibRTBase64_API[1])
+#define LibRTBase64_b64encode_internal (*(PyObject* (*)(PyObject *source, bool urlsafe)) LibRTBase64_API[2])
+#define LibRTBase64_b64decode_internal (*(PyObject* (*)(PyObject *source, bool urlsafe)) LibRTBase64_API[3])
+
+int import_librt_base64(void);
+
+#endif  // LIBRT_BASE64_API_H

mypyc/lib-rt/base64/librt_base64.h

@@ -7,44 +7,4 @@
 #define LIBRT_BASE64_API_VERSION 2
 #define LIBRT_BASE64_API_LEN 4
 
-extern void *LibRTBase64_API[LIBRT_BASE64_API_LEN];
-
-#define LibRTBase64_ABIVersion (*(int (*)(void)) LibRTBase64_API[0])
-#define LibRTBase64_APIVersion (*(int (*)(void)) LibRTBase64_API[1])
-#define LibRTBase64_b64encode_internal (*(PyObject* (*)(PyObject *source, bool urlsafe)) LibRTBase64_API[2])
-#define LibRTBase64_b64decode_internal (*(PyObject* (*)(PyObject *source, bool urlsafe)) LibRTBase64_API[3])
-
-static int
-import_librt_base64(void)
-{
-    PyObject *mod = PyImport_ImportModule("librt.base64");
-    if (mod == NULL)
-        return -1;
-    Py_DECREF(mod);  // we import just for the side effect of making the below work.
-    void *capsule = PyCapsule_Import("librt.base64._C_API", 0);
-    if (capsule == NULL)
-        return -1;
-    memcpy(LibRTBase64_API, capsule, sizeof(LibRTBase64_API));
-    if (LibRTBase64_ABIVersion() != LIBRT_BASE64_ABI_VERSION) {
-        char err[128];
-        snprintf(err, sizeof(err), "ABI version conflict for librt.base64, expected %d, found %d",
-            LIBRT_BASE64_ABI_VERSION,
-            LibRTBase64_ABIVersion()
-        );
-        PyErr_SetString(PyExc_ValueError, err);
-        return -1;
-    }
-    if (LibRTBase64_APIVersion() < LIBRT_BASE64_API_VERSION) {
-        char err[128];
-        snprintf(err, sizeof(err),
-                 "API version conflict for librt.base64, expected %d or newer, found %d (hint: upgrade librt)",
-            LIBRT_BASE64_API_VERSION,
-            LibRTBase64_APIVersion()
-        );
-        PyErr_SetString(PyExc_ValueError, err);
-        return -1;
-    }
-    return 0;
-}
-
 #endif  // LIBRT_BASE64_H

mypyc/lib-rt/base64/librt_base64_api.c

@@ -0,0 +1,36 @@
+#include "librt_base64_api.h"
+
+void *LibRTBase64_API[LIBRT_BASE64_API_LEN] = {0};
+
+int
+import_librt_base64(void)
+{
+    PyObject *mod = PyImport_ImportModule("librt.base64");
+    if (mod == NULL)
+        return -1;
+    Py_DECREF(mod);  // we import just for the side effect of making the below work.
+    void *capsule = PyCapsule_Import("librt.base64._C_API", 0);
+    if (capsule == NULL)
+        return -1;
+    memcpy(LibRTBase64_API, capsule, sizeof(LibRTBase64_API));
+    if (LibRTBase64_ABIVersion() != LIBRT_BASE64_ABI_VERSION) {
+        char err[128];
+        snprintf(err, sizeof(err), "ABI version conflict for librt.base64, expected %d, found %d",
+            LIBRT_BASE64_ABI_VERSION,
+            LibRTBase64_ABIVersion()
+        );
+        PyErr_SetString(PyExc_ValueError, err);
+        return -1;
+    }
+    if (LibRTBase64_APIVersion() < LIBRT_BASE64_API_VERSION) {
+        char err[128];
+        snprintf(err, sizeof(err),
+                 "API version conflict for librt.base64, expected %d or newer, found %d (hint: upgrade librt)",
+            LIBRT_BASE64_API_VERSION,
+            LibRTBase64_APIVersion()
+        );
+        PyErr_SetString(PyExc_ValueError, err);
+        return -1;
+    }
+    return 0;
+}

mypyc/lib-rt/base64/librt_base64_static.c

@@ -8,7 +8,7 @@
 #include <Python.h>
 
 #include "mypyc_util.h"
-#include "strings/librt_strings.h"
+#include "strings/librt_strings_api.h"
 #include "strings/librt_strings_common.h"
 
 // BytesWriter: Length and capacity

mypyc/lib-rt/byteswriter_extra_ops.h

@@ -12,7 +12,7 @@
 // API version -- more recent versions must maintain backward compatibility, i.e.
 // we can add new features but not remove or change existing features (unless
 // ABI version is changed, but see the comment above).
- #define LIBRT_INTERNAL_API_VERSION 1
+#define LIBRT_INTERNAL_API_VERSION 1
 
 // Number of functions in the capsule API. If you add a new function, also increase
 // LIBRT_INTERNAL_API_VERSION.
@@ -44,73 +44,6 @@ static PyTypeObject *WriteBuffer_type_internal(void);
 static int NativeInternal_API_Version(void);
 static PyObject *extract_symbol_internal(PyObject *data);
 
-#else
-
-extern void *NativeInternal_API[LIBRT_INTERNAL_API_LEN];
-
-#define ReadBuffer_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[0])
-#define WriteBuffer_internal (*(PyObject* (*)(void)) NativeInternal_API[1])
-#define WriteBuffer_getvalue_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[2])
-#define write_bool_internal (*(char (*)(PyObject *source, char value)) NativeInternal_API[3])
-#define read_bool_internal (*(char (*)(PyObject *source)) NativeInternal_API[4])
-#define write_str_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[5])
-#define read_str_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[6])
-#define write_float_internal (*(char (*)(PyObject *source, double value)) NativeInternal_API[7])
-#define read_float_internal (*(double (*)(PyObject *source)) NativeInternal_API[8])
-#define write_int_internal (*(char (*)(PyObject *source, CPyTagged value)) NativeInternal_API[9])
-#define read_int_internal (*(CPyTagged (*)(PyObject *source)) NativeInternal_API[10])
-#define write_tag_internal (*(char (*)(PyObject *source, uint8_t value)) NativeInternal_API[11])
-#define read_tag_internal (*(uint8_t (*)(PyObject *source)) NativeInternal_API[12])
-#define NativeInternal_ABI_Version (*(int (*)(void)) NativeInternal_API[13])
-#define write_bytes_internal (*(char (*)(PyObject *source, PyObject *value)) NativeInternal_API[14])
-#define read_bytes_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[15])
-#define cache_version_internal (*(uint8_t (*)(void)) NativeInternal_API[16])
-#define ReadBuffer_type_internal (*(PyTypeObject* (*)(void)) NativeInternal_API[17])
-#define WriteBuffer_type_internal (*(PyTypeObject* (*)(void)) NativeInternal_API[18])
-#define NativeInternal_API_Version (*(int (*)(void)) NativeInternal_API[19])
-#define extract_symbol_internal (*(PyObject* (*)(PyObject *source)) NativeInternal_API[20])
-
-static int
-import_librt_internal(void)
-{
-    PyObject *mod = PyImport_ImportModule("librt.internal");
-    if (mod == NULL)
-        return -1;
-    Py_DECREF(mod);  // we import just for the side effect of making the below work.
-    void *capsule = PyCapsule_Import("librt.internal._C_API", 0);
-    if (capsule == NULL)
-        return -1;
-    memcpy(NativeInternal_API, capsule, sizeof(NativeInternal_API));
-    if (NativeInternal_ABI_Version() != LIBRT_INTERNAL_ABI_VERSION) {
-        char err[128];
-        snprintf(err, sizeof(err), "ABI version conflict for librt.internal, expected %d, found %d",
-            LIBRT_INTERNAL_ABI_VERSION,
-            NativeInternal_ABI_Version()
-        );
-        PyErr_SetString(PyExc_ValueError, err);
-        return -1;
-    }
-    if (NativeInternal_API_Version() < LIBRT_INTERNAL_API_VERSION) {
-        char err[128];
-        snprintf(err, sizeof(err),
-                 "API version conflict for librt.internal, expected %d or newer, found %d (hint: upgrade librt)",
-            LIBRT_INTERNAL_API_VERSION,
-            NativeInternal_API_Version()
-        );
-        PyErr_SetString(PyExc_ValueError, err);
-        return -1;
-    }
-    return 0;
-}
-
 #endif
 
-static inline bool CPyReadBuffer_Check(PyObject *obj) {
-    return Py_TYPE(obj) == ReadBuffer_type_internal();
-}
-
-static inline bool CPyWriteBuffer_Check(PyObject *obj) {
-    return Py_TYPE(obj) == WriteBuffer_type_internal();
-}
-
 #endif  // LIBRT_INTERNAL_H

mypyc/lib-rt/internal/librt_internal.h

@@ -0,0 +1,36 @@
+#include "librt_internal_api.h"
+
+void *NativeInternal_API[LIBRT_INTERNAL_API_LEN] = {0};
+
+int
+import_librt_internal(void)
+{
+    PyObject *mod = PyImport_ImportModule("librt.internal");
+    if (mod == NULL)
+        return -1;
+    Py_DECREF(mod);  // we import just for the side effect of making the below work.
+    void *capsule = PyCapsule_Import("librt.internal._C_API", 0);
+    if (capsule == NULL)
+        return -1;
+    memcpy(NativeInternal_API, capsule, sizeof(NativeInternal_API));
+    if (NativeInternal_ABI_Version() != LIBRT_INTERNAL_ABI_VERSION) {
+        char err[128];
+        snprintf(err, sizeof(err), "ABI version conflict for librt.internal, expected %d, found %d",
+            LIBRT_INTERNAL_ABI_VERSION,
+            NativeInternal_ABI_Version()
+        );
+        PyErr_SetString(PyExc_ValueError, err);
+        return -1;
+    }
+    if (NativeInternal_API_Version() < LIBRT_INTERNAL_API_VERSION) {
+        char err[128];
+        snprintf(err, sizeof(err),
+                 "API version conflict for librt.internal, expected %d or newer, found %d (hint: upgrade librt)",
+            LIBRT_INTERNAL_API_VERSION,
+            NativeInternal_API_Version()
+        );
+        PyErr_SetString(PyExc_ValueError, err);
+        return -1;
+    }
+    return 0;
+}