refactor: move the shared core of the Python bindings into sqlbuild-python-base

Cargo.toml

@@ -22,6 +22,7 @@ members = [
     "crates/sqlbuild-refactor",
     "crates/sqlbuild-project",
     "crates/sqlbuild-cli",
+    "crates/sqlbuild-python-base",
     "crates/sqlbuild-python",
 ]
 resolver = "2"
@@ -53,9 +54,10 @@ order = [
     "sqlbuild-refactor",
     "sqlbuild-project",
     "sqlbuild-cli",
+    "sqlbuild-python-base",
     "sqlbuild-python",
 ]
-python-boundary = "sqlbuild-python"
+python-boundary = ["sqlbuild-python-base", "sqlbuild-python"]
 polyglot-floor = "sqlbuild-analysis"
 
 [workspace.package]
@@ -174,6 +176,7 @@ sqlbuild-model-config = { path = "crates/sqlbuild-model-config" }
 sqlbuild-orchestration = { path = "crates/sqlbuild-orchestration" }
 sqlbuild-process-ownership = { path = "crates/sqlbuild-process-ownership" }
 sqlbuild-project = { path = "crates/sqlbuild-project" }
+sqlbuild-python-base = { path = "crates/sqlbuild-python-base" }
 sqlbuild-pysource = { path = "crates/sqlbuild-pysource" }
 sqlbuild-recall = { path = "crates/sqlbuild-recall" }
 sqlbuild-scopes = { path = "crates/sqlbuild-scopes" }

crates/README.md

@@ -1,8 +1,9 @@
 # Native crates
 
 SQLBuild's native code is a Cargo workspace that builds one Python extension module,
-`sqlbuild._native`, and the `sqb`/`sqlbuild` executables in the `sqlbuild-cli` wheel. Only `sqlbuild-python` knows about Python; every other crate is plain Rust
-that can be tested and benchmarked with `cargo test` alone.
+`sqlbuild._native`, and the `sqb`/`sqlbuild` executables in the `sqlbuild-cli` wheel. Only the
+`sqlbuild-python*` binding crates know about Python; every other crate is plain Rust that can be
+tested and benchmarked with `cargo test` alone.
 
 | Crate | Responsibility |
 |---|---|
@@ -26,14 +27,15 @@ that can be tested and benchmarked with `cargo test` alone.
 | `sqlbuild-refactor` | Native rename/move plans, contract-adoption edits, MODEL/SCHEMA/YAML text edits and atomic staging/commit with rollback. Shared scope placement delegates to the native scope implementation; user Python source inspection remains at the Python AST boundary. |
 | `sqlbuild-project` | The compiled project as plain Rust: `CompiledProject` with the producers' facts and each resource's records (model inputs and schemas, compiled models and their analysis, sources, functions, audits and seeds), and what those records answer without Python, such as lineage rows, resolved namespaces and dependency keys. Values the host owns, such as user config and discovered files, and the views it builds lazily live in one slot per record, whose types `sqlbuild-python` supplies through the `ProjectHost` trait; reads that need Python stay in the bindings as extension traits. |
 | `sqlbuild-cli` | The `sqb` and `sqlbuild` executables: `--version`, unchanged compiles replayed from the shared reuse store with Python's exact streams, exit code and compute log capture, and a handoff of every other invocation to the installed `sqlbuild-python` script. Built as its own wheel (`crates/sqlbuild-cli/pyproject.toml`). |
-| `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. |
+| `sqlbuild-python-base` | The base of the PyO3 bindings: native record, enum and error classes with their specs, registration, construction and field reads; keyword calls, imports, conversions and panics at the Python boundary; host entry calls; and the process allocator. |
+| `sqlbuild-python` | The `_native` module: its Python classes and functions and the conversions from Python objects, on top of `sqlbuild-python-base`. |
 
 ## Layering
 
 Dependencies point one way, from the top of this graph to the bottom:
 
 ```text
-sqlbuild-python
+sqlbuild-python -> sqlbuild-python-base
   -> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments
   -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host
   -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core
@@ -52,7 +54,7 @@ in `[workspace.metadata.native-layers]` in the workspace `Cargo.toml`, and
 
 - a crate depends on a crate that is not in an earlier layer, or a workspace crate is missing
   from the order;
-- any crate other than `sqlbuild-python` reaches `pyo3`, directly or through another package;
+- a crate not listed in `python-boundary` reaches `pyo3`, directly or through another package;
 - a crate below `sqlbuild-analysis` reaches polyglot.
 
 The same check fails when the `sqlbuild-cli` and `sqlbuild-python` versions, the `pyproject.toml`
@@ -63,7 +65,8 @@ entry carries a version other than its manifest's.
 `main/` entry points, `models.rs`, `types.rs` and `constants.rs`, not through `_helpers/`.
 
 Inside `sqlbuild-python`, the custom rule XSB075 orders the `bindings/_helpers` modules in layers
-that follow the planned crate split: shared Python plumbing (`infra`), then the glue of the lower
+that follow the planned crate split: shared Python plumbing (`infra`, part of which is already
+`sqlbuild-python-base`, which Cargo keeps below every other layer), then the glue of the lower
 crates (`front`), the compiled project (`project`), planning and compilation as siblings (`plan`,
 `compile`), command output (`output`), command handlers (`commands`) and registration
 (`registry`). A module may import only from lower layers; `plan` and `compile` may not import each
@@ -88,10 +91,10 @@ empty: every import points down, and a new upward import fails the check.
   `sqlbuild._native_host`.
 
 Staying-Python entry points (adapter, user code, executor and runtime, third-party and public
-facade) are reached through the host table and `native_host::host::entry`, which imports only
-`sqlbuild._native_host`. Classes, enums, errors and functions `sqlbuild._native` registers are
-reached through the typed registry (`records::registry` with `RecordId`, `ErrorId` and
-`NativeAttr`), never by module path.
+facade) are reached through the host table and `host_entry` in `sqlbuild-python-base`, which
+imports only `sqlbuild._native_host`. Classes, enums, errors and functions `sqlbuild._native` registers are
+reached through the typed registry of `sqlbuild-python-base` (`records` with `RecordId`,
+`ErrorId` and `NativeAttr`), never by module path.
 
 ## Compiler
 

crates/sqlbuild-python-base/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "sqlbuild-python-base"
+description = "The shared base of the sqlbuild._native bindings: native records, Python calls and host entries."
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+publish.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+mimalloc.workspace = true
+pyo3.workspace = true
+serde_json.workspace = true
+sqlbuild-analysis.workspace = true
+sqlbuild-config.workspace = true
+sqlbuild-core.workspace = true
+sqlbuild-discovery.workspace = true
+sqlbuild-host.workspace = true
+sqlbuild-rules.workspace = true
+
+[lints]
+workspace = true

crates/sqlbuild-python/src/bindings/_helpers/boundary/config_values.rs → crates/sqlbuild-python-base/src/boundary/_helpers/config_values.rs

@@ -1,6 +1,6 @@
 //! Loaded configuration values as the Python objects `tomllib` builds.
 
-use crate::bindings::_helpers::records::fields::field as native_field;
+use crate::records::main::fields::field::field as native_field;
 use pyo3::IntoPyObjectExt;
 use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyModule, PyResult, Python};
 use pyo3::types::{IntoPyDict, PyDict, PyDictMethods, PyList, PyListMethods};

crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs → crates/sqlbuild-python-base/src/boundary/_helpers/mod.rs

@@ -5,7 +5,6 @@ mod allocator;
 pub(crate) mod call_arguments;
 pub(crate) mod config_values;
 pub(crate) mod expected_errors;
-pub(crate) mod keyword_request;
 pub(crate) mod oracles;
 pub(crate) mod panics;
 mod positional_records;

crates/sqlbuild-python/src/bindings/_helpers/boundary/panics.rs → crates/sqlbuild-python-base/src/boundary/_helpers/panics.rs

@@ -2,16 +2,31 @@
 
 use pyo3::Python;
 use pyo3::exceptions::{PyRuntimeError, PyValueError};
+use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult};
 use pyo3::{PyErr, create_exception};
 use sqlbuild_core::constants::PANIC_MESSAGE;
 use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic;
 use sqlbuild_core::panics::main::is_compiler_panic::is_compiler_panic;
 use std::panic::{AssertUnwindSafe, catch_unwind};
 
-use crate::bindings::types::CompilerDetach;
+use crate::types::CompilerDetach;
 
 create_exception!(_native, NativeCompilerError, PyRuntimeError);
 
+/// The named compiler error for an internal native failure.
+pub(crate) fn native_compiler_error(message: impl Into<String>) -> PyErr {
+    NativeCompilerError::new_err(message.into())
+}
+
+/// Add `NativeCompilerError` to the extension module.
+pub(crate) fn register_native_compiler_error(module: &Bound<'_, PyModule>) -> PyResult<()> {
+    module.add(
+        "NativeCompilerError",
+        module.py().get_type::<NativeCompilerError>(),
+    )
+}
+
+/// A panic message as `NativeCompilerError`, anything else as `ValueError`.
 pub(crate) fn compiler_error(error: impl std::fmt::Display) -> PyErr {
     let message = error.to_string();
     if is_compiler_panic(&message) {
@@ -21,10 +36,12 @@ pub(crate) fn compiler_error(error: impl std::fmt::Display) -> PyErr {
     }
 }
 
+/// The same as `compiler_error`.
 pub(crate) fn value_error(error: impl std::fmt::Display) -> PyErr {
     compiler_error(error)
 }
 
+/// Run `operation`, turning an unwinding panic into `NativeCompilerError`.
 pub(crate) fn compiler_guard<T>(
     operation: impl FnOnce() -> pyo3::PyResult<T>,
 ) -> pyo3::PyResult<T> {

crates/sqlbuild-python/src/bindings/_helpers/boundary/positional.rs → crates/sqlbuild-python-base/src/boundary/_helpers/positional.rs

@@ -10,15 +10,15 @@ macro_rules! positional_fields {
         )+
     };
     (@extract $name:ident { $($field:ident : $ty:ty),+ $(,)? }) => {
-        impl $crate::bindings::types::FromPositional for $name {
+        impl $crate::types::FromPositional for $name {
             fn from_positional(value: &pyo3::Bound<'_, pyo3::PyAny>) -> pyo3::PyResult<Self> {
                 let ($($field,)+): ($($ty,)+) = pyo3::types::PyAnyMethods::extract(value)?;
                 Ok($name { $($field),+ })
             }
         }
     };
     (@into $name:ident { $($field:ident : $ty:ty),+ $(,)? }) => {
-        impl $crate::bindings::types::IntoPositional for $name {
+        impl $crate::types::IntoPositional for $name {
             fn into_positional<'py>(
                 self,
                 py: pyo3::Python<'py>,
@@ -31,6 +31,7 @@ macro_rules! positional_fields {
 }
 
 /// Declare a struct naming the positions of a tuple Python sends (`extract`) or receives (`into`).
+#[macro_export]
 macro_rules! positional_record {
     (
         $($mode:ident),+;
@@ -41,7 +42,7 @@ macro_rules! positional_record {
         $vis struct $name<$lifetime> $body
 
         $(
-            positional_record!(
+            $crate::positional_record!(
                 @$mode $name <$lifetime> $body
             );
         )+
@@ -55,7 +56,7 @@ macro_rules! positional_record {
         $vis struct $name $body
 
         $(
-            positional_record!(
+            $crate::positional_record!(
                 @$mode $name $body
             );
         )+

crates/sqlbuild-python/src/bindings/_helpers/boundary/python_calls.rs → crates/sqlbuild-python-base/src/boundary/_helpers/python_calls.rs

@@ -3,10 +3,9 @@
 use std::collections::HashMap;
 use std::sync::{Mutex, PoisonError};
 
-use pyo3::exceptions::PyTypeError;
 use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python};
 use pyo3::sync::PyOnceLock;
-use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString};
+use pyo3::types::{PyDict, PyDictMethods};
 
 /// Standard-library modules whose attributes tests and users never replace, so lookups are kept.
 const STABLE_MODULES: &[&str] = &[
@@ -100,94 +99,6 @@ pub(crate) fn within<'py, T>(
     }
 }
 
-/// The transient status line, when one is shown.
-pub(crate) struct Status<'py>(pub(crate) Bound<'py, PyAny>);
-
-impl Status<'_> {
-    pub(crate) fn start(&self, message: &str) -> PyResult<()> {
-        if !self.0.is_none() {
-            self.0.call_method1("start", (message,))?;
-        }
-        Ok(())
-    }
-
-    pub(crate) fn complete(&self, message: &str) -> PyResult<()> {
-        if !self.0.is_none() {
-            call_with(
-                &self.0.getattr("complete")?,
-                &[("message", PyString::new(self.0.py(), message).into_any())],
-            )?;
-        }
-        Ok(())
-    }
-}
-
-/// Keyword-only arguments of a driver entry point, checked as Python checks a signature.
-pub(crate) struct KeywordArguments<'py> {
-    values: Bound<'py, PyDict>,
-}
-
-impl<'py> KeywordArguments<'py> {
-    /// Accept `options` for `function`, rejecting missing `required` and unknown names.
-    pub(crate) fn read(
-        py: Python<'py>,
-        function: &str,
-        options: Option<Bound<'py, PyDict>>,
-        (required, optional): (&[&str], &[&str]),
-    ) -> PyResult<Self> {
-        let values: Bound<'py, PyDict> = options.unwrap_or_else(|| PyDict::new(py));
-        for name in values.keys() {
-            let name: String = name.extract()?;
-            if !required.contains(&name.as_str()) && !optional.contains(&name.as_str()) {
-                return Err(PyTypeError::new_err(format!(
-                    "{function}() got an unexpected keyword argument '{name}'"
-                )));
-            }
-        }
-        for name in required {
-            if !values.contains(*name)? {
-                return Err(PyTypeError::new_err(format!(
-                    "{function}() missing 1 required keyword-only argument: '{name}'"
-                )));
-            }
-        }
-        Ok(Self { values })
-    }
-
-    /// Whether the argument `name` was passed.
-    pub(crate) fn contains(&self, name: &str) -> PyResult<bool> {
-        self.values.contains(name)
-    }
-
-    /// The argument `name`, or `None` when it was not passed.
-    pub(crate) fn value(&self, name: &str) -> PyResult<Bound<'py, PyAny>> {
-        let py: Python<'py> = self.values.py();
-        Ok(self
-            .values
-            .get_item(name)?
-            .unwrap_or_else(|| py.None().into_bound(py)))
-    }
-
-    /// The argument `name` as a truth value, `False` when it was not passed.
-    pub(crate) fn flag(&self, name: &str) -> PyResult<bool> {
-        self.value(name)?.is_truthy()
-    }
-
-    /// The argument `name`, or `False` when it was not passed.
-    pub(crate) fn value_or_false(&self, name: &str) -> PyResult<Bound<'py, PyAny>> {
-        let py: Python<'py> = self.values.py();
-        Ok(self
-            .values
-            .get_item(name)?
-            .unwrap_or_else(|| PyBool::new(py, false).to_owned().into_any()))
-    }
-
-    /// Whether `name` was passed.
-    pub(crate) fn passed(&self, name: &str) -> PyResult<bool> {
-        self.values.contains(name)
-    }
-}
-
 /// The `contextvars.ContextVar(name, default=None)` kept in `cell`, created once per interpreter.
 pub(crate) fn context_variable<'py>(
     py: Python<'py>,

crates/sqlbuild-python-base/src/boundary/main/call_argument.rs

@@ -0,0 +1,18 @@
+//! The argument `name`, passed by keyword or at `position`.
+
+use crate::boundary::_helpers::call_arguments;
+use pyo3::prelude::Bound;
+use pyo3::prelude::PyAny;
+use pyo3::prelude::PyResult;
+use pyo3::types::PyDict;
+use pyo3::types::PyTuple;
+
+/// The argument `name`, passed by keyword or at `position`.
+#[inline]
+pub fn call_argument<'py>(
+    (args, kwargs): (&Bound<'py, PyTuple>, Option<&Bound<'py, PyDict>>),
+    name: &str,
+    position: usize,
+) -> PyResult<Bound<'py, PyAny>> {
+    call_arguments::call_argument((args, kwargs), name, position)
+}