perf!: share Python identity work per discovery pass and keep compile phases out of sinks

benchmarks/duckshop/edge_cases/orchestration/python_identity_class_shapes.py

@@ -0,0 +1,51 @@
+"""Classes that `inspect` locates by parsing: shadowed, nested, local, decorated and trailing."""
+
+import functools
+
+if True:
+
+    class Shadowed:
+        def first(self):
+            return 1
+
+
+class Shadowed:  # noqa: F811
+    def second(self):
+        return 2
+
+
+class Outer:
+    class Inner:
+        class Deepest:
+            pass
+
+    def method(self):
+        class Local:
+            pass
+
+        return Local
+
+
+@functools.total_ordering
+class Decorated:
+    def __lt__(self, other):
+        return False
+
+    def __eq__(self, other):
+        return True
+
+
+class Trailing:
+    value = 1
+    # trailing comment
+
+
+def factory():
+    class Made:
+        pass
+
+    return Made
+
+
+LocalMade = factory()
+OuterLocal = Outer().method()

crates/sqlbuild-host/src/entries/_helpers/user_code.rs

@@ -149,6 +149,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[
         kind: HostKind::Function,
         edge: HostEdge::UserCode,
     },
+    HostEntry {
+        name: "python_identity_pass",
+        kind: HostKind::Function,
+        edge: HostEdge::UserCode,
+    },
     HostEntry {
         name: "python_node_identity",
         kind: HostKind::Function,

crates/sqlbuild-host/src/entries/models.rs

@@ -157,6 +157,7 @@ pub enum UserCodeHost {
     ActiveMacroBridge,
     LoadCustomMaterializations,
     PythonNodeDependencyKey,
+    PythonIdentityPass,
     PythonNodeIdentity,
     BuildHookIdentities,
     PythonNodeKind,

crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs

@@ -7,7 +7,7 @@ use pyo3::exceptions::PyRuntimeError;
 use pyo3::prelude::{
     Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python,
 };
-use pyo3::types::{PyBool, PyString, PyTuple, PyTupleMethods};
+use pyo3::types::{PyBool, PyList, PyString, PyTuple, PyTupleMethods};
 use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction};
 use sqlbuild_cli::dispatch::main::closing_status::closing_status;
 use sqlbuild_cli::dispatch::main::event_export_warning::event_export_warning;
@@ -32,6 +32,9 @@ use crate::bindings::_helpers::result_output::terminal_index::terminal_event_ind
 const OBSERVABILITY_LOGGER: &str = "sqlbuild.cli.observability";
 const DEBUG_COMMAND: &str = "debug";
 const LINEAGE_COMMAND: &str = "lineage";
+/// Compile's phase operations stay out of project lifecycle sinks; its invocation events do not.
+const COMPILE_COMMAND: &str = "compile";
+const COMPILE_WITHHELD_KINDS: &[&str] = &["operation"];
 const INVOCATION_COMPLETED: &str = "invocation_completed";
 const INVOCATION_FAILED: &str = "invocation_failed";
 /// What an incomplete-export notice says is the only thing affected.
@@ -369,12 +372,18 @@ impl Observed {
                     timeout.extract::<f64>()?
                 };
                 let timeout: Bound<'_, PyAny> = timeout.into_bound_py_any(py)?;
+                let withheld: &[&str] = if native_field(args, "command")?.eq(COMPILE_COMMAND)? {
+                    COMPILE_WITHHELD_KINDS
+                } else {
+                    &[]
+                };
                 let delivery: Bound<'_, PyAny> = EventExporterDispatcher::new(
                     py,
                     Some(keywords(
                         py,
                         &[
                             ("shutdown_timeout_seconds", timeout),
+                            ("withheld_kinds", PyList::new(py, withheld)?.into_any()),
                             (
                                 "failure_callback",
                                 wrap_pyfunction!(log_exporter_failure, py)?.into_any(),

crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs

@@ -39,9 +39,9 @@ use crate::bindings::_helpers::command_runtime::warehouses::{
     CommandScope, WarehouseInputs, apply_warehouse, command_scope, find_authored, resolve_warehouse,
 };
 use crate::bindings::_helpers::compile_attachment::target::build_effective_target_namespace;
-use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs;
 use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name;
 use crate::bindings::_helpers::diff_report::query::NAMESPACELESS_ADAPTERS;
+use crate::bindings::_helpers::discovery::configuration::discover_configured_providers;
 use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy;
 use crate::bindings::_helpers::native_host::host::redirect_stdout_to_stderr;
 use crate::bindings::_helpers::native_host::provider_settings::{
@@ -280,19 +280,8 @@ fn build_report(
 ) -> PyResult<DebugReport> {
     let project_dir_object: Bound<'_, PyAny> =
         imported(py, "pathlib", "Path")?.call1((project_dir,))?;
-    let discovered_inputs: Bound<'_, PyAny> = discover_project_inputs(
-        py,
-        Some(keywords(
-            py,
-            &[
-                ("project_dir", project_dir_object.clone()),
-                (
-                    "secret_policy",
-                    PyString::new(py, SecretPolicy::Defer.name()).into_any(),
-                ),
-            ],
-        )?),
-    )?;
+    let discovered_inputs: Bound<'_, PyAny> =
+        discover_configured_providers(py, &project_dir_object, SecretPolicy::Defer)?;
     let files: DebugConfigFiles = debug_config_files(project_dir);
     let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name(
         &native_field(&discovered_inputs, "project_config")?,

crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs

@@ -7,34 +7,68 @@ use sqlbuild_host::entries::models::ExecutorHost;
 use crate::bindings::_helpers::boundary::python_calls::{call_with, within};
 use crate::bindings::_helpers::compile_driver::config_validation::{
     ProjectConfiguration, reject_invalid_connection_configuration,
+    reject_invalid_project_configuration,
 };
+use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes;
 use crate::bindings::_helpers::discovery::project_config::{
-    load_local_config_file, load_project_config_file,
+    load_local_config_file, load_project_config_file, warn_deprecated_settings,
 };
+use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy;
 use crate::bindings::_helpers::native_host::host;
 use crate::bindings::_helpers::records::construct::new_record_with;
+use crate::bindings::_helpers::records::fields::field as native_field;
 use crate::bindings::_helpers::records::ids::RecordId;
 
 /// `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle.
 pub(crate) fn discover_project_configuration<'py>(
     py: Python<'py>,
     project_dir: &Bound<'py, PyAny>,
 ) -> PyResult<Bound<'py, PyAny>> {
-    configuration_inputs(py, project_dir, false)
+    configuration_inputs(py, project_dir, |_| Ok(()))
 }
 
 /// The validated configuration without a discovery lifecycle, which reading two files never needs.
 pub(crate) fn discover_connection_configuration<'py>(
     py: Python<'py>,
     project_dir: &Bound<'py, PyAny>,
 ) -> PyResult<Bound<'py, PyAny>> {
-    loaded_configuration(py, project_dir, true)
+    loaded_configuration(py, project_dir, reject_invalid_connection_configuration)
+}
+
+/// The configuration with discovery's checks and warnings, and the provider classes, without importing any other project Python.
+pub(crate) fn discover_configured_providers<'py>(
+    py: Python<'py>,
+    project_dir: &Bound<'py, PyAny>,
+    secret_policy: SecretPolicy,
+) -> PyResult<Bound<'py, PyAny>> {
+    let configuration: Bound<'py, PyAny> = loaded_configuration(py, project_dir, |config| {
+        reject_invalid_project_configuration(config)?;
+        warn_deprecated_settings(py, project_dir)
+    })?;
+    let providers: Bound<'py, PyAny> =
+        discover_provider_classes(project_dir.clone(), secret_policy.name())?;
+    new_record_with(
+        py,
+        RecordId::DiscoveredProjectInputs,
+        &[
+            ("project_dir", project_dir.clone()),
+            (
+                "project_config",
+                native_field(&configuration, "project_config")?,
+            ),
+            (
+                "local_config",
+                native_field(&configuration, "local_config")?,
+            ),
+            ("providers", providers),
+        ],
+    )
 }
 
 fn configuration_inputs<'py>(
     py: Python<'py>,
     project_dir: &Bound<'py, PyAny>,
-    validate_connections: bool,
+    checks: impl FnOnce(&ProjectConfiguration<'_, '_>) -> PyResult<()>,
 ) -> PyResult<Bound<'py, PyAny>> {
     let lifecycle: Bound<'py, PyAny> = call_with(
         &host::entry(py, ExecutorHost::OperationLifecycle)?,
@@ -47,25 +81,24 @@ fn configuration_inputs<'py>(
         ],
     )?;
     within(&lifecycle, |_| {
-        loaded_configuration(py, project_dir, validate_connections)
+        loaded_configuration(py, project_dir, checks)
     })
 }
 
+/// Both configuration files, after `checks` accepts them.
 fn loaded_configuration<'py>(
     py: Python<'py>,
     project_dir: &Bound<'py, PyAny>,
-    validate_connections: bool,
+    checks: impl FnOnce(&ProjectConfiguration<'_, '_>) -> PyResult<()>,
 ) -> PyResult<Bound<'py, PyAny>> {
     let project_config: Bound<'py, PyAny> =
         load_project_config_file(py, project_dir)?.into_bound(py);
     let local_config: Bound<'py, PyAny> = load_local_config_file(py, project_dir)?.into_bound(py);
-    if validate_connections {
-        reject_invalid_connection_configuration(&ProjectConfiguration {
-            project_dir,
-            project_config: &project_config,
-            local_config: &local_config,
-        })?;
-    }
+    checks(&ProjectConfiguration {
+        project_dir,
+        project_config: &project_config,
+        local_config: &local_config,
+    })?;
     new_record_with(
         py,
         RecordId::DiscoveredProjectInputs,

crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs

@@ -420,7 +420,8 @@ pub(crate) fn discover_python_node_functions<'py>(
                 ("project_dir", project.path.clone()),
             ],
         )?;
-        let discovered_nodes: PyResult<()> = buckets.discover_nodes(&project, &providers);
+        let discovered_nodes: PyResult<()> =
+            host::within_identity_pass(py, || buckets.discover_nodes(&project, &providers));
         call_with(
             &shim(py, UserCodeHost::RestorePythonRootModules)?,
             &[("saved_modules", saved)],

crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs

@@ -140,6 +140,8 @@ pub(crate) struct Shared {
 #[pyclass(module = "sqlbuild._native", frozen)]
 pub(crate) struct EventExporterDispatcher {
     shared: Arc<Shared>,
+    /// Event kinds this command never exports, such as compile's phase operations.
+    withheld_kinds: Vec<String>,
     thread: Py<PyAny>,
     notification_thread: Option<Py<PyAny>>,
     health_thread: Option<Py<PyAny>>,
@@ -664,9 +666,16 @@ impl EventExporterDispatcher {
                     "health_interval_seconds",
                     "failure_callback",
                     "summary_callback",
+                    "withheld_kinds",
                 ],
             ),
         )?;
+        let withheld: Bound<'_, PyAny> = options.value("withheld_kinds")?;
+        let withheld_kinds: Vec<String> = if withheld.is_none() {
+            Vec::new()
+        } else {
+            withheld.extract()?
+        };
         let limits: Limits = limits(&options)?;
         let exporters: Bound<'_, PyAny> = options.value("exporters")?;
         let (bound_exporters, filters) = if exporters.is_none() {
@@ -755,6 +764,7 @@ impl EventExporterDispatcher {
         thread.call_method0("start")?;
         Ok(Self {
             shared,
+            withheld_kinds,
             thread: thread.unbind(),
             notification_thread: notification_thread.map(Bound::unbind),
             health_thread: health_thread.map(Bound::unbind),
@@ -777,6 +787,9 @@ impl EventExporterDispatcher {
     fn enqueue(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> {
         let event_type: String = native_field(event, "event_type")?.extract()?;
         let policy: ExportPolicy = export_policy(py, &event_type)?;
+        if self.withheld_kinds.contains(&policy.kind) {
+            return Ok(());
+        }
         let shared = &self.shared;
         let _held = shared.hold(py)?;
         let mut state = shared.locked()?;

crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs

@@ -197,11 +197,14 @@ pub(crate) fn python_node_graph<'py>(
         providers,
     };
     let mut nodes: Vec<Bound<'py, PyAny>> = Vec::new();
-    for (collection, kind, documented) in KINDS {
-        for discovered in native_field(discovered_inputs, collection)?.try_iter()? {
-            nodes.push(python_node(&vocabulary, (kind, documented), &discovered?)?);
+    host::within_identity_pass(py, || {
+        for (collection, kind, documented) in KINDS {
+            for discovered in native_field(discovered_inputs, collection)?.try_iter()? {
+                nodes.push(python_node(&vocabulary, (kind, documented), &discovered?)?);
+            }
         }
-    }
+        Ok(())
+    })?;
     let inventory: Vec<InventoryNode> = nodes
         .iter()
         .zip(upstream_names(py, &nodes, &vocabulary.sql_ref)?)

crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs

@@ -8,9 +8,11 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python};
 use pyo3::sync::PyOnceLock;
 use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods};
 use sqlbuild_host::entries::constants::{HOST_ENTRIES, HOST_MODULE, HOST_TARGETS_FUNCTION};
-use sqlbuild_host::entries::models::HostId;
+use sqlbuild_host::entries::models::{HostId, UserCodeHost};
 use sqlbuild_host::entries::types::HostEntries;
 
+use crate::bindings::_helpers::boundary::python_calls::within;
+
 /// One entry's target module path and attribute name.
 type Target = (Py<PyString>, Py<PyString>);
 
@@ -127,3 +129,12 @@ pub(crate) fn log_warning_message(py: Python<'_>, logger: &str, message: &str) -
         .call_method1("warning", (message,))
         .map(|_| ())
 }
+
+/// Run `body` inside one Python-node identity pass, so the identities it builds share source work.
+pub(crate) fn within_identity_pass<'py, T>(
+    py: Python<'py>,
+    body: impl FnOnce() -> PyResult<T>,
+) -> PyResult<T> {
+    let identity_pass: Bound<'py, PyAny> = entry(py, UserCodeHost::PythonIdentityPass)?.call0()?;
+    within(&identity_pass, |_| body())
+}

fensu.toml

@@ -269,7 +269,7 @@ reason = "Native code resolves these through the sqlbuild._native_host name tabl
 
 [[targets.python.dead_code.roots]]
 modules = ["sqlbuild.compiler.python_nodes._helpers.user_inventory"]
-symbols = ["python_node_dependency_key", "python_node_identity"]
+symbols = ["python_identity_pass", "python_node_dependency_key", "python_node_identity"]
 reason = "Native code resolves these through the sqlbuild._native_host name table."
 
 [[targets.python.dead_code.roots]]
@@ -1177,7 +1177,8 @@ max_positional_args = 1
 
 [[targets.python.threshold_overrides]]
 paths = ["src/sqlbuild/__init__.py"]
-thresholds = { max_crate_code_lines = 89457 }
+# Raised from 89,457 for the shared Python identity pass (identity_pass.py, object_source.py).
+thresholds = { max_crate_code_lines = 89626 }
 reason = "The runtime package is above the code-line limit; this caps it at its current size until it is split into packages by area. Lower the cap as code moves out."
 
 [targets.python.rule_options.XSB074]
@@ -1646,7 +1647,8 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X
 
 [[targets.rust.threshold_overrides]]
 paths = ["crates/sqlbuild-python/Cargo.toml"]
-thresholds = { max_crate_code_lines = 217769 }
+# Raised from 217,769 for the identity-pass hooks, the sink operation filter and the checks-only debug loader.
+thresholds = { max_crate_code_lines = 217820 }
 reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out."
 
 [[targets.rust.threshold_overrides]]

python-boundary.toml

@@ -457,6 +457,16 @@ path = "compiler/python_nodes/_helpers/identity.py"
 rule = 2
 note = "Identity hashing over live user functions (source and globals)."
 
+[[stays]]
+path = "compiler/python_nodes/_helpers/object_source.py"
+rule = 2
+note = "Identity hashing over live user functions (source and globals)."
+
+[[stays]]
+path = "compiler/python_nodes/classes/identity_pass.py"
+rule = 2
+note = "Identity hashing over live user functions (source and globals)."
+
 [[stays]]
 path = "compiler/macro_bridge/_helpers/store_environment.py"
 rule = 2