From ea6297c33b9496bc7f2ca0a4c847f8ad81a98050 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 15 Nov 2026 05:55:35 +0000 Subject: [PATCH 1/6] feat: add SQBRPYTHON rules for orchestration sensors and calendars - Compile: a sensor or calendar declared inside a `_`-prefixed folder under orchestration/sensors/ or orchestration/calendars/ now fails with an error naming the file, since such folders are private support. - SQBRPYTHON016-020: module paths, private support, module surface, import direction and no docstring on sensor and calendar functions. - Provider and orchestration rules share parsing and import resolution. - Evict nested namespace packages before their parents between projects in one interpreter, so a project importing orchestration.* no longer breaks the next compile. - Docs and DuckShop fixtures for each rule. --- .../jobs/inventory_job.toml | 3 + .../inventory/_helpers/on_orders_sensor.py | 10 + .../jobs/inventory_job.toml | 3 + .../sensors/inventory/_helpers/selection.py | 3 + .../sensors/inventory/on_orders_sensor.py | 12 + .../jobs/restock_shipments_job.toml | 3 + .../sensors/fulfillment/_constants.py | 3 + .../sensors/fulfillment/_helpers/reasons.py | 6 + .../sensors/fulfillment/shipment_backlog.py | 18 ++ .../python_rules/expected_findings.json | 18 +- .../orchestration/jobs/stock_count_job.toml | 3 + .../sensors/fulfillment/_helpers/hours.py | 3 + .../sensors/inventory/_helpers/feed_window.py | 5 + .../sensors/inventory/_lib/age.py | 3 + .../orchestration/sensors/stock_feed.py | 18 ++ .../python/inventory/stock_count/node.py | 4 +- .../_helpers/inventory_files.rs | 7 +- .../_helpers/python_modules.rs | 28 +- .../src/orchestration_files/constants.rs | 2 + .../tests/test_orchestration_inventory.rs | 48 ++++ .../engine/tests/test_incremental_rules.rs | 5 + .../src/python_rules/_helpers/units/mod.rs | 1 + .../_helpers/units/orchestration.rs | 243 ++++++++++++++++ .../python_rules/_helpers/units/project.rs | 84 +++++- .../python_rules/_helpers/units/providers.rs | 62 +---- .../src/python_rules/constants.rs | 90 +++++- .../main/evaluate_python_rules.rs | 6 + .../sqlbuild-rules/src/python_rules/models.rs | 9 + .../sqlbuild-rules/src/python_rules/tests.rs | 2 + .../tests/test_orchestration_rules.rs | 261 ++++++++++++++++++ .../tests/test_python_catalogue.rs | 6 +- fensu.toml | 4 + .../references/docs/concepts/orchestration.md | 10 +- .../docs/concepts/rules/python-rules.md | 42 +++ .../_helpers/filesystem/project_modules.py | 8 +- .../sqlbuild/cli/commands/main/test_rules.py | 2 +- .../compiler/compiled_project/_test_types.py | 7 + .../test_orchestration_snapshot.py | 70 +++++ .../docs/docs/concepts/orchestration.mdx | 10 +- .../docs/docs/concepts/rules/python-rules.mdx | 41 +++ 40 files changed, 1074 insertions(+), 89 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/jobs/inventory_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/sensors/inventory/_helpers/on_orders_sensor.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/private_support_orders/jobs/inventory_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/_helpers/selection.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/on_orders_sensor.py create mode 100644 benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/jobs/restock_shipments_job.toml create mode 100644 benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_constants.py create mode 100644 benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_helpers/reasons.py create mode 100644 benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/shipment_backlog.py create mode 100644 benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/jobs/stock_count_job.toml create mode 100644 benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/fulfillment/_helpers/hours.py create mode 100644 benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_helpers/feed_window.py create mode 100644 benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_lib/age.py create mode 100644 benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/stock_feed.py create mode 100644 crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs create mode 100644 crates/sqlbuild-rules/src/python_rules/tests/test_orchestration_rules.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/jobs/inventory_job.toml b/benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/jobs/inventory_job.toml new file mode 100644 index 000000000..6d273e69a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/jobs/inventory_job.toml @@ -0,0 +1,3 @@ +name = "inventory_job" +description = "Destination inventory." +select = "stg_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/sensors/inventory/_helpers/on_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/sensors/inventory/_helpers/on_orders_sensor.py new file mode 100644 index 000000000..b0eae0644 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/sensors/inventory/_helpers/on_orders_sensor.py @@ -0,0 +1,10 @@ +"""A sensor declared inside a private support folder fails compile instead of never running.""" + +from sqlbuild.orchestration import NodeEventKind, node_event, sensor + +on_orders = sensor( + "on_orders_sensor", + description="Build inventory for every genuine orders rebuild.", + when=node_event(select="orders orders*", kinds=[NodeEventKind.SUCCESS]), + run="inventory_job", +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/jobs/inventory_job.toml b/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/jobs/inventory_job.toml new file mode 100644 index 000000000..6d273e69a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/jobs/inventory_job.toml @@ -0,0 +1,3 @@ +name = "inventory_job" +description = "Destination inventory." +select = "stg_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/_helpers/selection.py b/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/_helpers/selection.py new file mode 100644 index 000000000..d7e21a644 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/_helpers/selection.py @@ -0,0 +1,3 @@ +"""Private support: modules in a _-prefixed folder declare nothing.""" + +ORDERS_SELECTION: str = "orders orders*" diff --git a/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/on_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/on_orders_sensor.py new file mode 100644 index 000000000..9dcfcbb4e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/on_orders_sensor.py @@ -0,0 +1,12 @@ +"""A sensor in its domain folder reads its selection from a private support folder.""" + +from orchestration.sensors.inventory._helpers.selection import ORDERS_SELECTION + +from sqlbuild.orchestration import NodeEventKind, node_event, sensor + +on_orders = sensor( + "on_orders_sensor", + description="Build inventory for every genuine orders rebuild.", + when=node_event(select=ORDERS_SELECTION, kinds=[NodeEventKind.SUCCESS]), + run="inventory_job", +) diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/jobs/restock_shipments_job.toml b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/jobs/restock_shipments_job.toml new file mode 100644 index 000000000..6b22853d5 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/jobs/restock_shipments_job.toml @@ -0,0 +1,3 @@ +name = "restock_shipments_job" +description = "Rebuild shipments." +select = "shipments" diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_constants.py b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_constants.py new file mode 100644 index 000000000..e3b657540 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_constants.py @@ -0,0 +1,3 @@ +"""Shipment backlog settings.""" + +BACKLOG_REASON: str = "No shipment backlog" diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_helpers/reasons.py b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_helpers/reasons.py new file mode 100644 index 000000000..b518cf30d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_helpers/reasons.py @@ -0,0 +1,6 @@ +"""Skip reasons for the fulfillment sensors.""" + + +def prefixed(reason: str) -> str: + """Prefix a skip reason with its domain.""" + return f"fulfillment: {reason}" diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/shipment_backlog.py b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/shipment_backlog.py new file mode 100644 index 000000000..0a9c1bdbe --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/shipment_backlog.py @@ -0,0 +1,18 @@ +"""Skip restocking while the shipment backlog is empty.""" + +from orchestration.sensors.fulfillment._constants import BACKLOG_REASON +from orchestration.sensors.fulfillment._helpers.reasons import prefixed + +from sqlbuild.orchestration import SensorContext, SkipReason, sensor + +_REASON_SUFFIX: str = "." + + +@sensor( + "shipment_backlog_sensor", + description="Rebuild shipments when the backlog grows.", + run="restock_shipments_job", + minimum_interval_seconds=60, +) +def shipment_backlog(ctx: SensorContext) -> SkipReason: + return SkipReason(prefixed(BACKLOG_REASON) + _REASON_SUFFIX) diff --git a/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json b/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json index f7ae9dfa5..d5233f56c 100644 --- a/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json +++ b/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json @@ -2,6 +2,13 @@ "select": ["SQBRPYTHON", "SQBRPYTHON306"], "findings": [ ["SQBRPYTHON101", "hooks/python/alert.py", 4, 11], + ["SQBRPYTHON301", "orchestration/sensors/inventory/_helpers/feed_window.py", 3, 1], + ["SQBRPYTHON017", "orchestration/sensors/inventory/_helpers/feed_window.py", 3, 43], + ["SQBRPYTHON017", "orchestration/sensors/inventory/_lib/age.py", 1, 1], + ["SQBRPYTHON016", "orchestration/sensors/stock_feed.py", 1, 1], + ["SQBRPYTHON019", "orchestration/sensors/stock_feed.py", 3, 8], + ["SQBRPYTHON018", "orchestration/sensors/stock_feed.py", 7, 1], + ["SQBRPYTHON020", "orchestration/sensors/stock_feed.py", 16, 5], ["SQBRPYTHON015", "providers/carrier_api/provider.py", 3, 8], ["SQBRPYTHON014", "providers/carrier_api/provider.py", 8, 5], ["SQBRPYTHON013", "providers/stock_feed.py", 1, 1], @@ -16,11 +23,12 @@ ["SQBRPYTHON306", "python/inventory/restock/main.py", 4, 1], ["SQBRPYTHON306", "python/inventory/restock/main.py", 9, 1], ["SQBRPYTHON003", "python/inventory/stock_count/node.py", 1, 1], - ["SQBRPYTHON004", "python/inventory/stock_count/node.py", 5, 1], - ["SQBRPYTHON605", "python/inventory/stock_count/node.py", 5, 1], - ["SQBRPYTHON005", "python/inventory/stock_count/node.py", 9, 5], - ["SQBRPYTHON007", "python/inventory/stock_count/node.py", 9, 5], - ["SQBRPYTHON011", "python/inventory/stock_count/node.py", 9, 5], + ["SQBRPYTHON008", "python/inventory/stock_count/node.py", 3, 62], + ["SQBRPYTHON004", "python/inventory/stock_count/node.py", 7, 1], + ["SQBRPYTHON605", "python/inventory/stock_count/node.py", 7, 1], + ["SQBRPYTHON005", "python/inventory/stock_count/node.py", 11, 5], + ["SQBRPYTHON007", "python/inventory/stock_count/node.py", 11, 5], + ["SQBRPYTHON011", "python/inventory/stock_count/node.py", 11, 5], ["SQBRPYTHON001", "python/report.py", 1, 1], ["SQBRPYTHON614", "python/report.py", 1, 1] ] diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/jobs/stock_count_job.toml b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/jobs/stock_count_job.toml new file mode 100644 index 000000000..7d91d3f88 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/jobs/stock_count_job.toml @@ -0,0 +1,3 @@ +name = "stock_count_job" +description = "Count stock." +select = "stock_count" diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/fulfillment/_helpers/hours.py b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/fulfillment/_helpers/hours.py new file mode 100644 index 000000000..528014737 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/fulfillment/_helpers/hours.py @@ -0,0 +1,3 @@ +"""Shipment cut-off hours, private to the fulfillment sensors.""" + +CUTOFF_HOURS: int = 4 diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_helpers/feed_window.py b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_helpers/feed_window.py new file mode 100644 index 000000000..d1841f8ca --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_helpers/feed_window.py @@ -0,0 +1,5 @@ +"""Feed window that reads support private to another domain.""" + +from ...fulfillment._helpers.hours import CUTOFF_HOURS + +FEED_WINDOW_HOURS: int = CUTOFF_HOURS * 2 diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_lib/age.py b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_lib/age.py new file mode 100644 index 000000000..320f377af --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_lib/age.py @@ -0,0 +1,3 @@ +"""Private support in a folder other than _helpers/.""" + +FEED_AGE_HOURS: int = 6 diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/stock_feed.py b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/stock_feed.py new file mode 100644 index 000000000..b6998d066 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/stock_feed.py @@ -0,0 +1,18 @@ +"""A sensor outside a domain folder that documents itself twice and reads a node module.""" + +import python.report + +from sqlbuild.orchestration import SensorContext, SkipReason, sensor + +STALE_REASON: str = "Stock feed is fresh" + + +@sensor( + "stock_feed_stale_sensor", + description="Count stock when the supplier feed goes stale.", + run="stock_count_job", + minimum_interval_seconds=60, +) +def stock_feed_stale(ctx: SensorContext) -> SkipReason: + """Count stock when the supplier feed goes stale.""" + return SkipReason(f"{STALE_REASON} {python.report.__name__}") diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/stock_count/node.py b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/stock_count/node.py index 1e0c4bf07..4d0159b67 100644 --- a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/stock_count/node.py +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/stock_count/node.py @@ -1,5 +1,7 @@ """Count stock in every warehouse.""" +from orchestration.sensors.fulfillment._helpers.hours import CUTOFF_HOURS + from sqlbuild.tasks import TaskContext, task WAREHOUSES: tuple[str, ...] = ("north", "south") @@ -8,4 +10,4 @@ @task def stock_count(context: TaskContext) -> int: """Count stock in every warehouse.""" - return len(WAREHOUSES) + return len(WAREHOUSES) + CUTOFF_HOURS diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs index 57c463b88..5d8ba7447 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs @@ -157,13 +157,8 @@ pub(crate) fn read_python_modules( (SENSORS_DIRECTORY, PythonDeclarationKind::Sensor), ] { for path in role_files(tree, role, PYTHON_FILE_SUFFIX)? { - let stem: &str = path - .rsplit('/') - .next() - .and_then(|name| name.strip_suffix(PYTHON_FILE_SUFFIX)) - .unwrap_or_default(); let declared: Option = - module_declaration(&read(tree, &path)?, stem, kind, python_version) + module_declaration(&read(tree, &path)?, &path, kind, python_version) .map_err(|error| failure(&path, error))?; if let Some(mut module) = declared { module.path = path.clone(); diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs index d299717f2..18129c54a 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs @@ -5,8 +5,9 @@ use sqlbuild_pysource::orchestration_modules::models::{ LiteralArgument, ModuleDeclarations, PythonDeclaration, PythonDeclarationKind, SourcePosition, }; +use crate::constants::PYTHON_FILE_SUFFIX; use crate::orchestration_files::constants::{ - CALENDAR_KIND, PRIVATE_PREFIX, SENSOR_KIND, TOML_JOBS_HELP, + CALENDAR_KIND, PRIVATE_PREFIX, ROLE_FOLDER_DEPTH, SENSOR_KIND, TOML_JOBS_HELP, }; use crate::orchestration_files::errors::FileError; use crate::orchestration_files::models::{FilePosition, NamedModule, Positioned}; @@ -55,13 +56,21 @@ fn literal(argument: LiteralArgument, what: &str) -> Result, } } -/// The one declaration a public module in a `kind` folder makes; `None` for a private support module. +/// The one declaration a public module at `path` in a `kind` folder makes; `None` for a private support module. pub(crate) fn module_declaration( text: &str, - stem: &str, + path: &str, kind: PythonDeclarationKind, python_version: (u8, u8), ) -> Result, FileError> { + let (folders, file_name) = path.rsplit_once('/').unwrap_or(("", path)); + let stem: &str = file_name + .strip_suffix(PYTHON_FILE_SUFFIX) + .unwrap_or(file_name); + let private_folder: Option<&str> = folders + .split('/') + .skip(ROLE_FOLDER_DEPTH) + .find(|folder| folder.starts_with(PRIVATE_PREFIX)); let start = SourcePosition { line: 1, column: 1 }; let Some(ModuleDeclarations { declarations, @@ -98,7 +107,7 @@ pub(crate) fn module_declaration( None, )); } - let private: bool = stem.starts_with(PRIVATE_PREFIX); + let private: bool = stem.starts_with(PRIVATE_PREFIX) || private_folder.is_some(); let mut declarations = declarations.into_iter(); let first: Option = declarations.next(); if let Some(second) = declarations.next() { @@ -120,6 +129,17 @@ pub(crate) fn module_declaration( )), )); }; + if let Some(folder) = private_folder { + return Err(error( + declaration.position, + format!( + "declares a {noun} inside the private folder {folder}/; declarations can't live in a private folder" + ), + Some(format!( + "move the module out of {folder}/ into its domain folder, or move the {noun} into a module there" + )), + )); + } if private { return Err(error( declaration.position, diff --git a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs index c722549fb..304139ac9 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs @@ -54,6 +54,8 @@ pub(crate) const BYTECODE_DIRECTORY: &str = "__pycache__"; /// The prefix of hidden files and folders, and of private support modules. pub(crate) const HIDDEN_PREFIX: char = '.'; pub(crate) const PRIVATE_PREFIX: char = '_'; +/// The folders of a module path before the ones below its role folder: `orchestration` and the role. +pub(crate) const ROLE_FOLDER_DEPTH: usize = 2; /// Every folder `orchestration/` may hold. pub const ROLE_DIRECTORIES: [&str; 4] = [ JOBS_DIRECTORY, diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs index 0e63db921..7ee11f7aa 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs @@ -39,6 +39,46 @@ fn given_projects_when_inventorying_then_reads_each_kind_and_names_them() { ], expected_empty: false, }, + InventoryTestCase { + description: "a module inside a private folder is support and declares nothing", + files: [ + valid(), + vec![( + "orchestration/sensors/inventory/_helpers/feed_age.py", + "def feed_age() -> int:\n return 0\n", + )], + ] + .concat(), + expected_names: vec![ + named( + "orders_export_job", + "job", + "orchestration/jobs/orders/export.toml", + ), + named( + "orders_nightly_job_group", + "job group", + "orchestration/job_groups/orders/nightly.toml", + ), + named( + "trading_days_calendar", + "calendar", + "orchestration/calendars/trading_days.py", + ), + named( + "stock_stale_sensor", + "sensor", + "orchestration/sensors/inventory/stock.py", + ), + ], + expected_modules: vec![ + "orchestration/calendars/_holidays.py", + "orchestration/calendars/trading_days.py", + "orchestration/sensors/inventory/_helpers/feed_age.py", + "orchestration/sensors/inventory/stock.py", + ], + expected_empty: false, + }, InventoryTestCase { description: "no orchestration folder", files: vec![("models/orders.sql", "select 1")], @@ -177,6 +217,14 @@ fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_ "declare one sensor, or start the file name with `_` to make it a private support module", ), }, + InventoryErrorTestCase { + description: "a sensor declared inside a private folder", + files: with_file("orchestration/sensors/inventory/_helpers/stock.py", SENSOR), + expected_message: "orchestration/sensors/inventory/_helpers/stock.py:3:9: declares a sensor inside the private folder _helpers/; declarations can't live in a private folder", + expected_help: Some( + "move the module out of _helpers/ into its domain folder, or move the sensor into a module there", + ), + }, InventoryErrorTestCase { description: "removed Python job API", files: with_file( diff --git a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs index e6eee93ec..bc7070ad4 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs @@ -62,6 +62,11 @@ const REVIEWED_SCOPES: &[(&str, RuleScope)] = &[ ("SQBRPYTHON013", RuleScope::Python), ("SQBRPYTHON014", RuleScope::Python), ("SQBRPYTHON015", RuleScope::Python), + ("SQBRPYTHON016", RuleScope::Python), + ("SQBRPYTHON017", RuleScope::Python), + ("SQBRPYTHON018", RuleScope::Python), + ("SQBRPYTHON019", RuleScope::Python), + ("SQBRPYTHON020", RuleScope::Python), ("SQBRPYTHON101", RuleScope::Python), ("SQBRPYTHON102", RuleScope::Python), ("SQBRPYTHON103", RuleScope::Python), diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs index 4be254067..ff681aad6 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs @@ -2,5 +2,6 @@ pub(crate) mod imports; pub(crate) mod layout; pub(crate) mod modules; pub(crate) mod nodes; +pub(crate) mod orchestration; pub(crate) mod project; pub(crate) mod providers; diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs new file mode 100644 index 000000000..6a1f75d1a --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs @@ -0,0 +1,243 @@ +//! Sensor and calendar modules under orchestration/: where they sit, what they hold and what they import. + +use std::collections::BTreeSet; + +use sqlbuild_pysource::node_units::models::{UnitPosition, UnitStatementKind}; +use sqlbuild_pysource::orchestration_modules::main::orchestration_module_declarations::orchestration_module_declarations; + +use crate::models::{Fault, PythonSource}; +use crate::python_rules::_helpers::engine::faults::catalogue_fault; +use crate::python_rules::_helpers::units::project::{ + files_naming, nearest_unit, project_modules, rejected_import_faults, unit_fault, unit_folders, +}; +use crate::python_rules::constants::{ + INIT_FILE, ORCHESTRATION_GENERIC_MODULES, ORCHESTRATION_IMPORT_DIRECTION_CODE, + ORCHESTRATION_MODULE_PATHS_CODE, ORCHESTRATION_MODULE_SURFACE_CODE, + ORCHESTRATION_NO_DOCSTRING_CODE, ORCHESTRATION_PRIVATE_SUPPORT_CODE, + ORCHESTRATION_PYTHON_ROLES, ORCHESTRATION_ROOT, ORCHESTRATION_RULE_CODES, + ORCHESTRATION_SUPPORT_FILES, ORCHESTRATION_SUPPORT_FOLDER, PACKAGE_ROOT, PRIVATE_PREFIX, + PROVIDERS_ROOT, SHARED_UTILITIES_PACKAGE, +}; +use crate::python_rules::models::ParsedFile; + +const OTHER_STATEMENT: &str = + "a statement that is not an import, a private constant or the declaration"; + +/// Faults of the selected orchestration rules among `codes`. +pub(crate) fn orchestration_faults( + sources: &[PythonSource], + codes: &BTreeSet<&str>, + python_version: (u8, u8), +) -> Vec { + if !ORCHESTRATION_RULE_CODES + .iter() + .any(|code| codes.contains(code)) + { + return Vec::new(); + } + let mut faults: Vec = sources.iter().filter_map(layout_fault).collect(); + let files: Vec> = files_naming(sources, ORCHESTRATION_ROOT, python_version); + let modules: BTreeSet = project_modules(sources); + let units: BTreeSet = unit_folders(sources); + for file in &files { + if role_folders(&file.parts).is_some_and(|rest| !is_private(rest)) { + faults.extend(surface_faults(file, python_version)); + } + faults.extend(import_faults(file, &modules, &units)); + } + faults.retain(|fault| codes.contains(fault.code.as_str())); + faults +} + +/// The path parts below `orchestration//` for a sensor or calendar file; `None` for any other file. +fn role_folders<'p>(parts: &'p [&'p str]) -> Option<&'p [&'p str]> { + match parts { + [root, role, rest @ ..] + if *root == ORCHESTRATION_ROOT && ORCHESTRATION_PYTHON_ROLES.contains(role) => + { + Some(rest) + } + _ => None, + } +} + +/// Whether a file below a role folder is private support: its name or a folder above it starts with `_`. +fn is_private(rest: &[&str]) -> bool { + rest.iter().any(|part| part.starts_with(PRIVATE_PREFIX)) +} + +/// Where a sensor or calendar file sits wrongly: outside a domain folder, nested, generic, or odd support. +fn layout_fault(source: &PythonSource) -> Option { + let path: &str = source.relative_path.as_str(); + let parts: Vec<&str> = path.split('/').collect(); + let rest: &[&str] = role_folders(&parts)?; + let role: String = format!("{ORCHESTRATION_ROOT}/{}", parts[1]); + let (code, detail): (&str, String) = match rest { + [file] if *file == INIT_FILE => return None, + [file] if file.starts_with(PRIVATE_PREFIX) => ( + ORCHESTRATION_PRIVATE_SUPPORT_CODE, + format!("{path} is private support outside a domain folder of {role}/"), + ), + [_] => ( + ORCHESTRATION_MODULE_PATHS_CODE, + format!("{path} sits directly in {role}/, not in a domain folder"), + ), + [domain, ..] if domain.starts_with(PRIVATE_PREFIX) => ( + ORCHESTRATION_PRIVATE_SUPPORT_CODE, + format!("{path} is in private folder {domain}/ outside a domain folder of {role}/"), + ), + [_, file] if *file == INIT_FILE => return None, + [_, file] if file.starts_with(PRIVATE_PREFIX) => { + if ORCHESTRATION_SUPPORT_FILES.contains(file) { + return None; + } + ( + ORCHESTRATION_PRIVATE_SUPPORT_CODE, + format!("{path} is not _models.py, _types.py, _constants.py or _exceptions.py"), + ) + } + [_, file] => { + if !ORCHESTRATION_GENERIC_MODULES.contains(file) { + return None; + } + ( + ORCHESTRATION_MODULE_PATHS_CODE, + format!("{file} is too generic; name the module for its capability"), + ) + } + [_, folder, ..] if *folder == ORCHESTRATION_SUPPORT_FOLDER => return None, + [domain, folder, ..] if folder.starts_with(PRIVATE_PREFIX) => ( + ORCHESTRATION_PRIVATE_SUPPORT_CODE, + format!("{path} is in {folder}/, but {domain}/'s only private folder is _helpers/"), + ), + [domain, ..] => ( + ORCHESTRATION_MODULE_PATHS_CODE, + format!("{path} is nested below its domain folder {domain}/"), + ), + [] => return None, + }; + catalogue_fault(code, path, (1, 1), &detail) +} + +/// A declaration module's statements beyond imports, private constants and the declaration, and its docstring. +fn surface_faults(file: &ParsedFile<'_>, python_version: (u8, u8)) -> Vec { + let declaration_lines: Vec = + orchestration_module_declarations(file.text, python_version) + .map(|module| module.declarations) + .unwrap_or_default() + .iter() + .map(|declaration| declaration.position.line) + .collect(); + let statements = &file.module.statements; + let mut faults: Vec = Vec::new(); + for (index, statement) in statements.iter().enumerate() { + let end: usize = statements + .get(index + 1) + .map_or(usize::MAX, |next| next.position.line); + let declares: bool = declaration_lines + .iter() + .any(|line| (statement.position.line..end).contains(line)); + let (code, detail, position): (&str, String, UnitPosition) = match &statement.kind { + UnitStatementKind::Docstring | UnitStatementKind::Import => continue, + UnitStatementKind::Function(function) if declares && function.docstring => ( + ORCHESTRATION_NO_DOCSTRING_CODE, + format!( + "'{}' has a docstring; its description is description=", + function.name + ), + function.position, + ), + _ if declares => continue, + UnitStatementKind::Assignment(names) => match public_name(names) { + Some(detail) => ( + ORCHESTRATION_MODULE_SURFACE_CODE, + detail, + statement.position, + ), + None => continue, + }, + UnitStatementKind::Function(function) => ( + ORCHESTRATION_MODULE_SURFACE_CODE, + format!("function '{}' belongs in _helpers/", function.name), + function.position, + ), + UnitStatementKind::Class(class) => ( + ORCHESTRATION_MODULE_SURFACE_CODE, + format!("class '{}' belongs in _models.py or _helpers/", class.name), + class.position, + ), + UnitStatementKind::Other => ( + ORCHESTRATION_MODULE_SURFACE_CODE, + OTHER_STATEMENT.to_owned(), + statement.position, + ), + }; + faults.extend(unit_fault(code, file.path, position, &detail)); + } + faults +} + +/// Why an assignment does not belong in a declaration module; `None` for a private constant. +fn public_name(names: &[String]) -> Option { + if names.is_empty() { + return Some(OTHER_STATEMENT.to_owned()); + } + let name: &String = names + .iter() + .find(|name| !name.starts_with(PRIVATE_PREFIX))?; + Some(format!("public constant '{name}' belongs in _constants.py")) +} + +/// Imports of another domain's private support, and imports across the orchestration boundary. +fn import_faults( + file: &ParsedFile<'_>, + modules: &BTreeSet, + units: &BTreeSet, +) -> Vec { + rejected_import_faults(file, modules, |module| { + boundary_detail(&file.parts, module, units) + }) +} + +/// Why a file at `parts` may not import `module`; node units and providers are left to 008 and 015. +fn boundary_detail( + parts: &[&str], + module: &str, + units: &BTreeSet, +) -> Option<(&'static str, String)> { + let imported: Vec<&str> = module.split('.').collect(); + let Some(rest) = role_folders(parts) else { + let in_unit: bool = + parts.first() == Some(&PACKAGE_ROOT) && nearest_unit(&parts[1..], units).is_some(); + let outside: bool = parts.first() != Some(&ORCHESTRATION_ROOT) + && parts.first() != Some(&PROVIDERS_ROOT) + && !in_unit + && imported.first() == Some(&ORCHESTRATION_ROOT); + return outside.then(|| { + ( + ORCHESTRATION_IMPORT_DIRECTION_CODE, + format!("but only {ORCHESTRATION_ROOT}/ imports from {ORCHESTRATION_ROOT}/"), + ) + }); + }; + match imported.as_slice() { + [root, utils, ..] if *root == PACKAGE_ROOT && *utils == SHARED_UTILITIES_PACKAGE => None, + [root, ..] if *root == PACKAGE_ROOT => Some(( + ORCHESTRATION_IMPORT_DIRECTION_CODE, + format!( + "but sensors and calendars import only {PACKAGE_ROOT}/{SHARED_UTILITIES_PACKAGE}/ from {PACKAGE_ROOT}/" + ), + )), + [root, role, domain, support @ ..] + if *root == ORCHESTRATION_ROOT + && is_private(support) + && (Some(role) != parts.get(1) || Some(domain) != rest.first()) => + { + Some(( + ORCHESTRATION_PRIVATE_SUPPORT_CODE, + format!("which is private to {ORCHESTRATION_ROOT}/{role}/{domain}/"), + )) + } + _ => None, + } +} diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/project.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/project.rs index 8299d9621..26a085421 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/units/project.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/project.rs @@ -22,7 +22,7 @@ use crate::python_rules::constants::{ NODE_MODULE, PACKAGE_ROOT, ROLE_DIRECTORIES, SHARED_UTILITIES_PACKAGE, UNIT_NODE_FILE, UNIT_RULE_CODES, }; -use crate::python_rules::models::{UnitFile, UnitProject}; +use crate::python_rules::models::{ParsedFile, UnitFile, UnitProject}; /// Faults of the selected unit rules among `codes`. pub(crate) fn unit_faults( @@ -60,10 +60,7 @@ fn unit_project(sources: &[PythonSource], python_version: (u8, u8)) -> UnitProje (root == PACKAGE_ROOT).then(|| (source, rest.split('/').collect())) }) .collect(); - let units: BTreeSet = package - .iter() - .filter_map(|(_, parts)| unit_folder(parts)) - .collect(); + let units: BTreeSet = unit_folders(sources); let files: Vec> = package .par_iter() .filter_map(|(source, parts)| { @@ -91,15 +88,82 @@ fn unit_project(sources: &[PythonSource], python_version: (u8, u8)) -> UnitProje .iter() .map(|source| source.relative_path.as_str()) .collect(), - modules: sources - .iter() - .flat_map(|source| dotted_names(&source.relative_path)) - .collect(), + modules: project_modules(sources), } } +/// Every dotted module the project's Python files define, packages included. +pub(crate) fn project_modules(sources: &[PythonSource]) -> BTreeSet { + sources + .iter() + .flat_map(|source| dotted_names(&source.relative_path)) + .collect() +} + +/// The files under `root/` or whose text names it, parsed; a file that does not parse is left out. +pub(crate) fn files_naming<'a>( + sources: &'a [PythonSource], + root: &str, + python_version: (u8, u8), +) -> Vec> { + sources + .par_iter() + .filter(|source| { + source.relative_path.split('/').next() == Some(root) || source.text.contains(root) + }) + .filter_map(|source| { + Some(ParsedFile { + path: source.relative_path.as_str(), + parts: source.relative_path.split('/').collect(), + text: source.text.as_str(), + module: node_unit_module(&source.text, python_version)?, + }) + }) + .collect() +} + +/// Each import in `file` of a project module that `reason` rejects, as a fault of its rule and detail. +pub(crate) fn rejected_import_faults( + file: &ParsedFile<'_>, + modules: &BTreeSet, + reason: impl Fn(&str) -> Option<(&'static str, String)>, +) -> Vec { + let folder: &[&str] = &file.parts[..file.parts.len() - 1]; + file.module + .imports + .iter() + .filter_map(|import| { + let module: String = absolute_module(import, folder, modules)?; + if !modules.contains(&module) { + return None; + } + let (code, detail): (&str, String) = reason(&module)?; + unit_fault( + code, + file.path, + import.position, + &format!("{} imports {module}, {detail}", file.path), + ) + }) + .collect() +} + +/// The unit folders under `python/`, each relative to it. +pub(crate) fn unit_folders(sources: &[PythonSource]) -> BTreeSet { + sources + .iter() + .filter_map(|source| { + let (root, rest) = source.relative_path.split_once('/')?; + let parts: Vec<&str> = rest.split('/').collect(); + (root == PACKAGE_ROOT) + .then(|| unit_folder(&parts)) + .flatten() + }) + .collect() +} + /// The deepest unit folder among the file's folders. -fn nearest_unit(parts: &[&str], units: &BTreeSet) -> Option { +pub(crate) fn nearest_unit(parts: &[&str], units: &BTreeSet) -> Option { (1..parts.len()) .rev() .map(|depth| parts[..depth].join("/")) diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs index 971166a87..b88f4e2db 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs @@ -2,37 +2,28 @@ use std::collections::{BTreeMap, BTreeSet}; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; - -use sqlbuild_pysource::node_units::main::node_unit_module::node_unit_module; -use sqlbuild_pysource::node_units::models::{ - UnitClass, UnitModule, UnitPosition, UnitStatementKind, -}; +use sqlbuild_pysource::node_units::models::{UnitClass, UnitPosition, UnitStatementKind}; use crate::models::{Fault, PythonSource}; use crate::python_rules::_helpers::engine::faults::catalogue_fault; use crate::python_rules::_helpers::engine::placement::package_anchor; -use crate::python_rules::_helpers::units::project::{absolute_module, dotted_names, unit_fault}; +use crate::python_rules::_helpers::units::project::{ + absolute_module, dotted_names, files_naming, project_modules, rejected_import_faults, + unit_fault, +}; use crate::python_rules::constants::{ INIT_FILE, ORCHESTRATION_ROOT, PACKAGE_ROOT, PRIVATE_PREFIX, PROVIDER_BASE, PROVIDER_FILE, PROVIDER_IMPORT_BOUNDARIES_CODE, PROVIDER_LAYOUT_CODE, PROVIDER_MODULE_SHAPE_CODE, PROVIDER_PRIVATE_FOLDERS, PROVIDER_ROLE_FILES, PROVIDER_RULE_CODES, PROVIDERS_ROOT, SHARED_UTILITIES_PACKAGE, STAR_IMPORT, }; +use crate::python_rules::models::ParsedFile; const FILE_START: UnitPosition = UnitPosition { line: 1, column: 1 }; const OTHER_STATEMENT: &str = "a statement that is not an import, a constant or the provider class"; /// The package Provider is imported from. const SQLBUILD_PACKAGE: &str = "sqlbuild"; -/// One parsed project file and its path parts. -#[derive(Debug)] -struct ParsedFile<'a> { - path: &'a str, - parts: Vec<&'a str>, - module: UnitModule, -} - /// Faults of the selected provider rules among `codes`. pub(crate) fn provider_faults( sources: &[PythonSource], @@ -43,24 +34,8 @@ pub(crate) fn provider_faults( return Vec::new(); } let mut faults: Vec = provider_layout_faults(sources); - let files: Vec> = sources - .par_iter() - .filter(|source| { - source.relative_path.split('/').next() == Some(PROVIDERS_ROOT) - || source.text.contains(PROVIDERS_ROOT) - }) - .filter_map(|source| { - Some(ParsedFile { - path: source.relative_path.as_str(), - parts: source.relative_path.split('/').collect(), - module: node_unit_module(&source.text, python_version)?, - }) - }) - .collect(); - let modules: BTreeSet = sources - .iter() - .flat_map(|source| dotted_names(&source.relative_path)) - .collect(); + let files: Vec> = files_naming(sources, PROVIDERS_ROOT, python_version); + let modules: BTreeSet = project_modules(sources); let classes: ProviderClasses = provider_classes(&files, &modules); for file in &files { if matches!(file.parts.as_slice(), [root, _, file_name] if *root == PROVIDERS_ROOT && *file_name == PROVIDER_FILE) @@ -357,24 +332,9 @@ fn import_faults(file: &ParsedFile<'_>, modules: &BTreeSet) -> Vec Some(capability), _ => None, }; - let folder: &[&str] = &file.parts[..file.parts.len() - 1]; - file.module - .imports - .iter() - .filter_map(|import| { - let module: String = absolute_module(import, folder, modules)?; - if !modules.contains(&module) { - return None; - } - let detail: String = boundary_detail(&module, capability)?; - unit_fault( - PROVIDER_IMPORT_BOUNDARIES_CODE, - file.path, - import.position, - &format!("{} imports {module}, {detail}", file.path), - ) - }) - .collect() + rejected_import_faults(file, modules, |module| { + boundary_detail(module, capability).map(|detail| (PROVIDER_IMPORT_BOUNDARIES_CODE, detail)) + }) } /// Why a file in provider `capability` (or outside providers/ when `None`) may not import `module`. diff --git a/crates/sqlbuild-rules/src/python_rules/constants.rs b/crates/sqlbuild-rules/src/python_rules/constants.rs index 77649efd2..988c75309 100644 --- a/crates/sqlbuild-rules/src/python_rules/constants.rs +++ b/crates/sqlbuild-rules/src/python_rules/constants.rs @@ -5,7 +5,7 @@ use fensu_native::rules::constants as fensu; use crate::python_rules::models::{PythonFiles, PythonRuleSpec}; /// Every `SQBRPYTHON` rule; numbers are never reused, and new rules append to their band. -pub(crate) const PYTHON_RULES: [PythonRuleSpec; 77] = [ +pub(crate) const PYTHON_RULES: [PythonRuleSpec; 82] = [ PythonRuleSpec { code: "SQBRPYTHON001", fensu_code: None, @@ -186,6 +186,66 @@ pub(crate) const PYTHON_RULES: [PythonRuleSpec; 77] = [ message: "a provider's _helpers/ and _classes/ are private to it, and providers import only python/utils/ from the runtime package and nothing from orchestration/", remediation: "Import a provider through provider.py or its role files; move code a provider shares with nodes into python/utils//.", }, + PythonRuleSpec { + code: "SQBRPYTHON016", + fensu_code: None, + slug: "orchestration-module-paths", + family: "orchestration", + owner: "file", + applies: PythonFiles::All, + enabled_by_default: true, + options: &[], + message: "sensor and calendar modules sit at orchestration///.py and are named for their capability", + remediation: "Move the module into a domain folder directly under orchestration/sensors/ or orchestration/calendars/, and name it for what it checks instead of main.py, definitions.py, sensors.py, calendars.py or processing.py.", + }, + PythonRuleSpec { + code: "SQBRPYTHON017", + fensu_code: None, + slug: "orchestration-private-support", + family: "orchestration", + owner: "file", + applies: PythonFiles::All, + enabled_by_default: true, + options: &[], + message: "a sensor or calendar domain folder's private support is _models.py, _types.py, _constants.py, _exceptions.py and _helpers/, imported only from that folder", + remediation: "Keep support beside its declarations in one of the support files or _helpers/; move code another domain needs into python/utils// or a provider.", + }, + PythonRuleSpec { + code: "SQBRPYTHON018", + fensu_code: None, + slug: "orchestration-module-surface", + family: "orchestration", + owner: "file", + applies: PythonFiles::All, + enabled_by_default: true, + options: &[], + message: "a sensor or calendar module holds imports, private constants and its one declaration", + remediation: "Move functions and classes into the folder's _helpers/ or _models.py, and public constants into _constants.py.", + }, + PythonRuleSpec { + code: "SQBRPYTHON019", + fensu_code: None, + slug: "orchestration-import-direction", + family: "orchestration", + owner: "file", + applies: PythonFiles::All, + enabled_by_default: true, + options: &[], + message: "sensor and calendar code imports providers, python/utils/ and its own folder's support, and nothing outside orchestration/ imports from it", + remediation: "Read node results through the sensor's context instead of importing node units, share code through python/utils// or a provider, and keep other code from importing orchestration/.", + }, + PythonRuleSpec { + code: "SQBRPYTHON020", + fensu_code: None, + slug: "orchestration-no-docstring", + family: "orchestration", + owner: "file", + applies: PythonFiles::All, + enabled_by_default: true, + options: &[], + message: "sensor and calendar functions have no docstring; their description is the declaration's description=", + remediation: "Remove the docstring and keep the text in description=.", + }, PythonRuleSpec { code: "SQBRPYTHON101", fensu_code: Some(fensu::PARAMETER_ANNOTATION_CODE), @@ -1097,6 +1157,34 @@ pub(crate) const PROVIDER_PRIVATE_FOLDERS: [&str; 2] = ["_helpers", "_classes"]; pub(crate) const PROVIDER_BASE: &str = "Provider"; /// The folder holding orchestration declarations, imported as `orchestration`. pub(crate) const ORCHESTRATION_ROOT: &str = "orchestration"; +pub(crate) const ORCHESTRATION_MODULE_PATHS_CODE: &str = "SQBRPYTHON016"; +pub(crate) const ORCHESTRATION_PRIVATE_SUPPORT_CODE: &str = "SQBRPYTHON017"; +pub(crate) const ORCHESTRATION_MODULE_SURFACE_CODE: &str = "SQBRPYTHON018"; +pub(crate) const ORCHESTRATION_IMPORT_DIRECTION_CODE: &str = "SQBRPYTHON019"; +pub(crate) const ORCHESTRATION_NO_DOCSTRING_CODE: &str = "SQBRPYTHON020"; +/// The orchestration rules, which read sensor and calendar modules and their importers. +pub(crate) const ORCHESTRATION_RULE_CODES: [&str; 5] = [ + ORCHESTRATION_MODULE_PATHS_CODE, + ORCHESTRATION_PRIVATE_SUPPORT_CODE, + ORCHESTRATION_MODULE_SURFACE_CODE, + ORCHESTRATION_IMPORT_DIRECTION_CODE, + ORCHESTRATION_NO_DOCSTRING_CODE, +]; +/// The orchestration folders that hold Python sensor and calendar modules. +pub(crate) const ORCHESTRATION_PYTHON_ROLES: [&str; 2] = ["sensors", "calendars"]; +/// Module names too generic to say which capability a sensor or calendar checks. +pub(crate) const ORCHESTRATION_GENERIC_MODULES: [&str; 5] = [ + "main.py", + "definitions.py", + "sensors.py", + "calendars.py", + "processing.py", +]; +/// The private support files a sensor or calendar domain folder may hold. +pub(crate) const ORCHESTRATION_SUPPORT_FILES: [&str; 4] = + ["_models.py", "_types.py", "_constants.py", "_exceptions.py"]; +/// The private support folder a sensor or calendar domain folder may hold. +pub(crate) const ORCHESTRATION_SUPPORT_FOLDER: &str = "_helpers"; /// The prefix of a private name. pub(crate) const PRIVATE_PREFIX: &str = "_"; /// The name a `from module import *` binds. diff --git a/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs b/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs index 103cc7fe3..2917a095b 100644 --- a/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs +++ b/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs @@ -12,6 +12,7 @@ use crate::python_rules::_helpers::engine::layout::layout_faults; use crate::python_rules::_helpers::engine::placement::python_targets; use crate::python_rules::_helpers::engine::programs::Programs; use crate::python_rules::_helpers::engine::project_plane::loaded_project; +use crate::python_rules::_helpers::units::orchestration::orchestration_faults; use crate::python_rules::_helpers::units::project::unit_faults; use crate::python_rules::_helpers::units::providers::provider_faults; use crate::python_rules::models::{ @@ -55,6 +56,11 @@ pub(crate) fn evaluate_python_rules( &codes, request.python_version, )); + evaluation.faults.extend(orchestration_faults( + &request.python_sources, + &codes, + request.python_version, + )); evaluation.faults.sort_by(|left, right| { (&left.path, left.line, left.column, &left.code).cmp(&( &right.path, diff --git a/crates/sqlbuild-rules/src/python_rules/models.rs b/crates/sqlbuild-rules/src/python_rules/models.rs index 80628f54f..9f598d2cd 100644 --- a/crates/sqlbuild-rules/src/python_rules/models.rs +++ b/crates/sqlbuild-rules/src/python_rules/models.rs @@ -102,6 +102,15 @@ pub(crate) struct CacheSplit<'a> { pub stale: Vec>, } +/// One parsed project file, its path parts from the project root, and its text. +#[derive(Debug)] +pub(crate) struct ParsedFile<'a> { + pub path: &'a str, + pub parts: Vec<&'a str>, + pub text: &'a str, + pub module: sqlbuild_pysource::node_units::models::UnitModule, +} + /// One parsed module under `python/` and the unit folder it sits in. #[derive(Debug)] pub(crate) struct UnitFile<'a> { diff --git a/crates/sqlbuild-rules/src/python_rules/tests.rs b/crates/sqlbuild-rules/src/python_rules/tests.rs index ae23fb200..ffb2bc69d 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests.rs @@ -4,6 +4,8 @@ mod evaluate_python_rules; mod helpers; #[path = "tests/test_node_units.rs"] mod node_units; +#[path = "tests/test_orchestration_rules.rs"] +mod orchestration_rules; #[path = "tests/test_provider_rules.rs"] mod provider_rules; #[path = "tests/test_python_cache.rs"] diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_orchestration_rules.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_orchestration_rules.rs new file mode 100644 index 000000000..d2dc267d4 --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_orchestration_rules.rs @@ -0,0 +1,261 @@ +use serde_json::json; + +use crate::python_rules::tests::helpers::{owned_faults, python_rule_faults}; +use crate::python_rules::tests::test_types::PythonRunTestCase; + +const ORCHESTRATION_RULES: [&str; 5] = [ + "SQBRPYTHON016", + "SQBRPYTHON017", + "SQBRPYTHON018", + "SQBRPYTHON019", + "SQBRPYTHON020", +]; + +#[test] +fn given_sensor_and_calendar_modules_when_evaluating_orchestration_rules_then_reports_each_broken_convention() + { + let test_cases = [ + PythonRunTestCase { + description: "domain folders with one declaration per module, private support and allowed imports report nothing", + files: &[ + ("orchestration/sensors/__init__.py", ""), + ( + "orchestration/sensors/inventory/stock_feed.py", + r#"from sqlbuild.orchestration import sensor + +from orchestration.sensors.inventory._constants import STALE_AFTER +from python.utils.retries import ATTEMPTS +from providers.carrier_api.provider import CarrierApi + +from ._helpers.feed_age import feed_age + +_LIMIT: int = 3 + + +@sensor("stock_feed_stale_sensor", description="Stale feed.", run="restock_job") +def stock_feed_stale(ctx, carrier_api: CarrierApi): + return feed_age(ctx) > STALE_AFTER * ATTEMPTS * _LIMIT +"#, + ), + ( + "orchestration/sensors/inventory/_constants.py", + "STALE_AFTER = 3\n", + ), + ( + "orchestration/sensors/inventory/_helpers/feed_age.py", + "def feed_age(ctx):\n return 0\n", + ), + ( + "orchestration/calendars/finance/month_end.py", + r#"from sqlbuild.orchestration import calendar + +month_end = calendar("month_end_calendar", description="Month end.") +"#, + ), + ("python/utils/retries.py", "ATTEMPTS = 2\n"), + ("providers/carrier_api/provider.py", ""), + ], + config: json!({ "select": ORCHESTRATION_RULES }), + expected_faults: vec![], + }, + PythonRunTestCase { + description: "layout: no domain folder, a generic name, nesting, and support outside a domain or not in the allowed files", + files: &[ + ("orchestration/sensors/stock_feed.py", ""), + ("orchestration/sensors/inventory/sensors.py", ""), + ("orchestration/sensors/inventory/sub/feed.py", ""), + ("orchestration/sensors/inventory/_constants.py", ""), + ("orchestration/sensors/inventory/_helpers/feed_age.py", ""), + ("orchestration/sensors/inventory/_utils.py", ""), + ("orchestration/sensors/inventory/_lib/age.py", ""), + ("orchestration/sensors/_shared.py", ""), + ("orchestration/sensors/_shared/age.py", ""), + ("orchestration/calendars/__init__.py", ""), + ], + config: json!({ "select": ["SQBRPYTHON016", "SQBRPYTHON017"] }), + expected_faults: vec![ + ("SQBRPYTHON017", "orchestration/sensors/_shared.py", 1, 1), + ( + "SQBRPYTHON017", + "orchestration/sensors/_shared/age.py", + 1, + 1, + ), + ( + "SQBRPYTHON017", + "orchestration/sensors/inventory/_lib/age.py", + 1, + 1, + ), + ( + "SQBRPYTHON017", + "orchestration/sensors/inventory/_utils.py", + 1, + 1, + ), + ( + "SQBRPYTHON016", + "orchestration/sensors/inventory/sensors.py", + 1, + 1, + ), + ( + "SQBRPYTHON016", + "orchestration/sensors/inventory/sub/feed.py", + 1, + 1, + ), + ("SQBRPYTHON016", "orchestration/sensors/stock_feed.py", 1, 1), + ], + }, + PythonRunTestCase { + description: "surface: a public constant, a mixed assignment, a function, a class, a loop and a docstring on the sensor", + files: &[( + "orchestration/sensors/inventory/stock_feed.py", + r#""""Stock feed sensor.""" + +from sqlbuild.orchestration import sensor + +_STALE_AFTER = 3 +STALE = 4 +_A, B.x = 1, 2 + + +def helper(): + pass + + +class Feed: + pass + + +@sensor("stock_feed_stale_sensor", description="Stale feed.", run="restock_job") +def stock_feed_stale(ctx): + """Doc.""" + return None + + +for row in []: + pass +"#, + )], + config: json!({ "select": ["SQBRPYTHON018", "SQBRPYTHON020"] }), + expected_faults: vec![ + ( + "SQBRPYTHON018", + "orchestration/sensors/inventory/stock_feed.py", + 6, + 1, + ), + ( + "SQBRPYTHON018", + "orchestration/sensors/inventory/stock_feed.py", + 7, + 1, + ), + ( + "SQBRPYTHON018", + "orchestration/sensors/inventory/stock_feed.py", + 10, + 5, + ), + ( + "SQBRPYTHON018", + "orchestration/sensors/inventory/stock_feed.py", + 14, + 7, + ), + ( + "SQBRPYTHON020", + "orchestration/sensors/inventory/stock_feed.py", + 19, + 5, + ), + ( + "SQBRPYTHON018", + "orchestration/sensors/inventory/stock_feed.py", + 24, + 1, + ), + ], + }, + PythonRunTestCase { + description: "imports: another domain's support, a node unit from a sensor or its helpers, and orchestration/ from outside it except providers and node units", + files: &[ + ( + "orchestration/sensors/orders/refund_spike.py", + r#"from orchestration.sensors.inventory._helpers.feed_age import feed_age +from python.orders.daily.main import main +from python.utils.retries import ATTEMPTS +from providers.carrier_api.provider import CarrierApi +from ._helpers.window import window +"#, + ), + ("orchestration/sensors/orders/_helpers/window.py", ""), + ("orchestration/sensors/inventory/_helpers/feed_age.py", ""), + ( + "orchestration/sensors/inventory/_helpers/age.py", + "from ...orders._helpers.window import window\nfrom python.orders.daily.main import main\n", + ), + ("python/orders/weekly/node.py", ""), + ( + "python/orders/weekly/main.py", + "from orchestration.sensors.orders._helpers.window import window\n", + ), + ( + "python/orders/daily/main.py", + "from orchestration.sensors.orders._helpers.window import window\n", + ), + ("python/utils/retries.py", ""), + ( + "providers/carrier_api/provider.py", + "import orchestration.sensors.orders.refund_spike\n", + ), + ( + "hooks/python/notify.py", + "import orchestration.sensors.orders.refund_spike\n", + ), + ], + config: json!({ "select": ["SQBRPYTHON008", "SQBRPYTHON017", "SQBRPYTHON019"] }), + expected_faults: vec![ + ("SQBRPYTHON019", "hooks/python/notify.py", 1, 8), + ( + "SQBRPYTHON017", + "orchestration/sensors/inventory/_helpers/age.py", + 1, + 39, + ), + ( + "SQBRPYTHON019", + "orchestration/sensors/inventory/_helpers/age.py", + 2, + 38, + ), + ( + "SQBRPYTHON017", + "orchestration/sensors/orders/refund_spike.py", + 1, + 63, + ), + ( + "SQBRPYTHON019", + "orchestration/sensors/orders/refund_spike.py", + 2, + 38, + ), + ("SQBRPYTHON019", "python/orders/daily/main.py", 1, 58), + ("SQBRPYTHON008", "python/orders/weekly/main.py", 1, 58), + ], + }, + ]; + for test_case in test_cases { + let faults = python_rule_faults(test_case.files, test_case.config.clone()); + + assert_eq!( + faults, + Ok(owned_faults(&test_case.expected_faults)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs index 95ce1c8d8..b646fed56 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs @@ -14,14 +14,14 @@ use crate::rules::models::BuiltinOptionKind; fn given_the_catalogue_when_listing_python_rules_then_each_has_its_own_project_wide_code() { let test_cases = [ CatalogueTestCase { - description: "15 SQLBuild layout, unit and provider rules and 62 Fensu core rules have distinct codes", + description: "20 SQLBuild layout, unit, provider and orchestration rules and 62 Fensu core rules have distinct codes", count: distinct_python_codes, - expected_count: 77, + expected_count: 82, }, CatalogueTestCase { description: "every Python rule reports against the project in the python family", count: project_wide_python_rules, - expected_count: 77, + expected_count: 82, }, ]; for test_case in test_cases { diff --git a/fensu.toml b/fensu.toml index 3f1d83277..ee699d914 100644 --- a/fensu.toml +++ b/fensu.toml @@ -3016,3 +3016,7 @@ reason = "Each is a short stack walk over a JSON syntax tree with its own stop a [[dupes.allowlist]] paths = ["crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs"] reason = "Field lookups on unrelated value types in different crates: project configuration takes the first matching field of a built object, while rule headers take the last entry of an authored map so a repeated key overrides earlier ones." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs", "crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs"] +reason = "Each is one rule family's entry point: return early when no rule is selected, then run its own layout, module and import checks. Parsing, project modules and import resolution share files_naming, project_modules and rejected_import_faults." diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index da2a6a9d3..0f38b9505 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -44,8 +44,12 @@ sqb build nightly_orders_job # build what the job builds, locally - Subfolders are allowed, such as `orchestration/jobs/finance/revenue_job.toml`. - A name must end with its folder's suffix. Names are unique across the whole project, with models, sources and every other resource. -- A Python module whose file name starts with `_` is a private helper and declares nothing. +- A Python module whose file name starts with `_`, or that sits in a folder starting with `_` under + `orchestration/calendars/` or `orchestration/sensors/` (such as `sensors/inventory/_helpers/`), is + private support and declares nothing. Declaring a calendar or sensor there fails compilation. - Any other file under `orchestration/`, or a folder not in the table, is a `D017` error. +- The [Python Rules](rules/python-rules.md#laying-out-sensors-and-calendars) add a + domain-folder layout for sensor and calendar modules. ## Declare a job @@ -138,7 +142,7 @@ another system. The code location evaluates it when it publishes and again befor out. ```python -# orchestration/calendars/fulfillment_calendar.py +# orchestration/calendars/fulfillment/cutoffs.py from sqlbuild.orchestration import calendar @calendar(name="fulfillment_calendar", description="Upcoming fulfillment times", horizon="1d") @@ -159,7 +163,7 @@ returns timezone-aware datetimes inside it. Other parameters name A sensor starts a job or job group when something happens, named by `run`. ```python -# orchestration/sensors/on_new_orders_sensor.py +# orchestration/sensors/orders/new_orders.py from sqlbuild.orchestration import sensor, source_new_data on_new_orders = sensor( diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md index 78938aa8f..3c547cb16 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md @@ -13,6 +13,7 @@ Online: https://sqlbuild.com/docs/concepts/rules/python-rules/ - Laying out `python/` - Writing nodes as units - Laying out providers +- Laying out sensors and calendars - Main modules and entry modules - Unreachable definitions - Rule options @@ -189,6 +190,37 @@ A capability folder may also hold `types.py`, `constants.py`, `exceptions.py`, ` `Provider` through an alias or a base class in `_classes/`. A class whose base comes from another package is never reported as not being a provider. +## Laying out sensors and calendars + +`SQBRPYTHON016` to `SQBRPYTHON020` keep each sensor and calendar module in a domain folder, named +for what it checks, with its private support beside it: + +```text +orchestration/ + sensors/ + inventory/ + stock_feed.py # one sensor + _constants.py + _helpers/ + feed_age.py + calendars/ + finance/ + month_end.py # one calendar +``` + +A domain folder's private support is `_models.py`, `_types.py`, `_constants.py`, +`_exceptions.py` and `_helpers/`. These are reported: + +| Code | Finding | Fix | +|---|---|---| +| `SQBRPYTHON016` | A module directly in `sensors/` or `calendars/`, nested below its domain folder, or named `main.py`, `definitions.py`, `sensors.py`, `calendars.py` or `processing.py` | Move it to `//.py` | +| `SQBRPYTHON017` | Other private files or folders, private support outside a domain folder, or an import of another domain's private support | Use the support files or `_helpers/`; share code through `python/utils/` or a provider | +| `SQBRPYTHON018` | A sensor or calendar module holding a function, a class, a public constant or another statement besides its declaration | Move it to `_helpers/`, `_models.py` or `_constants.py` | +| `SQBRPYTHON019` | A sensor or calendar importing `python/` outside `utils/`, or code outside `orchestration/` importing it | Read node results through the sensor's context; share code through `python/utils/` or a provider | +| `SQBRPYTHON020` | A sensor or calendar function with a docstring | Keep the text in `description=` only | + +Providers importing `orchestration/` are reported by `SQBRPYTHON015` instead. + ## Main modules and entry modules Some Rules apply only to a domain's main modules: files under a `main/` folder, and a unit's @@ -278,6 +310,16 @@ remediation. | `SQBRPYTHON014` | provider-module-shape | All | yes | `provider.py` holds imports, private constants and one public `Provider` subclass | | `SQBRPYTHON015` | provider-import-boundaries | All | yes | A provider's `_helpers/` and `_classes/` are private to it; providers import only `python/utils/` from `python/` and nothing from `orchestration/` | +### Sensors and calendars + +| Code | Name | Files | Default | Checks | +|---|---|---|---|---| +| `SQBRPYTHON016` | orchestration-module-paths | All | yes | Sensor and calendar modules sit at `orchestration///.py` with a specific name | +| `SQBRPYTHON017` | orchestration-private-support | All | yes | A domain folder's private support is its support files and `_helpers/`, imported only from that folder | +| `SQBRPYTHON018` | orchestration-module-surface | All | yes | A sensor or calendar module holds imports, private constants and its one declaration | +| `SQBRPYTHON019` | orchestration-import-direction | All | yes | Sensors and calendars import only providers, `python/utils/` and their own support; nothing outside `orchestration/` imports them | +| `SQBRPYTHON020` | orchestration-no-docstring | All | yes | Sensor and calendar functions have no docstring | + ### Type annotations | Code | Name | Files | Default | Checks | diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py index 7813ad8e6..f8b157d1d 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py @@ -362,11 +362,17 @@ def _exec_project_module( return module +def _deepest_modules_first() -> list[tuple[str, ModuleType]]: + """Loaded modules, children first: a namespace package reads its parent's path.""" + + return sorted(sys.modules.items(), key=lambda item: item[0].count("."), reverse=True) + + def _evict_stale_project_package_modules(*, root_module: str, project_dir: Path) -> None: root_path: Path = (project_dir / root_module).resolve() module_name: str module: ModuleType - for module_name, module in tuple(sys.modules.items()): + for module_name, module in _deepest_modules_first(): if module_name != root_module and not module_name.startswith(f"{root_module}."): continue module_file: object = getattr(module, "__file__", None) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index d86e8149c..07d2a5d75 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -3520,7 +3520,7 @@ def test_given_project_python_when_running_python_rules_then_findings_match_comp _PYTHON_ANNOTATED_ALERT, ), ), - expected_hits_and_misses=((0, 1), (0, 14), (13, 1), (0, 14), (15, 0)), + expected_hits_and_misses=((0, 1), (0, 18), (17, 1), (0, 18), (19, 0)), expected_python_cache=(False, True, True, True, True), ) ], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 406627370..75ee9b840 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -323,6 +323,13 @@ class SnapshotDiscoveryTestCase: expected_trigger_declared_at: dict[str, dict[str, object] | None] = field(default_factory=dict) +@dataclass(frozen=True) +class SnapshotProcessReuseTestCase: + description: str + fixtures: tuple[str, ...] + expected_sensors: tuple[tuple[str, ...], ...] + + @dataclass(frozen=True) class SnapshotDiscoveryRejectionTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index aefebb42f..b75347c39 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -36,6 +36,7 @@ SnapshotExpressionRangeTestCase, SnapshotIdentityTestCase, SnapshotLineageTestCase, + SnapshotProcessReuseTestCase, SnapshotProvidersTestCase, SnapshotRejectionTestCase, SnapshotScheduleTestCase, @@ -1301,6 +1302,28 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject } }, ), + SnapshotDiscoveryTestCase( + description="DuckShop sensors read support modules in a private folder of their domain", + files={**duckshop_orchestration("private_support_orders")}, + expected_jobs=("inventory_job",), + expected_sensors={ + "on_orders_sensor": { + "description": "Build inventory for every genuine orders rebuild.", + "job_key": "inventory_job", + "when": { + "kind": "node_event", + "selection": ["model:orders"], + "kinds": ["success"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + "declared_at": { + "path": "orchestration/sensors/inventory/on_orders_sensor.py", + "line": 7, + }, + } + }, + ), SnapshotDiscoveryTestCase( description="failure and recovery selectors preserve their individual transition kinds", files={ @@ -1597,6 +1620,15 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects "Failed to import orchestration file orchestration/sensors/broken_orders_sensor.py" ), ), + SnapshotDiscoveryRejectionTestCase( + description="a DuckShop sensor declared in a private folder fails instead of never running", + files=duckshop_orchestration("private_folder_sensor"), + expected_error=ValueError, + expected_message=( + "orchestration/sensors/inventory/_helpers/on_orders_sensor.py.*" + "declares a sensor inside the private folder _helpers/" + ), + ), SnapshotDiscoveryRejectionTestCase( description="DuckShop settings cannot replace the selection", files={**duckshop_orchestration("invalid_settings")}, @@ -1871,6 +1903,44 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho ) +@pytest.mark.parametrize( + "test_case", + [ + SnapshotProcessReuseTestCase( + description="a project whose sensor imports orchestration/ leaves the next project compilable", + fixtures=("private_support_orders", "node_orders", "private_support_orders"), + expected_sensors=(("on_orders_sensor",), ("on_orders_sensor",), ("on_orders_sensor",)), + ), + ], + ids=lambda case: case.description, +) +def test_given_projects_compiled_in_one_process_when_one_imports_orchestration_then_each_compiles( + tmp_path: Path, test_case: SnapshotProcessReuseTestCase +) -> None: + sensors: list[tuple[str, ...]] = [] + for index, fixture in enumerate(test_case.fixtures): + project_dir: Path = tmp_path / f"project_{index}" + write_files( + project_dir=project_dir, files=TYPE_PROOF_FILES | duckshop_orchestration(fixture) + ) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + publication: dict[str, Any] = json.loads( + compile_snapshot( + project=build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ), + metadata_json=json.dumps(METADATA), + project_dir=project_dir, + discovered_inputs=discovered, + ).to_json() + ) + sensors.append(tuple(publication["documents"].get("triggers", {}))) + + assert tuple(sensors) == test_case.expected_sensors + + @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index 61c8651bd..30acbc814 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -30,8 +30,12 @@ sqb build nightly_orders_job # build what the job builds, locally - Subfolders are allowed, such as `orchestration/jobs/finance/revenue_job.toml`. - A name must end with its folder's suffix. Names are unique across the whole project, with models, sources and every other resource. -- A Python module whose file name starts with `_` is a private helper and declares nothing. +- A Python module whose file name starts with `_`, or that sits in a folder starting with `_` under + `orchestration/calendars/` or `orchestration/sensors/` (such as `sensors/inventory/_helpers/`), is + private support and declares nothing. Declaring a calendar or sensor there fails compilation. - Any other file under `orchestration/`, or a folder not in the table, is a `D017` error. +- The [Python Rules](/docs/concepts/rules/python-rules#laying-out-sensors-and-calendars) add a + domain-folder layout for sensor and calendar modules. ## Declare a job @@ -124,7 +128,7 @@ another system. The code location evaluates it when it publishes and again befor out. ```python -# orchestration/calendars/fulfillment_calendar.py +# orchestration/calendars/fulfillment/cutoffs.py from sqlbuild.orchestration import calendar @@ -146,7 +150,7 @@ returns timezone-aware datetimes inside it. Other parameters name A sensor starts a job or job group when something happens, named by `run`. ```python -# orchestration/sensors/on_new_orders_sensor.py +# orchestration/sensors/orders/new_orders.py from sqlbuild.orchestration import sensor, source_new_data on_new_orders = sensor( diff --git a/website/src/content/docs/docs/concepts/rules/python-rules.mdx b/website/src/content/docs/docs/concepts/rules/python-rules.mdx index 5e2c3263b..a41fa220f 100644 --- a/website/src/content/docs/docs/concepts/rules/python-rules.mdx +++ b/website/src/content/docs/docs/concepts/rules/python-rules.mdx @@ -174,6 +174,37 @@ A capability folder may also hold `types.py`, `constants.py`, `exceptions.py`, ` `Provider` through an alias or a base class in `_classes/`. A class whose base comes from another package is never reported as not being a provider. +## Laying out sensors and calendars + +`SQBRPYTHON016` to `SQBRPYTHON020` keep each sensor and calendar module in a domain folder, named +for what it checks, with its private support beside it: + +```text +orchestration/ + sensors/ + inventory/ + stock_feed.py # one sensor + _constants.py + _helpers/ + feed_age.py + calendars/ + finance/ + month_end.py # one calendar +``` + +A domain folder's private support is `_models.py`, `_types.py`, `_constants.py`, +`_exceptions.py` and `_helpers/`. These are reported: + +| Code | Finding | Fix | +|---|---|---| +| `SQBRPYTHON016` | A module directly in `sensors/` or `calendars/`, nested below its domain folder, or named `main.py`, `definitions.py`, `sensors.py`, `calendars.py` or `processing.py` | Move it to `//.py` | +| `SQBRPYTHON017` | Other private files or folders, private support outside a domain folder, or an import of another domain's private support | Use the support files or `_helpers/`; share code through `python/utils/` or a provider | +| `SQBRPYTHON018` | A sensor or calendar module holding a function, a class, a public constant or another statement besides its declaration | Move it to `_helpers/`, `_models.py` or `_constants.py` | +| `SQBRPYTHON019` | A sensor or calendar importing `python/` outside `utils/`, or code outside `orchestration/` importing it | Read node results through the sensor's context; share code through `python/utils/` or a provider | +| `SQBRPYTHON020` | A sensor or calendar function with a docstring | Keep the text in `description=` only | + +Providers importing `orchestration/` are reported by `SQBRPYTHON015` instead. + ## Main modules and entry modules Some Rules apply only to a domain's main modules: files under a `main/` folder, and a unit's @@ -263,6 +294,16 @@ remediation. | `SQBRPYTHON014` | provider-module-shape | All | yes | `provider.py` holds imports, private constants and one public `Provider` subclass | | `SQBRPYTHON015` | provider-import-boundaries | All | yes | A provider's `_helpers/` and `_classes/` are private to it; providers import only `python/utils/` from `python/` and nothing from `orchestration/` | +### Sensors and calendars + +| Code | Name | Files | Default | Checks | +|---|---|---|---|---| +| `SQBRPYTHON016` | orchestration-module-paths | All | yes | Sensor and calendar modules sit at `orchestration///.py` with a specific name | +| `SQBRPYTHON017` | orchestration-private-support | All | yes | A domain folder's private support is its support files and `_helpers/`, imported only from that folder | +| `SQBRPYTHON018` | orchestration-module-surface | All | yes | A sensor or calendar module holds imports, private constants and its one declaration | +| `SQBRPYTHON019` | orchestration-import-direction | All | yes | Sensors and calendars import only providers, `python/utils/` and their own support; nothing outside `orchestration/` imports them | +| `SQBRPYTHON020` | orchestration-no-docstring | All | yes | Sensor and calendar functions have no docstring | + ### Type annotations | Code | Name | Files | Default | Checks | From 580ed2838c12e1be0468b5d6feb9aadd2f02d46b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:18:28 +0000 Subject: [PATCH 2/6] feat: run commands without provider secrets they don't need A provider whose required secret settings are unset no longer stops every command. Static commands (compile, dag, format, scope, lineage, rules run) still fail with D012 naming the variable, unless --allow-missing-secrets, SQLBUILD_ALLOW_MISSING_SECRETS or the local allow_missing_secrets setting allows it. The project file rejects that setting. build, load and check read secrets only for the providers the selected nodes and sinks use, and fail before anything runs when one is missing. Providers are instantiated lazily in the runtime session. sqb debug reports how many providers are ready and lists missing secrets (--verbose lists every one). Location health reports missing_secrets and exits 1, and the Providers page marks unset secret settings. --- clients/typescript/src/schema.d.ts | 9 + .../sqlbuild-cli/src/argv/command_table.json | 84 ++++ crates/sqlbuild-cli/src/argv/constants.rs | 3 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 3 +- .../src/native_commands/_helpers/execution.rs | 35 +- .../native_commands/main/_location_work.rs | 2 +- .../main/run_location_invocation.rs | 2 +- .../native_commands/test_location_health.rs | 22 +- .../src/project/_helpers/documents.rs | 20 +- .../src/project/_helpers/sections.rs | 17 +- .../sqlbuild-config/src/project/constants.rs | 4 +- .../debug_report/main/debug_provider_lines.rs | 58 ++- .../debug_report/main/format_debug_json.rs | 6 +- .../sqlbuild-core/src/debug_report/models.rs | 7 + .../src/debug_report/tests/helpers.rs | 17 +- .../debug_report/tests/test_debug_report.rs | 61 ++- .../src/debug_report/tests/test_types.rs | 11 +- .../src/declarations/models.rs | 3 + .../platform/main/report_location_health.rs | 1 + .../src/platform/models.rs | 24 +- .../tests/api/test_providers.rs | 6 +- .../_helpers/build_command/command.rs | 58 +++ .../_helpers/build_command/invocation.rs | 5 + .../bindings/_helpers/check/check_command.rs | 7 + .../src/bindings/_helpers/cli/dispatch.rs | 8 + .../_helpers/command_runtime/invocation.rs | 12 +- .../compile_command/runtime_identity.rs | 2 + .../_helpers/compile_driver/discovery.rs | 56 ++- .../_helpers/contract_records/constants.rs | 1 + .../_helpers/debug_command/run_debug.rs | 33 +- .../bindings/_helpers/discovery/constants.rs | 10 + .../_helpers/discovery/imported_providers.rs | 108 ++++- .../src/bindings/_helpers/discovery/mod.rs | 1 + .../_helpers/discovery/provider_secrets.rs | 182 ++++++++ .../_helpers/event_exporting/scope.rs | 58 ++- .../bindings/_helpers/load/load_command.rs | 36 +- .../_helpers/location_command/command.rs | 2 +- .../_helpers/location_command/constants.rs | 6 +- .../location_command/snapshot_providers.rs | 63 +-- .../_helpers/native_host/constants.rs | 4 + .../src/bindings/_helpers/native_host/mod.rs | 1 + .../native_host/orchestration_sensors.rs | 7 +- .../native_host/orchestration_triggers.rs | 7 +- .../_helpers/native_host/provider_settings.rs | 150 +++++++ .../_helpers/query_command/project.rs | 16 + .../sqlbuild/references/docs/cli/compile.md | 1 + .../sqlbuild/references/docs/cli/dag.md | 1 + .../sqlbuild/references/docs/cli/debug.md | 14 +- .../sqlbuild/references/docs/cli/format.md | 1 + .../sqlbuild/references/docs/cli/lineage.md | 1 + .../sqlbuild/references/docs/cli/location.md | 4 + .../sqlbuild/references/docs/cli/rules.md | 2 +- .../sqlbuild/references/docs/cli/scope.md | 1 + .../docs/concepts/project-configuration.md | 1 + .../docs/concepts/python-nodes/providers.md | 57 ++- src/sqlbuild/_native.pyi | 6 +- .../discovery/main/runtime_extensions.py | 4 +- src/sqlbuild/provider/_helpers/session.py | 20 +- src/sqlbuild/provider/classes/session.py | 30 +- .../cli/commands/main/debug/test_debug.py | 16 +- .../commands/main/providers/_test_types.py | 18 + .../main/providers/test_event_exporters.py | 96 ++++ .../main/providers/test_provider_secrets.py | 411 ++++++++++++++++++ .../test_orchestration_snapshot.py | 8 + .../cli/commands/main/entry/test_main.py | 5 +- ui/src/lib/api/http-locations.test.ts | 7 + ui/src/lib/api/http-locations.ts | 11 +- ui/src/lib/api/http-providers.test.ts | 4 +- ui/src/lib/api/http-providers.ts | 2 +- ui/src/lib/api/mock/platform.ts | 3 +- ui/src/lib/api/types.ts | 2 +- ui/src/lib/api/types/platform.ts | 2 + ui/src/lib/platform/labels.ts | 2 + ui/src/routes/providers/[name]/+page.svelte | 7 +- website/src/content/docs/docs/cli/compile.mdx | 1 + website/src/content/docs/docs/cli/dag.mdx | 1 + website/src/content/docs/docs/cli/debug.mdx | 14 +- website/src/content/docs/docs/cli/format.mdx | 1 + website/src/content/docs/docs/cli/lineage.mdx | 1 + .../src/content/docs/docs/cli/location.mdx | 4 + website/src/content/docs/docs/cli/rules.mdx | 2 +- website/src/content/docs/docs/cli/scope.mdx | 1 + .../docs/concepts/project-configuration.mdx | 1 + .../docs/concepts/python-nodes/providers.mdx | 57 ++- 84 files changed, 1863 insertions(+), 185 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_host/provider_settings.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/providers/test_provider_secrets.py diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index be0db5ce1..15e3989cf 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1929,6 +1929,13 @@ export interface components { healthy: boolean; /** @enum {string} */ kind: "connection"; + } | { + /** Format: int64 */ + generation: number; + /** @enum {string} */ + kind: "provider_secrets"; + /** Format: int64 */ + missing: number; } | { /** Format: int64 */ elapsed_millis: number; @@ -2571,6 +2578,8 @@ export interface components { description?: string | null; /** @description The environment variable the setting is read from. */ env: string; + /** @description Whether this required secret is unset in the location's environment. */ + missing?: boolean; name: string; required: boolean; /** @description Whether the annotation is a pydantic secret type. */ diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index fdf96c060..f92498e5a 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1500,6 +1500,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--allow-missing-secrets" + ], + "dest": "allow_missing_secrets", + "nargs": 0, + "const": true, + "default": false, + "help": "Compile without provider secrets: skip the check for missing secret settings, still validating the others", + "group": 1 + }, { "kind": "store_true", "options": [ @@ -1639,6 +1651,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--allow-missing-secrets" + ], + "dest": "allow_missing_secrets", + "nargs": 0, + "const": true, + "default": false, + "help": "Compile without provider secrets: skip the check for missing secret settings, still validating the others", + "group": 1 + }, { "kind": "store_true", "options": [ @@ -3688,6 +3712,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--allow-missing-secrets" + ], + "dest": "allow_missing_secrets", + "nargs": 0, + "const": true, + "default": false, + "help": "Compile without provider secrets: skip the check for missing secret settings, still validating the others", + "group": 1 + }, { "kind": "store_true", "options": [ @@ -5243,6 +5279,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--allow-missing-secrets" + ], + "dest": "allow_missing_secrets", + "nargs": 0, + "const": true, + "default": false, + "help": "Compile without provider secrets: skip the check for missing secret settings, still validating the others", + "group": 1 + }, { "kind": "store_true", "options": [ @@ -5353,6 +5401,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--verbose" + ], + "dest": "verbose", + "nargs": 0, + "const": true, + "default": false, + "help": "List every provider and secret setting, not only the missing ones", + "group": 1 + }, { "kind": "store_true", "options": [ @@ -5900,6 +5960,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--allow-missing-secrets" + ], + "dest": "allow_missing_secrets", + "nargs": 0, + "const": true, + "default": false, + "help": "Compile without provider secrets: skip the check for missing secret settings, still validating the others", + "group": 1 + }, { "kind": "store_true", "options": [ @@ -8158,6 +8230,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--allow-missing-secrets" + ], + "dest": "allow_missing_secrets", + "nargs": 0, + "const": true, + "default": false, + "help": "Compile without provider secrets: skip the check for missing secret settings, still validating the others", + "group": 1 + }, { "kind": "store_true", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 5d238abd5..6669feaeb 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -73,7 +73,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 187] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 188] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -86,6 +86,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 187] = [ ("skip_dbt_debug", NamespaceDefault::Bool(false)), ("no_sql_validation", NamespaceDefault::Bool(false)), ("no_cache", NamespaceDefault::Bool(false)), + ("allow_missing_secrets", NamespaceDefault::Bool(false)), ("defer_to", NamespaceDefault::None), ("defer_clone_from", NamespaceDefault::None), ("defer_sources_to", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 43487a369..fbeb74f65 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -504,12 +504,13 @@ pub(crate) const COST_ROUTE: Route = routed( REQUEST_SHAPE, ); -const DEBUG_ARGUMENTS: [Field; 5] = [ +const DEBUG_ARGUMENTS: [Field; 6] = [ PROJECT_DIR, NO_COLOR, same("no_connection"), SELECTED_TARGET, JSON_OUTPUT, + same("verbose"), ]; pub(crate) const DEBUG_ROUTE: Route = routed( "run_debug", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 9fd9f3af5..035882d3e 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -812,7 +812,7 @@ pub(crate) async fn check_published_location( "Checking location connection with sqb debug...\n", )])); tokio::pin!(stopped); - let Some((connected, files)) = + let Some((connected, missing, files)) = check_location_connection(options, script, stopped.as_mut()).await? else { return Ok(None); @@ -821,6 +821,18 @@ pub(crate) async fn check_published_location( result = &mut stopped => { result?; return Ok(None); }, () = report_location_health(options, bearer, session_id, LocationHealth::Connection { generation, healthy: connected }) => {}, } + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = report_location_health(options, bearer, session_id, LocationHealth::ProviderSecrets { generation, missing }) => {}, + } + if missing > 0 { + drop(write_output(&[( + Stream::Stderr, + &format!( + "{missing} provider secret(s) are unset in this location's environment; runs that use them fail. Run sqb debug for the list.\n" + ), + )])); + } drop(write_output(&[( Stream::Stderr, if connected { @@ -832,12 +844,12 @@ pub(crate) async fn check_published_location( Ok(Some(files)) } -/// Capture the ordinary debug check's exact local file paths without contaminating publication stdout. +/// Capture the ordinary debug check's exact local file paths and missing provider secret count without contaminating publication stdout. pub(crate) async fn check_location_connection( options: &LocationServe, script: &Path, stopped: impl Future>, -) -> Result)>, ControlError> { +) -> Result)>, ControlError> { let mut child = location_child(options, script) .args(["debug", "--target", &options.target, "--json"]) .stdout(Stdio::piped()) @@ -856,20 +868,27 @@ pub(crate) async fn check_location_connection( }, result = async { tokio::try_join!(child.wait(), stdout.read_to_end(&mut output)) } => result?.0.success(), }; - let files: Vec = match serde_json::from_slice::(&output) - .and_then(|report| serde_json::from_value(report["local_files"].clone())) - { - Ok(files) => files, + let (files, missing): (Vec, u64) = match serde_json::from_slice::( + &output, + ) + .and_then(|report| { + Ok(( + serde_json::from_value(report["local_files"].clone())?, + report["missing_secrets"].as_u64().unwrap_or_default(), + )) + }) { + Ok(report) => report, Err(_) => { drop(write_output(&[( Stream::Stderr, "Location connection check did not produce a valid report; retaining the published snapshot.\n", )])); - return Ok(Some((false, Vec::new()))); + return Ok(Some((false, 0, Vec::new()))); } }; Ok(Some(( connected, + missing, location_database_files(&options.project_dir, files), ))) } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index 8b40bcfe0..67ebe7e7d 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -139,7 +139,7 @@ async fn perform_work( LocationWork::SyncWarehouse {} => Some((LocationWorkOutcome::SyncStarted {}, Vec::new())), LocationWork::CheckConnection {} => check_location_connection(options, script, expires) .await? - .map(|(healthy, files)| (LocationWorkOutcome::Connection { healthy }, files)), + .map(|(healthy, _, files)| (LocationWorkOutcome::Connection { healthy }, files)), LocationWork::EvaluateSensor { evaluation } => { evaluate_location_sensor(options, script, &evaluation, expires) .await? diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs index 49ca0c930..ed7947b94 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs @@ -91,7 +91,7 @@ fn location_health(namespace: &Namespace) -> i32 { ])); i32::from(matches!( state, - LocationState::Broken | LocationState::FallingBack + LocationState::Broken | LocationState::FallingBack | LocationState::MissingSecrets )) } Err(error) => { diff --git a/crates/sqlbuild-cli/tests/native_commands/test_location_health.rs b/crates/sqlbuild-cli/tests/native_commands/test_location_health.rs index 4d32b485a..92b4c5e03 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_location_health.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_location_health.rs @@ -13,7 +13,7 @@ use crate::helpers::{api_ready, start_control}; use crate::test_types::LocationHealthTestCase; #[test] -fn given_location_health_reports_when_checking_health_then_down_or_falling_back_locations_exit_nonzero() +fn given_location_health_reports_when_checking_health_then_down_falling_back_or_missing_secret_locations_exit_nonzero() { let test_cases = [ LocationHealthTestCase { @@ -60,6 +60,26 @@ fn given_location_health_reports_when_checking_health_then_down_or_falling_back_ expected_state: "falling_back", expected_exit: 1, }, + LocationHealthTestCase { + description: "a provider secret is unset, even with a failed connection check", + reports: &[ + r#"{"kind":"connection","generation":2,"healthy":false}"#, + r#"{"kind":"provider_secrets","generation":2,"missing":1}"#, + ], + other_target_connections: 0, + expected_state: "missing_secrets", + expected_exit: 1, + }, + LocationHealthTestCase { + description: "a later load found every provider secret set", + reports: &[ + r#"{"kind":"provider_secrets","generation":2,"missing":2}"#, + r#"{"kind":"provider_secrets","generation":3,"missing":0}"#, + ], + other_target_connections: 0, + expected_state: "healthy", + expected_exit: 0, + }, LocationHealthTestCase { description: "the warehouse connection check failed", reports: &[r#"{"kind":"connection","generation":2,"healthy":false}"#], diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 09e0abbf4..fe516eaeb 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -24,8 +24,9 @@ use crate::project::_helpers::targets::{ }; use crate::project::_helpers::values::is_truthy; use crate::project::constants::{ - COLUMN_CONTRACT_MODES, LEGACY_COLUMN_CONTRACT_MODE_KEY, LEGACY_EVENT_EXPORTERS_KEY, - LOCAL_CONFIG_FILENAME, QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY, + ALLOW_MISSING_SECRETS_KEY, COLUMN_CONTRACT_MODES, LEGACY_COLUMN_CONTRACT_MODE_KEY, + LEGACY_EVENT_EXPORTERS_KEY, LOCAL_CONFIG_FILENAME, QUERY_CHANGE_TRACKING_KEY, + REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY, }; use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; use crate::project::models::{ @@ -184,6 +185,21 @@ fn identity(payload: &Entries) -> Loaded { } fn sections(payload: &Entries) -> Loaded { + if get(payload, "settings") + .and_then(ConfigValue::as_map) + .is_some_and(|settings| has(settings, ALLOW_MISSING_SECRETS_KEY)) + { + return Err(ConfigFailure::at( + PROJECT_FILE, + format!( + "cannot set [settings] {ALLOW_MISSING_SECRETS_KEY}; it applies to one machine, \ + so only {LOCAL_CONFIG_FILENAME} can set it" + ), + ) + .with_help(format!( + "move it to {LOCAL_CONFIG_FILENAME}, or set SQLBUILD_ALLOW_MISSING_SECRETS=1 in CI" + ))); + } Ok(vec![ ( "settings", diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 32947c975..5d966d192 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -10,6 +10,11 @@ use crate::project::_helpers::fields::{ use crate::project::_helpers::keys::{Place, reject_removed_virtual_keys, validate_allowed_keys}; use crate::project::_helpers::targets::seconds_value; use crate::project::_helpers::values::{int_below, python_equal}; +use crate::project::constants::{ + ALLOW_MISSING_SECRETS_KEY, DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, + DELTA_COMPILE_KEY, DISABLED_VALUE, SCENARIO_TIMEOUT_KEY, TEST_TIMEOUT_KEY, + UNIT_TEST_MAX_REAL_MODELS_KEY, +}; use crate::project::constants::{ AUDIT_SEVERITIES, CHECK_DECLARED_COLUMNS_KEY, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES, CONCURRENCY_KEY, DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, @@ -26,10 +31,6 @@ use crate::project::constants::{ CHANGE_COUNTS, CHANGE_REACHES, CHANGES_KEYS, CONTROL_KEYS, MIN_SYNC_INTERVAL_SECONDS, SYNC_INTERVAL_HELP, SYNC_INTERVAL_OFF, }; -use crate::project::constants::{ - DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DELTA_COMPILE_KEY, DISABLED_VALUE, - SCENARIO_TIMEOUT_KEY, TEST_TIMEOUT_KEY, UNIT_TEST_MAX_REAL_MODELS_KEY, -}; use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ChangesConfig, ConfigFile, ControlConfig, ReviewConfig}; use crate::project::types::{Entries, Fields, Loaded}; @@ -276,6 +277,14 @@ pub(crate) fn load_settings(payload: Option<&ConfigValue>, file: ConfigFile) -> true, )?)), ), + ( + ALLOW_MISSING_SECRETS_KEY, + Built::Value(ConfigValue::Bool(optional_bool( + mapping, + ALLOW_MISSING_SECRETS_KEY, + false, + )?)), + ), ]; let mut fields = fields; for (key, seconds_key, default) in [ diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 5bc122f08..8c7ec909f 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -47,9 +47,10 @@ pub(crate) const UNIT_TEST_MAX_REAL_MODELS_KEY: &str = "unit_test_max_real_model pub(crate) const DEFAULT_UNIT_TEST_MAX_REAL_MODELS: i64 = 10; pub(crate) const TEST_TIMEOUT_KEY: &str = "test_timeout"; pub(crate) const DELTA_COMPILE_KEY: &str = "delta_compile"; +pub(crate) const ALLOW_MISSING_SECRETS_KEY: &str = "allow_missing_secrets"; pub(crate) const SCENARIO_TIMEOUT_KEY: &str = "scenario_timeout"; pub(crate) const DEFAULT_TEST_TIMEOUT: &str = "30s"; -pub(crate) const SETTINGS_KEYS: [&str; 15] = [ +pub(crate) const SETTINGS_KEYS: [&str; 16] = [ "sql_analysis", "require_sql_analysis", "query_change_tracking", @@ -65,6 +66,7 @@ pub(crate) const SETTINGS_KEYS: [&str; 15] = [ "test_timeout", "scenario_timeout", "delta_compile", + "allow_missing_secrets", ]; pub(crate) const DEFAULTS_KEYS: [&str; 33] = [ "append_cursor_inclusive", diff --git a/crates/sqlbuild-core/src/debug_report/main/debug_provider_lines.rs b/crates/sqlbuild-core/src/debug_report/main/debug_provider_lines.rs index 0e411d654..aad527755 100644 --- a/crates/sqlbuild-core/src/debug_report/main/debug_provider_lines.rs +++ b/crates/sqlbuild-core/src/debug_report/main/debug_provider_lines.rs @@ -2,9 +2,9 @@ use crate::debug_report::models::{DebugLine, DebugProvider, DebugStatus}; -/// The provider count followed by one line per provider. +/// How many providers have every secret set, then the missing secrets, or with `verbose` every provider's secrets. #[must_use] -pub fn debug_provider_lines(providers: &[DebugProvider]) -> Vec { +pub fn debug_provider_lines(providers: &[DebugProvider], verbose: bool) -> Vec { if providers.is_empty() { return vec![DebugLine::checked( "providers", @@ -12,17 +12,53 @@ pub fn debug_provider_lines(providers: &[DebugProvider]) -> Vec { (DebugStatus::Ok, None), )]; } + let ready: usize = providers + .iter() + .filter(|provider| has_every_secret(provider)) + .count(); + let all_ready: bool = ready == providers.len(); let mut lines: Vec = vec![DebugLine::checked( "providers", - &providers.len().to_string(), - (DebugStatus::Ok, Some("discovered")), + &format!("{ready} of {} ready", providers.len()), + if all_ready { + (DebugStatus::Ok, Some("secrets set")) + } else { + (DebugStatus::Warn, Some("missing secrets")) + }, )]; - lines.extend(providers.iter().map(|provider| { - DebugLine::checked( - &provider.name, - &format!("{}:{}", provider.relative_path, provider.class_name), - (DebugStatus::Ok, Some("valid settings")), - ) - })); + for provider in providers { + if verbose && provider.secrets.is_empty() { + lines.push(DebugLine::checked( + &provider.name, + "(no secrets)", + (DebugStatus::Ok, Some("ok")), + )); + } + for (variable, set) in &provider.secrets { + if *set && !verbose { + continue; + } + lines.push(DebugLine::checked( + &provider.name, + variable, + if *set { + (DebugStatus::Ok, Some("set")) + } else { + (DebugStatus::Warn, Some("missing")) + }, + )); + } + } + if !verbose && !all_ready { + lines.push(DebugLine::checked( + "more", + "sqb debug --verbose lists every provider", + (DebugStatus::Skip, None), + )); + } lines } + +fn has_every_secret(provider: &DebugProvider) -> bool { + provider.secrets.iter().all(|(_, set)| *set) +} diff --git a/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs index 9eb5e8408..d4a6aece7 100644 --- a/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs +++ b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs @@ -2,7 +2,7 @@ use crate::debug_report::models::{DebugLine, DebugReport}; use crate::json::main::dumps::dumps; -use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; +use crate::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; fn line_value(line: &DebugLine) -> JsonValue { let mut fields: Vec<(String, JsonValue)> = vec![ @@ -42,6 +42,10 @@ pub fn format_debug_json(report: &DebugReport) -> String { lines_value(&report.configuration), ), ("providers".to_owned(), lines_value(&report.providers)), + ( + "missing_secrets".to_owned(), + JsonValue::Integer(JsonInteger::from(report.missing_secrets as u64)), + ), ("connection".to_owned(), lines_value(&report.connection)), ( "local_files".to_owned(), diff --git a/crates/sqlbuild-core/src/debug_report/models.rs b/crates/sqlbuild-core/src/debug_report/models.rs index 144079f7d..71b592f3f 100644 --- a/crates/sqlbuild-core/src/debug_report/models.rs +++ b/crates/sqlbuild-core/src/debug_report/models.rs @@ -6,6 +6,8 @@ use std::path::PathBuf; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DebugStatus { Ok, + /// Worth fixing, but the report still succeeds, such as a provider secret this machine lacks. + Warn, Error, Skip, } @@ -16,6 +18,7 @@ impl DebugStatus { pub fn label(self) -> &'static str { match self { Self::Ok => "OK", + Self::Warn => "WARN", Self::Error => "ERROR", Self::Skip => "SKIP", } @@ -38,6 +41,8 @@ pub struct DebugReport { pub configuration: Vec, pub providers: Vec, pub connection: Vec, + /// How many provider secret settings the environment leaves unset. + pub missing_secrets: usize, /// Exact local database paths, not the shortened display values in connection lines. pub local_files: Vec, } @@ -118,6 +123,8 @@ pub struct DebugProvider { pub name: String, pub relative_path: String, pub class_name: String, + /// Each secret setting's environment variable and whether the environment sets it. + pub secrets: Vec<(String, bool)>, } /// One connection setting value, as Python would print it. diff --git a/crates/sqlbuild-core/src/debug_report/tests/helpers.rs b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs index 240efc55d..f1582adba 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/helpers.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs @@ -1,8 +1,11 @@ -use crate::debug_report::models::{ConnectionValue, DebugLine, DebugReport, DebugStatus}; +use crate::debug_report::models::{ + ConnectionValue, DebugLine, DebugProvider, DebugReport, DebugStatus, +}; pub(super) fn sample_report() -> DebugReport { DebugReport { local_files: vec!["/repo/warehouse.db".into()], + missing_secrets: 0, runtime: vec![DebugLine::plain("sqlbuild version", "0.2.1")], configuration: vec![DebugLine::checked( "project file", @@ -40,3 +43,15 @@ pub(super) fn text(value: &str) -> ConnectionValue { pub(super) fn line_summary(line: &DebugLine) -> (&str, &str, Option<&str>) { (&line.label, &line.message, line.status_message.as_deref()) } + +pub(super) fn provider(name: &str, secrets: &[(&str, bool)]) -> DebugProvider { + DebugProvider { + name: name.to_owned(), + relative_path: format!("providers/{name}/provider.py"), + class_name: "Provider".to_owned(), + secrets: secrets + .iter() + .map(|(variable, set)| ((*variable).to_owned(), *set)) + .collect(), + } +} diff --git a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs index 5fb74ee02..c90dd3f04 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs @@ -1,14 +1,15 @@ use crate::cli_text::models::CliStyle; use crate::debug_report::main::debug_connection_lines::debug_connection_lines; use crate::debug_report::main::debug_failure::debug_failure; +use crate::debug_report::main::debug_provider_lines::debug_provider_lines; use crate::debug_report::main::format_debug_json::format_debug_json; use crate::debug_report::main::format_debug_text::format_debug_text; use crate::debug_report::models::{ ConnectionCheck, ConnectionValue, DebugFailure, DebugLine, DebugReport, DebugStatus, }; -use crate::debug_report::tests::helpers::{line_summary, sample_report, text}; +use crate::debug_report::tests::helpers::{line_summary, provider, sample_report, text}; use crate::debug_report::tests::test_types::{ - ConnectionLinesTestCase, DebugFailureTestCase, FormatTestCase, + ConnectionLinesTestCase, DebugFailureTestCase, FormatTestCase, ProviderLinesTestCase, }; /// Expectations recorded from the retired Python formatter. @@ -18,7 +19,7 @@ fn given_debug_report_when_formatting_then_matches_python_outputs() { description: "renders diagnostics sections and json sections", report: sample_report(), expected_text: "\nSQLBuild Diagnostics\n\nRuntime:\n sqlbuild version: 0.2.1\n\nConfiguration:\n project file: /repo/sqlbuild_project.toml [OK found and valid]\n\nProviders:\n providers: 1 [OK discovered]\n marker_provider: providers/marker.py:MarkerProvider [OK valid settings]\n\nConnection:\n token: ****\n connection test: [ERROR authentication failed]\n query test: [SKIP connection failed]\n\n", - expected_json: r#"{"configuration": [{"label": "project file", "message": "/repo/sqlbuild_project.toml", "status": "OK", "status_message": "found and valid"}], "connection": [{"label": "token", "message": "****"}, {"label": "connection test", "message": "", "status": "ERROR", "status_message": "authentication failed"}, {"label": "query test", "message": "", "status": "SKIP", "status_message": "connection failed"}], "local_files": ["/repo/warehouse.db"], "providers": [{"label": "providers", "message": "1", "status": "OK", "status_message": "discovered"}, {"label": "marker_provider", "message": "providers/marker.py:MarkerProvider", "status": "OK", "status_message": "valid settings"}], "runtime": [{"label": "sqlbuild version", "message": "0.2.1"}], "success": false}"#, + expected_json: r#"{"configuration": [{"label": "project file", "message": "/repo/sqlbuild_project.toml", "status": "OK", "status_message": "found and valid"}], "connection": [{"label": "token", "message": "****"}, {"label": "connection test", "message": "", "status": "ERROR", "status_message": "authentication failed"}, {"label": "query test", "message": "", "status": "SKIP", "status_message": "connection failed"}], "local_files": ["/repo/warehouse.db"], "missing_secrets": 0, "providers": [{"label": "providers", "message": "1", "status": "OK", "status_message": "discovered"}, {"label": "marker_provider", "message": "providers/marker.py:MarkerProvider", "status": "OK", "status_message": "valid settings"}], "runtime": [{"label": "sqlbuild version", "message": "0.2.1"}], "success": false}"#, expected_color_fragments: &[ "\x1b[34m\x1b[1mSQLBuild Diagnostics\x1b[0m", "\x1b[1mRuntime:\x1b[0m", @@ -156,3 +157,57 @@ fn given_a_debug_report_when_its_failure_is_read_then_the_first_failed_check_is_ ); } } + +#[test] +fn given_providers_with_and_without_secrets_when_listing_then_missing_secrets_lead_and_verbose_lists_all() + { + let providers = vec![ + provider("slack_alerts", &[("SLACK_WEBHOOK_URL", false)]), + provider( + "partner_sftp", + &[("PARTNER_SFTP_PASSWORD", true), ("PARTNER_SFTP_KEY", false)], + ), + provider("storefront_api", &[]), + ]; + let test_cases = [ + ProviderLinesTestCase { + description: "default lists only the missing secrets, then a hint", + providers: providers.clone(), + verbose: false, + expected_lines: vec![ + ("providers", "1 of 3 ready", Some("missing secrets")), + ("slack_alerts", "SLACK_WEBHOOK_URL", Some("missing")), + ("partner_sftp", "PARTNER_SFTP_KEY", Some("missing")), + ("more", "sqb debug --verbose lists every provider", None), + ], + }, + ProviderLinesTestCase { + description: "verbose lists every provider and secret", + providers, + verbose: true, + expected_lines: vec![ + ("providers", "1 of 3 ready", Some("missing secrets")), + ("slack_alerts", "SLACK_WEBHOOK_URL", Some("missing")), + ("partner_sftp", "PARTNER_SFTP_PASSWORD", Some("set")), + ("partner_sftp", "PARTNER_SFTP_KEY", Some("missing")), + ("storefront_api", "(no secrets)", Some("ok")), + ], + }, + ProviderLinesTestCase { + description: "every secret set prints one line", + providers: vec![provider("partner_sftp", &[("PARTNER_SFTP_PASSWORD", true)])], + verbose: false, + expected_lines: vec![("providers", "1 of 1 ready", Some("secrets set"))], + }, + ]; + for test_case in test_cases { + let lines = debug_provider_lines(&test_case.providers, test_case.verbose); + + assert_eq!( + lines.iter().map(line_summary).collect::>(), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/debug_report/tests/test_types.rs b/crates/sqlbuild-core/src/debug_report/tests/test_types.rs index 685417e0c..2e539a46e 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/test_types.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/test_types.rs @@ -1,4 +1,6 @@ -use crate::debug_report::models::{ConnectionCheck, ConnectionValue, DebugFailure, DebugReport}; +use crate::debug_report::models::{ + ConnectionCheck, ConnectionValue, DebugFailure, DebugProvider, DebugReport, +}; pub(super) struct FormatTestCase { pub(super) description: &'static str, @@ -20,3 +22,10 @@ pub(super) struct DebugFailureTestCase { pub(super) report: DebugReport, pub(super) expected_failure: Option, } + +pub(super) struct ProviderLinesTestCase { + pub(super) description: &'static str, + pub(super) providers: Vec, + pub(super) verbose: bool, + pub(super) expected_lines: Vec<(&'static str, &'static str, Option<&'static str>)>, +} diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 8af0a3282..56f9c1218 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -391,6 +391,9 @@ pub struct ProviderSetting { /// The environment variable the setting is read from. pub env: String, pub description: Option, + /// Whether this required secret is unset in the location's environment. + #[serde(default)] + pub missing: bool, } /// Code that injects a provider: a node (`loader`, `task`, `asset`, `check`, `hook` or `materialization`, keyed `kind:name`), an `event_exporter` or a `sink`. diff --git a/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs index 7d2dad4bc..06b6761a5 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs @@ -26,6 +26,7 @@ pub async fn report_location_health( LocationHealth::Heartbeat {} => None, LocationHealth::CompileFailed { generation } | LocationHealth::Connection { generation, .. } + | LocationHealth::ProviderSecrets { generation, .. } | LocationHealth::WarehouseSync { generation, .. } => Some(generation), }; if generation.is_some_and(|generation| generation < 1) { diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index efcacecef..626b9b821 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -43,6 +43,11 @@ pub enum LocationHealth { generation: i64, healthy: bool, }, + /// How many provider secret settings the location's environment leaves unset. + ProviderSecrets { + generation: i64, + missing: u64, + }, WarehouseSync { generation: i64, location: WarehouseStateLocation, @@ -96,12 +101,14 @@ pub enum LocationState { Degraded, /// The last load failed, so runs use the previous snapshot. FallingBack, + /// A provider secret is unset in the location's environment, so runs that use the provider fail. + MissingSecrets, /// No instance has checked in for 90 seconds. Broken, } impl LocationState { - /// `target`'s state at `now` from observations that may cover several targets: down without a recent check-in, then a failed load, then a failed connection. + /// `target`'s state at `now` from observations that may cover several targets: down without a recent check-in, then a failed load, then missing provider secrets, then a failed connection. #[must_use] pub fn of( observations: &[LocationHealthObservation], @@ -121,6 +128,7 @@ impl LocationState { } let mut failed: Option<(DateTime, i64)> = None; let mut connection: Option<(DateTime, i64, bool)> = None; + let mut secrets: Option<(DateTime, u64)> = None; for observation in own { let seen = observation.received_at; match observation.report.health { @@ -135,15 +143,21 @@ impl LocationState { } if connection.is_none_or(|(at, _, _)| seen > at) => { connection = Some((seen, generation, healthy)); } + LocationHealth::ProviderSecrets { missing, .. } + if secrets.is_none_or(|(at, _)| seen > at) => + { + secrets = Some((seen, missing)); + } _ => {} } } - match (failed, connection) { - (Some((_, failed)), Some((_, connected, _))) if failed >= connected => { + match (failed, connection, secrets) { + (Some((_, failed)), Some((_, connected, _)), _) if failed >= connected => { Self::FallingBack } - (Some(_), None) => Self::FallingBack, - (_, Some((_, _, false))) => Self::Degraded, + (Some(_), None, _) => Self::FallingBack, + (_, _, Some((_, missing))) if missing > 0 => Self::MissingSecrets, + (_, Some((_, _, false)), _) => Self::Degraded, _ => Self::Healthy, } } diff --git a/crates/sqlbuild-orchestration/tests/api/test_providers.rs b/crates/sqlbuild-orchestration/tests/api/test_providers.rs index 01e151856..72a747f78 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_providers.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_providers.rs @@ -91,7 +91,7 @@ fn given_published_providers_when_people_read_them_then_jobs_using_their_nodes_a "description": "Client for the fulfillment service.", "path": "providers/fulfillment.py", "settings": [ - {"name": "api_key", "type": "SecretStr", "required": true, "secret": true, "env": "API_KEY", "description": null}, + {"name": "api_key", "type": "SecretStr", "required": true, "secret": true, "env": "API_KEY", "description": null, "missing": true}, {"name": "endpoint", "type": "str", "required": false, "secret": false, "env": "ENDPOINT", "description": "Base URL"}, ], "used_by": [ @@ -113,8 +113,8 @@ fn given_published_providers_when_people_read_them_then_jobs_using_their_nodes_a "description": "Client for the fulfillment service.", "path": "providers/fulfillment.py", "settings": [ - {"name": "api_key", "type": "SecretStr", "required": true, "secret": true, "env": "API_KEY", "description": null}, - {"name": "endpoint", "type": "str", "required": false, "secret": false, "env": "ENDPOINT", "description": "Base URL"}, + {"name": "api_key", "type": "SecretStr", "required": true, "secret": true, "env": "API_KEY", "description": null, "missing": true}, + {"name": "endpoint", "type": "str", "required": false, "secret": false, "env": "ENDPOINT", "description": "Base URL", "missing": false}, ], "used_by": [ {"kind": "hook", "name": "notify_fulfillment"}, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 98cff67d7..b42d11672 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -19,8 +19,14 @@ use crate::bindings::_helpers::build_command::planning::{ compile_build_plan, enforce_build_plan_policies, run_defer_clone_prephase, write_compile_target, write_plan_text, }; +use crate::bindings::_helpers::build_command::support::{frozenset, tuple}; +use crate::bindings::_helpers::check::check_core::relevant_check_functions; use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::diagnostics::trackers::BuildPhaseTimingTracker; +use crate::bindings::_helpers::discovery::provider_secrets::{ + consumer_uses, provider_uses, require_used_secrets, sink_uses, +}; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::observations::source_capture::{ arm_source_capture, take_source_capture, @@ -69,9 +75,61 @@ fn run_cached_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> built } +/// The post-build check functions that run if every planned node succeeds. +fn planned_check_functions<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, +) -> PyResult>> { + if !native_field(&context.request, "run_audits")?.extract::()? { + return Ok(Vec::new()); + } + let py: Python<'py> = context.py(); + let plan: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?; + let mut dependencies: Vec> = Vec::new(); + for entry in native_field(pipeline, "python_plan_entries")?.try_iter()? { + dependencies.push(native_field(&entry?, "name")?); + } + let mut completed: Vec> = Vec::new(); + for entries in ["model_entries", "seed_entries", "source_load_entries"] { + for entry in native_field(&plan, entries)?.try_iter()? { + completed.push(native_field(&entry?, "name")?); + } + } + for load in native_field(&plan, "source_load_entries")?.try_iter()? { + dependencies.push(native_field(&load?, "loader")?); + } + let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; + relevant_check_functions( + &discovered, + &python_node_graph(&discovered)?, + native_field(&context.request, "exclude")?, + (&frozenset(py, &dependencies)?, &frozenset(py, &completed)?), + ) +} + +/// Fail before anything runs when planned nodes, their post-build checks or sinks lack provider secrets. +fn require_plan_secrets(context: &BuildContext<'_>, pipeline: &Bound<'_, PyAny>) -> PyResult<()> { + let discovered: Bound<'_, PyAny> = context.get("discovered_inputs")?; + let mut uses: Vec<(String, String)> = provider_uses( + &native_field(&native_field(pipeline, "plan_output")?, "provider_usages")?, + None, + )?; + uses.extend(consumer_uses(&native_field( + pipeline, + "python_plan_entries", + )?)?); + uses.extend(consumer_uses(&tuple( + context.py(), + &planned_check_functions(context, pipeline)?, + )?)?); + uses.extend(sink_uses(&discovered)?); + require_used_secrets(&discovered, &uses) +} + /// Plan and run the build, sharing one relation metadata cache from planning to the run's end. fn run_captured_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { let pipeline: Bound<'_, PyAny> = compile_build_plan(context)?; + require_plan_secrets(context, &pipeline)?; if let Some(worker) = &context.worker { worker.enforce_plan_writers(&pipeline)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index a2a5d45df..2f6d753cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -24,6 +24,7 @@ use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::{new_record, new_record_with}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -124,6 +125,10 @@ pub(crate) fn resolve_build_context<'py>( &[ ("project_dir", project_dir.clone()), ("extract_output_column_locations", boolean(py, false)), + ( + "secret_policy", + pyo3::types::PyString::new(py, SecretPolicy::Defer.name()).into_any(), + ), ], )?), ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index f9b8a4d5c..ce9feea89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -10,6 +10,9 @@ use crate::bindings::_helpers::compile_driver::relation_targets::build_project_r use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; +use crate::bindings::_helpers::discovery::provider_secrets::{ + consumer_uses, require_used_secrets, sink_uses, +}; use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; use crate::bindings::_helpers::graph::execution_gates::sql_ref_key; use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; @@ -661,6 +664,10 @@ pub(crate) fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> P header.set_item("use_color", native_field(&invocation, "use_color")?)?; write_execution_header(py, Some(header))?; let prepared: Prepared<'_> = prepare(request, &invocation)?; + let discovered: Bound<'_, PyAny> = native_field(&invocation, "discovered_inputs")?; + let mut uses: Vec<(String, String)> = consumer_uses(&prepared.check_functions)?; + uses.extend(sink_uses(&discovered)?); + require_used_secrets(&discovered, &uses)?; let session: Bound<'_, PyAny> = call_with( &host::entry(py, UserCodeHost::BuildProviderSession)?, &[( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 5f6e49b0b..234f01bd6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -18,6 +18,7 @@ use crate::bindings::_helpers::dbt_command::init::run_dbt_init_command; use crate::bindings::_helpers::debug_command::run_debug::run_debug_command; use crate::bindings::_helpers::diff_report::command::run_diff_command; use crate::bindings::_helpers::diff_report::support::cli_error; +use crate::bindings::_helpers::discovery::provider_secrets::MissingSecretsScope; use crate::bindings::_helpers::format_command::command::run_format_command; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::init_command::command::run_init_command; @@ -76,6 +77,9 @@ const RAW_QUERY_ATTRIBUTES: [&str; 4] = [ "right_query_file", ]; +/// The namespace attribute of `--allow-missing-secrets`. +const ALLOW_MISSING_SECRETS_DEST: &str = "allow_missing_secrets"; + /// One invocation being dispatched: its namespace and the values derived from it once. pub(crate) struct Invocation<'py> { args: Bound<'py, PyAny>, @@ -448,6 +452,10 @@ fn routed_command<'py>( ) -> PyResult> { let py: Python<'py> = args.py(); let invocation = Invocation::new(args)?; + let _secrets = MissingSecretsScope::enter( + args.getattr_opt(ALLOW_MISSING_SECRETS_DEST)? + .is_some_and(|allowed| allowed.is_truthy().unwrap_or(false)), + ); match command_dispatch(&command_facts(args)?) { Dispatch::Nothing => 0.into_bound_py_any(py), Dispatch::Fail(failure) => Err(user_error(py, failure)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index 45f161496..0f0db48fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; @@ -81,7 +82,16 @@ fn invocation_context<'py>( Some(discovered) => discovered.clone(), None => discover_project_inputs( py, - Some(keywords(py, &[("project_dir", project_dir.clone())])?), + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ( + "secret_policy", + pyo3::types::PyString::new(py, SecretPolicy::Defer.name()).into_any(), + ), + ], + )?), )?, }; let adapter_context: Bound<'py, PyAny> = resolve_adapter_connection_context( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs index 0f9299991..64245f373 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs @@ -8,6 +8,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::discovery::provider_secrets::invocation_allows_missing_secrets; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; @@ -86,6 +87,7 @@ pub(crate) fn invocation_identity<'py>( for field in REQUEST_CHOICES { payload.set_item(field, native_field(request, field)?)?; } + payload.set_item("allow_missing_secrets", invocation_allows_missing_secrets())?; payload.set_item("cwd", py.import("os")?.getattr("getcwd")?.call0()?)?; payload.set_item("use_color", use_color)?; payload.set_item("select", listed(native_field(request, "select")?)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index c565a917a..f78bbff89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -14,6 +14,9 @@ use crate::bindings::_helpers::compile_driver::config_validation::{ }; use crate::bindings::_helpers::compile_driver::integration_loaders::integration_loader_functions; use crate::bindings::_helpers::compile_driver::stage_work::{PROJECT_DISCOVERIES, count_stage}; +use crate::bindings::_helpers::discovery::constants::{ + ALLOW_MISSING_SECRETS_ENV, ALLOW_MISSING_SECRETS_SETTING, TRUE_ENV_VALUES, +}; use crate::bindings::_helpers::discovery::declaration_records::{ discover_adapter_file, discover_audit_files, discover_constant_files, discover_enum_files, discover_macro_files, discover_model_schema_files, discover_seed_files, @@ -31,6 +34,9 @@ use crate::bindings::_helpers::discovery::model_records::{ use crate::bindings::_helpers::discovery::project_config::{ load_local_config_file, load_project_config_file, warn_deprecated_settings, }; +use crate::bindings::_helpers::discovery::provider_secrets::{ + SecretPolicy, invocation_allows_missing_secrets, +}; use crate::bindings::_helpers::discovery::python_function_files::discover_python_function_files; use crate::bindings::_helpers::discovery::python_paths::reject_unsupported_python_paths; use crate::bindings::_helpers::discovery::snapshot::in_snapshot; @@ -186,6 +192,27 @@ struct DiscoveryRequest { project_dir: Py, sql_analysis_enabled_override: Py, extract_output_column_locations: bool, + secret_policy: SecretPolicy, +} + +/// Whether `--allow-missing-secrets`, `SQLBUILD_ALLOW_MISSING_SECRETS` or the local `allow_missing_secrets` setting lets a static command run without provider secrets. +fn missing_secrets_allowed(local_config: &Bound<'_, PyAny>) -> PyResult { + if invocation_allows_missing_secrets() { + return Ok(true); + } + if std::env::var(ALLOW_MISSING_SECRETS_ENV) + .is_ok_and(|value| TRUE_ENV_VALUES.contains(&value.trim().to_ascii_lowercase().as_str())) + { + return Ok(true); + } + Ok( + native_field(local_config, "setting_overrides")?.contains(ALLOW_MISSING_SECRETS_SETTING)? + && native_field( + &native_field(local_config, "settings")?, + ALLOW_MISSING_SECRETS_SETTING, + )? + .is_truthy()?, + ) } /// Load all raw project inputs from disk before semantic resolution. @@ -204,15 +231,22 @@ pub(crate) fn discover_project_inputs<'py>( &[ "sql_analysis_enabled_override", "extract_output_column_locations", + "secret_policy", ], ), )?; count_stage(py, PROJECT_DISCOVERIES, 1)?; + let secret_policy: SecretPolicy = if arguments.passed("secret_policy")? { + SecretPolicy::parse(&arguments.value("secret_policy")?.extract::()?)? + } else { + SecretPolicy::Require + }; let request: DiscoveryRequest = DiscoveryRequest { project_dir: arguments.value("project_dir")?.unbind(), sql_analysis_enabled_override: arguments.value("sql_analysis_enabled_override")?.unbind(), extract_output_column_locations: !arguments.passed("extract_output_column_locations")? || arguments.flag("extract_output_column_locations")?, + secret_policy, }; within(&lifecycle(py, "project_discovery")?, |_| { let assembled: Bound<'py, PyAny> = @@ -263,6 +297,13 @@ fn assembled_inputs<'py>( } else { native_field(&project_config, "settings")?.getattr("sql_analysis")? }; + let secret_policy: SecretPolicy = if request.secret_policy == SecretPolicy::Require + && missing_secrets_allowed(&local_config)? + { + SecretPolicy::Allow + } else { + request.secret_policy + }; let discovered: Bound<'py, PyAny> = discovered_inputs( py, (project_dir, &project_config, &local_config), @@ -270,6 +311,7 @@ fn assembled_inputs<'py>( sql_analysis_enabled.is_truthy()?, request.extract_output_column_locations, ), + secret_policy, )?; host::entry(py, UserCodeHost::ValidateDiscoveredInputs)?.call1((&discovered,))?; configure_discovered_event_exporters(&discovered)?; @@ -309,6 +351,7 @@ fn build_discovered_project_inputs<'py>( !arguments.passed("extract_output_column_locations")? || arguments.flag("extract_output_column_locations")?, ), + SecretPolicy::Require, ) } @@ -321,6 +364,7 @@ fn discovered_inputs<'py>( &Bound<'py, PyAny>, ), flags: (bool, bool), + secret_policy: SecretPolicy, ) -> PyResult> { let declarations: Bound<'py, PyDict> = within( &lifecycle(py, "discovery_declaration_parse")?, @@ -338,7 +382,7 @@ fn discovered_inputs<'py>( let python: Bound<'py, PyDict> = within( &lifecycle(py, "discovery_python_import")?, |python_lifecycle| { - let python: Bound<'py, PyDict> = imported_python(project_dir)?; + let python: Bound<'py, PyDict> = imported_python(project_dir, secret_policy)?; completed(python_lifecycle, imported_python_paths(&python)?)?; Ok(python) }, @@ -392,10 +436,14 @@ fn declaration_files<'py>( } /// Providers, then every kind of imported project Python that may use them. -fn imported_python<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { +fn imported_python<'py>( + project_dir: &Bound<'py, PyAny>, + secret_policy: SecretPolicy, +) -> PyResult> { let py: Python<'py> = project_dir.py(); - let providers: Bound<'py, PyAny> = discover_provider_classes(project_dir.clone()) - .and_then(|value| value.into_bound_py_any(py))?; + let providers: Bound<'py, PyAny> = + discover_provider_classes(project_dir.clone(), secret_policy.name()) + .and_then(|value| value.into_bound_py_any(py))?; let python: Bound<'py, PyDict> = PyDict::new(py); python.set_item("providers", &providers)?; for (field, discoverer) in PYTHON_DISCOVERERS { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs index eadc8f712..e6224c7dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -214,6 +214,7 @@ const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("scenario_timeout", FieldDefault::None), FieldSpec::new("scenario_timeout_seconds", FieldDefault::None), FieldSpec::new("delta_compile", FieldDefault::True), + FieldSpec::new("allow_missing_secrets", FieldDefault::False), ], }; const R_SCOPES_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 1ebb3f2c6..95983bfa4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -42,7 +42,11 @@ use crate::bindings::_helpers::compile_attachment::target::build_effective_targe use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::diff_report::query::NAMESPACELESS_ADAPTERS; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::native_host::host::redirect_stdout_to_stderr; +use crate::bindings::_helpers::native_host::provider_settings::{ + missing_secrets, secret_variables, +}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::fields::field as native_field; @@ -56,6 +60,7 @@ const DEBUG_OPTIONS: (&[&str], &[&str]) = ( "no_connection", "selected_target", "json_output", + "verbose", ], ); @@ -167,7 +172,16 @@ fn providers(discovered_inputs: &Bound<'_, PyAny>) -> PyResult = Vec::new(); for provider in native_field(discovered_inputs, "providers")?.try_iter()? { let provider: Bound<'_, PyAny> = provider?; + let provider_class: Bound<'_, PyAny> = native_field(&provider, "provider_class")?; + let missing: Vec = missing_secrets(&provider_class)?; providers.push(DebugProvider { + secrets: secret_variables(&provider_class)? + .into_iter() + .map(|variable| { + let set: bool = !missing.contains(&variable); + (variable, set) + }) + .collect(), name: native_field(&provider, "name")?.str()?.extract()?, relative_path: native_field(&provider, "relative_path")?.str()?.extract()?, class_name: native_field(&provider, "provider_class")? @@ -262,7 +276,7 @@ fn effective_connection<'py>( fn build_report( py: Python<'_>, project_dir: &PathBuf, - (selected_target, check_connection): (Bound<'_, PyAny>, bool), + (selected_target, check_connection, verbose): (Bound<'_, PyAny>, bool, bool), ) -> PyResult { let project_dir_object: Bound<'_, PyAny> = imported(py, "pathlib", "Path")?.call1((project_dir,))?; @@ -270,7 +284,13 @@ fn build_report( py, Some(keywords( py, - &[("project_dir", project_dir_object.clone())], + &[ + ("project_dir", project_dir_object.clone()), + ( + "secret_policy", + PyString::new(py, SecretPolicy::Defer.name()).into_any(), + ), + ], )?), )?; let files: DebugConfigFiles = debug_config_files(project_dir); @@ -359,6 +379,7 @@ fn build_report( }; let warehouses: Vec = warehouse_checks(&context, (&command_scope, &base_connection))?; + let providers: Vec = providers(&context.discovered_inputs)?; let check: ConnectionCheck = if check_connection { connection_check(&context.adapter, &connection_config)? } else { @@ -368,7 +389,12 @@ fn build_report( local_files: local_database_files(&facts.adapter_name, &connection_config)?, runtime: debug_runtime_lines(&runtime(py)?), configuration: debug_configuration_lines(&facts, &warehouses), - providers: debug_provider_lines(&providers(&context.discovered_inputs)?), + missing_secrets: providers + .iter() + .flat_map(|provider| &provider.secrets) + .filter(|(_, set)| !set) + .count(), + providers: debug_provider_lines(&providers, verbose), connection: debug_connection_lines(&connection_settings(&connection_config)?, &check), }) } @@ -398,6 +424,7 @@ pub(crate) fn run_debug_command( ( arguments.value("selected_target")?, !arguments.flag("no_connection")?, + arguments.passed("verbose")? && arguments.flag("verbose")?, ), ) }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs index ef9a10eba..9e24509e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs @@ -9,3 +9,13 @@ pub(crate) const SQL_HOOK_OUTPUT_FIELDS: [&str; 6] = [ "kwargs", "description", ]; + +/// The flag of the static commands that lets them compile without provider secrets. +pub(crate) const ALLOW_MISSING_SECRETS_FLAG: &str = "--allow-missing-secrets"; +/// The environment variable that sets `--allow-missing-secrets`. +pub(crate) const ALLOW_MISSING_SECRETS_ENV: &str = "SQLBUILD_ALLOW_MISSING_SECRETS"; +/// The `[settings]` key of `sqlbuild_local.toml` that sets `--allow-missing-secrets`. +pub(crate) const ALLOW_MISSING_SECRETS_SETTING: &str = "allow_missing_secrets"; + +/// Environment values that switch a boolean `SQLBUILD_*` variable on. +pub(crate) const TRUE_ENV_VALUES: [&str; 4] = ["1", "on", "true", "yes"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs index d3eefb90c..702b84ed2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs @@ -3,7 +3,7 @@ use std::collections::HashSet; use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyType}; @@ -12,11 +12,14 @@ use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; use crate::bindings::_helpers::compile_context::input_reads::record_settings_class_read; use crate::bindings::_helpers::contracts::resource_identities::validate_resource_identity; +use crate::bindings::_helpers::discovery::constants::ALLOW_MISSING_SECRETS_FLAG; use crate::bindings::_helpers::discovery::imported_modules::{ ProjectDir, discovered, discovery_error, discovery_error_from, module_functions, provider_by_name, provider_usages, shim, tuple_of, }; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::provider_settings::{ProviderState, provider_state}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get_or; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; @@ -26,7 +29,6 @@ const SINK_ERROR: ErrorId = ErrorId::EventExporterDiscovery; /// The provider name kept free so a sink's `RecallWriter` parameter can never be mistaken for a provider. const RESERVED_PROVIDER_NAME: &str = "recall_writer"; const PROVIDER_INPUT_ERROR: UserCodeHost = UserCodeHost::ProviderInputError; -const VALIDATION_ERROR: (&str, &str) = ("pydantic", "ValidationError"); const SINK_DEFINITION: UserCodeHost = UserCodeHost::GetLifecycleEventSinkDefinition; const ROOT_LOCATION: &str = ""; const DEFAULT_MESSAGE: &str = "invalid value"; @@ -108,41 +110,82 @@ fn validation_details(error: &Bound<'_, PyAny>) -> PyResult { Ok(details.join("\n")) } -/// An instance of the provider class, reporting invalid settings with each field's error. +/// An instance of the provider class, with missing secrets an error unless `policy` defers or allows them. fn provider_instance<'py>( provider_class: &Bound<'py, PyAny>, (provider_name, relative_path): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { + policy: SecretPolicy, +) -> PyResult<(Bound<'py, PyAny>, Vec)> { let py: Python<'py> = provider_class.py(); - provider_class.call0().or_else(|error| { - let validation_error: Bound<'py, PyAny> = - imported(py, VALIDATION_ERROR.0, VALIDATION_ERROR.1)?; - if !error.value(py).is_instance(&validation_error)? { - return Err(error); + let missing: Vec = match provider_state(provider_class)? { + ProviderState::Ready(instance) => return Ok((instance, Vec::new())), + ProviderState::MissingSecrets(missing) => missing, + ProviderState::Invalid(error) => { + let message: String = format!( + "Provider '{}' in {} has invalid settings:\n{}", + provider_name.str()?, + relative_path.str()?, + validation_details(error.value(py))? + ); + return Err(discovery_error_from(py, (PROVIDER_ERROR, message), error)); } - let message: String = format!( - "Provider '{}' in {} has invalid settings:\n{}", - provider_name.str()?, - relative_path.str()?, - validation_details(error.value(py))? + }; + if policy == SecretPolicy::Require { + let variables: String = missing.join(", "); + let noun: &str = if missing.len() == 1 { + "secret" + } else { + "secrets" + }; + let error: PyErr = discovery_error( + py, + PROVIDER_ERROR, + format!( + "provider '{}' ({}) is missing {noun} {variables}", + provider_name.str()?, + relative_path.str()? + ), ); - Err(discovery_error_from(py, (PROVIDER_ERROR, message), error)) - }) + error.value(py).setattr( + "help", + format!("set {variables}, or pass {ALLOW_MISSING_SECRETS_FLAG} where secrets aren't available"), + )?; + return Err(error); + } + Ok((provider_class.call_method0("model_construct")?, missing)) } /// Provider classes under providers/. #[pyfunction] -#[pyo3(signature = (*, project_dir))] +#[pyo3(signature = (*, project_dir, secret_policy = SecretPolicy::Require.name()))] pub(crate) fn discover_provider_classes<'py>( project_dir: Bound<'py, PyAny>, + secret_policy: &str, ) -> PyResult> { let py: Python<'py> = project_dir.py(); + let policy: SecretPolicy = SecretPolicy::parse(secret_policy)?; + let mut allowed: Vec = Vec::new(); let project = ProjectDir { path: project_dir }; if let Some(cached) = project.cached_extensions()? { let providers: Bound<'py, PyAny> = cached.get_item(0)?; for provider in providers.try_iter()? { - record_settings_class_read(&native_field(&provider?, "provider_class")?)?; + let provider: Bound<'py, PyAny> = provider?; + let provider_class: Bound<'py, PyAny> = native_field(&provider, "provider_class")?; + record_settings_class_read(&provider_class)?; + if policy == SecretPolicy::Defer { + continue; + } + let name: Bound<'py, PyAny> = native_field(&provider, "name")?; + let (_, missing) = provider_instance( + &provider_class, + (&name, &native_field(&provider, "relative_path")?), + policy, + )?; + if !missing.is_empty() { + allowed.push(name.extract()?); + } } + report_allowed(py, policy, &allowed)?; return Ok(providers); } if !project @@ -188,8 +231,11 @@ pub(crate) fn discover_provider_classes<'py>( } seen.push((name_text, file_path.clone())); record_settings_class_read(&provider_class)?; - let settings: Bound<'py, PyAny> = - provider_instance(&provider_class, (&name, &relative_path))?; + let (settings, missing) = + provider_instance(&provider_class, (&name, &relative_path), policy)?; + if !missing.is_empty() { + allowed.push(name.extract()?); + } providers.push(discovered( py, RecordId::DiscoveredProvider, @@ -203,9 +249,31 @@ pub(crate) fn discover_provider_classes<'py>( )?); } } + report_allowed(py, policy, &allowed)?; tuple_of(py, providers) } +/// Note on stderr which providers run with missing secrets because the command allows them. +fn report_allowed(py: Python<'_>, policy: SecretPolicy, allowed: &[String]) -> PyResult<()> { + if policy != SecretPolicy::Allow || allowed.is_empty() { + return Ok(()); + } + let noun: &str = if allowed.len() == 1 { + "provider has" + } else { + "providers have" + }; + host::write_console( + py, + "stderr", + &format!( + "{} {noun} missing secrets (allowed): {}\n", + allowed.len(), + allowed.join(", ") + ), + ) +} + /// Sink modules imported and their lifecycle declarations collected, without providers. #[pyfunction] #[pyo3(signature = (*, project_dir))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 4fe23096f..a83fee866 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -23,6 +23,7 @@ pub(crate) mod project_config; pub(crate) mod project_facts; pub(crate) mod project_files; pub(crate) mod project_validation; +pub(crate) mod provider_secrets; pub(crate) mod python_function_files; pub(crate) mod python_hook_files; pub(crate) mod python_module_files; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs new file mode 100644 index 000000000..64d1f0feb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs @@ -0,0 +1,182 @@ +//! A provider class's setting fields and which of its secret settings the environment leaves unset. + +use std::cell::Cell; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; + +use crate::bindings::_helpers::native_host::provider_settings::missing_secrets; +use crate::bindings::_helpers::records::construct::new_annotated_error; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::ids::ErrorId; + +/// Discovered collections of sinks that run with every command. +const SINK_COLLECTIONS: [&str; 2] = ["event_exporters", "command_output_sinks"]; + +thread_local! { + /// Whether the invocation running on this thread passed `--allow-missing-secrets`. + static INVOCATION_ALLOWS_MISSING_SECRETS: Cell = const { Cell::new(false) }; +} + +/// `--allow-missing-secrets` for one invocation, restored when the invocation ends. +pub(crate) struct MissingSecretsScope { + previous: bool, +} + +impl MissingSecretsScope { + pub(crate) fn enter(allowed: bool) -> Self { + Self { + previous: INVOCATION_ALLOWS_MISSING_SECRETS.replace(allowed), + } + } +} + +impl Drop for MissingSecretsScope { + fn drop(&mut self) { + INVOCATION_ALLOWS_MISSING_SECRETS.set(self.previous); + } +} + +/// Whether the running invocation passed `--allow-missing-secrets`. +pub(crate) fn invocation_allows_missing_secrets() -> bool { + INVOCATION_ALLOWS_MISSING_SECRETS.get() +} + +/// What discovery does when a provider's required secrets are unset. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SecretPolicy { + /// Fail with D012: static commands by default. + Require, + /// Continue and list the providers on stderr: static commands that opted out. + Allow, + /// Continue silently; the command checks the providers it uses before running anything. + Defer, +} + +impl SecretPolicy { + /// The keyword value that selects the policy. + pub(crate) const fn name(self) -> &'static str { + match self { + Self::Require => "require", + Self::Allow => "allow", + Self::Defer => "defer", + } + } + + pub(crate) fn parse(value: &str) -> PyResult { + [Self::Require, Self::Allow, Self::Defer] + .into_iter() + .find(|policy| policy.name() == value) + .ok_or_else(|| PyValueError::new_err(format!("unknown secret policy '{value}'"))) + } +} + +/// `(provider, consumer)` for each usage, named by the usage's own `consumer_name` or else by `consumer`. +pub(crate) fn provider_uses( + usages: &Bound<'_, PyAny>, + consumer: Option<&str>, +) -> PyResult> { + usages + .try_iter()? + .map(|usage| usage_pair(&usage?, consumer)) + .collect() +} + +fn usage_pair(usage: &Bound<'_, PyAny>, consumer: Option<&str>) -> PyResult<(String, String)> { + let provider: String = field(usage, "provider_name")?.extract()?; + let name: String = match consumer { + Some(name) => name.to_owned(), + None => field(usage, "consumer_name")?.extract()?, + }; + Ok((provider, name)) +} + +/// `(provider, consumer)` for each provider the named consumers, such as checks or loaders, inject. +pub(crate) fn consumer_uses(consumers: &Bound<'_, PyAny>) -> PyResult> { + let mut uses: Vec<(String, String)> = Vec::new(); + for consumer in consumers.try_iter()? { + let consumer: Bound<'_, PyAny> = consumer?; + let name: String = field(&consumer, "name")?.extract()?; + uses.extend(provider_uses( + &field(&consumer, "provider_usages")?, + Some(&name), + )?); + } + Ok(uses) +} + +/// The providers the project's lifecycle and command-output sinks use, which run with every command. +pub(crate) fn sink_uses(discovered: &Bound<'_, PyAny>) -> PyResult> { + let mut uses: Vec<(String, String)> = Vec::new(); + for collection in SINK_COLLECTIONS { + if let Some(sinks) = discovered.getattr_opt(collection)? { + uses.extend(consumer_uses(&sinks)?); + } + } + Ok(uses) +} + +/// Fail before anything runs when a provider in `uses` lacks required secrets, listing each with what needs it. +pub(crate) fn require_used_secrets( + discovered: &Bound<'_, PyAny>, + uses: &[(String, String)], +) -> PyResult<()> { + let py: Python<'_> = discovered.py(); + let mut consumers: Vec<(String, Vec)> = Vec::new(); + for (provider, consumer) in uses { + match consumers.iter_mut().find(|(name, _)| name == provider) { + Some((_, names)) if !names.contains(consumer) => names.push(consumer.clone()), + Some(_) => {} + None => consumers.push((provider.clone(), vec![consumer.clone()])), + } + } + let mut rows: Vec<[String; 3]> = Vec::new(); + for provider in field(discovered, "providers")?.try_iter()? { + let provider: Bound<'_, PyAny> = provider?; + let name: String = field(&provider, "name")?.extract()?; + let Some((_, names)) = consumers.iter().find(|(used, _)| *used == name) else { + continue; + }; + let needed_by: String = match names.as_slice() { + [only] => only.clone(), + [first, rest @ ..] => format!("{first}, +{}", rest.len()), + [] => String::new(), + }; + for variable in missing_secrets(&field(&provider, "provider_class")?)? { + rows.push([name.clone(), variable, format!("({needed_by})")]); + } + } + if rows.is_empty() { + return Ok(()); + } + let widths: [usize; 2] = [0, 1].map(|column| column_width(&rows, column)); + let lines: Vec = rows + .iter() + .map(|[provider, variable, needed_by]| { + format!( + " {provider: usize { + rows.iter() + .map(|row| row[column].len()) + .max() + .unwrap_or_default() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs index 3c86c9b39..5379e7b86 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs @@ -1,5 +1,6 @@ //! `EventExporterCommandScope`: typed sink binding, its provider session and the active command scope. +use std::collections::HashMap; use std::sync::{Mutex, MutexGuard}; use pyo3::exceptions::PyRuntimeError; @@ -17,6 +18,7 @@ use crate::bindings::_helpers::boundary::python_calls::{ use crate::bindings::_helpers::event_exporting::constants::SCOPE_CONTEXT_VARIABLE; use crate::bindings::_helpers::event_exporting::models::BoundEventExporter; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::provider_settings::{ProviderState, provider_state}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; @@ -77,15 +79,58 @@ fn provider_arguments<'py>( Ok(arguments) } -/// Bound lifecycle exporters and command-output sinks over `session`'s provider instances. +/// Each discovered provider whose required secret settings are unset, with their variables. +fn providers_missing_secrets( + providers: &Bound<'_, PyAny>, +) -> PyResult>> { + let mut missing: HashMap> = HashMap::new(); + for provider in providers.try_iter()? { + let provider: Bound<'_, PyAny> = provider?; + if let ProviderState::MissingSecrets(variables) = + provider_state(&native_field(&provider, "provider_class")?)? + { + missing.insert(native_field(&provider, "name")?.extract()?, variables); + } + } + Ok(missing) +} + +/// Whether `declaration` can bind; when a provider it takes is missing secrets, a notice says it is left unbound. +fn bindable( + declaration: &Bound<'_, PyAny>, + missing: &HashMap>, +) -> PyResult { + for usage in native_field(declaration, "provider_usages")?.try_iter()? { + let provider: String = native_field(&usage?, "provider_name")?.extract()?; + if let Some(variables) = missing.get(&provider) { + host::write_console( + declaration.py(), + "stderr", + &format!( + "sink '{}' is not bound: provider '{provider}' is missing {}\n", + native_field(declaration, "name")?.str()?, + variables.join(", ") + ), + )?; + return Ok(false); + } + } + Ok(true) +} + +/// Bound lifecycle exporters and command-output sinks over `session`, except those using `missing` providers. fn bindings<'py>( session: &Bound<'py, PyAny>, (event_exporters, command_output_sinks): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + missing: &HashMap>, ) -> PyResult<(Bound<'py, PyTuple>, Bound<'py, PyTuple>)> { let py: Python<'py> = session.py(); let mut exporters: Vec> = Vec::new(); for exporter in event_exporters.try_iter()? { let exporter: Bound<'py, PyAny> = exporter?; + if !bindable(&exporter, missing)? { + continue; + } let arguments: Bound<'py, PyDict> = provider_arguments(session, &exporter)?; exporters.push( Bound::new( @@ -109,6 +154,9 @@ fn bindings<'py>( let mut sinks: Vec> = Vec::new(); for sink in command_output_sinks.try_iter()? { let sink: Bound<'py, PyAny> = sink?; + if !bindable(&sink, missing)? { + continue; + } let arguments: Bound<'py, PyDict> = provider_arguments(session, &sink)?; sinks.push(call_with( &sink_class, @@ -162,6 +210,7 @@ impl EventExporterCommandScope { let empty = PyTuple::empty(py).into_any(); return self.bind_dispatcher(py, (empty.clone(), empty)); } + let missing: HashMap> = providers_missing_secrets(&providers)?; let session: Bound<'_, PyAny> = call_with( &host::entry(py, UserCodeHost::BuildProviderSession)?, &[ @@ -170,8 +219,11 @@ impl EventExporterCommandScope { ], )?; session.call_method0("hold_open")?; - let (exporters, sinks) = match bindings(&session, (&event_exporters, &command_output_sinks)) - { + let (exporters, sinks) = match bindings( + &session, + (&event_exporters, &command_output_sinks), + &missing, + ) { Ok(bound) => bound, Err(error) => { session.call_method0("release")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index beae4031d..8fc5d16e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -29,6 +29,9 @@ use crate::bindings::_helpers::compile_attachment::effective_config::build_effec use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; +use crate::bindings::_helpers::discovery::provider_secrets::{ + SecretPolicy, consumer_uses, require_used_secrets, sink_uses, +}; use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, }; @@ -85,6 +88,27 @@ fn write(stream: &Bound<'_, PyAny>, value: &str) -> PyResult<()> { Ok(()) } +/// Fail before loading anything when the selected sources' loaders or the sinks need provider secrets that are unset. +fn require_loader_secrets( + discovered: &Bound<'_, PyAny>, + selected: &[Bound<'_, PyAny>], +) -> PyResult<()> { + let mut loaders: Vec = Vec::new(); + for entry in selected { + let loader: Bound<'_, PyAny> = native_field(entry, "loader")?; + if !loader.is_none() { + loaders.push(loader.extract()?); + } + } + let mut uses: Vec<(String, String)> = + consumer_uses(&native_field(discovered, "loader_functions")?)? + .into_iter() + .filter(|(_, loader)| loaders.contains(loader)) + .collect(); + uses.extend(sink_uses(discovered)?); + require_used_secrets(discovered, &uses) +} + fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult> { let project_dir: Bound<'py, PyAny> = match native_field(request, "project_dir")? { directory if !directory.is_none() => directory, @@ -92,7 +116,16 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = discover_project_inputs( py, - Some(keywords(py, &[("project_dir", project_dir.clone())])?), + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ( + "secret_policy", + PyString::new(py, SecretPolicy::Defer.name()).into_any(), + ), + ], + )?), )?; let select: Bound<'py, PyAny> = native_field(request, "select")?; let exclude: Bound<'py, PyAny> = native_field(request, "exclude")?; @@ -122,6 +155,7 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult( let project = project_adapter( py, Some(arguments.value("project_dir")?), - ProjectInputs::Discovered, + ProjectInputs::DiscoveredDeferringSecrets, )?; status(py, "Location project discovered. Compiling snapshot...")?; let (snapshot, blocked) = compile_snapshot(py, arguments, (input, &project), None)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index fb1c70fdf..27ea6aed4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -19,10 +19,6 @@ pub(crate) const COMMAND_IMPLEMENTATIONS: [(RunCommand, &str); 5] = [ pub(crate) const MODEL_NODE_KIND: &str = "model"; pub(crate) const SOURCE_NODE_KIND: &str = "source"; -/// `env_prefix_target` values under which pydantic-settings prefixes alias names too. -pub(crate) const ALIAS_PREFIX_TARGET: &str = "alias"; -pub(crate) const ALL_PREFIX_TARGET: &str = "all"; - /// DuckDB's message when another process, such as a running build, holds the database file. pub(crate) const DUCKDB_LOCK_CONFLICT: &str = "Conflicting lock is held"; @@ -36,3 +32,5 @@ pub(crate) const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; /// A SQL unit test's published kind; a frozen run selects it back with `test:`. pub(crate) const SQL_TEST_NODE_KIND: &str = "sql_test"; +/// The setting-row field the published providers document leaves out: secret matching reads it, the UI does not. +pub(crate) const SNAPSHOT_HIDDEN_SETTING: &str = "alias"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs index 55d170d62..9b5f318e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs @@ -4,15 +4,14 @@ use std::collections::{BTreeMap, BTreeSet}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use serde_json::{Value, json}; -use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::compiled_project::snapshot_boundary::relative_path; -use crate::bindings::_helpers::compiled_project::values::plain_value; -use crate::bindings::_helpers::discovery::imported_modules::shim; use crate::bindings::_helpers::location_command::constants::{ - ALIAS_PREFIX_TARGET, ALL_PREFIX_TARGET, MODEL_NODE_KIND, SOURCE_NODE_KIND, + MODEL_NODE_KIND, SNAPSHOT_HIDDEN_SETTING, SOURCE_NODE_KIND, +}; +use crate::bindings::_helpers::native_host::provider_settings::{ + missing_secrets, provider_class_settings, setting_rows, }; use crate::bindings::_helpers::records::fields::field; @@ -46,7 +45,6 @@ pub(crate) fn providers_document( discovered_inputs: &Bound<'_, PyAny>, nodes: &BTreeMap, ) -> PyResult> { - let py = discovered_inputs.py(); let providers = field(discovered_inputs, "providers")?; if providers.len()? == 0 { return Ok(None); @@ -68,21 +66,29 @@ pub(crate) fn providers_document( } } } - let read_class = shim(py, UserCodeHost::ProviderSettings)?; let mut rows: Vec = Vec::new(); for provider in providers.try_iter()? { let provider = provider?; let name: String = field(&provider, "name")?.extract()?; - let class = plain_value(&call_with( - &read_class, - &[("provider_class", field(&provider, "provider_class")?)], - )?)?; + let provider_class = field(&provider, "provider_class")?; + let class = provider_class_settings(&provider_class)?; + let missing: Vec = missing_secrets(&provider_class)?; + let mut settings: Vec = setting_rows(&class); + for setting in &mut settings { + let unset = setting["env"] + .as_str() + .is_some_and(|env| missing.iter().any(|name| name == env)); + setting["missing"] = Value::Bool(unset); + if let Some(row) = setting.as_object_mut() { + row.remove(SNAPSHOT_HIDDEN_SETTING); + } + } rows.push(json!({ "name": name, "class": class["class"], "description": class["description"], "path": relative_path(&provider)?, - "settings": settings(&class), + "settings": settings, "nodes": running.remove(&name).unwrap_or_default(), "used_by": used_by.remove(&name).unwrap_or_default(), })); @@ -134,36 +140,3 @@ fn calls_hook(definition: &Value, hook: &str) -> bool { } false } - -/// Declared fields with the environment variable each is read from, which pydantic-settings derives from the prefix unless an alias names it. -fn settings(class: &Value) -> Vec { - let prefix = class["env_prefix"].as_str().unwrap_or_default(); - let case_sensitive = class["case_sensitive"].as_bool().unwrap_or_default(); - let target = class["env_prefix_target"].as_str(); - let alias_prefix = match target { - Some(ALIAS_PREFIX_TARGET | ALL_PREFIX_TARGET) => prefix, - _ => "", - }; - let name_prefix = match target { - Some(ALIAS_PREFIX_TARGET) => "", - _ => prefix, - }; - let mut rows: Vec = Vec::new(); - for field in class["fields"].as_array().into_iter().flatten() { - let name = field["name"].as_str().unwrap_or_default(); - let type_name = field["type"].as_str().unwrap_or_default(); - let env = field["alias"].as_str().map_or_else( - || format!("{name_prefix}{name}"), - |alias| format!("{alias_prefix}{alias}"), - ); - rows.push(json!({ - "name": name, - "type": type_name, - "required": field["required"], - "secret": type_name.contains("Secret"), - "env": if case_sensitive { env } else { env.to_uppercase() }, - "description": field["description"], - })); - } - rows -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index 2f12728a8..9de61aa2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -19,3 +19,7 @@ pub(crate) const SENSOR_CLASS: &str = "Sensor"; /// Pages one bounded state-history SELECT streams before the scan continues from its cursor. pub(crate) const HISTORY_STREAM_PAGES: usize = 50; + +/// `env_prefix_target` values under which pydantic-settings prefixes alias names too. +pub(crate) const ALIAS_PREFIX_TARGET: &str = "alias"; +pub(crate) const ALL_PREFIX_TARGET: &str = "all"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 5a6e327e6..48803fe01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -13,6 +13,7 @@ pub(crate) mod orchestration_declarations; pub(crate) mod orchestration_environment; pub(crate) mod orchestration_sensors; pub(crate) mod orchestration_triggers; +pub(crate) mod provider_settings; pub(crate) mod registration; pub(crate) mod resource_costs_state; pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index edb486b35..640f5d6be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::compiled_project::snapshot_declarations::python_d use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::native_host::constants::SENSOR_CONTEXT_PARAMETER; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_environment::{ @@ -70,8 +71,10 @@ pub(crate) fn evaluate_sensor( .import("sqlbuild.orchestration")? .getattr("SensorContext")? .call((), Some(&fields))?; - let providers = - discover_provider_classes(imported(py, "pathlib", "Path")?.call1((project_dir,))?)?; + let providers = discover_provider_classes( + imported(py, "pathlib", "Path")?.call1((project_dir,))?, + SecretPolicy::Defer.name(), + )?; let by_name = provider_by_name(&providers)?; let session = call_with( &entry(py, UserCodeHost::BuildProviderSession)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index a76334202..ab265c388 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -20,6 +20,7 @@ use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::constants::{ CALENDAR_WINDOW_PARAMETER, SOURCE_NEW_DATA_CONDITION, SOURCE_STALE_CONDITION, @@ -156,8 +157,10 @@ pub(crate) fn compile_calendars( if requests.is_empty() { return Ok(definitions); } - let providers = - discover_provider_classes(imported(py, "pathlib", "Path")?.call1((project_dir,))?)?; + let providers = discover_provider_classes( + imported(py, "pathlib", "Path")?.call1((project_dir,))?, + SecretPolicy::Defer.name(), + )?; let by_name = provider_by_name(&providers)?; let session = call_with( &entry(py, UserCodeHost::BuildProviderSession)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/provider_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/provider_settings.rs new file mode 100644 index 000000000..b0d1905c5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/provider_settings.rs @@ -0,0 +1,150 @@ +//! A provider class's declared settings and whether the environment sets its required secrets. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use serde_json::{Value, json}; +use sqlbuild_host::entries::models::UserCodeHost; + +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; +use crate::bindings::_helpers::native_host::constants::{ALIAS_PREFIX_TARGET, ALL_PREFIX_TARGET}; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::mapping_get_or; + +const VALIDATION_ERROR: (&str, &str) = ("pydantic", "ValidationError"); +const MISSING_ERROR_TYPE: &str = "missing"; + +/// How a provider class instantiates from the environment. +pub(crate) enum ProviderState<'py> { + Ready(Bound<'py, PyAny>), + /// Only required secret settings are unset: their environment variables, sorted. + MissingSecrets(Vec), + /// Another setting is invalid: pydantic's validation error. + Invalid(PyErr), +} + +/// The class's description and declared setting fields, read without any setting value. +pub(crate) fn provider_class_settings(provider_class: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = provider_class.py(); + let settings: Bound<'_, PyAny> = call_with( + &host::entry(py, UserCodeHost::ProviderSettings)?, + &[("provider_class", provider_class.clone())], + )?; + let text: String = imported(py, "json", "dumps")? + .call1((settings,))? + .extract()?; + serde_json::from_str(&text).map_err(|error| PyValueError::new_err(error.to_string())) +} + +/// Declared fields with the environment variable each is read from, which pydantic-settings derives from the prefix unless an alias names it. +pub(crate) fn setting_rows(class: &Value) -> Vec { + let prefix = class["env_prefix"].as_str().unwrap_or_default(); + let case_sensitive = class["case_sensitive"].as_bool().unwrap_or_default(); + let target = class["env_prefix_target"].as_str(); + let alias_prefix = match target { + Some(ALIAS_PREFIX_TARGET | ALL_PREFIX_TARGET) => prefix, + _ => "", + }; + let name_prefix = match target { + Some(ALIAS_PREFIX_TARGET) => "", + _ => prefix, + }; + let mut rows: Vec = Vec::new(); + for field in class["fields"].as_array().into_iter().flatten() { + let name = field["name"].as_str().unwrap_or_default(); + let type_name = field["type"].as_str().unwrap_or_default(); + let env = field["alias"].as_str().map_or_else( + || format!("{name_prefix}{name}"), + |alias| format!("{alias_prefix}{alias}"), + ); + rows.push(json!({ + "name": name, + "alias": field["alias"], + "type": type_name, + "required": field["required"], + "secret": type_name.contains("Secret"), + "env": if case_sensitive { env } else { env.to_uppercase() }, + "description": field["description"], + })); + } + rows +} + +/// The environment variables of the class's secret settings. +pub(crate) fn secret_variables(provider_class: &Bound<'_, PyAny>) -> PyResult> { + Ok(setting_rows(&provider_class_settings(provider_class)?) + .iter() + .filter(|row| row["secret"].as_bool().unwrap_or_default()) + .filter_map(|row| row["env"].as_str().map(str::to_owned)) + .collect()) +} + +/// Instantiate the class, telling unset required secrets apart from other invalid settings. +pub(crate) fn provider_state<'py>( + provider_class: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = provider_class.py(); + let error: PyErr = match provider_class.call0() { + Ok(instance) => return Ok(ProviderState::Ready(instance)), + Err(error) => error, + }; + if !error + .value(py) + .is_instance(&imported(py, VALIDATION_ERROR.0, VALIDATION_ERROR.1)?)? + { + return Err(error); + } + let rows: Vec = setting_rows(&provider_class_settings(provider_class)?); + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("include_input", false)?; + let mut missing: Vec = Vec::new(); + for item in error + .value(py) + .call_method("errors", (), Some(&options))? + .try_iter()? + { + let item: Bound<'py, PyAny> = item?; + let kind: String = mapping_get_or(&item, "type", "")?.str()?.extract()?; + let location: Vec = mapping_get_or(&item, "loc", ())? + .try_iter()? + .map(|part| part?.str()?.extract::()) + .collect::>>()?; + let secret: Option<&str> = match location.as_slice() { + [field] if kind == MISSING_ERROR_TYPE => secret_env(&rows, field), + _ => None, + }; + let Some(env) = secret else { + return Ok(ProviderState::Invalid(error)); + }; + missing.push(env.to_owned()); + } + if missing.is_empty() { + return Ok(ProviderState::Invalid(error)); + } + missing.sort(); + missing.dedup(); + Ok(ProviderState::MissingSecrets(missing)) +} + +/// The environment variables of the class's required secrets the environment leaves unset; an invalid setting is an error. +pub(crate) fn missing_secrets(provider_class: &Bound<'_, PyAny>) -> PyResult> { + match provider_state(provider_class)? { + ProviderState::Ready(_) => Ok(Vec::new()), + ProviderState::MissingSecrets(missing) => Ok(missing), + ProviderState::Invalid(error) => Err(error), + } +} + +/// The environment variable of the secret setting pydantic reports as `field`, by its name or alias. +fn secret_env<'a>(rows: &'a [Value], field: &str) -> Option<&'a str> { + rows.iter() + .filter(|row| row["secret"].as_bool().unwrap_or_default()) + .find(|row| { + row["name"].as_str() == Some(field) + || row["alias"].as_str() == Some(field) + || row["env"] + .as_str() + .is_some_and(|env| env.eq_ignore_ascii_case(field)) + }) + .and_then(|row| row["env"].as_str()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs index fbbeb3b01..1d20d2386 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -8,6 +8,7 @@ use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::discovery::configuration::discover_connection_configuration; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::records::fields::field as native_field; /// The project directory, defaulting to the working directory, with its inputs and adapter. @@ -22,6 +23,8 @@ pub(crate) struct ProjectAdapter<'py> { pub(crate) enum ProjectInputs { /// Every declaration file and the imported project Python. Discovered, + /// As `Discovered`, leaving missing provider secrets to the runs that use the providers. + DiscoveredDeferringSecrets, /// Only the project and local configuration, with its connection settings validated. Configuration, } @@ -40,6 +43,19 @@ pub(crate) fn project_adapter<'py>( py, Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?, + ProjectInputs::DiscoveredDeferringSecrets => discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ( + "secret_policy", + pyo3::types::PyString::new(py, SecretPolicy::Defer.name()).into_any(), + ), + ], + )?), + )?, ProjectInputs::Configuration => discover_connection_configuration(py, &project_dir)?, }; let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 48b1f1d84..e7b3dc61e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -66,6 +66,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--exclude` | Remove resources from the selected scope | | `--no-sql-analysis` | Turn off SQL syntax, binding, type inference and semantic checks (`--no-sql-validation` is an alias) | | `--no-cache` | Compile in full, without reusing earlier compiles or the Rules cache; see [Forcing a fresh compile](#forcing-a-fresh-compile) | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](../concepts/python-nodes/providers.md#secrets) | | `--defer-to` | Resolve unselected model references against another target | | `--json` | Print the full compile report as JSON | | `--manifest` | Also write `target/manifest.json` with project metadata | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md index 7e126a40c..dd9ae1bcf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md @@ -30,6 +30,7 @@ sqb --project-dir dag [flags] |------|-------------| | `--json` | Print the full DAG artifact as JSON to stdout | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](../concepts/python-nodes/providers.md#secrets) | | `--vars` | JSON object of project variable overrides | Without `--json`, the command prints a summary: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md index 4f53bb48a..95697cdcf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md @@ -24,7 +24,14 @@ The command reports four groups of checks: **Configuration** - Finds and validates `sqlbuild_project.toml`, loads `sqlbuild_local.toml` if present, resolves the adapter, active target, connection, database, and schema. For Snowflake it also shows the warehouse each [command group](../concepts/adapters/snowflake.md#command-group-warehouses) connects with and where that value came from. -**Providers** - Lists the [providers](../concepts/python-nodes/providers.md) the project defines. +**Providers** - Counts the [providers](../concepts/python-nodes/providers.md) whose [secrets](../concepts/python-nodes/providers.md#secrets) are all set, and lists each missing secret as `WARN`. When every secret is set it prints one line. `--verbose` lists every provider and secret. + +```text +Providers: + providers: 2 of 3 ready [WARN missing secrets] + slack_alerts: SLACK_WEBHOOK_URL [WARN missing] + more: sqb debug --verbose lists every provider [SKIP] +``` **Connection** - Displays connection settings as `sqb debug` itself uses them, with the `query` warehouse group applied (secrets are masked), attempts to connect to the warehouse, and runs `SELECT 1` to verify query execution. @@ -33,7 +40,8 @@ The command reports four groups of checks: | Flag | Description | |------|-------------| | `--no-connection` | Skip the connection and query tests. Useful for validating config without warehouse access. | -| `--json` | Output results as JSON instead of formatted text. | +| `--json` | Output results as JSON instead of formatted text, with `missing_secrets` counting the unset provider secrets. | +| `--verbose` | List every provider and secret setting, set or missing. | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/adapters/snowflake.md#command-group-warehouses) and the connection warehouse | ## Example @@ -81,7 +89,7 @@ Connection: ## Exit codes -Returns `0` when all checks pass, `1` when any check fails (e.g. connection refused, invalid config). +Returns `0` when all checks pass, `1` when any check fails (e.g. connection refused, invalid config). A missing provider secret is a warning and doesn't fail the command. A failed run is recorded in [Recall](../concepts/observability/recall.md) with the first failed check as its cause: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md index 1c09acc5b..297fc5c28 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -27,6 +27,7 @@ sqb format [flags] | `--fix` | Also apply compiler-verified, meaning-preserving Rule fixes before layout formatting | | `--json` | Print structured results | | `--no-cache` | Format every file instead of skipping files a previous run found clean | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](../concepts/python-nodes/providers.md#secrets) | | `--select`, `-s` | Format selected models | | `--select-file` | Read selectors from a file | | `--exclude` | Exclude models from the selected scope | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md index 710838a91..8762e344d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md @@ -44,6 +44,7 @@ Exactly one of a positional target or `--select` is required. | `--json` | Shorthand for `--format json`. | | `--mode` | Column lineage analysis mode: `rich` (default) or `fast`. | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias). | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](../concepts/python-nodes/providers.md#secrets) | | `--select`, `-s` | Select resources using the same selector syntax as `sqb compile`, `sqb plan` and `sqb build`, for example `path:models/marts`, `tag:finance` or `stg_*`. | | `--exclude` | Exclude resources from the selection. | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md index e9e56aa8f..448a5c8a4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md @@ -34,6 +34,7 @@ sqb location health --server https://control.example.com --location-id "$LOCATIO | --- | --- | --- | | `healthy` | An instance checked in within 90 seconds and its last load published | `0` | | `degraded` | The warehouse connection check after the last load failed | `0` | +| `missing_secrets` | A provider [secret](../concepts/python-nodes/providers.md#secrets) is unset in the location's environment, so runs that use it fail | `1` | | `falling_back` | The last load failed, so runs use the previous snapshot | `1` | | `broken` | No instance has checked in for 90 seconds | `1` | @@ -51,3 +52,6 @@ healthcheck: A `falling_back` location still runs scheduled jobs from its last good snapshot. Restarting it does not fix the failed load; fix the project and deploy it again. + +A `missing_secrets` location publishes and runs jobs whose nodes have their secrets. Set the +missing variables in its environment and restart it; `sqb debug` in the location lists them. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md index 590199fdb..fcb239757 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md @@ -34,7 +34,7 @@ Lifecycle progress is written to stderr so stdout remains valid JSON. - `run`: compile and evaluate only one exact code or family prefix. - `skills`: install guidance derived from configured Rules; `--check` verifies freshness without writing. -`run` accepts normal SQLBuild model selectors. Native built-ins execute before custom Python Rules. +`run` accepts normal SQLBuild model selectors and `--allow-missing-secrets`, which runs without provider [secrets](../concepts/python-nodes/providers.md#secrets). Native built-ins execute before custom Python Rules. ## Exit codes diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md index 7106ff1ba..98c66c22f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md @@ -85,6 +85,7 @@ These identities are for inspection. SQL authored inside the owning model contin | `--kind KIND` | Keep `macro`, `enum`, or `constant` declarations. Repeat to include multiple kinds. | | `--match GLOB` | Match declaration names or qualified identities with a deterministic, case-sensitive glob. | | `--used-only` | Keep declarations used by the target. | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](../concepts/python-nodes/providers.md#secrets) | | `--include-nearby` | Include bounded nearby declarations that are unavailable to the target. | | `--nearby-depth N` | Set filesystem proximity for nearby descendant and sibling discovery (default: `1`). | | `--dependency-depth N` | Expand dependencies from declarations in the used section by `N` edges (default: `0`). | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index db0ae5c2e..151c8dc5b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -516,6 +516,7 @@ delta_compile = true | `test_timeout` | `30s` | Time limit for each SQL unit test, or `"disabled"` | | `scenario_timeout` | none | Time limit for each scenario, or `"disabled"` | | `delta_compile` | `true` | After an edit to model SQL or SQL test files only, compile just the changed files against the previous compile. See [Compiling after an edit](../cli/compile.md#compiling-after-an-edit). | +| `allow_missing_secrets` | `false` | Let `compile`, `rules run`, `format`, `dag`, `lineage` and `scope` run without provider secrets. `sqlbuild_local.toml` only. See [Secrets](python-nodes/providers.md#secrets). | ### Unit test and scenario limits diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md index c9be71b78..827a3b736 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -15,6 +15,7 @@ Online: https://sqlbuild.com/docs/concepts/python-nodes/providers/ - Using providers via context - Lifecycle - Configuration from environment variables +- Secrets - Discovery rules - Plan output - Providers in the web UI @@ -150,8 +151,8 @@ Both approaches (parameter injection and `ctx.providers`) are equivalent. Parame Providers follow a lazy setup, reverse-teardown lifecycle scoped to the command invocation: -1. **Discovery** - on compile, SQLBuild discovers all `Provider` subclasses under `providers/` and validates their settings (from environment variables or field defaults). -2. **Lazy setup** - `setup(ctx)` is called the first time a provider is accessed during a build, not at startup. Providers that are never used are never set up. +1. **Discovery** - on compile, SQLBuild discovers all `Provider` subclasses under `providers/` and validates their settings (from environment variables or field defaults). Missing [secrets](#secrets) follow the command's rules. +2. **Lazy setup** - the first time a provider is accessed during a build, SQLBuild reads its settings and calls `setup(ctx)`. Providers that are never used are never set up. 3. **Teardown** - after the command completes, `teardown()` is called on all providers that were set up, in reverse setup order. Teardown runs even if the build failed. ```python @@ -183,13 +184,61 @@ class SlackNotifier(Provider): See the [pydantic-settings documentation](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) for advanced configuration like custom env prefixes, `.env` file support, and nested settings. +## Secrets + +Declare a secret as a field typed `SecretStr`. Laptops and CI jobs can then compile without production secrets, and builds check only the secrets their selection needs. + +```python +from pydantic import SecretStr + +class SlackAlerts(Provider): + """Posts alerts to Slack.""" + + channel: str = "#data-alerts" + slack_webhook_url: SecretStr # reads SLACK_WEBHOOK_URL +``` + +| Commands | When a required secret is unset | +| --- | --- | +| `compile`, `rules run`, `format`, `dag`, `lineage`, `scope` | Error `D012`, unless missing secrets are allowed | +| `build`, `load`, `check`, and runs on `sqb server` workers | Error before anything runs, only when a selected node or a sink uses the provider | +| `sqb debug` | Lists the missing secrets as warnings | +| `sqb location health` | Reports `missing_secrets` and exits `1` | + +```text +$ sqb compile +error[D012]: provider 'slack_alerts' (providers/slack_alerts/provider.py) is missing secret SLACK_WEBHOOK_URL + = help: set SLACK_WEBHOOK_URL, or pass --allow-missing-secrets where secrets aren't available +``` + +Allow missing secrets for the static commands in any of these ways: + +| Where | Setting | +| --- | --- | +| Command line | `--allow-missing-secrets` | +| Environment | `SQLBUILD_ALLOW_MISSING_SECRETS=1` | +| `sqlbuild_local.toml` | `allow_missing_secrets = true` under `[settings]` | + +The command then lists the providers it skipped on stderr, such as +`1 provider has missing secrets (allowed): slack_alerts`. Settings that aren't secrets are still +validated. Execution commands accept none of these. + +```text +$ sqb build --select python/orders +error[D012]: selected nodes need secrets that are missing: + slack_alerts SLACK_WEBHOOK_URL (partner_export_alert) +nothing was run +``` + +Run [`sqb debug`](../../cli/debug.md) to see which secrets this machine has. + ## Discovery rules - Provider classes are discovered from `.py` files under `providers/` recursively - Files named `__init__.py` or starting with `_` are skipped - Each concrete (non-abstract) subclass of `Provider` is registered - Provider names must be globally unique across project resources -- Settings are validated at discovery time. Missing required fields (without environment variables set) raise a discovery error immediately, not at runtime +- Settings are validated at discovery time. A missing or invalid setting that isn't a secret is a `D012` error; missing secrets follow [Secrets](#secrets) ## Plan output @@ -210,7 +259,7 @@ provider with its class and docstring. A provider's page shows: | Tab | Shows | | --- | --- | -| Settings | Each setting's type, environment variable and whether it is required; secret settings are marked | +| Settings | Each setting's type, environment variable and whether it is required; secret settings are marked, and flagged **missing** when the location's environment doesn't set them | | Used by | The loaders, tasks, hooks and other code that take the provider, and the jobs that run that code | Setting values, defaults included, are never published. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index cc6d6d1f4..e50261379 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5175,7 +5175,9 @@ def discover_hook_functions( def discover_materialization_files( *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] | None = None ) -> tuple[DiscoveredMaterializationFile, ...]: ... -def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, ...]: ... +def discover_provider_classes( + *, project_dir: Path, secret_policy: str = ... +) -> tuple[DiscoveredProvider, ...]: ... def discover_event_exporter_declarations( *, project_dir: Path ) -> tuple[DiscoveredEventExporterDeclaration, ...]: ... @@ -12023,6 +12025,7 @@ class SettingsConfig(NativeRecord): scenario_timeout: str | None scenario_timeout_seconds: int | None delta_compile: bool + allow_missing_secrets: bool def __init__( self, sql_analysis: bool = ..., @@ -12042,6 +12045,7 @@ class SettingsConfig(NativeRecord): scenario_timeout: str | None = ..., scenario_timeout_seconds: int | None = ..., delta_compile: bool = ..., + allow_missing_secrets: bool = ..., ) -> None: ... @property def sql_validation(self) -> bool: ... diff --git a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py index 197605d5e..773d40dce 100644 --- a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py +++ b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py @@ -52,7 +52,9 @@ def discover_runtime_extensions(*, project_dir: Path) -> DiscoveredRuntimeExtens ) if not declarations and not output_declarations: return DiscoveredRuntimeExtensions() - providers: tuple[DiscoveredProvider, ...] = discover_provider_classes(project_dir=project_dir) + providers: tuple[DiscoveredProvider, ...] = discover_provider_classes( + project_dir=project_dir, secret_policy="defer" + ) exporters: tuple[DiscoveredEventExporter, ...] = bind_event_exporter_declarations( declarations=declarations, project_dir=project_dir, diff --git a/src/sqlbuild/provider/_helpers/session.py b/src/sqlbuild/provider/_helpers/session.py index 732740b55..b61f991da 100644 --- a/src/sqlbuild/provider/_helpers/session.py +++ b/src/sqlbuild/provider/_helpers/session.py @@ -9,7 +9,6 @@ if TYPE_CHECKING: from sqlbuild._native import EventExporterCommandScope - from sqlbuild.provider.classes.provider import Provider def build_provider_session( @@ -18,20 +17,23 @@ def build_provider_session( setup_context: object | None = None, allow_shared: bool = True, ) -> ProviderSession: - """Build a runtime provider session with fresh provider instances.""" + """Build a runtime provider session that reads each provider's settings on first use.""" if allow_shared: from sqlbuild._native import current_event_exporter_command_scope scope: EventExporterCommandScope | None = current_event_exporter_command_scope() if scope is not None and scope.provider_session is not None: - expected_names: tuple[str, ...] = tuple( + expected_names: frozenset[str] = frozenset( provider.name for provider in discovered_providers ) - if scope.provider_session.keys() == expected_names: + if frozenset(scope.provider_session.keys()) == expected_names: return scope.provider_session - providers: dict[str, Provider] = { - discovered_provider.name: discovered_provider.provider_class() - for discovered_provider in discovered_providers - } - return ProviderSession(providers, setup_context=setup_context) + return ProviderSession( + {}, + setup_context=setup_context, + deferred={ + discovered_provider.name: discovered_provider.provider_class + for discovered_provider in discovered_providers + }, + ) diff --git a/src/sqlbuild/provider/classes/session.py b/src/sqlbuild/provider/classes/session.py index 07e836710..81462cb14 100644 --- a/src/sqlbuild/provider/classes/session.py +++ b/src/sqlbuild/provider/classes/session.py @@ -25,8 +25,14 @@ def __init__( providers: Mapping[str, Provider] | Iterable[tuple[str, Provider]], *, setup_context: object | None = None, + deferred: Mapping[str, type[Provider]] | None = None, ) -> None: self._providers: dict[str, Provider] = dict(providers) + self._deferred: dict[str, type[Provider]] = { + name: provider_class + for name, provider_class in (deferred or {}).items() + if name not in self._providers + } self._setup_context: object | None = setup_context self._container: ProviderContainer = ProviderContainer(self) self._setup_names: list[str] = [] @@ -41,7 +47,7 @@ def providers(self) -> ProviderContainer: return self._container def __contains__(self, name: object) -> bool: - return isinstance(name, str) and name in self._providers + return isinstance(name, str) and (name in self._providers or name in self._deferred) def __enter__(self) -> ProviderSession: return self @@ -56,12 +62,14 @@ def __exit__(self, exc_type: object, exc: object, traceback: object) -> bool: return False def keys(self) -> tuple[str, ...]: - return tuple(self._providers) + return (*self._providers, *self._deferred) def get(self, name: str) -> Provider: with self._lifecycle_lock: if self._is_closed: raise ProviderLookupError(f"Provider session is closed; cannot access '{name}'") + if name in self._deferred: + self._providers[name] = self._instantiate(name=name) provider: Provider | None = self._providers.get(name) if provider is None: available: str = ", ".join(self.keys()) or "none" @@ -111,6 +119,24 @@ def close(self, *, force: bool = False) -> None: if errors: raise ProviderTeardownError("\n".join(errors)) + def _instantiate(self, *, name: str) -> Provider: + """Read a deferred provider's settings from the environment on its first use.""" + + from pydantic import ValidationError + + provider_class: type[Provider] = self._deferred.pop(name) + try: + return provider_class() + except ValidationError as error: + details: list[str] = [] + for item in error.errors(include_input=False): + location: str = ".".join(str(part) for part in item["loc"]) + details.append(f"{location}: {item['msg']}") + raise ProviderSetupError( + f"Provider '{name}' ({provider_class.__name__}) has missing or invalid " + f"settings: {'; '.join(details)}" + ) from error + def _setup_provider(self, *, name: str, provider: Provider) -> None: try: provider.setup(self._setup_context) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py index 1f508ca12..ce2573e7b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py @@ -315,11 +315,7 @@ def test_given_local_default_and_explicit_target_when_debugging_json_then_report DebugCliTestCase( description="reports discovered providers", command=("debug", "--no-connection"), - expected_stdout_fragment=( - "Providers:\n" - " providers: 1 [OK discovered]\n" - " marker_provider: providers/marker.py:MarkerProvider [OK valid settings]\n" - ), + expected_stdout_fragment=("Providers:\n providers: 1 of 1 ready [OK secrets set]\n"), ) ], ids=lambda case: case.description, @@ -369,15 +365,9 @@ class MarkerProvider(Provider): expected_check_tail=[ { "label": "providers", - "message": "1", - "status": "OK", - "status_message": "discovered", - }, - { - "label": "marker_provider", - "message": "providers/marker.py:MarkerProvider", + "message": "1 of 1 ready", "status": "OK", - "status_message": "valid settings", + "status_message": "secrets set", }, ], ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py index a9430b328..d957f3c98 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py @@ -131,3 +131,21 @@ class SlowSinkExitE2ETestCase: description: str command: tuple[str, ...] expected_marker_entries: tuple[str, ...] + + +@dataclass(frozen=True) +class ProviderSecretE2ETestCase: + description: str + command: tuple[str, ...] + env: dict[str, str] + extra_repo_files: dict[str, str] + expected_exit_code: int + expected_output_fragments: tuple[str, ...] + expected_marker_entries: tuple[str, ...] + + +@dataclass(frozen=True) +class SharedSinkProviderE2ETestCase: + description: str + unused_provider_name: str + expected_setups: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py index 2da770b29..3dc164b6c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py @@ -15,6 +15,7 @@ FilteredSinkFailureE2ETestCase, NoExporterCommandE2ETestCase, ReadOnlyProviderE2ETestCase, + SharedSinkProviderE2ETestCase, SlowSinkExitE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -535,3 +536,98 @@ def publish_lifecycle(event: LifecycleEvent, destination: Destination) -> None: tuple(marker_path.read_text(encoding="utf-8").splitlines()) == test_case.expected_marker_entries ) + + +SHARED_PROVIDER: str = dedent( + """ + import os + from pathlib import Path + + from sqlbuild.providers import Provider + + + class {class_name}(Provider): + '''Test provider {class_name}.''' + + def setup(self, ctx): + del ctx + with Path(os.environ["SETUP_LOG_PATH"]).open("a", encoding="utf-8") as log: + log.write("{name}\\n") + """ +).strip() + + +@pytest.mark.parametrize( + "test_case", + ( + SharedSinkProviderE2ETestCase( + description="the shared provider sorts after one the sink does not use", + unused_provider_name="a_first", + expected_setups=("z_shared",), + ), + SharedSinkProviderE2ETestCase( + description="the shared provider sorts before one the sink does not use", + unused_provider_name="zz_last", + expected_setups=("z_shared",), + ), + ), + ids=lambda case: case.description, +) +def test_given_sink_and_task_sharing_a_provider_when_building_then_it_is_set_up_once( + test_case: SharedSinkProviderE2ETestCase, + tmp_path: Path, +) -> None: + setup_log: Path = tmp_path / "setups.log" + setup_log.write_text("", encoding="utf-8") + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shared_sink_provider_project", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "shared_sink_provider_project"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "shared_sink_provider_project.duckdb"\n' + ), + f"providers/{test_case.unused_provider_name}.py": SHARED_PROVIDER.format( + class_name="UnusedProvider", name=test_case.unused_provider_name + ) + + "\n", + "providers/z_shared.py": SHARED_PROVIDER.format( + class_name="SharedProvider", name="z_shared" + ) + + "\n", + "sinks/events.py": dedent( + """ + from providers.z_shared import SharedProvider + from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink + + + @lifecycle_event_sink + def export_event(event: LifecycleEvent, shared_provider: SharedProvider) -> None: + del event, shared_provider + """ + ).strip() + + "\n", + "python/tasks/uses_shared.py": dedent( + """ + from providers.z_shared import SharedProvider + from sqlbuild.tasks import task + + + @task + def uses_shared(ctx, shared_provider: SharedProvider): + '''Test task uses_shared.''' + del ctx, shared_provider + """ + ).strip() + + "\n", + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", "uses_shared"), + project_dir=project_dir, + env={"SETUP_LOG_PATH": str(setup_log)}, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert tuple(setup_log.read_text(encoding="utf-8").splitlines()) == test_case.expected_setups diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_provider_secrets.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_provider_secrets.py new file mode 100644 index 000000000..25494f864 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_provider_secrets.py @@ -0,0 +1,411 @@ +"""E2E tests for providers whose secrets are not set.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.providers._test_types import ( + ProviderCommandDiagnosticE2ETestCase, + ProviderSecretE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + run_sqb, +) + +SECRET_VARIABLE: str = "SECRETS_E2E_API_KEY" + +PROJECT_TOML: str = ( + 'name = "provider_secret_project"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "provider_secret_project.duckdb"\n' +) + + +MARKING_NODE: str = ( + dedent( + """ + import os + from pathlib import Path + + from providers.api import ApiProvider + from sqlbuild.{decorator}s import {decorator} + + + @{decorator}{arguments} + def {name}(ctx{parameter}): + '''Test node {name}.''' + with Path(os.environ["MARKER_PATH"]).open("a", encoding="utf-8") as marker: + marker.write("{name}\\n") + """ + ).strip() + + "\n{body}" +) +USES_API: str = ", api_provider: ApiProvider" + + +PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": PROJECT_TOML, + "providers/api.py": dedent( + f""" + from pydantic import Field, SecretStr + from sqlbuild.providers import Provider + + + class ApiProvider(Provider): + '''Test provider ApiProvider.''' + api_key: SecretStr = Field(validation_alias="{SECRET_VARIABLE}") + region: str = "eu" + """ + ).strip() + + "\n", + "python/tasks/uses_api.py": MARKING_NODE.format( + decorator="task", arguments="", name="uses_api", parameter=USES_API, body="" + ), + "python/tasks/no_api.py": MARKING_NODE.format( + decorator="task", arguments="", name="no_api", parameter="", body="" + ), + "python/checks/api_check.py": MARKING_NODE.format( + decorator="check", + arguments="(depends_on=())", + name="api_check", + parameter=USES_API, + body=" return True\n", + ), + "python/loaders/raw_api_events.py": MARKING_NODE.format( + decorator="loader", + arguments="", + name="raw_api_events", + parameter=USES_API, + body=' return [{"event_id": 1}]\n', + ), + "sources/raw.yml": dedent( + """ + sources: + - name: raw_api_events + description: Test source raw_api_events. + managed: true + write_strategy: table + columns: + - name: event_id + type: INTEGER + """ + ).strip() + + "\n", +} + +SECRET_FREE_SINK: dict[str, str] = { + "providers/event_log.py": dedent( + """ + from sqlbuild.providers import Provider + + + class EventLog(Provider): + '''Test provider EventLog.''' + region: str = "eu" + """ + ).strip() + + "\n", + "sinks/events.py": dedent( + """ + from providers.event_log import EventLog + from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink + + + @lifecycle_event_sink + def log_event(event: LifecycleEvent, event_log: EventLog) -> None: + del event, event_log + """ + ).strip() + + "\n", +} +API_SINK: dict[str, str] = { + "sinks/events.py": dedent( + """ + from providers.api import ApiProvider + from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink + + + @lifecycle_event_sink + def export_event(event: LifecycleEvent, api_provider: ApiProvider) -> None: + del event, api_provider + """ + ).strip() + + "\n", +} +POST_BUILD_CHECK: dict[str, str] = { + "python/checks/no_api_check.py": dedent( + """ + import os + from pathlib import Path + + from providers.api import ApiProvider + from python.tasks.no_api import no_api + from sqlbuild.checks import check + + + @check(depends_on=(no_api,)) + def no_api_check(ctx, api_provider: ApiProvider): + '''Test check no_api_check.''' + with Path(os.environ["MARKER_PATH"]).open("a", encoding="utf-8") as marker: + marker.write("no_api_check\\n") + return True + """ + ).strip() + + "\n", +} + +MISSING_ERROR: str = ( + f"provider 'api_provider' (providers/api.py) is missing secret {SECRET_VARIABLE}" +) +ALLOWED_NOTICE: str = "1 provider has missing secrets (allowed): api_provider" +SELECTION_ERROR: str = "selected nodes need secrets that are missing:" +UNBOUND_NOTICE: str = ( + f"sink 'export_event' is not bound: provider 'api_provider' is missing {SECRET_VARIABLE}" +) +BUILD_WITH_CHECKS: tuple[str, ...] = ("--no-color", "build", "--no-tests", "--select") +BUILD: tuple[str, ...] = ("--no-color", "build", "--no-tests", "--no-audits", "--select") + + +@pytest.mark.parametrize( + "test_case", + [ + ProviderSecretE2ETestCase( + description="compile fails and names the variable", + command=("--no-color", "compile"), + env={}, + extra_repo_files={}, + expected_exit_code=1, + expected_output_fragments=(MISSING_ERROR, "--allow-missing-secrets"), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="compile passes with the flag", + command=("--no-color", "compile", "--allow-missing-secrets"), + env={}, + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=(ALLOWED_NOTICE,), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="compile passes with the environment variable", + command=("--no-color", "compile"), + env={"SQLBUILD_ALLOW_MISSING_SECRETS": "true"}, + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=(ALLOWED_NOTICE,), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="compile passes with the local setting", + command=("--no-color", "compile"), + env={}, + extra_repo_files={"sqlbuild_local.toml": "[settings]\nallow_missing_secrets = true\n"}, + expected_exit_code=0, + expected_output_fragments=(ALLOWED_NOTICE,), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="build of a selection that does not use the provider runs", + command=(*BUILD, "no_api"), + env={}, + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=("Completed successfully",), + expected_marker_entries=("no_api",), + ), + ProviderSecretE2ETestCase( + description="build of a selection that uses the provider runs nothing", + command=(*BUILD, "no_api", "uses_api"), + env={}, + extra_repo_files={}, + expected_exit_code=1, + expected_output_fragments=(SELECTION_ERROR, SECRET_VARIABLE, "nothing was run"), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="build of a selection that uses the provider runs once the secret is set", + command=(*BUILD, "uses_api"), + env={SECRET_VARIABLE: "token"}, + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=("Completed successfully",), + expected_marker_entries=("uses_api",), + ), + ProviderSecretE2ETestCase( + description="check that uses the provider runs nothing", + command=("--no-color", "check", "--select", "api_check"), + env={}, + extra_repo_files={}, + expected_exit_code=1, + expected_output_fragments=(SELECTION_ERROR, SECRET_VARIABLE, "nothing was run"), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="load that uses the provider runs nothing", + command=("--no-color", "load", "--select", "raw_api_events"), + env={}, + extra_repo_files={}, + expected_exit_code=1, + expected_output_fragments=(SELECTION_ERROR, SECRET_VARIABLE, "nothing was run"), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="build whose post-build check uses the provider runs nothing", + command=(*BUILD_WITH_CHECKS, "no_api"), + env={}, + extra_repo_files=POST_BUILD_CHECK, + expected_exit_code=1, + expected_output_fragments=(SELECTION_ERROR, SECRET_VARIABLE, "nothing was run"), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="build whose post-build check uses the provider runs it once the secret is set", + command=(*BUILD_WITH_CHECKS, "no_api"), + env={SECRET_VARIABLE: "token"}, + extra_repo_files=POST_BUILD_CHECK, + expected_exit_code=0, + expected_output_fragments=("Completed successfully",), + expected_marker_entries=("no_api", "no_api_check"), + ), + ProviderSecretE2ETestCase( + description="compile with a sink still fails on another provider's missing secret", + command=("--no-color", "compile"), + env={}, + extra_repo_files=SECRET_FREE_SINK, + expected_exit_code=1, + expected_output_fragments=(MISSING_ERROR,), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="compile with a sink and the flag notes the allowed provider", + command=("--no-color", "compile", "--allow-missing-secrets"), + env={}, + extra_repo_files=SECRET_FREE_SINK, + expected_exit_code=0, + expected_output_fragments=(ALLOWED_NOTICE,), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="compile whose sink's provider is missing the secret fails with the coded error", + command=("--no-color", "compile"), + env={}, + extra_repo_files=API_SINK, + expected_exit_code=1, + expected_output_fragments=(MISSING_ERROR,), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="compile with the flag leaves the sink unbound", + command=("--no-color", "compile", "--allow-missing-secrets"), + env={}, + extra_repo_files=API_SINK, + expected_exit_code=0, + expected_output_fragments=(UNBOUND_NOTICE, ALLOWED_NOTICE), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="build whose sink's provider is missing the secret runs nothing", + command=(*BUILD, "no_api"), + env={}, + extra_repo_files=API_SINK, + expected_exit_code=1, + expected_output_fragments=(SELECTION_ERROR, "export_event", "nothing was run"), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="debug with a sink lists the missing secret", + command=("--no-color", "debug", "--no-connection"), + env={}, + extra_repo_files=API_SINK, + expected_exit_code=0, + expected_output_fragments=(f" api_provider: {SECRET_VARIABLE} [WARN missing]\n",), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="debug lists the missing secret", + command=("--no-color", "debug", "--no-connection"), + env={}, + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=( + " providers: 0 of 1 ready [WARN missing secrets]\n" + f" api_provider: {SECRET_VARIABLE} [WARN missing]\n", + ), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="verbose debug lists set secrets", + command=("--no-color", "debug", "--no-connection", "--verbose"), + env={SECRET_VARIABLE: "token"}, + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=( + " providers: 1 of 1 ready [OK secrets set]\n" + f" api_provider: {SECRET_VARIABLE} [OK set]\n", + ), + expected_marker_entries=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_provider_secret_unset_when_running_commands_then_only_users_of_it_fail( + test_case: ProviderSecretE2ETestCase, + tmp_path: Path, +) -> None: + marker_path: Path = tmp_path / "marker.log" + marker_path.write_text("", encoding="utf-8") + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="provider_secret_project", + repo_files={**PROJECT_FILES, **test_case.extra_repo_files}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, + project_dir=project_dir, + env={"MARKER_PATH": str(marker_path), **test_case.env}, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_output_fragments: + assert fragment in output, output + marker_entries: tuple[str, ...] = tuple(marker_path.read_text(encoding="utf-8").splitlines()) + assert marker_entries == test_case.expected_marker_entries + + +@pytest.mark.parametrize( + "test_case", + [ + ProviderCommandDiagnosticE2ETestCase( + description="the project file cannot allow missing secrets", + command=("--no-color", "compile"), + expected_error_fragment="cannot set [settings] allow_missing_secrets", + ) + ], + ids=lambda case: case.description, +) +def test_given_project_file_allows_missing_secrets_when_compiling_then_fails( + test_case: ProviderCommandDiagnosticE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="provider_secret_project", + repo_files={ + **PROJECT_FILES, + "sqlbuild_project.toml": PROJECT_TOML + "\n[settings]\nallow_missing_secrets = true\n", + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + assert result.returncode != 0 + assert test_case.expected_error_fragment in result.stdout + result.stderr diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index b75347c39..d21ca1661 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -484,6 +484,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": True, "env": "FULFILLMENT_API_KEY", "description": None, + "missing": False, }, { "name": "backup_key", @@ -492,6 +493,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": True, "env": "FULFILLMENT_BACKUP_KEY", "description": None, + "missing": False, }, { "name": "regions", @@ -500,6 +502,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": False, "env": "FULFILLMENT_REGIONS", "description": None, + "missing": False, }, { "name": "compression", @@ -508,6 +511,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": False, "env": "FULFILLMENT_COMPRESSION", "description": None, + "missing": False, }, { "name": "endpoint", @@ -516,6 +520,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": False, "env": "FULFILLMENT_ENDPOINT", "description": "Base URL", + "missing": False, }, { "name": "timeout_seconds", @@ -524,6 +529,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": False, "env": "SHIP_TIMEOUT", "description": None, + "missing": False, }, ], "used_by": [ @@ -568,6 +574,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": False, "env": "REGION", "description": None, + "missing": False, }, { "name": "token", @@ -576,6 +583,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": False, "env": "INVENTORY_TOKEN", "description": None, + "missing": False, }, ], "used_by": [], diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 299b15387..1d2445a75 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -1289,9 +1289,10 @@ def run_debug( no_connection: bool, selected_target: str | None, json_output: bool, + verbose: bool, ) -> int: del selected_target - received_args.append((project_dir, no_color, no_connection, json_output)) + received_args.append((project_dir, no_color, no_connection, json_output, verbose)) return test_case.expected_exit_code exit_code: int = _main_with_dependencies( @@ -1300,7 +1301,7 @@ def run_debug( ) assert exit_code == test_case.expected_exit_code - assert received_args == [(None, True, True, True)] + assert received_args == [(None, True, True, True, False)] @pytest.mark.parametrize( diff --git a/ui/src/lib/api/http-locations.test.ts b/ui/src/lib/api/http-locations.test.ts index 0a4342f57..1b03b9d81 100644 --- a/ui/src/lib/api/http-locations.test.ts +++ b/ui/src/lib/api/http-locations.test.ts @@ -25,6 +25,13 @@ describe('location health', () => { expect(healthOf([seen(5, { kind: 'connection', generation: 4, healthy: true }), failed], now).health).toBe('healthy'); }); + it('reports missing provider secrets ahead of a failed connection until a newer report finds none', () => { + const missing = seen(10, { kind: 'provider_secrets', generation: 2, missing: 1 }); + const unhealthy = seen(10, { kind: 'connection', generation: 2, healthy: false }); + expect(healthOf([missing, unhealthy], now).health).toBe('missing_secrets'); + expect(healthOf([seen(5, { kind: 'provider_secrets', generation: 3, missing: 0 }), missing], now).health).toBe('healthy'); + }); + it('needs attention when the latest connection check failed', () => { const observations = [seen(5, { kind: 'connection', generation: 2, healthy: false }), seen(60, { kind: 'connection', generation: 1, healthy: true })]; expect(healthOf(observations, now).health).toBe('degraded'); diff --git a/ui/src/lib/api/http-locations.ts b/ui/src/lib/api/http-locations.ts index a276a4516..bce103588 100644 --- a/ui/src/lib/api/http-locations.ts +++ b/ui/src/lib/api/http-locations.ts @@ -11,19 +11,22 @@ function latestOf(observations: Wire[' return observations.find((o) => o.report.health.kind === kind) as (Wire['LocationHealthObservation'] & { report: { health: Extract } }) | undefined; } -/** Health from the newest observations: down without a recent check-in, then load failures, then connection failures. */ +/** Health from the newest observations: down without a recent check-in, then load failures, then missing provider secrets, then connection failures. */ export function healthOf(observations: Wire['LocationHealthObservation'][], now: number): { health: LocationHealth; liveInstances: number; lastSeenAt: string | null } { const live = new Set(observations.filter((o) => now - Date.parse(o.received_at) <= LIVE_WINDOW_MS).map((o) => o.report.session_id)); const lastSeenAt = observations.reduce((latest, o) => (!latest || o.received_at > latest ? o.received_at : latest), null); const failed = latestOf(observations, 'compile_failed'); const connection = latestOf(observations, 'connection'); + const secrets = latestOf(observations, 'provider_secrets'); const health: LocationHealth = !live.size ? 'broken' : failed && (!connection || failed.report.health.generation >= connection.report.health.generation) ? 'falling_back' - : connection && !connection.report.health.healthy - ? 'degraded' - : 'healthy'; + : secrets && secrets.report.health.missing > 0 + ? 'missing_secrets' + : connection && !connection.report.health.healthy + ? 'degraded' + : 'healthy'; return { health, liveInstances: live.size, lastSeenAt }; } diff --git a/ui/src/lib/api/http-providers.test.ts b/ui/src/lib/api/http-providers.test.ts index b2e319b58..4c40d63e7 100644 --- a/ui/src/lib/api/http-providers.test.ts +++ b/ui/src/lib/api/http-providers.test.ts @@ -14,7 +14,7 @@ describe('toProviders', () => { class: 'providers.fulfillment.FulfillmentApi', description: 'Client for the fulfillment service.', path: 'providers/fulfillment.py', - settings: [{ name: 'api_key', type: 'SecretStr', required: true, secret: true, env: 'FULFILLMENT_API_KEY', description: null }], + settings: [{ name: 'api_key', type: 'SecretStr', required: true, secret: true, env: 'FULFILLMENT_API_KEY', description: null, missing: true }], used_by: [ { kind: 'task', name: 'publish_orders' }, { kind: 'sink', name: 'order_events' }, @@ -26,7 +26,7 @@ describe('toProviders', () => { }; const [provider] = toProviders(wire); expect(provider.className).toBe('providers.fulfillment.FulfillmentApi'); - expect(provider.settings[0]).toEqual({ name: 'api_key', type: 'SecretStr', required: true, secret: true, env: 'FULFILLMENT_API_KEY', description: null }); + expect(provider.settings[0]).toEqual({ name: 'api_key', type: 'SecretStr', required: true, secret: true, env: 'FULFILLMENT_API_KEY', description: null, missing: true }); expect(provider.usedBy.map((c) => `${c.kind}:${c.name}`)).toEqual(['task:publish_orders', 'sink:order_events', 'event_exporter:run_events']); expect(provider.jobs).toEqual(['order_exports']); }); diff --git a/ui/src/lib/api/http-providers.ts b/ui/src/lib/api/http-providers.ts index 773d59cdc..1d343173b 100644 --- a/ui/src/lib/api/http-providers.ts +++ b/ui/src/lib/api/http-providers.ts @@ -9,7 +9,7 @@ export function toProviders(wire: Wire['ProvidersRead']): ProviderConnection[] { className: p.class, description: p.description, sourceFile: p.path, - settings: p.settings.map((s) => ({ name: s.name, type: s.type, required: s.required, secret: s.secret, env: s.env, description: s.description ?? null })), + settings: p.settings.map((s) => ({ name: s.name, type: s.type, required: s.required, secret: s.secret, env: s.env, description: s.description ?? null, missing: s.missing ?? false })), usedBy: p.used_by.map((c) => ({ kind: c.kind as ProviderConsumer['kind'], name: c.name })), jobs: p.jobs ?? [] })); diff --git a/ui/src/lib/api/mock/platform.ts b/ui/src/lib/api/mock/platform.ts index 6a18bedfa..a22aa2065 100644 --- a/ui/src/lib/api/mock/platform.ts +++ b/ui/src/lib/api/mock/platform.ts @@ -376,6 +376,7 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { secret: type.includes('Secret'), env, description, + missing: false, ...extra }); const providers: ProviderConnection[] = [ @@ -430,7 +431,7 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { description: 'Incoming webhook of the team chat room.', sourceFile: 'providers/chat.py', settings: [ - setting('url', 'SecretStr', 'CHAT_WEBHOOK_URL', 'Webhook URL', { required: true }), + setting('url', 'SecretStr', 'CHAT_WEBHOOK_URL', 'Webhook URL', { required: true, missing: true }), setting('timeout_seconds', 'int', 'CHAT_WEBHOOK_TIMEOUT_SECONDS', 'Request timeout') ], usedBy: [], diff --git a/ui/src/lib/api/types.ts b/ui/src/lib/api/types.ts index d627752c5..03ceda96b 100644 --- a/ui/src/lib/api/types.ts +++ b/ui/src/lib/api/types.ts @@ -81,7 +81,7 @@ export type LocationTarget = { connectionSummary: string | null; }; -export type LocationHealth = 'healthy' | 'degraded' | 'falling_back' | 'broken'; +export type LocationHealth = 'healthy' | 'degraded' | 'missing_secrets' | 'falling_back' | 'broken'; /** A project deployed against exactly one target. Staging and dev are separate locations. */ export type Location = { /** Opaque server-generated UUID; the name is a mutable label. */ diff --git a/ui/src/lib/api/types/platform.ts b/ui/src/lib/api/types/platform.ts index 581f53722..52b68cdb7 100644 --- a/ui/src/lib/api/types/platform.ts +++ b/ui/src/lib/api/types/platform.ts @@ -251,6 +251,8 @@ export type ProviderSetting = { /** The environment variable the setting is read from. */ env: string; description: string | null; + /** Whether this required secret is unset in the location's environment. */ + missing: boolean; }; /** Python code that injects a provider. None of it is a catalog node, so it has no page to open. */ diff --git a/ui/src/lib/platform/labels.ts b/ui/src/lib/platform/labels.ts index f94438962..0283b3dbf 100644 --- a/ui/src/lib/platform/labels.ts +++ b/ui/src/lib/platform/labels.ts @@ -20,6 +20,7 @@ import type { export const healthTone: Record = { healthy: 'var(--sb-success)', degraded: 'var(--sb-warning)', + missing_secrets: 'var(--sb-error)', falling_back: 'var(--sb-warning)', broken: 'var(--sb-error)' }; @@ -27,6 +28,7 @@ export const healthTone: Record = { export const healthLabel: Record = { healthy: 'Healthy', degraded: 'Needs attention', + missing_secrets: 'Missing provider secrets', falling_back: 'Running previous version', broken: 'Not starting scheduled runs' }; diff --git a/ui/src/routes/providers/[name]/+page.svelte b/ui/src/routes/providers/[name]/+page.svelte index 9bc9cac8b..5f871cf64 100644 --- a/ui/src/routes/providers/[name]/+page.svelte +++ b/ui/src/routes/providers/[name]/+page.svelte @@ -63,7 +63,7 @@ {#if f.description}
{f.description}
{/if} {f.type} - {f.env} + {f.env}{#if f.missing}missing{/if} {f.required ? 'Required' : 'Optional'} {/each} @@ -128,4 +128,9 @@ color: var(--muted-foreground); font-size: 11px; } + .missing { + margin-left: 6px; + color: var(--sb-error); + font-size: 11px; + } diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index ed2f9227a..ee1d7602f 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -50,6 +50,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--exclude` | Remove resources from the selected scope | | `--no-sql-analysis` | Turn off SQL syntax, binding, type inference and semantic checks (`--no-sql-validation` is an alias) | | `--no-cache` | Compile in full, without reusing earlier compiles or the Rules cache; see [Forcing a fresh compile](#forcing-a-fresh-compile) | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](/docs/concepts/python-nodes/providers#secrets) | | `--defer-to` | Resolve unselected model references against another target | | `--json` | Print the full compile report as JSON | | `--manifest` | Also write `target/manifest.json` with project metadata | diff --git a/website/src/content/docs/docs/cli/dag.mdx b/website/src/content/docs/docs/cli/dag.mdx index aacd2cb35..dbefae2d5 100644 --- a/website/src/content/docs/docs/cli/dag.mdx +++ b/website/src/content/docs/docs/cli/dag.mdx @@ -21,6 +21,7 @@ sqb --project-dir dag [flags] |------|-------------| | `--json` | Print the full DAG artifact as JSON to stdout | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](/docs/concepts/python-nodes/providers#secrets) | | `--vars` | JSON object of project variable overrides | Without `--json`, the command prints a summary: diff --git a/website/src/content/docs/docs/cli/debug.mdx b/website/src/content/docs/docs/cli/debug.mdx index 488c1e6f4..2905c8980 100644 --- a/website/src/content/docs/docs/cli/debug.mdx +++ b/website/src/content/docs/docs/cli/debug.mdx @@ -23,7 +23,14 @@ The command reports four groups of checks: **Configuration** - Finds and validates `sqlbuild_project.toml`, loads `sqlbuild_local.toml` if present, resolves the adapter, active target, connection, database, and schema. For Snowflake it also shows the warehouse each [command group](/docs/concepts/adapters/snowflake#command-group-warehouses) connects with and where that value came from. -**Providers** - Lists the [providers](/docs/concepts/python-nodes/providers) the project defines. +**Providers** - Counts the [providers](/docs/concepts/python-nodes/providers) whose [secrets](/docs/concepts/python-nodes/providers#secrets) are all set, and lists each missing secret as `WARN`. When every secret is set it prints one line. `--verbose` lists every provider and secret. + +```text +Providers: + providers: 2 of 3 ready [WARN missing secrets] + slack_alerts: SLACK_WEBHOOK_URL [WARN missing] + more: sqb debug --verbose lists every provider [SKIP] +``` **Connection** - Displays connection settings as `sqb debug` itself uses them, with the `query` warehouse group applied (secrets are masked), attempts to connect to the warehouse, and runs `SELECT 1` to verify query execution. @@ -32,7 +39,8 @@ The command reports four groups of checks: | Flag | Description | |------|-------------| | `--no-connection` | Skip the connection and query tests. Useful for validating config without warehouse access. | -| `--json` | Output results as JSON instead of formatted text. | +| `--json` | Output results as JSON instead of formatted text, with `missing_secrets` counting the unset provider secrets. | +| `--verbose` | List every provider and secret setting, set or missing. | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/adapters/snowflake#command-group-warehouses) and the connection warehouse | ## Example @@ -80,7 +88,7 @@ Connection: ## Exit codes -Returns `0` when all checks pass, `1` when any check fails (e.g. connection refused, invalid config). +Returns `0` when all checks pass, `1` when any check fails (e.g. connection refused, invalid config). A missing provider secret is a warning and doesn't fail the command. A failed run is recorded in [Recall](/docs/concepts/observability/recall) with the first failed check as its cause: diff --git a/website/src/content/docs/docs/cli/format.mdx b/website/src/content/docs/docs/cli/format.mdx index e46f501a4..ea9b8a207 100644 --- a/website/src/content/docs/docs/cli/format.mdx +++ b/website/src/content/docs/docs/cli/format.mdx @@ -18,6 +18,7 @@ sqb format [flags] | `--fix` | Also apply compiler-verified, meaning-preserving Rule fixes before layout formatting | | `--json` | Print structured results | | `--no-cache` | Format every file instead of skipping files a previous run found clean | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](/docs/concepts/python-nodes/providers#secrets) | | `--select`, `-s` | Format selected models | | `--select-file` | Read selectors from a file | | `--exclude` | Exclude models from the selected scope | diff --git a/website/src/content/docs/docs/cli/lineage.mdx b/website/src/content/docs/docs/cli/lineage.mdx index e1fd442f0..664d900e9 100644 --- a/website/src/content/docs/docs/cli/lineage.mdx +++ b/website/src/content/docs/docs/cli/lineage.mdx @@ -34,6 +34,7 @@ Exactly one of a positional target or `--select` is required. | `--json` | Shorthand for `--format json`. | | `--mode` | Column lineage analysis mode: `rich` (default) or `fast`. | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias). | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](/docs/concepts/python-nodes/providers#secrets) | | `--select`, `-s` | Select resources using the same selector syntax as `sqb compile`, `sqb plan` and `sqb build`, for example `path:models/marts`, `tag:finance` or `stg_*`. | | `--exclude` | Exclude resources from the selection. | diff --git a/website/src/content/docs/docs/cli/location.mdx b/website/src/content/docs/docs/cli/location.mdx index c40cac7fb..e1717bcf7 100644 --- a/website/src/content/docs/docs/cli/location.mdx +++ b/website/src/content/docs/docs/cli/location.mdx @@ -33,6 +33,7 @@ sqb location health --server https://control.example.com --location-id "$LOCATIO | --- | --- | --- | | `healthy` | An instance checked in within 90 seconds and its last load published | `0` | | `degraded` | The warehouse connection check after the last load failed | `0` | +| `missing_secrets` | A provider [secret](/docs/concepts/python-nodes/providers#secrets) is unset in the location's environment, so runs that use it fail | `1` | | `falling_back` | The last load failed, so runs use the previous snapshot | `1` | | `broken` | No instance has checked in for 90 seconds | `1` | @@ -50,3 +51,6 @@ healthcheck: A `falling_back` location still runs scheduled jobs from its last good snapshot. Restarting it does not fix the failed load; fix the project and deploy it again. + +A `missing_secrets` location publishes and runs jobs whose nodes have their secrets. Set the +missing variables in its environment and restart it; `sqb debug` in the location lists them. diff --git a/website/src/content/docs/docs/cli/rules.mdx b/website/src/content/docs/docs/cli/rules.mdx index 98088c8cc..f28a6b402 100644 --- a/website/src/content/docs/docs/cli/rules.mdx +++ b/website/src/content/docs/docs/cli/rules.mdx @@ -33,7 +33,7 @@ Lifecycle progress is written to stderr so stdout remains valid JSON. - `run`: compile and evaluate only one exact code or family prefix. - `skills`: install guidance derived from configured Rules; `--check` verifies freshness without writing. -`run` accepts normal SQLBuild model selectors. Native built-ins execute before custom Python Rules. +`run` accepts normal SQLBuild model selectors and `--allow-missing-secrets`, which runs without provider [secrets](/docs/concepts/python-nodes/providers#secrets). Native built-ins execute before custom Python Rules. ## Exit codes diff --git a/website/src/content/docs/docs/cli/scope.mdx b/website/src/content/docs/docs/cli/scope.mdx index ad9fee78f..39ae1ed19 100644 --- a/website/src/content/docs/docs/cli/scope.mdx +++ b/website/src/content/docs/docs/cli/scope.mdx @@ -68,6 +68,7 @@ These identities are for inspection. SQL authored inside the owning model contin | `--kind KIND` | Keep `macro`, `enum`, or `constant` declarations. Repeat to include multiple kinds. | | `--match GLOB` | Match declaration names or qualified identities with a deterministic, case-sensitive glob. | | `--used-only` | Keep declarations used by the target. | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](/docs/concepts/python-nodes/providers#secrets) | | `--include-nearby` | Include bounded nearby declarations that are unavailable to the target. | | `--nearby-depth N` | Set filesystem proximity for nearby descendant and sibling discovery (default: `1`). | | `--dependency-depth N` | Expand dependencies from declarations in the used section by `N` edges (default: `0`). | diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 5951af7c2..991be9205 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -491,6 +491,7 @@ delta_compile = true | `test_timeout` | `30s` | Time limit for each SQL unit test, or `"disabled"` | | `scenario_timeout` | none | Time limit for each scenario, or `"disabled"` | | `delta_compile` | `true` | After an edit to model SQL or SQL test files only, compile just the changed files against the previous compile. See [Compiling after an edit](/docs/cli/compile#compiling-after-an-edit). | +| `allow_missing_secrets` | `false` | Let `compile`, `rules run`, `format`, `dag`, `lineage` and `scope` run without provider secrets. `sqlbuild_local.toml` only. See [Secrets](/docs/concepts/python-nodes/providers#secrets). | ### Unit test and scenario limits diff --git a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx index d8f140c41..b9b6efd79 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx @@ -133,8 +133,8 @@ Both approaches (parameter injection and `ctx.providers`) are equivalent. Parame Providers follow a lazy setup, reverse-teardown lifecycle scoped to the command invocation: -1. **Discovery** - on compile, SQLBuild discovers all `Provider` subclasses under `providers/` and validates their settings (from environment variables or field defaults). -2. **Lazy setup** - `setup(ctx)` is called the first time a provider is accessed during a build, not at startup. Providers that are never used are never set up. +1. **Discovery** - on compile, SQLBuild discovers all `Provider` subclasses under `providers/` and validates their settings (from environment variables or field defaults). Missing [secrets](#secrets) follow the command's rules. +2. **Lazy setup** - the first time a provider is accessed during a build, SQLBuild reads its settings and calls `setup(ctx)`. Providers that are never used are never set up. 3. **Teardown** - after the command completes, `teardown()` is called on all providers that were set up, in reverse setup order. Teardown runs even if the build failed. ```python @@ -166,13 +166,62 @@ class SlackNotifier(Provider): See the [pydantic-settings documentation](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) for advanced configuration like custom env prefixes, `.env` file support, and nested settings. +## Secrets + +Declare a secret as a field typed `SecretStr`. Laptops and CI jobs can then compile without production secrets, and builds check only the secrets their selection needs. + +```python +from pydantic import SecretStr + + +class SlackAlerts(Provider): + """Posts alerts to Slack.""" + + channel: str = "#data-alerts" + slack_webhook_url: SecretStr # reads SLACK_WEBHOOK_URL +``` + +| Commands | When a required secret is unset | +| --- | --- | +| `compile`, `rules run`, `format`, `dag`, `lineage`, `scope` | Error `D012`, unless missing secrets are allowed | +| `build`, `load`, `check`, and runs on `sqb server` workers | Error before anything runs, only when a selected node or a sink uses the provider | +| `sqb debug` | Lists the missing secrets as warnings | +| `sqb location health` | Reports `missing_secrets` and exits `1` | + +```text +$ sqb compile +error[D012]: provider 'slack_alerts' (providers/slack_alerts/provider.py) is missing secret SLACK_WEBHOOK_URL + = help: set SLACK_WEBHOOK_URL, or pass --allow-missing-secrets where secrets aren't available +``` + +Allow missing secrets for the static commands in any of these ways: + +| Where | Setting | +| --- | --- | +| Command line | `--allow-missing-secrets` | +| Environment | `SQLBUILD_ALLOW_MISSING_SECRETS=1` | +| `sqlbuild_local.toml` | `allow_missing_secrets = true` under `[settings]` | + +The command then lists the providers it skipped on stderr, such as +`1 provider has missing secrets (allowed): slack_alerts`. Settings that aren't secrets are still +validated. Execution commands accept none of these. + +```text +$ sqb build --select python/orders +error[D012]: selected nodes need secrets that are missing: + slack_alerts SLACK_WEBHOOK_URL (partner_export_alert) +nothing was run +``` + +Run [`sqb debug`](/docs/cli/debug/) to see which secrets this machine has. + ## Discovery rules - Provider classes are discovered from `.py` files under `providers/` recursively - Files named `__init__.py` or starting with `_` are skipped - Each concrete (non-abstract) subclass of `Provider` is registered - Provider names must be globally unique across project resources -- Settings are validated at discovery time. Missing required fields (without environment variables set) raise a discovery error immediately, not at runtime +- Settings are validated at discovery time. A missing or invalid setting that isn't a secret is a `D012` error; missing secrets follow [Secrets](#secrets) ## Plan output @@ -193,7 +242,7 @@ provider with its class and docstring. A provider's page shows: | Tab | Shows | | --- | --- | -| Settings | Each setting's type, environment variable and whether it is required; secret settings are marked | +| Settings | Each setting's type, environment variable and whether it is required; secret settings are marked, and flagged **missing** when the location's environment doesn't set them | | Used by | The loaders, tasks, hooks and other code that take the provider, and the jobs that run that code | Setting values, defaults included, are never published. From 63a67835dc6385e863e55135eb419a634b18131f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 15 Nov 2026 08:42:14 +0000 Subject: [PATCH 3/6] feat: run project Python tests with sqb test and check them with SQBRPYTHON rules sqb test now runs the project's pytest modules under tests/python/ and tests/providers/ after its SQL tests, from the project directory with --import-mode=importlib. --sql and --python pick one kind. Selecting a Python node runs the tests in its mirrored folder; selecting only models or unit tests runs none. The command fails when either kind fails, and fails early when Python tests exist but pytest is not installed. sqb test --since, --inspect, review and ad-hoc runs stay SQL-only, and sqb build never runs Python tests. With --json, pytest writes to stderr. - SQBRPYTHON021: a test sits in the folder mirroring the code it tests, and test_.py has a .py or / beside that code. - SQBRPYTHON701-724: the Fensu test-shape rules (table-driven dataclass cases, naming, parametrize ids), applied only to Python tests. - Docs for sqb test, sqb build, testing, project layout, custom rule tests and the Python Rules reference, plus DuckShop fixtures. --- .../python/utils/currency/_test_types.py | 12 + .../python/utils/currency/test_amounts.py | 22 ++ .../python_rules/expected_findings.json | 13 +- .../python/inventory/test_restock_window.py | 10 + .../sqlbuild-cli/src/argv/command_table.json | 29 ++ crates/sqlbuild-cli/src/argv/constants.rs | 4 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 4 +- .../src/python_paths/_helpers/python_files.rs | 6 +- .../src/python_paths/constants.rs | 2 + .../src/python_paths/main/mod.rs | 1 + .../python_paths/main/python_test_paths.rs | 14 + .../main/supported_python_paths.rs | 2 +- .../main/unsupported_python_paths.rs | 2 +- .../src/python_paths/tests.rs | 2 + .../tests/test_python_test_paths.rs | 51 +++ .../src/python_paths/tests/test_types.rs | 7 + .../_helpers/compile_driver/static_context.rs | 22 ++ .../src/bindings/_helpers/native_host/host.rs | 21 + .../bindings/_helpers/records/constants.rs | 2 + .../_helpers/review_command/execution.rs | 2 + .../bindings/_helpers/sql_test_command/mod.rs | 1 + .../_helpers/sql_test_command/python_tests.rs | 272 +++++++++++++ .../_helpers/sql_test_command/test_command.rs | 172 ++++++-- .../src/engine/_helpers/memo.rs | 2 + .../engine/tests/test_incremental_rules.rs | 25 ++ crates/sqlbuild-rules/src/models.rs | 3 + .../python_rules/_helpers/engine/contexts.rs | 6 +- .../python_rules/_helpers/engine/execution.rs | 6 +- .../src/python_rules/_helpers/engine/mod.rs | 2 + .../_helpers/engine/observations.rs | 9 + .../python_rules/_helpers/engine/passes.rs | 58 +++ .../python_rules/_helpers/engine/placement.rs | 17 +- .../_helpers/engine/source_files.rs | 22 ++ .../src/python_rules/_helpers/units/mod.rs | 1 + .../_helpers/units/python_tests.rs | 65 +++ .../src/python_rules/constants.rs | 309 ++++++++++++++- .../main/evaluate_python_rules.rs | 44 +-- .../src/python_rules/main/mod.rs | 1 + .../src/python_rules/main/python_sources.rs | 14 +- .../python_rules/main/python_test_sources.rs | 28 ++ .../sqlbuild-rules/src/python_rules/models.rs | 4 + .../sqlbuild-rules/src/python_rules/tests.rs | 2 + .../tests/test_python_catalogue.rs | 6 +- .../tests/test_python_test_rules.rs | 237 +++++++++++ fensu.toml | 2 +- .../sqlbuild/references/docs/CONTENTS.md | 2 +- .../sqlbuild/references/docs/cli/build.md | 2 +- .../sqlbuild/references/docs/cli/test.md | 42 +- .../docs/concepts/project-configuration.md | 9 +- .../custom-rules/testing-and-determinism.md | 4 +- .../docs/concepts/rules/python-rules.md | 95 ++++- .../references/docs/concepts/testing.md | 3 + src/sqlbuild/_native.pyi | 4 + .../cli/commands/main/test/_test_types.py | 25 ++ .../commands/main/test/test_python_tests.py | 374 ++++++++++++++++++ website/src/content/docs/docs/cli/build.mdx | 2 +- website/src/content/docs/docs/cli/test.mdx | 41 +- .../docs/concepts/project-configuration.mdx | 9 +- .../custom-rules/testing-and-determinism.mdx | 4 +- .../docs/docs/concepts/rules/python-rules.mdx | 96 ++++- .../content/docs/docs/concepts/testing.mdx | 3 + 61 files changed, 2142 insertions(+), 109 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/_test_types.py create mode 100644 benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/test_amounts.py create mode 100644 benchmarks/duckshop/edge_cases/python_rules/violations/tests/python/inventory/test_restock_window.py create mode 100644 crates/sqlbuild-discovery/src/python_paths/main/python_test_paths.rs create mode 100644 crates/sqlbuild-discovery/src/python_paths/tests/test_python_test_paths.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs create mode 100644 crates/sqlbuild-rules/src/python_rules/_helpers/engine/passes.rs create mode 100644 crates/sqlbuild-rules/src/python_rules/_helpers/engine/source_files.rs create mode 100644 crates/sqlbuild-rules/src/python_rules/_helpers/units/python_tests.rs create mode 100644 crates/sqlbuild-rules/src/python_rules/main/python_test_sources.rs create mode 100644 crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/_test_types.py b/benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/_test_types.py new file mode 100644 index 000000000..6448f05b0 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/_test_types.py @@ -0,0 +1,12 @@ +"""Test cases for carrier money formatting.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CentsTextTestCase: + """One amount in cents and its carrier text.""" + + description: str + cents: int + expected_text: str diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/test_amounts.py b/benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/test_amounts.py new file mode 100644 index 000000000..ce9ebf710 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/test_amounts.py @@ -0,0 +1,22 @@ +"""Tests for carrier money formatting.""" + +import pytest +from python.utils.currency import amounts +from tests.python.utils.currency._test_types import CentsTextTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + CentsTextTestCase( + description="whole amounts keep two decimals", cents=1200, expected_text="12.00" + ), + CentsTextTestCase(description="single cents are padded", cents=1205, expected_text="12.05"), + ], + ids=lambda case: case.description, +) +def test_given_cents_when_formatting_then_returns_carrier_text( + test_case: CentsTextTestCase, +) -> None: + """Format cents as carrier text.""" + assert amounts.cents_text(test_case.cents) == test_case.expected_text diff --git a/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json b/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json index d5233f56c..ec30392e7 100644 --- a/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json +++ b/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json @@ -30,6 +30,17 @@ ["SQBRPYTHON007", "python/inventory/stock_count/node.py", 11, 5], ["SQBRPYTHON011", "python/inventory/stock_count/node.py", 11, 5], ["SQBRPYTHON001", "python/report.py", 1, 1], - ["SQBRPYTHON614", "python/report.py", 1, 1] + ["SQBRPYTHON614", "python/report.py", 1, 1], + ["SQBRPYTHON021", "tests/python/inventory/test_restock_window.py", 1, 1], + ["SQBRPYTHON710", "tests/python/inventory/test_restock_window.py", 1, 1], + ["SQBRPYTHON722", "tests/python/inventory/test_restock_window.py", 6, 35], + ["SQBRPYTHON712", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON714", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON715", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON716", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON718", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON719", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON724", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON704", "tests/python/inventory/test_restock_window.py", 9, 5] ] } diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/tests/python/inventory/test_restock_window.py b/benchmarks/duckshop/edge_cases/python_rules/violations/tests/python/inventory/test_restock_window.py new file mode 100644 index 000000000..b1257ee1a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/tests/python/inventory/test_restock_window.py @@ -0,0 +1,10 @@ +"""Tests for a restock window that has no module.""" + +import pytest + + +@pytest.mark.parametrize("case", [{"days": 3, "expected": True}]) +def test_window(case: dict[str, object]) -> None: + """Check the restock window.""" + if case["days"]: + assert case["expected"] diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index f92498e5a..cc1c12d0d 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -2885,6 +2885,9 @@ "title": "options" } ], + "mutex_groups": [ + false + ], "actions": [ { "kind": "help", @@ -2994,6 +2997,32 @@ "help": "with --since, show why each test was selected", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--sql" + ], + "dest": "sql_only", + "nargs": 0, + "const": true, + "default": false, + "help": "Run only SQL tests", + "mutex_group": 0, + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--python" + ], + "dest": "python_only", + "nargs": 0, + "const": true, + "default": false, + "help": "Run only Python tests under tests/python/ and tests/providers/", + "mutex_group": 0, + "group": 1 + }, { "kind": "store", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 6669feaeb..fc1ffb0ad 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -73,7 +73,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 188] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 190] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -96,6 +96,8 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 188] = [ ("inspect", NamespaceDefault::Bool(false)), ("since", NamespaceDefault::None), ("explain", NamespaceDefault::Bool(false)), + ("sql_only", NamespaceDefault::Bool(false)), + ("python_only", NamespaceDefault::Bool(false)), ("review_against", NamespaceDefault::None), ("review_downstream", NamespaceDefault::None), ("review_no_clone", NamespaceDefault::Bool(false)), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index fbeb74f65..237838934 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -213,7 +213,7 @@ pub(crate) const FRESHNESS_ROUTE: Route = routed( REQUEST_KEYWORD_SHAPE, ); -const TEST_FIELDS: [Field; 15] = [ +const TEST_FIELDS: [Field; 17] = [ PROJECT_DIR, NO_SQL_VALIDATION, NO_COLOR, @@ -229,6 +229,8 @@ const TEST_FIELDS: [Field; 15] = [ same("test_timeout"), same("since"), same("explain"), + same("sql_only"), + same("python_only"), ]; const TEST_ARGUMENTS: [Field; 1] = [request(class("TestCommandRequest"), &TEST_FIELDS)]; pub(crate) const TEST_ROUTE: Route = routed( diff --git a/crates/sqlbuild-discovery/src/python_paths/_helpers/python_files.rs b/crates/sqlbuild-discovery/src/python_paths/_helpers/python_files.rs index 1c700e44e..9c59eda73 100644 --- a/crates/sqlbuild-discovery/src/python_paths/_helpers/python_files.rs +++ b/crates/sqlbuild-discovery/src/python_paths/_helpers/python_files.rs @@ -6,10 +6,10 @@ use std::path::Path; use crate::constants::PYTHON_FILE_SUFFIX; use crate::python_paths::constants::{HIDDEN_PREFIX, IGNORED_ROOTS, PYTHON_CACHE_DIRECTORY}; -/// The relative parts of every Python file below `project_dir`, sorted part by part. -pub(crate) fn python_files(project_dir: &Path) -> Vec> { +/// The relative parts of every Python file in the `start` folder below `project_dir`, sorted part by part. +pub(crate) fn python_files(project_dir: &Path, start: &[&str]) -> Vec> { let mut files: Vec> = Vec::new(); - let mut pending: Vec> = vec![Vec::new()]; + let mut pending: Vec> = vec![start.iter().map(|part| (*part).to_owned()).collect()]; while let Some(parts) = pending.pop() { let directory = parts .iter() diff --git a/crates/sqlbuild-discovery/src/python_paths/constants.rs b/crates/sqlbuild-discovery/src/python_paths/constants.rs index 4483a9f78..6069594a4 100644 --- a/crates/sqlbuild-discovery/src/python_paths/constants.rs +++ b/crates/sqlbuild-discovery/src/python_paths/constants.rs @@ -32,3 +32,5 @@ pub(crate) const GROUPED_PYTHON_HOOK_ROLES: [[&str; 3]; 2] = [ ]; /// The prefix of a hidden directory the walk skips. pub(crate) const HIDDEN_PREFIX: &str = "."; +/// The folders holding project Python tests, each mirroring the project folder its second part names. +pub(crate) const PYTHON_TEST_ROOTS: [[&str; 2]; 2] = [["tests", "providers"], ["tests", "python"]]; diff --git a/crates/sqlbuild-discovery/src/python_paths/main/mod.rs b/crates/sqlbuild-discovery/src/python_paths/main/mod.rs index 07d5f14de..a8412c27a 100644 --- a/crates/sqlbuild-discovery/src/python_paths/main/mod.rs +++ b/crates/sqlbuild-discovery/src/python_paths/main/mod.rs @@ -1,2 +1,3 @@ +pub mod python_test_paths; pub mod supported_python_paths; pub mod unsupported_python_paths; diff --git a/crates/sqlbuild-discovery/src/python_paths/main/python_test_paths.rs b/crates/sqlbuild-discovery/src/python_paths/main/python_test_paths.rs new file mode 100644 index 000000000..55f3ce38f --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/main/python_test_paths.rs @@ -0,0 +1,14 @@ +//! Project Python tests, which `sqb test` runs with pytest and discovery never imports. + +use std::path::Path; + +use crate::python_paths::_helpers::python_files::python_files; +use crate::python_paths::constants::PYTHON_TEST_ROOTS; + +/// Each Python file under `tests/providers/` or `tests/python/` as relative parts, sorted part by part. +pub fn python_test_paths(project_dir: &Path) -> Vec> { + PYTHON_TEST_ROOTS + .iter() + .flat_map(|root| python_files(project_dir, root)) + .collect() +} diff --git a/crates/sqlbuild-discovery/src/python_paths/main/supported_python_paths.rs b/crates/sqlbuild-discovery/src/python_paths/main/supported_python_paths.rs index 3c4a75691..e76d6e610 100644 --- a/crates/sqlbuild-discovery/src/python_paths/main/supported_python_paths.rs +++ b/crates/sqlbuild-discovery/src/python_paths/main/supported_python_paths.rs @@ -7,7 +7,7 @@ use crate::python_paths::_helpers::supported::is_supported; /// Each supported Python file below `project_dir` as relative parts, sorted part by part. pub fn supported_python_paths(project_dir: &Path) -> Vec> { - python_files(project_dir) + python_files(project_dir, &[]) .into_iter() .filter(|parts| is_supported(parts)) .collect() diff --git a/crates/sqlbuild-discovery/src/python_paths/main/unsupported_python_paths.rs b/crates/sqlbuild-discovery/src/python_paths/main/unsupported_python_paths.rs index 337529281..cf4b4dc7d 100644 --- a/crates/sqlbuild-discovery/src/python_paths/main/unsupported_python_paths.rs +++ b/crates/sqlbuild-discovery/src/python_paths/main/unsupported_python_paths.rs @@ -7,7 +7,7 @@ use crate::python_paths::_helpers::supported::is_supported; /// Each unsupported Python file below `project_dir` as relative parts, sorted part by part. pub fn unsupported_python_paths(project_dir: &Path) -> Vec> { - python_files(project_dir) + python_files(project_dir, &[]) .into_iter() .filter(|parts| !is_supported(parts)) .collect() diff --git a/crates/sqlbuild-discovery/src/python_paths/tests.rs b/crates/sqlbuild-discovery/src/python_paths/tests.rs index cf987b755..c7677845e 100644 --- a/crates/sqlbuild-discovery/src/python_paths/tests.rs +++ b/crates/sqlbuild-discovery/src/python_paths/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_python_test_paths.rs"] +mod python_test_paths; #[path = "tests/test_supported_python_paths.rs"] mod supported_python_paths; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-discovery/src/python_paths/tests/test_python_test_paths.rs b/crates/sqlbuild-discovery/src/python_paths/tests/test_python_test_paths.rs new file mode 100644 index 000000000..7922ba8e1 --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/tests/test_python_test_paths.rs @@ -0,0 +1,51 @@ +use crate::python_paths::main::python_test_paths::python_test_paths; +use crate::python_paths::main::supported_python_paths::supported_python_paths; +use crate::python_paths::tests::test_types::TestPathsTestCase; + +#[test] +fn given_project_tests_when_listing_python_test_paths_then_only_mirrored_test_roots_remain() { + let test_cases = [ + TestPathsTestCase { + description: "tests/python and tests/providers, skipping SQL test folders, caches and hidden folders", + files: &[ + "tests/python/orders/import/test_main.py", + "tests/providers/storefront/test_provider.py", + "tests/python/orders/_test_types.py", + "tests/rules/test_rule.py", + "tests/unit/orders.py", + "tests/python/__pycache__/test_main.py", + "tests/python/.hidden/test_main.py", + "python/orders/import/main.py", + ], + expected_tests: &[ + "tests/providers/storefront/test_provider.py", + "tests/python/orders/_test_types.py", + "tests/python/orders/import/test_main.py", + ], + }, + TestPathsTestCase { + description: "a project without Python tests", + files: &["python/orders/import/main.py"], + expected_tests: &[], + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("temporary project"); + for file in test_case.files { + let path = directory.path().join(file); + std::fs::create_dir_all(path.parent().expect("parent")).expect("folders"); + std::fs::write(&path, "").expect("file"); + } + let tests: Vec = python_test_paths(directory.path()) + .into_iter() + .map(|parts| parts.join("/")) + .collect(); + let supported: Vec> = supported_python_paths(directory.path()); + assert_eq!(tests, test_case.expected_tests, "{}", test_case.description); + assert!( + supported.iter().all(|parts| parts[0] != "tests"), + "{}: discovery never reads tests/", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/python_paths/tests/test_types.rs b/crates/sqlbuild-discovery/src/python_paths/tests/test_types.rs index 640c9b2ae..3679d3b8e 100644 --- a/crates/sqlbuild-discovery/src/python_paths/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/python_paths/tests/test_types.rs @@ -11,3 +11,10 @@ pub(super) struct SupportedPathsTestCase { pub(super) files: &'static [&'static str], pub(super) expected_supported: &'static [&'static str], } + +/// One project layout and its Python test files. +pub(super) struct TestPathsTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [&'static str], + pub(super) expected_tests: &'static [&'static str], +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs index 3ab1dc70e..396f3a9ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs @@ -10,7 +10,9 @@ use crate::bindings::_helpers::compile_driver::deferral::{ build_deferred_locations, resolve_deferred_target_config, }; use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; +use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; use crate::bindings::_helpers::compile_driver::selection_phases::unit_test_selectors_applied; +use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; use crate::bindings::_helpers::planner_entries::command_plans::{ resolve_static_command_scope, resolve_static_relation_context, }; @@ -104,6 +106,26 @@ pub(crate) fn compile_static_command_context<'py>( )?), )?; let project: Bound<'py, PyAny> = native_field(&compiled, "project")?; + let selective: bool = native_field(&options, "select")?.is_truthy()? + || native_field(&options, "exclude")?.is_truthy()?; + if selected_keys.is_none() + && selective + && native_field(&options, "resolve_python_run_selectors")?.is_truthy()? + { + let graph: Bound<'py, PyAny> = build_project_graph_from_compiled_project( + py, + Some(keywords(py, &[("project", project.clone())])?), + )?; + let selection: Bound<'py, PyAny> = run_selection_from_inputs( + ( + native_field(&options, "select")?.extract()?, + native_field(&options, "exclude")?.extract()?, + ), + &graph, + &discovered, + )?; + selected_keys = native_field(&selection, "sql_keys")?; + } let deferred: Bound<'py, PyAny> = deferred_locations((&discovered, &adapter), &options, &project)?; let scope: Bound<'py, PyAny> = resolve_static_command_scope( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs index 84d87dcdc..c372767b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs @@ -1,5 +1,7 @@ //! Resolve host entries: targets read once from `sqlbuild._native_host`, attributes read live. +use std::path::PathBuf; + use pyo3::PyErr; use pyo3::exceptions::PyImportError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -73,6 +75,25 @@ pub(crate) fn write_console(py: Python<'_>, stream: &str, text: &str) -> PyResul Ok(()) } +/// Flush the interpreter's live console `stream` so text written before a child process appears first. +pub(crate) fn flush_console(py: Python<'_>, stream: &str) -> PyResult<()> { + py.import("sys")?.getattr(stream)?.call_method0("flush")?; + Ok(()) +} + +/// The interpreter running SQLBuild, which child processes reuse. +pub(crate) fn python_executable(py: Python<'_>) -> PyResult { + py.import("sys")?.getattr("executable")?.extract() +} + +/// Whether this interpreter can import the top-level module `name`. +pub(crate) fn module_installed(py: Python<'_>, name: &str) -> PyResult { + Ok(!py + .import("importlib.util")? + .call_method1("find_spec", (name,))? + .is_none()) +} + /// Keep user prints off machine-readable stdout while running project code. pub(crate) fn redirect_stdout_to_stderr(py: Python<'_>) -> PyResult> { py.import("contextlib")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index caae4c98c..8647dd03c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -698,6 +698,8 @@ const TEST_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("test_timeout", FieldDefault::None), FieldSpec::new("since", FieldDefault::None), FieldSpec::new("explain", FieldDefault::False), + FieldSpec::new("sql_only", FieldDefault::False), + FieldSpec::new("python_only", FieldDefault::False), ], }; const REFACTOR_COMMAND_REQUEST: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs index c590e518e..53eba661c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs @@ -5,6 +5,7 @@ use std::path::{Path, PathBuf}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyBool; use serde_json::Value; use sqlbuild_analysis::review_command::constants::{ CLONE_START, FULL_REFRESH_START, SEED_KIND, UNIT_TEST_LABEL, @@ -289,6 +290,7 @@ pub(crate) fn run_changed_tests( ("json_output_path", path_object(py, &output)?), ("concurrency", native_field(inputs.request, "concurrency")?), ("since", text(py, &settings.since)), + ("sql_only", PyBool::new(py, true).to_owned().into_any()), ], )?; let ran: PyResult = quietly(inputs.output, || run_test_command(py, &request)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs index 5db811030..1751c40d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod python_tests; pub(crate) mod registration; pub(crate) mod test_command; pub(crate) mod timeouts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs new file mode 100644 index 000000000..8c1aab682 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs @@ -0,0 +1,272 @@ +//! The project's Python tests, which a local `sqb test` runs with pytest after its SQL tests. + +use std::collections::BTreeSet; +use std::os::fd::AsFd; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitStatus, Stdio}; +use std::time::Instant; + +use pyo3::exceptions::{PyOSError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyFrozenSetMethods; +use serde_json::{Value, json}; +use sqlbuild_core::cli_text::main::format_phase_line::format_phase_line; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_discovery::python_paths::main::python_test_paths::python_test_paths; + +use crate::bindings::_helpers::build_progress::terminal::{flush, write}; +use crate::bindings::_helpers::compile_driver::pipeline_entries::project_graph; +use crate::bindings::_helpers::graph::python_node_graph::{ + python_node_graph, run_selection_from_inputs, +}; +use crate::bindings::_helpers::graph::python_selection::python_node_names; +use crate::bindings::_helpers::native_host::host::{ + flush_console, module_installed, python_executable, +}; +use crate::bindings::_helpers::planner_selection::unit_tests::split_project_unit_test_selectors; +use crate::bindings::_helpers::records::construct::new_helped_error; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::ErrorId; + +/// The prefix of a Python test module's file name. +const TEST_MODULE_PREFIX: &str = "test_"; +const PYTEST_MODULE: &str = "pytest"; +/// The diagnostic code of a Python test run that cannot start. +pub(crate) const PYTHON_TESTS_INPUT_CODE: &str = "S000"; +/// The project folder whose `python/` and `providers/` folders mirror the project's. +const TESTS_FOLDER: &str = "tests"; +/// The discovered Python node collections whose folders a selection's tests mirror. +const NODE_COLLECTIONS: [&str; 4] = [ + "loader_functions", + "task_functions", + "asset_functions", + "check_functions", +]; +/// Lets tests import project and `tests.*` modules, and mirrored folders reuse module names. +const IMPORT_MODE: &str = "--import-mode=importlib"; +/// The JSON section reporting the Python tests a `sqb test` ran. +const PYTHON_TESTS_SECTION: &str = "python_tests"; +const PASSED_STATUS: &str = "success"; +const FAILED_STATUS: &str = "failed"; + +/// Whether the request selects or excludes anything. +pub(crate) fn selective(request: &Bound<'_, PyAny>) -> PyResult { + Ok(native_field(request, "select")?.is_truthy()? + || native_field(request, "exclude")?.is_truthy()?) +} + +/// Every `test_*.py` module under the project's `tests/python/` and `tests/providers/`. +pub(crate) fn python_test_modules(invocation: &Bound<'_, PyAny>) -> PyResult> { + let project_dir: PathBuf = native_field(invocation, "effective_project_dir")?.extract()?; + Ok(python_test_paths(&project_dir) + .into_iter() + .filter(|parts| { + parts + .last() + .is_some_and(|name| name.starts_with(TEST_MODULE_PREFIX)) + }) + .map(|parts| parts.join("/")) + .collect()) +} + +/// Whether a selector of the request reaches a discovered Python node, such as `+storefront_import` or `tag:ingest`. +pub(crate) fn names_python_node( + request: &Bound<'_, PyAny>, + discovered: &Bound<'_, PyAny>, +) -> PyResult { + let graph: Bound<'_, PyAny> = python_node_graph(discovered)?; + let mut selectors: Vec = native_field(request, "select")?.extract()?; + selectors.extend(native_field(request, "exclude")?.extract::>()?); + for selector in selectors { + if python_node_names(&graph, (vec![selector], Vec::new())) + .is_ok_and(|names| !PyFrozenSetMethods::is_empty(&names)) + { + return Ok(true); + } + } + Ok(false) +} + +/// The Python test `modules` a local `sqb test` runs: all, or those mirroring the selected Python nodes. +pub(crate) fn selected_python_tests( + request: &Bound<'_, PyAny>, + invocation: &Bound<'_, PyAny>, + (pipeline, modules): (Option<&Bound<'_, PyAny>>, Vec), +) -> PyResult> { + let py: Python<'_> = request.py(); + let selected: Vec = match pipeline { + Some(pipeline) if !modules.is_empty() && selective(request)? => { + let folders: BTreeSet = mirrored_folders( + request, + pipeline, + &native_field(invocation, "discovered_inputs")?, + )?; + let mut mirrored: Vec = Vec::new(); + for module in modules { + if folders.iter().any(|folder| module.starts_with(folder)) { + mirrored.push(module); + } + } + mirrored + } + _ => modules, + }; + if native_field(request, "python_only")?.is_truthy()? && selected.is_empty() { + return Err(new_helped_error( + py, + ErrorId::PlannerInput, + ("no Python tests found".to_owned(), PYTHON_TESTS_INPUT_CODE), + Some( + "add test_*.py modules under tests/python/ or tests/providers/, mirroring the code they test", + ), + )); + } + if !selected.is_empty() && !module_installed(py, PYTEST_MODULE)? { + return Err(new_helped_error( + py, + ErrorId::PlannerInput, + ( + "Python tests found but pytest isn't installed".to_owned(), + PYTHON_TESTS_INPUT_CODE, + ), + Some("install the dev dependencies, or run `sqb test --sql`"), + )); + } + Ok(selected) +} + +/// Run `modules` with this interpreter's pytest from the project folder; pytest's exit code. +pub(crate) fn run_python_tests( + request: &Bound<'_, PyAny>, + invocation: &Bound<'_, PyAny>, + modules: &[String], +) -> PyResult { + let py: Python<'_> = request.py(); + let stream: Bound<'_, PyAny> = native_field(invocation, "progress_stream")?; + let style: CliStyle = CliStyle { + use_color: native_field(invocation, "use_color")?.is_truthy()?, + }; + let project_dir: PathBuf = native_field(invocation, "effective_project_dir")?.extract()?; + let interpreter: PathBuf = python_executable(py)?; + let files: String = match modules.len() { + 1 => "1 file".to_owned(), + count => format!("{count} files"), + }; + write( + &stream, + &format!("\nRunning Python tests with pytest ({files})\n"), + )?; + flush(&stream)?; + flush_console(py, "stdout")?; + let mut command: Command = Command::new(&interpreter); + command + .args(["-m", PYTEST_MODULE, IMPORT_MODE]) + .args(modules) + .current_dir(&project_dir) + .env("PYTHONPATH", python_path(&project_dir)?) + .stdin(Stdio::null()); + if native_field(request, "json_output")?.is_truthy()? { + command.stdout(Stdio::from(std::io::stderr().as_fd().try_clone_to_owned()?)); + } + let started: Instant = Instant::now(); + let status: ExitStatus = py + .detach(|| command.status()) + .map_err(|error| PyOSError::new_err(format!("cannot run pytest: {error}")))?; + let elapsed: f64 = started.elapsed().as_secs_f64(); + let code: i32 = status.code().unwrap_or(1); + let line: String = if code == 0 { + format_phase_line( + style, + true, + ("Python tests passed", &format!("{files} ({elapsed:.2}s)")), + ) + } else { + format_phase_line( + style, + false, + ( + "Python tests failed", + &format!("pytest exit code {code} ({elapsed:.2}s)"), + ), + ) + }; + write(&stream, &format!("{line}\n"))?; + flush(&stream)?; + Ok(code) +} + +/// The `tests/` folders mirroring the Python nodes the request's resource selectors resolve to. +fn mirrored_folders( + request: &Bound<'_, PyAny>, + pipeline: &Bound<'_, PyAny>, + discovered: &Bound<'_, PyAny>, +) -> PyResult> { + let split: Bound<'_, PyAny> = split_project_unit_test_selectors( + &native_field(request, "select")?, + &native_field(request, "exclude")?, + discovered, + true, + )?; + let select: Bound<'_, PyAny> = native_field(&split, "select")?; + let exclude: Bound<'_, PyAny> = native_field(&split, "exclude")?; + if !select.is_truthy()? && native_field(request, "select")?.is_truthy()? { + return Ok(BTreeSet::new()); + } + let selection: Bound<'_, PyAny> = run_selection_from_inputs( + (select.extract()?, exclude.extract()?), + &project_graph(&native_field(pipeline, "project")?)?, + discovered, + )?; + let names: Bound<'_, PyAny> = native_field(&selection, "python_node_names")?; + let mut folders: BTreeSet = BTreeSet::new(); + for collection in NODE_COLLECTIONS { + for node in native_field(discovered, collection)?.try_iter()? { + let node: Bound<'_, PyAny> = node?; + if !names.contains(native_field(&node, "name")?)? { + continue; + } + let path: PathBuf = native_field(&node, "relative_path")?.extract()?; + if let Some(folder) = path.parent() { + folders.insert(format!("{TESTS_FOLDER}/{}/", folder.to_string_lossy())); + } + } + } + Ok(folders) +} + +/// The project folder ahead of any inherited `PYTHONPATH`. +fn python_path(project_dir: &Path) -> PyResult { + let inherited: Vec = std::env::var_os("PYTHONPATH") + .map(|value| std::env::split_paths(&value).collect()) + .unwrap_or_default(); + std::env::join_paths(std::iter::once(project_dir.to_path_buf()).chain(inherited)) + .map_err(|error| PyOSError::new_err(format!("cannot build PYTHONPATH: {error}"))) +} + +/// `payload` with a `python_tests` section for `modules` and pytest's exit `code`; a failure fails it. +pub(crate) fn with_python_section( + payload: &str, + modules: &[String], + code: i32, +) -> PyResult { + let invalid = |error: serde_json::Error| PyValueError::new_err(error.to_string()); + let mut document: Value = serde_json::from_str(payload).map_err(invalid)?; + let status: &str = if code == 0 { + PASSED_STATUS + } else { + FAILED_STATUS + }; + if let Value::Object(fields) = &mut document { + if code != 0 { + fields.insert("status".to_owned(), json!(FAILED_STATUS)); + } + fields.insert( + PYTHON_TESTS_SECTION.to_owned(), + json!({ "status": status, "exit_code": code, "files": modules }), + ); + } + Ok(format!( + "{}\n", + serde_json::to_string_pretty(&document).map_err(invalid)? + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 35accda3c..744efeed5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -1,4 +1,4 @@ -//! `sqb test`: compile the selected SQL unit tests, inspect or run them, and report. +//! `sqb test`: compile the selected SQL unit tests, inspect or run them, and report; locally, then run the Python tests. use std::collections::{BTreeSet, HashMap}; use std::path::PathBuf; @@ -50,7 +50,7 @@ use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::planner_entries::command_plans::build_test_command_plan; -use crate::bindings::_helpers::records::construct::error_class; +use crate::bindings::_helpers::records::construct::{error_class, new_helped_error}; use crate::bindings::_helpers::records::construct::{new_record_with, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced; @@ -59,6 +59,10 @@ use crate::bindings::_helpers::result_output::documents::test_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use crate::bindings::_helpers::result_output::resource_ids::sql_test_identity; +use crate::bindings::_helpers::sql_test_command::python_tests::{ + PYTHON_TESTS_INPUT_CODE, names_python_node, python_test_modules, run_python_tests, + selected_python_tests, selective, with_python_section, +}; use crate::bindings::_helpers::sql_test_command::timeouts::{ RunTimeout, TimeoutKind, resolve_run_timeout, }; @@ -304,6 +308,7 @@ fn unknown_case(py: Python<'_>, message: String) -> PyErr { fn compile_tests<'py>( request: &Bound<'py, PyAny>, invocation: &Bound<'py, PyAny>, + python_selectors: bool, ) -> PyResult> { let py: Python<'py> = request.py(); let discovered: Bound<'py, PyAny> = invocation.getattr("discovered_inputs")?; @@ -328,6 +333,10 @@ fn compile_tests<'py>( "accepts_unit_test_selectors", PyBool::new(py, true).to_owned().into_any(), ), + ( + "resolve_python_run_selectors", + PyBool::new(py, python_selectors).to_owned().into_any(), + ), ( "connection_config", invocation.getattr("connection_config")?, @@ -639,13 +648,12 @@ pub(crate) fn timeout_record<'py>( host::entry(py, ExecutorHost::RunItemTimeouts)?.call((), Some(&timeout.keywords(py)?)) } -/// Write the summary, failures and JSON output; returns the exit code. +/// Write the summary and failures; returns the exit code and the `--json` document. fn complete( - request: &Bound<'_, PyAny>, invocation: &Bound<'_, PyAny>, (results, since_json): (&[SqlTestExecutionResult>], Option<&str>), -) -> PyResult { - let py: Python<'_> = request.py(); +) -> PyResult<(i32, String)> { + let py: Python<'_> = invocation.py(); let mut failures: Vec = Vec::new(); for result in results { if result.outcome == SqlTestOutcome::Pass { @@ -672,18 +680,10 @@ fn complete( ),), )?; stream.call_method0("flush")?; - let payload: Bound<'_, PyAny> = PyString::new( - py, - &with_since_section(test_execution_json(py, results)?, since_json)?, - ) - .into_any(); - write_execution_json_output( - py, - payload, - request.getattr("json_output")?.extract()?, - request.getattr("json_output_path")?, - )?; - Ok(test_exit_code(passed, results.len())) + Ok(( + test_exit_code(passed, results.len()), + with_since_section(test_execution_json(py, results)?, since_json)?, + )) } /// Run one `sqb test` request. @@ -700,14 +700,105 @@ pub(crate) fn run_test<'py>( verified: Option>, ) -> PyResult { let (worker, discovered) = verified.unzip(); - let inspect_only: bool = request.getattr("inspect")?.is_truthy()?; + let inspect_only: bool = native_field(request, "inspect")?.is_truthy()?; + if native_field(request, "python_only")?.is_truthy()? && worker.is_some() { + return Err(new_helped_error( + py, + ErrorId::PlannerInput, + ( + "--python cannot run on a server location".to_owned(), + PYTHON_TESTS_INPUT_CODE, + ), + Some("runs on a server location run SQL tests only; run `sqb test --python` locally"), + )); + } + if native_field(request, "python_only")?.is_truthy()? + && (inspect_only || !native_field(request, "since")?.is_none()) + { + return Err(new_helped_error( + py, + ErrorId::PlannerInput, + ( + "--python cannot be combined with --inspect or --since".to_owned(), + PYTHON_TESTS_INPUT_CODE, + ), + Some( + "--inspect and --since apply to SQL tests only; run `sqb test --python` without them", + ), + )); + } let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( request, &record_class(py, RecordId::TestInvocation)?, (!inspect_only, discovered), )?; + let python: bool = worker.is_none() + && !inspect_only + && !native_field(request, "sql_only")?.is_truthy()? + && native_field(request, "since")?.is_none(); + let modules: Option> = if python { + Some(python_test_modules(&invocation)?) + } else { + None + }; + let resolve: bool = match &modules { + Some(modules) if !modules.is_empty() => { + names_python_node(request, &native_field(&invocation, "discovered_inputs")?)? + } + _ => false, + }; + let (sql_code, python_tests, payload): (i32, Vec, Option) = match modules { + Some(modules) + if native_field(request, "python_only")?.is_truthy()? && !selective(request)? => + { + ( + 0, + selected_python_tests(request, &invocation, (None, modules))?, + Some(test_execution_json(py, &[])?), + ) + } + modules => run_sql_tests(request, &invocation, worker, (modules, resolve))?, + }; + if python_tests.is_empty() { + write_json(request, payload)?; + return Ok(sql_code); + } + let python_code: i32 = run_python_tests(request, &invocation, &python_tests)?; + write_json( + request, + payload + .map(|payload| with_python_section(&payload, &python_tests, python_code)) + .transpose()?, + )?; + Ok(i32::from(sql_code != 0 || python_code != 0)) +} + +/// Write a `--json` document to stdout or `--json-output`, when there is one. +fn write_json(request: &Bound<'_, PyAny>, payload: Option) -> PyResult<()> { + let Some(payload) = payload else { + return Ok(()); + }; + let py: Python<'_> = request.py(); + write_execution_json_output( + py, + PyString::new(py, &payload).into_any(), + native_field(request, "json_output")?.extract()?, + native_field(request, "json_output_path")?, + )?; + Ok(()) +} + +/// Compile, inspect or run the selected SQL tests and pick the Python test `modules` to run after. +fn run_sql_tests<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, + worker: Option>, + (modules, resolve): (Option>, bool), +) -> PyResult<(i32, Vec, Option)> { + let py: Python<'py> = request.py(); + let inspect_only: bool = native_field(request, "inspect")?.is_truthy()?; let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; - let project_dir: Bound<'_, PyAny> = native_field(&invocation, "effective_project_dir")?; + let project_dir: Bound<'_, PyAny> = native_field(invocation, "effective_project_dir")?; let since: Option = native_field(request, "since")?.extract()?; let base: Option = match since.as_deref() { Some(since) => Some(base_comparison( @@ -720,7 +811,14 @@ pub(crate) fn run_test<'py>( )?), None => None, }; - let mut pipeline: Bound<'_, PyAny> = compile_tests(request, &invocation)?; + let mut pipeline: Bound<'_, PyAny> = compile_tests(request, invocation, resolve)?; + let python_tests: Vec = match modules { + Some(modules) => selected_python_tests(request, invocation, (Some(&pipeline), modules))?, + None => Vec::new(), + }; + if native_field(request, "python_only")?.is_truthy()? { + return Ok((0, python_tests, Some(test_execution_json(py, &[])?))); + } let mut reasons: HashMap> = HashMap::new(); let mut since_json: Option = None; if let Some(base) = base { @@ -729,7 +827,7 @@ pub(crate) fn run_test<'py>( ( &pipeline, &stream, - &native_field(&invocation, "discovered_inputs")?, + &native_field(invocation, "discovered_inputs")?, ), &project_dir.extract::()?, base, @@ -738,11 +836,30 @@ pub(crate) fn run_test<'py>( since_json = Some(json); } if inspect_only { - return inspect(&stream, &pipeline.getattr("plan_output")?, &reasons); + return Ok(( + inspect(&stream, &pipeline.getattr("plan_output")?, &reasons)?, + python_tests, + None, + )); + } + if !python_tests.is_empty() + && !pipeline + .getattr("plan_output")? + .getattr("test_entries")? + .is_truthy()? + { + return Ok(( + 0, + python_tests, + Some(with_since_section( + test_execution_json(py, &[])?, + since_json.as_deref(), + )?), + )); } let concurrency: i64 = resolve_concurrency( &request.getattr("concurrency")?, - &native_field(&invocation, "discovered_inputs")?, + &native_field(invocation, "discovered_inputs")?, &request.getattr("selected_target")?, )? .limit; @@ -763,17 +880,18 @@ pub(crate) fn run_test<'py>( TimeoutKind::Test, )?; let results: Vec>> = - run_tests(&invocation, &pipeline, (concurrency, &timeout, worker))?; + run_tests(invocation, &pipeline, (concurrency, &timeout, worker))?; write_test_runtime_target( ( - &native_field(&invocation, "effective_project_dir")? + &native_field(invocation, "effective_project_dir")? .call_method1("__truediv__", ("target",))?, &invocation.getattr("adapter")?, ), &pipeline.getattr("plan_output")?, &results, )?; - complete(request, &invocation, (&results, since_json.as_deref())) + let (code, payload): (i32, String) = complete(invocation, (&results, since_json.as_deref()))?; + Ok((code, python_tests, Some(payload))) } /// Add source and safe typed values only for parameterized test rows. diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs index cb8c5ca23..7a101e305 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -14,6 +14,7 @@ use crate::errors::RowsError; use crate::models::{EvaluateRequest, Model, RetainedRows, RowsEvaluation, RowsRequest}; use crate::orchestration_rules::main::orchestration_facts::orchestration_facts; use crate::python_rules::main::python_sources::python_sources; +use crate::python_rules::main::python_test_sources::python_test_sources; /// The shared key prefix of fields a request serializes but evaluation derives again. const DERIVED_FIELD_PREFIX: &str = ",\"empty_input_only"; @@ -60,6 +61,7 @@ pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result>()?; request.python_sources = python_sources(&request).map_err(RowsError::Rules)?; + request.python_test_sources = python_test_sources(&request).map_err(RowsError::Rules)?; request.orchestration = orchestration_facts(&request).map_err(RowsError::Rules)?; let mut request_json: Option> = None; let memo: Option<(String, PathBuf)> = match memo { diff --git a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs index bc7070ad4..7fd6a6967 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs @@ -67,6 +67,7 @@ const REVIEWED_SCOPES: &[(&str, RuleScope)] = &[ ("SQBRPYTHON018", RuleScope::Python), ("SQBRPYTHON019", RuleScope::Python), ("SQBRPYTHON020", RuleScope::Python), + ("SQBRPYTHON021", RuleScope::Python), ("SQBRPYTHON101", RuleScope::Python), ("SQBRPYTHON102", RuleScope::Python), ("SQBRPYTHON103", RuleScope::Python), @@ -129,6 +130,30 @@ const REVIEWED_SCOPES: &[(&str, RuleScope)] = &[ ("SQBRPYTHON625", RuleScope::Python), ("SQBRPYTHON626", RuleScope::Python), ("SQBRPYTHON627", RuleScope::Python), + ("SQBRPYTHON701", RuleScope::Python), + ("SQBRPYTHON702", RuleScope::Python), + ("SQBRPYTHON703", RuleScope::Python), + ("SQBRPYTHON704", RuleScope::Python), + ("SQBRPYTHON705", RuleScope::Python), + ("SQBRPYTHON706", RuleScope::Python), + ("SQBRPYTHON707", RuleScope::Python), + ("SQBRPYTHON708", RuleScope::Python), + ("SQBRPYTHON709", RuleScope::Python), + ("SQBRPYTHON710", RuleScope::Python), + ("SQBRPYTHON711", RuleScope::Python), + ("SQBRPYTHON712", RuleScope::Python), + ("SQBRPYTHON713", RuleScope::Python), + ("SQBRPYTHON714", RuleScope::Python), + ("SQBRPYTHON715", RuleScope::Python), + ("SQBRPYTHON716", RuleScope::Python), + ("SQBRPYTHON717", RuleScope::Python), + ("SQBRPYTHON718", RuleScope::Python), + ("SQBRPYTHON719", RuleScope::Python), + ("SQBRPYTHON720", RuleScope::Python), + ("SQBRPYTHON721", RuleScope::Python), + ("SQBRPYTHON722", RuleScope::Python), + ("SQBRPYTHON723", RuleScope::Python), + ("SQBRPYTHON724", RuleScope::Python), ("SQBRTEST101", RuleScope::Project), ("SQBRTEST102", RuleScope::Project), ("SQBRTEST103", RuleScope::Project), diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index e50fe328a..dfcdb43bf 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -773,6 +773,8 @@ pub struct EvaluateRequest { pub python_version: (u8, u8), /// The supported project Python files, filled from disk only when a Python rule is selected. pub python_sources: Vec, + /// The project's Python tests, filled from disk only when a Python test rule is selected. + pub python_test_sources: Vec, /// The job and sensor selections, filled from disk only when an orchestration rule is selected. pub orchestration: OrchestrationFacts, } @@ -825,6 +827,7 @@ impl Default for EvaluateRequest { rules_cache_path: String::new(), python_version: (3, 12), python_sources: vec![], + python_test_sources: vec![], orchestration: OrchestrationFacts::default(), } } diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/contexts.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/contexts.rs index 23e2d3a0e..8e646e4eb 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/contexts.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/contexts.rs @@ -11,7 +11,7 @@ use std::path::Path; use crate::models::{EvaluateRequest, RulesConfig}; use crate::python_rules::constants::{ - NAMING_CONTRACTS, NAMING_FAMILY, PACKAGE_ROOT, PROJECT_CONFIG_FILE, ROOT_SCOPE, + NAMING_CONTRACTS, NAMING_FAMILY, PACKAGE_ROOT, PROJECT_CONFIG_FILE, }; use crate::python_rules::models::{PythonRuleSpec, PythonTarget}; @@ -68,12 +68,12 @@ pub(crate) fn context_settings( /// The context of `target`, without thresholds, observations or the dead-code context. pub(crate) fn rule_context(target: &PythonTarget, settings: &ContextSettings) -> NativeRuleContext { let shared = NativeRuleContext { - scope: ROOT_SCOPE.to_owned(), + scope: target.scope.to_owned(), repository_path: target.repository_path.clone(), relative_parts: target.relative_parts.clone(), contracts: settings.contracts.clone(), repo_root: settings.repo_root.clone(), - scope_roots: vec![(ROOT_SCOPE.to_owned(), target.root.clone())], + scope_roots: vec![(target.scope.to_owned(), target.root.clone())], ..NativeRuleContext::default() }; if !target.package { diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/execution.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/execution.rs index 9ef5c21bd..8a8c81249 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/execution.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/execution.rs @@ -19,7 +19,7 @@ use crate::python_rules::_helpers::engine::file_cache::{answers_digest, file_ide use crate::python_rules::_helpers::engine::observations::{ observations, relative_queries, repository_index, }; -use crate::python_rules::constants::{PACKAGE_ROOT, PYTHON_RULES, ROOT_SCOPE}; +use crate::python_rules::constants::{PACKAGE_ROOT, PYTHON_RULES}; use crate::python_rules::models::{ CacheSplit, CallInputs, FilePlan, PythonCacheEntry, PythonEvaluation, PythonFiles, PythonRuleSpec, PythonTarget, RuleCall, @@ -258,7 +258,7 @@ fn file_rules<'a>( let applicable: Vec<&PythonRuleSpec> = rules .iter() .copied() - .filter(|rule| package || rule.applies == PythonFiles::All) + .filter(|rule| package || rule.applies != PythonFiles::Package) .collect(); let execution_targets: Vec = indexes .iter() @@ -290,7 +290,7 @@ fn execution_rule(rule: &PythonRuleSpec) -> NativeExecutionRule { fn execution_target(target: &PythonTarget) -> NativeExecutionTarget { NativeExecutionTarget { repository_path: target.repository_path.clone(), - scope: ROOT_SCOPE.to_owned(), + scope: target.scope.to_owned(), root: target.root.clone(), relative_parts: target.relative_parts.clone(), direct: true, diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/mod.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/mod.rs index 6685ab9f6..2d83f7e21 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/mod.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/mod.rs @@ -4,6 +4,8 @@ pub(crate) mod faults; pub(crate) mod file_cache; pub(crate) mod layout; pub(crate) mod observations; +pub(crate) mod passes; pub(crate) mod placement; pub(crate) mod programs; pub(crate) mod project_plane; +pub(crate) mod source_files; diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/observations.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/observations.rs index d34f67a6c..201e96f43 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/observations.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/observations.rs @@ -98,6 +98,15 @@ fn answer( Some(module) => programs.program(module)?.is_public_facade(&query.argument), None => false, })], + "dataclasses" => match programs.index_of(path) { + Some(module) => programs + .program(module)? + .dataclass_rows() + .iter() + .map(|row| row.name.clone()) + .collect(), + None => Vec::new(), + }, "package_anchor" => vec![bool_text( package_anchor(programs.sources, path).is_some_and(|anchor| anchor == query.argument), )], diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/passes.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/passes.rs new file mode 100644 index 000000000..7db5fe0ca --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/passes.rs @@ -0,0 +1,58 @@ +//! One Python rule pass: the selected Fensu-backed rules over project Python, or over its tests. + +use std::collections::BTreeMap; + +use fensu_native::NativeProjectPlane; + +use crate::models::{EvaluateRequest, PythonSource}; +use crate::python_rules::_helpers::engine::contexts::context_settings; +use crate::python_rules::_helpers::engine::execution::{evaluated_files, file_plans}; +use crate::python_rules::_helpers::engine::file_cache::project_digest; +use crate::python_rules::_helpers::engine::placement::{python_targets, test_targets}; +use crate::python_rules::_helpers::engine::programs::Programs; +use crate::python_rules::_helpers::engine::project_plane::loaded_project; +use crate::python_rules::models::{ + CallInputs, PythonCacheEntry, PythonEvaluation, PythonFiles, PythonRuleSpec, PythonTarget, +}; + +/// Cache-aware findings of the `selected` rules over project Python tests, or the rest of project Python. +pub(crate) fn evaluated_pass( + request: &EvaluateRequest, + (selected, tests): (&[&'static PythonRuleSpec], bool), + cached: &BTreeMap, + fingerprint: &str, +) -> Result { + let rules: Vec<&PythonRuleSpec> = selected + .iter() + .copied() + .filter(|rule| (rule.applies == PythonFiles::Tests) == tests) + .collect(); + let sources: &[PythonSource] = if tests { + &request.python_test_sources + } else { + &request.python_sources + }; + if rules.is_empty() || sources.is_empty() { + return Ok(PythonEvaluation::default()); + } + let programs = Programs::new(sources, request.python_version); + let targets: Vec = if tests { + test_targets(sources) + } else { + python_targets(sources) + }; + let (plane, entrypoints) = if tests { + (NativeProjectPlane::default(), Vec::new()) + } else { + loaded_project(&rules, &targets, &programs, request)? + }; + let inputs = CallInputs { + request, + settings: context_settings(request, &rules, entrypoints), + programs: &programs, + cached, + fingerprint, + project_digest: project_digest(sources), + }; + evaluated_files(file_plans(&targets, &rules, &inputs)?, &inputs, &plane) +} diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/placement.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/placement.rs index 4239ce8a5..750b2305c 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/placement.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/placement.rs @@ -2,7 +2,8 @@ use crate::models::PythonSource; use crate::python_rules::constants::{ - INIT_FILE, MAIN_ROLE, PACKAGE_ROOT, ROLE_DIRECTORIES, ROLE_FILES, UNIT_MAIN_FILE, + INIT_FILE, MAIN_ROLE, PACKAGE_ROOT, ROLE_DIRECTORIES, ROLE_FILES, ROOT_SCOPE, TEST_SCOPE, + UNIT_MAIN_FILE, }; use crate::python_rules::models::PythonTarget; @@ -11,6 +12,18 @@ pub(crate) fn python_targets(sources: &[PythonSource]) -> Vec { sources.iter().enumerate().map(placed).collect() } +/// Place every Python test under its own folder in Fensu's test scope. +pub(crate) fn test_targets(sources: &[PythonSource]) -> Vec { + sources + .iter() + .enumerate() + .map(|source| PythonTarget { + scope: TEST_SCOPE, + ..placed(source) + }) + .collect() +} + fn placed((source, python): (usize, &PythonSource)) -> PythonTarget { let parts: Vec = python.relative_path.split('/').map(str::to_owned).collect(); if parts.len() > 1 && parts[0] == PACKAGE_ROOT { @@ -32,6 +45,7 @@ fn placed((source, python): (usize, &PythonSource)) -> PythonTarget { is_entry_module: is_main_module && relative_parts.last().is_some_and(|name| name != INIT_FILE), relative_parts, + scope: ROOT_SCOPE, }; } let (root, file) = python @@ -47,6 +61,7 @@ fn placed((source, python): (usize, &PythonSource)) -> PythonTarget { role: None, is_main_module: false, is_entry_module: false, + scope: ROOT_SCOPE, } } diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/source_files.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/source_files.rs new file mode 100644 index 000000000..a118570ea --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/source_files.rs @@ -0,0 +1,22 @@ +//! Read one project Python file as a rule source, digested so the memo sees every edit. + +use std::path::Path; + +use crate::models::PythonSource; + +/// The project file at `relative_path`, with its digest. +pub(crate) fn read_source( + project_dir: &Path, + relative_path: String, +) -> Result { + let bytes: Vec = std::fs::read(project_dir.join(&relative_path)) + .map_err(|error| format!("{relative_path}: cannot read Python source: {error}"))?; + let digest: String = blake3::hash(&bytes).to_hex().to_string(); + let text: String = String::from_utf8(bytes) + .map_err(|_| format!("{relative_path}: Python source is not valid UTF-8"))?; + Ok(PythonSource { + relative_path, + digest, + text, + }) +} diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs index ff681aad6..3a3e3300c 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs @@ -5,3 +5,4 @@ pub(crate) mod nodes; pub(crate) mod orchestration; pub(crate) mod project; pub(crate) mod providers; +pub(crate) mod python_tests; diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/python_tests.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/python_tests.rs new file mode 100644 index 000000000..ed2447812 --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/python_tests.rs @@ -0,0 +1,65 @@ +//! Where project Python tests sit: each mirrors an existing folder, and each test module a module, under `python/` or `providers/`. + +use std::collections::BTreeSet; + +use crate::models::{Fault, PythonSource}; +use crate::python_rules::_helpers::engine::faults::catalogue_fault; +use crate::python_rules::constants::{PYTHON_TEST_PLACEMENT_CODE, TEST_FILE_PREFIX}; + +/// A fault on each Python test whose mirrored folder, or for a test module whose module, is not in the project. +pub(crate) fn python_test_faults( + sources: &[PythonSource], + tests: &[PythonSource], + codes: &BTreeSet<&str>, +) -> Vec { + if !codes.contains(PYTHON_TEST_PLACEMENT_CODE) { + return Vec::new(); + } + let paths: BTreeSet<&str> = sources + .iter() + .map(|source| source.relative_path.as_str()) + .collect(); + tests + .iter() + .filter_map(|test| { + let detail: String = missing_counterpart(&test.relative_path, &paths)?; + catalogue_fault( + PYTHON_TEST_PLACEMENT_CODE, + &test.relative_path, + (1, 1), + &detail, + ) + }) + .collect() +} + +/// What the test at `path`, such as `tests/python/orders/test_export.py`, mirrors that the project lacks. +fn missing_counterpart(path: &str, paths: &BTreeSet<&str>) -> Option { + let parts: Vec<&str> = path.split('/').collect(); + let [_, root, folders @ .., file] = parts.as_slice() else { + return None; + }; + if folders.is_empty() { + return Some(format!( + "it sits directly in tests/{root}/, not in a folder mirroring one under {root}/" + )); + } + let folder: String = format!("{root}/{}", folders.join("/")); + if !holds_files(paths, &folder) { + return Some(format!("{folder}/ does not exist")); + } + let module: &str = file.strip_prefix(TEST_FILE_PREFIX)?; + let tested: String = format!("{folder}/{module}"); + let package: &str = tested.trim_end_matches(".py"); + (!paths.contains(tested.as_str()) && !holds_files(paths, package)) + .then(|| format!("{tested} does not exist")) +} + +/// Whether any project Python file sits below `folder`. +fn holds_files(paths: &BTreeSet<&str>, folder: &str) -> bool { + let prefix: String = format!("{folder}/"); + paths + .range(prefix.as_str()..) + .next() + .is_some_and(|path| path.starts_with(&prefix)) +} diff --git a/crates/sqlbuild-rules/src/python_rules/constants.rs b/crates/sqlbuild-rules/src/python_rules/constants.rs index 988c75309..175edd738 100644 --- a/crates/sqlbuild-rules/src/python_rules/constants.rs +++ b/crates/sqlbuild-rules/src/python_rules/constants.rs @@ -1,11 +1,11 @@ -//! The Python rule codes: two SQLBuild layout rules, then Fensu's core rules in bands by Fensu family. +//! The Python rule codes: SQLBuild's own project rules, then Fensu's core rules in bands by Fensu family. use fensu_native::rules::constants as fensu; use crate::python_rules::models::{PythonFiles, PythonRuleSpec}; /// Every `SQBRPYTHON` rule; numbers are never reused, and new rules append to their band. -pub(crate) const PYTHON_RULES: [PythonRuleSpec; 82] = [ +pub(crate) const PYTHON_RULES: [PythonRuleSpec; 107] = [ PythonRuleSpec { code: "SQBRPYTHON001", fensu_code: None, @@ -246,6 +246,18 @@ pub(crate) const PYTHON_RULES: [PythonRuleSpec; 82] = [ message: "sensor and calendar functions have no docstring; their description is the declaration's description=", remediation: "Remove the docstring and keep the text in description=.", }, + PythonRuleSpec { + code: "SQBRPYTHON021", + fensu_code: None, + slug: "python-test-placement", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "Python tests must mirror an existing module under python/ or providers/", + remediation: "Move the test to tests/python/ or tests/providers/ at the path of the module it tests: tests/python/orders/storefront_import/test_main.py tests python/orders/storefront_import/main.py.", + }, PythonRuleSpec { code: "SQBRPYTHON101", fensu_code: Some(fensu::PARAMETER_ANNOTATION_CODE), @@ -1040,6 +1052,294 @@ pub(crate) const PYTHON_RULES: [PythonRuleSpec; 82] = [ message: "source files must stay below the configured line limit", remediation: "Split the file by a cohesive role or concern instead of extracting arbitrary numbered fragments.", }, + PythonRuleSpec { + code: "SQBRPYTHON701", + fensu_code: Some(fensu::TEST_INIT_MODULE_EMPTY_CODE), + slug: "test-init-module-empty", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test package __init__.py files must be empty or docstring-only", + remediation: "Remove declarations from __init__.py and import them from their owning module.", + }, + PythonRuleSpec { + code: "SQBRPYTHON702", + fensu_code: Some(fensu::TEST_ABSOLUTE_IMPORTS_CODE), + slug: "test-absolute-imports", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "tests must use absolute imports", + remediation: "Replace the relative import with the full tests or project package path.", + }, + PythonRuleSpec { + code: "SQBRPYTHON703", + fensu_code: Some(fensu::TEST_NO_TOP_LEVEL_HELPERS_CODE), + slug: "test-no-top-level-helpers", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test modules may contain only tests, imports, and declarations", + remediation: "Move reusable functions into the local helpers.py module.", + }, + PythonRuleSpec { + code: "SQBRPYTHON704", + fensu_code: Some(fensu::TEST_NO_IF_IN_TESTS_CODE), + slug: "test-no-conditionals", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "tests and local test helpers must not contain conditional control flow", + remediation: "Use parametrized cases when setup and assertions remain branch-free; otherwise split the behavior into separate test functions.", + }, + PythonRuleSpec { + code: "SQBRPYTHON705", + fensu_code: Some(fensu::TEST_PRIVATE_CONSTANT_ORDER_CODE), + slug: "test-private-constant-order", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "private test constants must appear before test functions", + remediation: "Move the private constant above the first test so module setup is visible before behavior.", + }, + PythonRuleSpec { + code: "SQBRPYTHON706", + fensu_code: Some(fensu::TEST_NO_COMPLEX_COMPREHENSIONS_CODE), + slug: "test-no-complex-comprehensions", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "nested or multi-generator comprehensions hide control flow and data shapes", + remediation: "Use simple statements with named intermediate values, or a named helper in helpers.py.", + }, + PythonRuleSpec { + code: "SQBRPYTHON707", + fensu_code: Some(fensu::TEST_TYPES_DESCRIPTION_CODE), + slug: "test-case-description", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test-case dataclasses must define a description field", + remediation: "Add description: str so parametrized cases explain the behavior they represent.", + }, + PythonRuleSpec { + code: "SQBRPYTHON708", + fensu_code: Some(fensu::TEST_TYPES_EXPECTED_FIELD_CODE), + slug: "test-case-expected-field", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test-case dataclasses must define at least one expected_ field", + remediation: "Name expected outcomes with an expected_ prefix and assert against them in the test.", + }, + PythonRuleSpec { + code: "SQBRPYTHON709", + fensu_code: Some(fensu::TEST_LOCAL_TEST_TYPES_IMPORT_CODE), + slug: "test-local-types-import", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "tests must import test-case types from their local _test_types.py", + remediation: "Move the dataclass beside the test and import it through its absolute tests path.", + }, + PythonRuleSpec { + code: "SQBRPYTHON710", + fensu_code: Some(fensu::TEST_LOCAL_TEST_TYPES_FILE_CODE), + slug: "test-local-types-file", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test directories must provide a local _test_types.py", + remediation: "Create _test_types.py beside the test and define its test-case dataclasses there.", + }, + PythonRuleSpec { + code: "SQBRPYTHON711", + fensu_code: Some(fensu::TEST_FILE_NAME_CODE), + slug: "test-file-name", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test modules must use a test_ filename", + remediation: "Rename the module to test_.py after the module it tests.", + }, + PythonRuleSpec { + code: "SQBRPYTHON712", + fensu_code: Some(fensu::TEST_FUNCTION_NAME_CODE), + slug: "test-function-name", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test functions must use test_given__when__then_", + remediation: "Rename the test so its precondition, action, and expected behavior are explicit.", + }, + PythonRuleSpec { + code: "SQBRPYTHON713", + fensu_code: Some(fensu::TEST_DATACLASS_PARAMETRIZE_CODE), + slug: "test-dataclass-parametrize", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "tests must use dataclass-backed pytest parameterization", + remediation: "Add @pytest.mark.parametrize with local test-case dataclass instances.", + }, + PythonRuleSpec { + code: "SQBRPYTHON714", + fensu_code: Some(fensu::TEST_ACCEPTS_TEST_CASE_CODE), + slug: "test-accepts-test-case", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "parametrized tests must accept a test_case argument", + remediation: "Name the parameter test_case and read inputs and expectations from it.", + }, + PythonRuleSpec { + code: "SQBRPYTHON715", + fensu_code: Some(fensu::TEST_CASE_ANNOTATION_CODE), + slug: "test-case-annotation", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test_case parameters must use a local test-case dataclass annotation", + remediation: "Annotate test_case with a dataclass imported from the local _test_types.py.", + }, + PythonRuleSpec { + code: "SQBRPYTHON716", + fensu_code: Some(fensu::TEST_EXPECTED_FIELD_ASSERTION_CODE), + slug: "test-expected-field-assertion", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "tests must assert against an expected_ field from test_case", + remediation: "Store the expected outcome on test_case and reference it in an assertion.", + }, + PythonRuleSpec { + code: "SQBRPYTHON717", + fensu_code: Some(fensu::TEST_PARAMETRIZE_ARGUMENTS_CODE), + slug: "test-parametrize-arguments", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest parametrize decorators must define parameter names and values", + remediation: "Pass both the parameter-name string and the case sequence to parametrize.", + }, + PythonRuleSpec { + code: "SQBRPYTHON718", + fensu_code: Some(fensu::TEST_PARAMETRIZE_TEST_CASE_CODE), + slug: "test-parametrize-test-case", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest parametrize must expose cases through the test_case parameter", + remediation: "Use \"test_case\" as the parametrize parameter name.", + }, + PythonRuleSpec { + code: "SQBRPYTHON719", + fensu_code: Some(fensu::TEST_PARAMETRIZE_IDS_CODE), + slug: "test-parametrize-ids", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest parametrize decorators must define readable case ids", + remediation: "Set ids=lambda case: case.description.", + }, + PythonRuleSpec { + code: "SQBRPYTHON720", + fensu_code: Some(fensu::TEST_INLINE_PARAMETRIZE_VALUES_CODE), + slug: "test-inline-parametrize-values", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest parametrize values must be a visible list, tuple, or local comprehension", + remediation: "Inline the case sequence in @pytest.mark.parametrize so its cases are visible beside the test.", + }, + PythonRuleSpec { + code: "SQBRPYTHON721", + fensu_code: Some(fensu::TEST_NONEMPTY_PARAMETRIZE_VALUES_CODE), + slug: "test-nonempty-parametrize-values", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest parametrize case sequences must not be empty", + remediation: "Add at least one behavior case or remove the test until a real case exists.", + }, + PythonRuleSpec { + code: "SQBRPYTHON722", + fensu_code: Some(fensu::TEST_NO_DICT_TEST_CASES_CODE), + slug: "test-no-dict-cases", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest cases must use typed dataclasses instead of dictionaries", + remediation: "Define a local frozen test-case dataclass and construct one instance per case.", + }, + PythonRuleSpec { + code: "SQBRPYTHON723", + fensu_code: Some(fensu::TEST_LOCAL_TEST_CASE_CONSTRUCTORS_CODE), + slug: "test-local-case-constructors", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest cases must construct dataclasses from the local _test_types.py", + remediation: "Parametrize with dataclasses imported from the local _test_types.py.", + }, + PythonRuleSpec { + code: "SQBRPYTHON724", + fensu_code: Some(fensu::TEST_DESCRIPTION_LAMBDA_IDS_CODE), + slug: "test-description-ids", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest case ids must come from each test case description", + remediation: "Use ids=lambda case: case.description so failures name the behavior.", + }, ]; /// The project folder that holds the runtime Python package, imported as `python`. @@ -1162,6 +1462,11 @@ pub(crate) const ORCHESTRATION_PRIVATE_SUPPORT_CODE: &str = "SQBRPYTHON017"; pub(crate) const ORCHESTRATION_MODULE_SURFACE_CODE: &str = "SQBRPYTHON018"; pub(crate) const ORCHESTRATION_IMPORT_DIRECTION_CODE: &str = "SQBRPYTHON019"; pub(crate) const ORCHESTRATION_NO_DOCSTRING_CODE: &str = "SQBRPYTHON020"; +pub(crate) const PYTHON_TEST_PLACEMENT_CODE: &str = "SQBRPYTHON021"; +/// Fensu's scope for test files, the only scope its test rules check. +pub(crate) const TEST_SCOPE: &str = "test"; +/// The prefix of a test module's file name; other files in a mirrored test folder need only the folder. +pub(crate) const TEST_FILE_PREFIX: &str = "test_"; /// The orchestration rules, which read sensor and calendar modules and their importers. pub(crate) const ORCHESTRATION_RULE_CODES: [&str; 5] = [ ORCHESTRATION_MODULE_PATHS_CODE, diff --git a/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs b/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs index 2917a095b..f5ddcd959 100644 --- a/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs +++ b/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs @@ -3,21 +3,14 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::models::{EvaluateRequest, RuleMetadata}; -use crate::python_rules::_helpers::engine::contexts::context_settings; -use crate::python_rules::_helpers::engine::execution::{ - evaluated_files, file_plans, selected_rules, -}; -use crate::python_rules::_helpers::engine::file_cache::project_digest; +use crate::python_rules::_helpers::engine::execution::selected_rules; use crate::python_rules::_helpers::engine::layout::layout_faults; -use crate::python_rules::_helpers::engine::placement::python_targets; -use crate::python_rules::_helpers::engine::programs::Programs; -use crate::python_rules::_helpers::engine::project_plane::loaded_project; +use crate::python_rules::_helpers::engine::passes::evaluated_pass; use crate::python_rules::_helpers::units::orchestration::orchestration_faults; use crate::python_rules::_helpers::units::project::unit_faults; use crate::python_rules::_helpers::units::providers::provider_faults; -use crate::python_rules::models::{ - CallInputs, PythonCacheEntry, PythonEvaluation, PythonRuleSpec, PythonTarget, -}; +use crate::python_rules::_helpers::units::python_tests::python_test_faults; +use crate::python_rules::models::{PythonCacheEntry, PythonEvaluation, PythonRuleSpec}; /// Sorted findings of the selected Python rules, reusing `cached` ones made under `fingerprint` that still hold. pub(crate) fn evaluate_python_rules( @@ -27,22 +20,14 @@ pub(crate) fn evaluate_python_rules( fingerprint: &str, ) -> Result { let codes: BTreeSet<&str> = selected.iter().map(|rule| rule.code.as_str()).collect(); - let rules: Vec<&PythonRuleSpec> = selected_rules(selected); - let mut evaluation = PythonEvaluation::default(); - if !rules.is_empty() && !request.python_sources.is_empty() { - let programs = Programs::new(&request.python_sources, request.python_version); - let targets: Vec = python_targets(&request.python_sources); - let (plane, entrypoints) = loaded_project(&rules, &targets, &programs, request)?; - let inputs = CallInputs { - request, - settings: context_settings(request, &rules, entrypoints), - programs: &programs, - cached, - fingerprint, - project_digest: project_digest(&request.python_sources), - }; - evaluation = evaluated_files(file_plans(&targets, &rules, &inputs)?, &inputs, &plane)?; - } + let rules: Vec<&'static PythonRuleSpec> = selected_rules(selected); + let mut evaluation: PythonEvaluation = + evaluated_pass(request, (&rules, false), cached, fingerprint)?; + let tests: PythonEvaluation = evaluated_pass(request, (&rules, true), cached, fingerprint)?; + evaluation.faults.extend(tests.faults); + evaluation.entries.extend(tests.entries); + evaluation.hits += tests.hits; + evaluation.misses += tests.misses; evaluation .faults .extend(layout_faults(&request.python_sources, &codes)); @@ -61,6 +46,11 @@ pub(crate) fn evaluate_python_rules( &codes, request.python_version, )); + evaluation.faults.extend(python_test_faults( + &request.python_sources, + &request.python_test_sources, + &codes, + )); evaluation.faults.sort_by(|left, right| { (&left.path, left.line, left.column, &left.code).cmp(&( &right.path, diff --git a/crates/sqlbuild-rules/src/python_rules/main/mod.rs b/crates/sqlbuild-rules/src/python_rules/main/mod.rs index b735fb0d5..17a3a7ea9 100644 --- a/crates/sqlbuild-rules/src/python_rules/main/mod.rs +++ b/crates/sqlbuild-rules/src/python_rules/main/mod.rs @@ -2,3 +2,4 @@ pub mod evaluate_python_rules; pub mod python_catalogue; pub mod python_rule_options; pub mod python_sources; +pub mod python_test_sources; diff --git a/crates/sqlbuild-rules/src/python_rules/main/python_sources.rs b/crates/sqlbuild-rules/src/python_rules/main/python_sources.rs index 1ec2fdbaa..807758102 100644 --- a/crates/sqlbuild-rules/src/python_rules/main/python_sources.rs +++ b/crates/sqlbuild-rules/src/python_rules/main/python_sources.rs @@ -5,6 +5,7 @@ use std::path::Path; use sqlbuild_discovery::python_paths::main::supported_python_paths::supported_python_paths; use crate::models::{EvaluateRequest, PythonSource}; +use crate::python_rules::_helpers::engine::source_files::read_source; use crate::rules::main::request_selects::request_selects; /// Every supported project Python file when the request selects a Python rule, otherwise none. @@ -18,16 +19,3 @@ pub(crate) fn python_sources(request: &EvaluateRequest) -> Result Result { - let bytes: Vec = std::fs::read(project_dir.join(&relative_path)) - .map_err(|error| format!("{relative_path}: cannot read Python source: {error}"))?; - let digest: String = blake3::hash(&bytes).to_hex().to_string(); - let text: String = String::from_utf8(bytes) - .map_err(|_| format!("{relative_path}: Python source is not valid UTF-8"))?; - Ok(PythonSource { - relative_path, - digest, - text, - }) -} diff --git a/crates/sqlbuild-rules/src/python_rules/main/python_test_sources.rs b/crates/sqlbuild-rules/src/python_rules/main/python_test_sources.rs new file mode 100644 index 000000000..1b411c927 --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/main/python_test_sources.rs @@ -0,0 +1,28 @@ +//! The project Python test files the selected Python test rules read. + +use std::path::Path; + +use sqlbuild_discovery::python_paths::main::python_test_paths::python_test_paths; + +use crate::models::{EvaluateRequest, PythonSource}; +use crate::python_rules::_helpers::engine::source_files::read_source; +use crate::python_rules::constants::PYTHON_RULES; +use crate::python_rules::models::PythonFiles; +use crate::rules::main::request_selects::request_selects; + +/// Every project Python test file when the request selects a Python test rule, otherwise none. +pub(crate) fn python_test_sources(request: &EvaluateRequest) -> Result, String> { + let test_rule = |code: &str| { + PYTHON_RULES + .iter() + .any(|rule| rule.code == code && rule.applies == PythonFiles::Tests) + }; + if !request_selects(request, |rule| test_rule(&rule.code))? { + return Ok(Vec::new()); + } + let project_dir: &Path = Path::new(&request.project_dir); + python_test_paths(project_dir) + .into_iter() + .map(|parts| read_source(project_dir, parts.join("/"))) + .collect() +} diff --git a/crates/sqlbuild-rules/src/python_rules/models.rs b/crates/sqlbuild-rules/src/python_rules/models.rs index 9f598d2cd..d228ee9ad 100644 --- a/crates/sqlbuild-rules/src/python_rules/models.rs +++ b/crates/sqlbuild-rules/src/python_rules/models.rs @@ -7,6 +7,8 @@ pub enum PythonFiles { Package, /// Every supported project Python file, including hooks, UDFs and orchestration modules. All, + /// Only the project's Python tests under `tests/python/` and `tests/providers/`. + Tests, } /// One `SQBRPYTHON` rule and the Fensu core rule it runs, if any. @@ -42,6 +44,8 @@ pub(crate) struct PythonTarget { pub role: Option, pub is_main_module: bool, pub is_entry_module: bool, + /// Fensu's scope for the file: `test` for project Python tests, else the root scope. + pub scope: &'static str, } /// One `evaluate_core_rules` call on a file: the Fensu codes it runs and the context they read. diff --git a/crates/sqlbuild-rules/src/python_rules/tests.rs b/crates/sqlbuild-rules/src/python_rules/tests.rs index ffb2bc69d..a28c2f837 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests.rs @@ -14,5 +14,7 @@ mod python_cache; mod python_catalogue; #[path = "tests/test_python_scaling.rs"] mod python_scaling; +#[path = "tests/test_python_test_rules.rs"] +mod python_test_rules; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs index b646fed56..d677ae2b8 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs @@ -14,14 +14,14 @@ use crate::rules::models::BuiltinOptionKind; fn given_the_catalogue_when_listing_python_rules_then_each_has_its_own_project_wide_code() { let test_cases = [ CatalogueTestCase { - description: "20 SQLBuild layout, unit, provider and orchestration rules and 62 Fensu core rules have distinct codes", + description: "21 SQLBuild layout, unit, provider, orchestration and test placement rules and 86 Fensu core and test rules have distinct codes", count: distinct_python_codes, - expected_count: 82, + expected_count: 107, }, CatalogueTestCase { description: "every Python rule reports against the project in the python family", count: project_wide_python_rules, - expected_count: 82, + expected_count: 107, }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs new file mode 100644 index 000000000..009100532 --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs @@ -0,0 +1,237 @@ +use serde_json::json; + +use crate::python_rules::tests::helpers::{owned_faults, python_rule_faults}; +use crate::python_rules::tests::test_types::PythonRunTestCase; + +const RUNTIME_FILES: [(&str, &str); 5] = [ + ( + "python/orders/storefront_import/node.py", + r#"from sqlbuild.tasks import TaskContext, task + +from python.orders.storefront_import.main import main + + +@task(description="Import storefront orders.") +def storefront_import(ctx: TaskContext) -> None: + return main(ctx) +"#, + ), + ( + "python/orders/storefront_import/main.py", + r#"from sqlbuild.tasks import PythonNodeResult, TaskContext + + +def main(ctx: TaskContext) -> PythonNodeResult: + return PythonNodeResult() +"#, + ), + ( + "python/orders/storefront_import/_helpers/row_mapping.py", + r#"def map_row(raw: dict[str, object]) -> dict[str, object]: + return {"status": str(raw["state"]).lower()} +"#, + ), + ( + "providers/storefront_api/provider.py", + r#"from sqlbuild.providers import Provider + + +class StorefrontApi(Provider): + region: str = "eu" +"#, + ), + ("providers/storefront_api/__init__.py", ""), +]; + +const CASE_TYPES: &str = r#"from dataclasses import dataclass + + +@dataclass(frozen=True) +class RowMappingCase: + description: str + raw: dict[str, object] + expected_status: str +"#; + +const ROW_MAPPING_TEST: &str = r#"import pytest + +from python.orders.storefront_import._helpers.row_mapping import map_row +from tests.python.orders.storefront_import._helpers._test_types import RowMappingCase + + +@pytest.mark.parametrize( + "test_case", + [ + RowMappingCase(description="paid order", raw={"state": "PAID"}, expected_status="paid"), + ], + ids=lambda case: case.description, +) +def test_given_raw_order_when_mapping_then_status_is_normalised(test_case: RowMappingCase) -> None: + assert map_row(test_case.raw)["status"] == test_case.expected_status +"#; + +const PROVIDER_CASE_TYPES: &str = r#"from dataclasses import dataclass + + +@dataclass(frozen=True) +class RegionCase: + description: str + region: str + expected_region: str +"#; + +const PROVIDER_TEST: &str = r#"import pytest + +from providers.storefront_api.provider import StorefrontApi +from tests.providers.storefront_api._test_types import RegionCase + + +@pytest.mark.parametrize( + "test_case", + [RegionCase(description="explicit region", region="us", expected_region="us")], + ids=lambda case: case.description, +) +def test_given_region_when_constructing_then_provider_keeps_it(test_case: RegionCase) -> None: + assert StorefrontApi(region=test_case.region).region == test_case.expected_region +"#; + +const BRANCHING_TEST: &str = r#"import pytest + +from .._test_types import RowMappingCase + + +def build_raw(state: str) -> dict[str, object]: + return {"state": state} + + +@pytest.mark.parametrize( + "test_case", + [RowMappingCase(description="paid order", raw={"state": "PAID"}, expected_status="paid")], + ids=lambda case: case.description, +) +def test_given_raw_order_when_mapping_then_status_is_normalised(test_case: RowMappingCase) -> None: + if test_case.raw: + assert build_raw("PAID") == test_case.raw + assert test_case.expected_status == "paid" +"#; + +#[test] +fn given_project_python_tests_when_evaluating_test_rules_then_reports_misplaced_and_unconventional_tests() + { + let test_cases = [ + PythonRunTestCase { + description: "tests mirroring a unit, its helpers and a provider, with dataclass cases, report nothing", + files: &[ + RUNTIME_FILES[0], + RUNTIME_FILES[1], + RUNTIME_FILES[2], + RUNTIME_FILES[3], + RUNTIME_FILES[4], + ( + "tests/python/orders/storefront_import/_helpers/_test_types.py", + CASE_TYPES, + ), + ( + "tests/python/orders/storefront_import/_helpers/test_row_mapping.py", + ROW_MAPPING_TEST, + ), + ( + "tests/providers/storefront_api/_test_types.py", + PROVIDER_CASE_TYPES, + ), + ( + "tests/providers/storefront_api/test_provider.py", + PROVIDER_TEST, + ), + ], + config: json!({"select": ["SQBRPYTHON021", "SQBRPYTHON701", "SQBRPYTHON702", "SQBRPYTHON703", "SQBRPYTHON704", "SQBRPYTHON705", "SQBRPYTHON706", "SQBRPYTHON707", "SQBRPYTHON708", "SQBRPYTHON709", "SQBRPYTHON710", "SQBRPYTHON711", "SQBRPYTHON712", "SQBRPYTHON713", "SQBRPYTHON714", "SQBRPYTHON715", "SQBRPYTHON716", "SQBRPYTHON717", "SQBRPYTHON718", "SQBRPYTHON719", "SQBRPYTHON720", "SQBRPYTHON721", "SQBRPYTHON722", "SQBRPYTHON723", "SQBRPYTHON724"]}), + expected_faults: vec![], + }, + PythonRunTestCase { + description: "tests for missing modules or folders, or outside any folder, fail placement", + files: &[ + RUNTIME_FILES[0], + RUNTIME_FILES[1], + ("tests/python/orders/_test_types.py", CASE_TYPES), + ("tests/python/orders/test_storefront.py", ROW_MAPPING_TEST), + ("tests/python/inventory/_test_types.py", CASE_TYPES), + ("tests/python/inventory/test_stock.py", ROW_MAPPING_TEST), + ("tests/python/_test_types.py", CASE_TYPES), + ("tests/python/test_everything.py", ROW_MAPPING_TEST), + ], + config: json!({"select": ["SQBRPYTHON021"]}), + expected_faults: vec![ + ("SQBRPYTHON021", "tests/python/_test_types.py", 1, 1), + ( + "SQBRPYTHON021", + "tests/python/inventory/_test_types.py", + 1, + 1, + ), + ( + "SQBRPYTHON021", + "tests/python/inventory/test_stock.py", + 1, + 1, + ), + ( + "SQBRPYTHON021", + "tests/python/orders/test_storefront.py", + 1, + 1, + ), + ("SQBRPYTHON021", "tests/python/test_everything.py", 1, 1), + ], + }, + PythonRunTestCase { + description: "a relative import, a top-level helper and a branch in a test break the test conventions", + files: &[ + RUNTIME_FILES[0], + RUNTIME_FILES[1], + RUNTIME_FILES[2], + ( + "tests/python/orders/storefront_import/_test_types.py", + CASE_TYPES, + ), + ( + "tests/python/orders/storefront_import/_helpers/_test_types.py", + CASE_TYPES, + ), + ( + "tests/python/orders/storefront_import/_helpers/test_row_mapping.py", + BRANCHING_TEST, + ), + ], + config: json!({"select": ["SQBRPYTHON702", "SQBRPYTHON703", "SQBRPYTHON704"]}), + expected_faults: vec![ + ( + "SQBRPYTHON702", + "tests/python/orders/storefront_import/_helpers/test_row_mapping.py", + 3, + 1, + ), + ( + "SQBRPYTHON703", + "tests/python/orders/storefront_import/_helpers/test_row_mapping.py", + 6, + 1, + ), + ( + "SQBRPYTHON704", + "tests/python/orders/storefront_import/_helpers/test_row_mapping.py", + 16, + 5, + ), + ], + }, + ]; + for test_case in test_cases { + let faults = python_rule_faults(test_case.files, test_case.config).expect("rules run"); + assert_eq!( + faults, + owned_faults(&test_case.expected_faults), + "{}", + test_case.description + ); + } +} diff --git a/fensu.toml b/fensu.toml index ee699d914..f6a70bb98 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2471,7 +2471,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=100", + "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=99", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 1f393a71a..6073b2a6e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -140,7 +140,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [sqb debug](cli/debug.md) (`cli/debug`) - Validate project configuration and test the warehouse connection. - [sqb compile](cli/compile.md) (`cli/compile`) - Compile models into resolved SQL, validate contracts, and write target artifacts - fully offline. - [sqb format](cli/format.md) (`cli/format`) - Apply canonical SQLBuild SQL formatting. -- [sqb test](cli/test.md) (`cli/test`) - Run SQL unit tests and multi-model tests in isolation. +- [sqb test](cli/test.md) (`cli/test`) - Run SQL unit tests, multi-model tests and the project's Python tests in isolation. - [sqb scenario](cli/scenario.md) (`cli/scenario`) - Run end-to-end scenario tests against the warehouse or locally with DuckDB. - [sqb lineage](cli/lineage.md) (`cli/lineage`) - Explore model and column-level dependency graphs from the command line. - [sqb scope](cli/scope.md) (`cli/scope`) - Inspect declaration visibility, usage, placement, and move impact offline. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index a716fb092..e7b91404d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -51,7 +51,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | Flag | Description | |------|-------------| | `--target` | Build against a configured target instead of the active/default target | -| `--no-tests` | Skip SQL unit tests | +| `--no-tests` | Skip SQL unit tests; a build never runs [Python tests](test.md#python-tests) | | `--no-audits` | Skip audits | | `--no-python` | Skip read-side Python tasks and assets (loader-side Python still runs for selected sources) | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md index 70ab7391c..59804b6c8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md @@ -2,7 +2,7 @@ # sqb test -> Run SQL unit tests and multi-model tests in isolation. +> Run SQL unit tests, multi-model tests and the project's Python tests in isolation. Online: https://sqlbuild.com/docs/cli/test/ @@ -10,11 +10,13 @@ Online: https://sqlbuild.com/docs/cli/test/ - Usage - Flags +- Python tests - Run only affected tests - Examples Runs SQL unit tests, independently reported parameterized cases, and multi-model tests without -building models. Useful for validating test logic independently. +building models, then the project's [Python tests](#python-tests) with pytest. Useful for +validating test logic independently. ## Usage @@ -36,6 +38,8 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--test-timeout ` | Time limit for each test in this run, such as `30s` or `2m`, or `disabled`; overrides `TEST(timeout ...)`, the target and `[settings] test_timeout` | | `--since ` | Run only the tests affected by changes since the merge base of `` and `HEAD` (see [Run only affected tests](#run-only-affected-tests)); `@base` reads `[git] base_ref` | | `--explain` | With `--since`, print why each selected test was selected | +| `--sql` | Run only SQL tests | +| `--python` | Run only [Python tests](#python-tests); cannot be combined with `--inspect` or `--since` | Test names are globally unique, so a bare test name selects exactly that test. `test:` is the explicit form and accepts the same glob patterns as names, such as `test:orders_*`. A bare pattern @@ -60,6 +64,40 @@ counted under `TIMEOUT=` and excluded from `FAIL=` in the summary line output, `summary.fail_count` still includes timed-out tests, and `summary.timeout_count` reports how many of them timed out. A timeout still fails the run: `sqb test` exits non-zero. +## Python tests + +Python tests are pytest modules named `test_*.py` under `tests/python/` and `tests/providers/`. Each +sits in the folder that mirrors the code it tests, so the tests of +`python/orders/storefront_import/node.py` live in `tests/python/orders/storefront_import/`. + +| Command | SQL unit tests | Python tests | +|---------|----------------|--------------| +| `sqb test` | Yes | Yes, when any exist; needs pytest installed | +| `sqb test --sql` | Yes | No | +| `sqb test --python` | No | Yes | +| `sqb test --since ` | The affected tests | No | +| `sqb build` | Yes | Never | + +pytest runs from the project directory with the same Python as SQLBuild and with +`--import-mode=importlib`, so tests import project code as `python.orders...` or `providers...`. +Its output appears after the SQL test summary, followed by `Python tests passed` or +`Python tests failed`. `sqb test` exits non-zero when either kind fails. + +With `--json`, pytest writes to stderr so stdout stays one JSON document. When Python tests ran, the +document has a `python_tests` object with `status` (`success` or `failed`), pytest's `exit_code` +and the test `files`, and the top-level `status` is `failed` when either kind failed. + +Selecting a Python node, such as `sqb test storefront_import`, runs the Python tests in that node's +mirrored folder. Selecting only models or unit tests runs no Python tests. When Python tests exist +but pytest is not installed, `sqb test` fails before running anything; install it, or run +`sqb test --sql`. + +[Ad-hoc runs](server.md#run-ad-hoc-commands) on a server run SQL tests only, and reject +`--python`. + +[Python Rules](../concepts/rules/python-rules.md#testing-python) check where Python tests sit and +how they are written. + ## Run only affected tests `--since ` runs only the tests a change can affect, which keeps CI on branch pushes from diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 151c8dc5b..15338d537 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -770,10 +770,11 @@ Code that extends SQLBuild itself keeps its own root: Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create an unofficial project structure. -Keep repository pytest tests outside the SQLBuild project's `tests/` directory, which is reserved for -SQLBuild SQL tests and scenarios. Orchestrator code such as Dagster or Rivers definitions lives outside -the SQLBuild project directory and points at it; see [Dagster](../integrations/dagster.md) and -[Rivers](../integrations/rivers.md). +The project's `tests/` directory holds SQL tests and scenarios, and pytest tests of project Python +under `tests/python/` and `tests/providers/`, which `sqb test` runs; see +[Python tests](../cli/test.md#python-tests). Orchestrator code such as Dagster or Rivers definitions +lives outside the SQLBuild project directory and points at it; see +[Dagster](../integrations/dagster.md) and [Rivers](../integrations/rivers.md). ## sqlbuild_local.toml diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md index af117ddc3..81dae29d9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md @@ -60,7 +60,9 @@ passes, the case fails with `custom rule is not deterministic`. The usual module-level state, such as a set of already-seen models, that carries over from one subject to the next. Compute findings from the subject and `ctx` alone. -Repository pytest files remain outside the SQLBuild project's SQL `tests/` directory. +Keep Rule tests under the project's `tests/` directory, outside `tests/python/` and +`tests/providers/`, such as in `tests/rules/`. `sqb test` runs only those two folders, so run Rule +tests with pytest. ## Parallel evaluation diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md index 3c547cb16..6b1be9af6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md @@ -14,6 +14,7 @@ Online: https://sqlbuild.com/docs/concepts/rules/python-rules/ - Writing nodes as units - Laying out providers - Laying out sensors and calendars +- Testing Python - Main modules and entry modules - Unreachable definitions - Rule options @@ -63,10 +64,12 @@ sqb rules run SQBRPYTHON The Python Rules read every supported project Python file: the runtime package under `python/` and the folders that extend SQLBuild, such as `hooks/python/`, `functions/python/`, `providers/`, `rules/` and `macros/`. See [Project Python](../project-configuration.md#project-python) -for the full list. Files under `tests/` are not checked. +for the full list. Under `tests/`, only the [Python tests](#testing-python) in `tests/python/` and +`tests/providers/` are checked, and only by the Rules marked `tests/`. Rules marked `python/` in the [reference](#rule-reference) check only the runtime package, because -they describe how its packages are organised. The others check every file. +they describe how its packages are organised. Rules marked `tests/` check only Python tests. The +others check every file except Python tests. ## Laying out `python/` @@ -221,6 +224,64 @@ A domain folder's private support is `_models.py`, `_types.py`, `_constants.py`, Providers importing `orchestration/` are reported by `SQBRPYTHON015` instead. +## Testing Python + +`sqb test` runs the pytest modules under `tests/python/` and `tests/providers/`; see +[Python tests](../../cli/test.md#python-tests). `SQBRPYTHON021` keeps each test beside a mirror of the +code it tests, and `SQBRPYTHON701` to `SQBRPYTHON724` keep tests table-driven: + +```text +python/ + orders/ + storefront_import/ + node.py + _helpers/ + row_mapping.py +tests/ + python/ + orders/ + storefront_import/ + _helpers/ + _test_types.py # the test-case dataclasses + test_row_mapping.py # tests row_mapping.py +``` + +```python +# tests/python/orders/storefront_import/_helpers/_test_types.py +from dataclasses import dataclass + +@dataclass(frozen=True) +class RowMappingTestCase: + description: str + status: str + expected_label: str +``` + +```python +# tests/python/orders/storefront_import/_helpers/test_row_mapping.py +import pytest + +from python.orders.storefront_import._helpers.row_mapping import status_label +from tests.python.orders.storefront_import._helpers._test_types import RowMappingTestCase + +@pytest.mark.parametrize( + "test_case", + [ + RowMappingTestCase(description="shipped orders", status="S", expected_label="shipped"), + RowMappingTestCase(description="open orders", status="O", expected_label="open"), + ], + ids=lambda case: case.description, +) +def test_given_a_status_when_mapping_then_returns_its_label(test_case: RowMappingTestCase) -> None: + assert status_label(test_case.status) == test_case.expected_label +``` + +| Code | Finding | Fix | +|---|---|---| +| `SQBRPYTHON021` | A test module whose folder does not mirror one under `python/` or `providers/`, or a `test_.py` with no `.py` or `/` package beside the code it tests | Move the test to the mirrored folder, or rename it for the module it tests | +| `SQBRPYTHON701` to `SQBRPYTHON711` | A test folder or module whose layout, imports, helpers, conditionals or test-case dataclasses break the pattern above | Follow the pattern above | +| `SQBRPYTHON712` to `SQBRPYTHON724` | A test function or `pytest.mark.parametrize` that is not named `test_given__when__then_`, or does not take dataclass cases through `test_case` with ids from each `description` | Follow the pattern above | + ## Main modules and entry modules Some Rules apply only to a domain's main modules: files under a `main/` folder, and a unit's @@ -320,6 +381,36 @@ remediation. | `SQBRPYTHON019` | orchestration-import-direction | All | yes | Sensors and calendars import only providers, `python/utils/` and their own support; nothing outside `orchestration/` imports them | | `SQBRPYTHON020` | orchestration-no-docstring | All | yes | Sensor and calendar functions have no docstring | +### Python tests + +| Code | Name | Files | Default | Checks | +|---|---|---|---|---| +| `SQBRPYTHON021` | python-test-placement | `tests/` | yes | Python tests must mirror an existing module under `python/` or `providers/` | +| `SQBRPYTHON701` | test-init-module-empty | `tests/` | yes | Test package `__init__.py` files must be empty or docstring-only | +| `SQBRPYTHON702` | test-absolute-imports | `tests/` | yes | Tests must use absolute imports | +| `SQBRPYTHON703` | test-no-top-level-helpers | `tests/` | yes | Test modules may contain only tests, imports, and declarations | +| `SQBRPYTHON704` | test-no-conditionals | `tests/` | yes | Tests and local test helpers must not contain conditional control flow | +| `SQBRPYTHON705` | test-private-constant-order | `tests/` | yes | Private test constants must appear before test functions | +| `SQBRPYTHON706` | test-no-complex-comprehensions | `tests/` | yes | Nested or multi-generator comprehensions hide control flow and data shapes | +| `SQBRPYTHON707` | test-case-description | `tests/` | yes | Test-case dataclasses must define a `description` field | +| `SQBRPYTHON708` | test-case-expected-field | `tests/` | yes | Test-case dataclasses must define at least one `expected_` field | +| `SQBRPYTHON709` | test-local-types-import | `tests/` | yes | Tests must import test-case types from their local `_test_types.py` | +| `SQBRPYTHON710` | test-local-types-file | `tests/` | yes | Test directories must provide a local `_test_types.py` | +| `SQBRPYTHON711` | test-file-name | `tests/` | yes | Test modules must use a `test_` filename | +| `SQBRPYTHON712` | test-function-name | `tests/` | yes | Test functions must use `test_given__when__then_` | +| `SQBRPYTHON713` | test-dataclass-parametrize | `tests/` | yes | Tests must use dataclass-backed pytest parameterization | +| `SQBRPYTHON714` | test-accepts-test-case | `tests/` | yes | Parametrized tests must accept a `test_case` argument | +| `SQBRPYTHON715` | test-case-annotation | `tests/` | yes | `test_case` parameters must use a local test-case dataclass annotation | +| `SQBRPYTHON716` | test-expected-field-assertion | `tests/` | yes | Tests must assert against an `expected_` field from `test_case` | +| `SQBRPYTHON717` | test-parametrize-arguments | `tests/` | yes | Pytest parametrize decorators must define parameter names and values | +| `SQBRPYTHON718` | test-parametrize-test-case | `tests/` | yes | Pytest parametrize must expose cases through the `test_case` parameter | +| `SQBRPYTHON719` | test-parametrize-ids | `tests/` | yes | Pytest parametrize decorators must define readable case ids | +| `SQBRPYTHON720` | test-inline-parametrize-values | `tests/` | yes | Pytest parametrize values must be a visible list, tuple, or local comprehension | +| `SQBRPYTHON721` | test-nonempty-parametrize-values | `tests/` | yes | Pytest parametrize case sequences must not be empty | +| `SQBRPYTHON722` | test-no-dict-cases | `tests/` | yes | Pytest cases must use typed dataclasses instead of dictionaries | +| `SQBRPYTHON723` | test-local-case-constructors | `tests/` | yes | Pytest cases must construct dataclasses from the local `_test_types.py` | +| `SQBRPYTHON724` | test-description-ids | `tests/` | yes | Pytest case ids must come from each test case `description` | + ### Type annotations | Code | Name | Files | Default | Checks | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index 067ba1cb3..77f54892c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -536,6 +536,9 @@ sqb test sqb test --select stg_orders ``` +`sqb test` also runs the project's pytest modules under `tests/python/` and `tests/providers/`; +`sqb build` never runs them. See [Python tests](../cli/test.md#python-tests). + ## More test features [Testing reference](testing-reference.md) covers macro, UDF and table function tests, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index e50261379..c40b89674 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -7102,6 +7102,8 @@ class TestCommandRequest(NativeRecord): test_timeout: str | None since: str | None explain: bool + sql_only: bool + python_only: bool def __init__( self, project_dir: Path | None = None, @@ -7119,6 +7121,8 @@ class TestCommandRequest(NativeRecord): test_timeout: str | None = None, since: str | None = None, explain: bool = False, + sql_only: bool = False, + python_only: bool = False, ) -> None: ... class RefactorCommandRequest(NativeRecord): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index f04798925..9f8e2bd4a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -613,3 +613,28 @@ class PythonUdfConnectionsE2ETestCase: database: str concurrency: int expected_stdout_fragment: str + + +@dataclass(frozen=True) +class PythonTestsE2ETestCase: + """One `sqb test` run over a project with SQL tests and Python tests.""" + + description: str + command: tuple[str, ...] + extra_repo_files: Mapping[str, str] + expected_exit_code: int + expected_output_fragments: tuple[str, ...] + expected_absent_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PythonTestsJsonE2ETestCase: + """One `sqb test --json` run and the document's overall and Python test status.""" + + description: str + command: tuple[str, ...] + extra_repo_files: Mapping[str, str] + expected_exit_code: int + expected_status: str + expected_python_status: str + expected_python_files: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py new file mode 100644 index 000000000..52396e05f --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py @@ -0,0 +1,374 @@ +"""E2E tests for the project Python tests `sqb test` runs with pytest.""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + PythonTestsE2ETestCase, + PythonTestsJsonE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + run_sqb, +) + +PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "python_tests_project"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "python_tests_project.duckdb"\n' + ), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\nSELECT status FROM __source("raw_orders")\n' + ), + "sources/raw_orders.yml": ( + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + ), + "tests/unit/test_orders.sql": ( + 'TEST (name "order_status");\n\n' + "WITH\n" + "__source__raw_orders AS (SELECT 'open' AS status),\n" + "__expected__orders AS (SELECT 'open' AS status)\n" + "SELECT 1\n" + ), + "python/orders/storefront_import/node.py": ( + "from sqlbuild.tasks import task\n\n\n" + "@task(description='Import storefront orders.', tags=['ingest'])\n" + "def storefront_import(ctx):\n return None\n" + ), + "python/orders/storefront_import/_helpers/row_mapping.py": ( + "def map_row(raw):\n return {'status': str(raw['state']).lower()}\n" + ), + "python/inventory/stock_sync/node.py": ( + "from sqlbuild.tasks import task\n\n\n" + "@task(description='Sync stock.')\n" + "def stock_sync(ctx):\n return None\n" + ), +} + +PASSING_PYTHON_TEST: dict[str, str] = { + "tests/python/orders/storefront_import/_helpers/test_row_mapping.py": ( + "from python.orders.storefront_import._helpers.row_mapping import map_row\n\n\n" + "def test_given_paid_order_when_mapping_then_status_is_paid():\n" + " assert map_row({'state': 'PAID'}) == {'status': 'paid'}\n" + ), +} + +FAILING_PYTHON_TEST: dict[str, str] = { + "tests/python/inventory/stock_sync/test_node.py": ( + "def test_given_stock_when_syncing_then_fails():\n assert False\n" + ), +} + +BROKEN_MODEL: dict[str, str] = { + "models/broken.sql": ( + 'MODEL (description "Test model broken.");\n\nSELECT missing_column FROM __ref("orders")\n' + ), +} + +PASSING_FILE: str = "tests/python/orders/storefront_import/_helpers/test_row_mapping.py" +FAILING_FILE: str = "tests/python/inventory/stock_sync/test_node.py" + +SQL_PASSED: str = "PASS=1 FAIL=0" +PYTHON_PASSED: str = "Python tests passed 1 file" +PYTHON_STARTED: str = "Running Python tests with pytest" +PYTEST_MISSING: str = "Python tests found but pytest isn't installed" + +HIDDEN_PYTEST_SITECUSTOMIZE: str = """ +import sys + + +class _HidePytest: + def __init__(self, finder): + self.finder = finder + + def find_spec(self, name, path=None, target=None): + return None if name.split(".")[0] == "pytest" else self.finder.find_spec(name, path, target) + + def __getattr__(self, name): + return getattr(self.finder, name) + + +sys.meta_path[:] = [_HidePytest(finder) for finder in sys.meta_path] +""" + + +@pytest.mark.parametrize( + "test_case", + [ + PythonTestsE2ETestCase( + description="without Python tests only SQL tests run", + command=("--no-color", "test"), + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=(PYTHON_STARTED,), + ), + PythonTestsE2ETestCase( + description="with Python tests both SQL and Python tests run", + command=("--no-color", "test"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED, PYTHON_PASSED, "1 passed"), + expected_absent_fragments=(), + ), + PythonTestsE2ETestCase( + description="a failing Python test fails the command after the SQL tests", + command=("--no-color", "test"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=1, + expected_output_fragments=( + SQL_PASSED, + "Python tests failed pytest exit code 1", + "1 failed, 1 passed", + ), + expected_absent_fragments=(), + ), + PythonTestsE2ETestCase( + description="--sql runs only SQL tests", + command=("--no-color", "test", "--sql"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=(PYTHON_STARTED,), + ), + PythonTestsE2ETestCase( + description="--python runs only Python tests", + command=("--no-color", "test", "--python"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=0, + expected_output_fragments=(PYTHON_PASSED,), + expected_absent_fragments=(SQL_PASSED,), + ), + PythonTestsE2ETestCase( + description="--python without Python tests is an error", + command=("--no-color", "test", "--python"), + extra_repo_files={}, + expected_exit_code=1, + expected_output_fragments=("no Python tests found",), + expected_absent_fragments=(SQL_PASSED,), + ), + PythonTestsE2ETestCase( + description="--select of a Python node runs only the tests in its mirrored folder", + command=("--no-color", "test", "--select", "storefront_import"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(PYTHON_PASSED, "1 passed"), + expected_absent_fragments=(SQL_PASSED, "failed"), + ), + PythonTestsE2ETestCase( + description="--select of a glob reaching a Python node runs its mirrored tests", + command=("--no-color", "test", "--select", "storefront_*"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(PYTHON_PASSED, "1 passed"), + expected_absent_fragments=("failed",), + ), + PythonTestsE2ETestCase( + description="--select of a tag on a Python node runs its mirrored tests", + command=("--no-color", "test", "--select", "tag:ingest"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(PYTHON_PASSED, "1 passed"), + expected_absent_fragments=("failed",), + ), + PythonTestsE2ETestCase( + description="--select of a Python node's path runs its mirrored tests", + command=("--no-color", "test", "--select", "python/orders/storefront_import"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(PYTHON_PASSED, "1 passed"), + expected_absent_fragments=("failed",), + ), + PythonTestsE2ETestCase( + description="--select of a glob reaching only models leaves an unrelated broken model unanalysed", + command=("--no-color", "test", "--select", "ord*"), + extra_repo_files={**PASSING_PYTHON_TEST, **BROKEN_MODEL}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=("B002", PYTHON_STARTED), + ), + PythonTestsE2ETestCase( + description="--select of a model runs its SQL tests and no Python tests", + command=("--no-color", "test", "--select", "orders"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=(PYTHON_STARTED,), + ), + PythonTestsE2ETestCase( + description="--select of a model leaves an unrelated broken model unanalysed", + command=("--no-color", "test", "--select", "orders"), + extra_repo_files={**PASSING_PYTHON_TEST, **BROKEN_MODEL}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=("B002", PYTHON_STARTED), + ), + PythonTestsE2ETestCase( + description="--select of a unit test runs it and no Python tests", + command=("--no-color", "test", "--select", "order_status"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=(PYTHON_STARTED,), + ), + PythonTestsE2ETestCase( + description="--python with --inspect is an error", + command=("--no-color", "test", "--python", "--inspect"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=1, + expected_output_fragments=("--python cannot be combined with --inspect or --since",), + expected_absent_fragments=(PYTHON_STARTED,), + ), + PythonTestsE2ETestCase( + description="build never runs Python tests", + command=("--no-color", "build", "--select", "storefront_import"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=("Completed successfully",), + expected_absent_fragments=(PYTHON_STARTED,), + ), + ], + ids=lambda case: case.description, +) +def test_given_project_python_tests_when_running_sqb_test_then_runs_the_requested_kinds( + test_case: PythonTestsE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_tests_project", + repo_files={**PROJECT_FILES, **test_case.extra_repo_files}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_output_fragments: + assert fragment in output, output + for fragment in test_case.expected_absent_fragments: + assert fragment not in output, output + + +@pytest.mark.parametrize( + "test_case", + [ + PythonTestsE2ETestCase( + description="without Python tests pytest is not needed", + command=("--no-color", "test"), + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=(PYTEST_MISSING,), + ), + PythonTestsE2ETestCase( + description="Python tests without pytest fail before any test runs", + command=("--no-color", "test"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=1, + expected_output_fragments=(PYTEST_MISSING, "sqb test --sql"), + expected_absent_fragments=(SQL_PASSED,), + ), + PythonTestsE2ETestCase( + description="--sql runs SQL tests without pytest", + command=("--no-color", "test", "--sql"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=(PYTEST_MISSING,), + ), + ], + ids=lambda case: case.description, +) +def test_given_pytest_missing_when_running_sqb_test_then_only_python_tests_need_it( + test_case: PythonTestsE2ETestCase, + tmp_path: Path, +) -> None: + hidden: Path = tmp_path / "hidden_pytest" + hidden.mkdir() + (hidden / "sitecustomize.py").write_text(HIDDEN_PYTEST_SITECUSTOMIZE, encoding="utf-8") + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_tests_project", + repo_files={**PROJECT_FILES, **test_case.extra_repo_files}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, + project_dir=project_dir, + env={"PYTHONPATH": os.pathsep.join((str(hidden), os.environ.get("PYTHONPATH", "")))}, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_output_fragments: + assert fragment in output, output + for fragment in test_case.expected_absent_fragments: + assert fragment not in output, output + + +@pytest.mark.parametrize( + "test_case", + [ + PythonTestsJsonE2ETestCase( + description="SQL and Python tests passing give one passing document", + command=("--no-color", "test", "--json"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=0, + expected_status="success", + expected_python_status="success", + expected_python_files=(PASSING_FILE,), + ), + PythonTestsJsonE2ETestCase( + description="a failing Python test fails the document whose SQL tests passed", + command=("--no-color", "test", "--json"), + extra_repo_files=FAILING_PYTHON_TEST, + expected_exit_code=1, + expected_status="failed", + expected_python_status="failed", + expected_python_files=(FAILING_FILE,), + ), + PythonTestsJsonE2ETestCase( + description="--python still writes the document with no SQL tests", + command=("--no-color", "test", "--json", "--python"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=0, + expected_status="success", + expected_python_status="success", + expected_python_files=(PASSING_FILE,), + ), + ], + ids=lambda case: case.description, +) +def test_given_json_output_when_running_python_tests_then_stdout_is_one_document_with_them( + test_case: PythonTestsJsonE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_tests_project", + repo_files={**PROJECT_FILES, **test_case.extra_repo_files}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + document: dict[str, object] = json.loads(result.stdout) + assert document["status"] == test_case.expected_status, document + assert document["python_tests"] == { + "status": test_case.expected_python_status, + "exit_code": test_case.expected_exit_code, + "files": list(test_case.expected_python_files), + }, document + assert "1 passed" in result.stderr or "1 failed" in result.stderr, result.stderr diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index d323e30c6..d1a145086 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -35,7 +35,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | Flag | Description | |------|-------------| | `--target` | Build against a configured target instead of the active/default target | -| `--no-tests` | Skip SQL unit tests | +| `--no-tests` | Skip SQL unit tests; a build never runs [Python tests](/docs/cli/test#python-tests) | | `--no-audits` | Skip audits | | `--no-python` | Skip read-side Python tasks and assets (loader-side Python still runs for selected sources) | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | diff --git a/website/src/content/docs/docs/cli/test.mdx b/website/src/content/docs/docs/cli/test.mdx index d810a8fec..67cfd4fd0 100644 --- a/website/src/content/docs/docs/cli/test.mdx +++ b/website/src/content/docs/docs/cli/test.mdx @@ -1,12 +1,13 @@ --- title: "sqb test" -description: "Run SQL unit tests and multi-model tests in isolation." +description: "Run SQL unit tests, multi-model tests and the project's Python tests in isolation." sidebar: label: "test" --- Runs SQL unit tests, independently reported parameterized cases, and multi-model tests without -building models. Useful for validating test logic independently. +building models, then the project's [Python tests](#python-tests) with pytest. Useful for +validating test logic independently. ## Usage @@ -28,6 +29,8 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--test-timeout ` | Time limit for each test in this run, such as `30s` or `2m`, or `disabled`; overrides `TEST(timeout ...)`, the target and `[settings] test_timeout` | | `--since ` | Run only the tests affected by changes since the merge base of `` and `HEAD` (see [Run only affected tests](#run-only-affected-tests)); `@base` reads `[git] base_ref` | | `--explain` | With `--since`, print why each selected test was selected | +| `--sql` | Run only SQL tests | +| `--python` | Run only [Python tests](#python-tests); cannot be combined with `--inspect` or `--since` | Test names are globally unique, so a bare test name selects exactly that test. `test:` is the explicit form and accepts the same glob patterns as names, such as `test:orders_*`. A bare pattern @@ -52,6 +55,40 @@ counted under `TIMEOUT=` and excluded from `FAIL=` in the summary line output, `summary.fail_count` still includes timed-out tests, and `summary.timeout_count` reports how many of them timed out. A timeout still fails the run: `sqb test` exits non-zero. +## Python tests + +Python tests are pytest modules named `test_*.py` under `tests/python/` and `tests/providers/`. Each +sits in the folder that mirrors the code it tests, so the tests of +`python/orders/storefront_import/node.py` live in `tests/python/orders/storefront_import/`. + +| Command | SQL unit tests | Python tests | +|---------|----------------|--------------| +| `sqb test` | Yes | Yes, when any exist; needs pytest installed | +| `sqb test --sql` | Yes | No | +| `sqb test --python` | No | Yes | +| `sqb test --since ` | The affected tests | No | +| `sqb build` | Yes | Never | + +pytest runs from the project directory with the same Python as SQLBuild and with +`--import-mode=importlib`, so tests import project code as `python.orders...` or `providers...`. +Its output appears after the SQL test summary, followed by `Python tests passed` or +`Python tests failed`. `sqb test` exits non-zero when either kind fails. + +With `--json`, pytest writes to stderr so stdout stays one JSON document. When Python tests ran, the +document has a `python_tests` object with `status` (`success` or `failed`), pytest's `exit_code` +and the test `files`, and the top-level `status` is `failed` when either kind failed. + +Selecting a Python node, such as `sqb test storefront_import`, runs the Python tests in that node's +mirrored folder. Selecting only models or unit tests runs no Python tests. When Python tests exist +but pytest is not installed, `sqb test` fails before running anything; install it, or run +`sqb test --sql`. + +[Ad-hoc runs](/docs/cli/server#run-ad-hoc-commands) on a server run SQL tests only, and reject +`--python`. + +[Python Rules](/docs/concepts/rules/python-rules#testing-python) check where Python tests sit and +how they are written. + ## Run only affected tests `--since ` runs only the tests a change can affect, which keeps CI on branch pushes from diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 991be9205..b43a5fbcc 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -745,10 +745,11 @@ Code that extends SQLBuild itself keeps its own root: Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create an unofficial project structure. -Keep repository pytest tests outside the SQLBuild project's `tests/` directory, which is reserved for -SQLBuild SQL tests and scenarios. Orchestrator code such as Dagster or Rivers definitions lives outside -the SQLBuild project directory and points at it; see [Dagster](/docs/integrations/dagster) and -[Rivers](/docs/integrations/rivers). +The project's `tests/` directory holds SQL tests and scenarios, and pytest tests of project Python +under `tests/python/` and `tests/providers/`, which `sqb test` runs; see +[Python tests](/docs/cli/test#python-tests). Orchestrator code such as Dagster or Rivers definitions +lives outside the SQLBuild project directory and points at it; see +[Dagster](/docs/integrations/dagster) and [Rivers](/docs/integrations/rivers). ## sqlbuild_local.toml diff --git a/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx b/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx index 8567dc956..289190d85 100644 --- a/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx +++ b/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx @@ -50,7 +50,9 @@ passes, the case fails with `custom rule is not deterministic`. The usual module-level state, such as a set of already-seen models, that carries over from one subject to the next. Compute findings from the subject and `ctx` alone. -Repository pytest files remain outside the SQLBuild project's SQL `tests/` directory. +Keep Rule tests under the project's `tests/` directory, outside `tests/python/` and +`tests/providers/`, such as in `tests/rules/`. `sqb test` runs only those two folders, so run Rule +tests with pytest. ## Parallel evaluation diff --git a/website/src/content/docs/docs/concepts/rules/python-rules.mdx b/website/src/content/docs/docs/concepts/rules/python-rules.mdx index a41fa220f..a7bf6e3a8 100644 --- a/website/src/content/docs/docs/concepts/rules/python-rules.mdx +++ b/website/src/content/docs/docs/concepts/rules/python-rules.mdx @@ -46,10 +46,12 @@ sqb rules run SQBRPYTHON The Python Rules read every supported project Python file: the runtime package under `python/` and the folders that extend SQLBuild, such as `hooks/python/`, `functions/python/`, `providers/`, `rules/` and `macros/`. See [Project Python](/docs/concepts/project-configuration#project-python) -for the full list. Files under `tests/` are not checked. +for the full list. Under `tests/`, only the [Python tests](#testing-python) in `tests/python/` and +`tests/providers/` are checked, and only by the Rules marked `tests/`. Rules marked `python/` in the [reference](#rule-reference) check only the runtime package, because -they describe how its packages are organised. The others check every file. +they describe how its packages are organised. Rules marked `tests/` check only Python tests. The +others check every file except Python tests. ## Laying out `python/` @@ -205,6 +207,66 @@ A domain folder's private support is `_models.py`, `_types.py`, `_constants.py`, Providers importing `orchestration/` are reported by `SQBRPYTHON015` instead. +## Testing Python + +`sqb test` runs the pytest modules under `tests/python/` and `tests/providers/`; see +[Python tests](/docs/cli/test#python-tests). `SQBRPYTHON021` keeps each test beside a mirror of the +code it tests, and `SQBRPYTHON701` to `SQBRPYTHON724` keep tests table-driven: + +```text +python/ + orders/ + storefront_import/ + node.py + _helpers/ + row_mapping.py +tests/ + python/ + orders/ + storefront_import/ + _helpers/ + _test_types.py # the test-case dataclasses + test_row_mapping.py # tests row_mapping.py +``` + +```python +# tests/python/orders/storefront_import/_helpers/_test_types.py +from dataclasses import dataclass + + +@dataclass(frozen=True) +class RowMappingTestCase: + description: str + status: str + expected_label: str +``` + +```python +# tests/python/orders/storefront_import/_helpers/test_row_mapping.py +import pytest + +from python.orders.storefront_import._helpers.row_mapping import status_label +from tests.python.orders.storefront_import._helpers._test_types import RowMappingTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + RowMappingTestCase(description="shipped orders", status="S", expected_label="shipped"), + RowMappingTestCase(description="open orders", status="O", expected_label="open"), + ], + ids=lambda case: case.description, +) +def test_given_a_status_when_mapping_then_returns_its_label(test_case: RowMappingTestCase) -> None: + assert status_label(test_case.status) == test_case.expected_label +``` + +| Code | Finding | Fix | +|---|---|---| +| `SQBRPYTHON021` | A test module whose folder does not mirror one under `python/` or `providers/`, or a `test_.py` with no `.py` or `/` package beside the code it tests | Move the test to the mirrored folder, or rename it for the module it tests | +| `SQBRPYTHON701` to `SQBRPYTHON711` | A test folder or module whose layout, imports, helpers, conditionals or test-case dataclasses break the pattern above | Follow the pattern above | +| `SQBRPYTHON712` to `SQBRPYTHON724` | A test function or `pytest.mark.parametrize` that is not named `test_given__when__then_`, or does not take dataclass cases through `test_case` with ids from each `description` | Follow the pattern above | + ## Main modules and entry modules Some Rules apply only to a domain's main modules: files under a `main/` folder, and a unit's @@ -304,6 +366,36 @@ remediation. | `SQBRPYTHON019` | orchestration-import-direction | All | yes | Sensors and calendars import only providers, `python/utils/` and their own support; nothing outside `orchestration/` imports them | | `SQBRPYTHON020` | orchestration-no-docstring | All | yes | Sensor and calendar functions have no docstring | +### Python tests + +| Code | Name | Files | Default | Checks | +|---|---|---|---|---| +| `SQBRPYTHON021` | python-test-placement | `tests/` | yes | Python tests must mirror an existing module under `python/` or `providers/` | +| `SQBRPYTHON701` | test-init-module-empty | `tests/` | yes | Test package `__init__.py` files must be empty or docstring-only | +| `SQBRPYTHON702` | test-absolute-imports | `tests/` | yes | Tests must use absolute imports | +| `SQBRPYTHON703` | test-no-top-level-helpers | `tests/` | yes | Test modules may contain only tests, imports, and declarations | +| `SQBRPYTHON704` | test-no-conditionals | `tests/` | yes | Tests and local test helpers must not contain conditional control flow | +| `SQBRPYTHON705` | test-private-constant-order | `tests/` | yes | Private test constants must appear before test functions | +| `SQBRPYTHON706` | test-no-complex-comprehensions | `tests/` | yes | Nested or multi-generator comprehensions hide control flow and data shapes | +| `SQBRPYTHON707` | test-case-description | `tests/` | yes | Test-case dataclasses must define a `description` field | +| `SQBRPYTHON708` | test-case-expected-field | `tests/` | yes | Test-case dataclasses must define at least one `expected_` field | +| `SQBRPYTHON709` | test-local-types-import | `tests/` | yes | Tests must import test-case types from their local `_test_types.py` | +| `SQBRPYTHON710` | test-local-types-file | `tests/` | yes | Test directories must provide a local `_test_types.py` | +| `SQBRPYTHON711` | test-file-name | `tests/` | yes | Test modules must use a `test_` filename | +| `SQBRPYTHON712` | test-function-name | `tests/` | yes | Test functions must use `test_given__when__then_` | +| `SQBRPYTHON713` | test-dataclass-parametrize | `tests/` | yes | Tests must use dataclass-backed pytest parameterization | +| `SQBRPYTHON714` | test-accepts-test-case | `tests/` | yes | Parametrized tests must accept a `test_case` argument | +| `SQBRPYTHON715` | test-case-annotation | `tests/` | yes | `test_case` parameters must use a local test-case dataclass annotation | +| `SQBRPYTHON716` | test-expected-field-assertion | `tests/` | yes | Tests must assert against an `expected_` field from `test_case` | +| `SQBRPYTHON717` | test-parametrize-arguments | `tests/` | yes | Pytest parametrize decorators must define parameter names and values | +| `SQBRPYTHON718` | test-parametrize-test-case | `tests/` | yes | Pytest parametrize must expose cases through the `test_case` parameter | +| `SQBRPYTHON719` | test-parametrize-ids | `tests/` | yes | Pytest parametrize decorators must define readable case ids | +| `SQBRPYTHON720` | test-inline-parametrize-values | `tests/` | yes | Pytest parametrize values must be a visible list, tuple, or local comprehension | +| `SQBRPYTHON721` | test-nonempty-parametrize-values | `tests/` | yes | Pytest parametrize case sequences must not be empty | +| `SQBRPYTHON722` | test-no-dict-cases | `tests/` | yes | Pytest cases must use typed dataclasses instead of dictionaries | +| `SQBRPYTHON723` | test-local-case-constructors | `tests/` | yes | Pytest cases must construct dataclasses from the local `_test_types.py` | +| `SQBRPYTHON724` | test-description-ids | `tests/` | yes | Pytest case ids must come from each test case `description` | + ### Type annotations | Code | Name | Files | Default | Checks | diff --git a/website/src/content/docs/docs/concepts/testing.mdx b/website/src/content/docs/docs/concepts/testing.mdx index 221b09aa9..c6e72e476 100644 --- a/website/src/content/docs/docs/concepts/testing.mdx +++ b/website/src/content/docs/docs/concepts/testing.mdx @@ -517,6 +517,9 @@ sqb test sqb test --select stg_orders ``` +`sqb test` also runs the project's pytest modules under `tests/python/` and `tests/providers/`; +`sqb build` never runs them. See [Python tests](/docs/cli/test#python-tests). + ## More test features [Testing reference](/docs/concepts/testing-reference) covers macro, UDF and table function tests, From 6189b08afde6314f6e7830c33db5e5f6491ba56d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 15 Nov 2026 09:05:51 +0000 Subject: [PATCH 4/6] feat: add SQBRPYTHON022 compile-time purity rule sqb compile imports project Python with no warehouse connection, so the code that runs then must not reach outside the machine. SQBRPYTHON022 checks calls at module scope, in class bodies and directly in @factory bodies, follows the project functions they call through their imports, and reports a call that reaches a subprocess, an HTTP request, a socket, a database or warehouse driver, or a cloud or SSH client. It reads source only and follows only calls it can resolve, so a call through a variable or an object is never reported. - approved_loader_boundaries lists exact project functions the walk does not enter; a glob fails configuration and a module name fails the run. - Findings are reported at the root call and name the external call and where it is. Results per function are memoized, with a scaling test over deep shared call chains. - Docs for the rule, its option and its reference row, and a DuckShop violation behind a branch compile never takes. --- .../python_rules/expected_findings.json | 4 + .../inventory/restock/_helpers/__init__.py | 0 .../inventory/restock/_helpers/thresholds.py | 13 + .../python/inventory/restock/constants.py | 5 + .../src/node_units/_helpers/deferred.rs | 57 +++ .../src/node_units/_helpers/mod.rs | 1 + .../src/node_units/main/deferred_spans.rs | 34 ++ .../src/node_units/main/mod.rs | 1 + .../src/node_units/models.rs | 15 + .../sqlbuild-pysource/src/node_units/tests.rs | 2 + .../src/node_units/tests/helpers.rs | 7 +- .../node_units/tests/test_deferred_spans.rs | 52 +++ .../src/node_units/tests/test_types.rs | 8 +- .../src/configuration/_helpers/loading.rs | 36 ++ .../engine/tests/test_incremental_rules.rs | 1 + .../python_rules/_helpers/engine/faults.rs | 2 +- .../_helpers/units/compile_purity.rs | 339 ++++++++++++++++++ .../src/python_rules/_helpers/units/mod.rs | 1 + .../src/python_rules/constants.rs | 100 +++++- .../main/evaluate_python_rules.rs | 4 + .../sqlbuild-rules/src/python_rules/tests.rs | 2 + .../src/python_rules/tests/helpers.rs | 37 ++ .../tests/test_compile_purity_rules.rs | 281 +++++++++++++++ .../tests/test_python_catalogue.rs | 4 +- .../python_rules/tests/test_python_scaling.rs | 35 +- .../src/python_rules/tests/test_types.rs | 16 + .../src/rules/main/builtin_options.rs | 7 + .../docs/concepts/rules/python-rules.md | 54 ++- .../sqlbuild/cli/commands/main/test_rules.py | 24 +- .../docs/docs/concepts/rules/python-rules.mdx | 53 ++- 30 files changed, 1182 insertions(+), 13 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/_helpers/__init__.py create mode 100644 benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/_helpers/thresholds.py create mode 100644 benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/constants.py create mode 100644 crates/sqlbuild-pysource/src/node_units/_helpers/deferred.rs create mode 100644 crates/sqlbuild-pysource/src/node_units/main/deferred_spans.rs create mode 100644 crates/sqlbuild-pysource/src/node_units/tests/test_deferred_spans.rs create mode 100644 crates/sqlbuild-rules/src/python_rules/_helpers/units/compile_purity.rs create mode 100644 crates/sqlbuild-rules/src/python_rules/tests/test_compile_purity_rules.rs diff --git a/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json b/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json index ec30392e7..6b7fa5c2c 100644 --- a/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json +++ b/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json @@ -19,6 +19,10 @@ ["SQBRPYTHON306", "python/inventory/main/publish.py", 4, 1], ["SQBRPYTHON614", "python/inventory/replenishment.py", 1, 1], ["SQBRPYTHON012", "python/inventory/replenishment.py", 7, 5], + ["SQBRPYTHON306", "python/inventory/restock/_helpers/thresholds.py", 6, 1], + ["SQBRPYTHON306", "python/inventory/restock/_helpers/thresholds.py", 9, 1], + ["SQBRPYTHON306", "python/inventory/restock/constants.py", 5, 1], + ["SQBRPYTHON022", "python/inventory/restock/constants.py", 5, 38], ["SQBRPYTHON617", "python/inventory/restock/main.py", 1, 1], ["SQBRPYTHON306", "python/inventory/restock/main.py", 4, 1], ["SQBRPYTHON306", "python/inventory/restock/main.py", 9, 1], diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/_helpers/__init__.py b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/_helpers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/_helpers/thresholds.py b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/_helpers/thresholds.py new file mode 100644 index 000000000..da04d662d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/_helpers/thresholds.py @@ -0,0 +1,13 @@ +"""Read restock thresholds, from the stock service when asked to refresh.""" + +import json +import urllib.request + +_DEFAULT_THRESHOLDS: dict[str, int] = {"north": 10, "south": 5} + + +def load_thresholds(refresh: bool = False) -> dict[str, int]: + """Return each warehouse's restock threshold.""" + if refresh: + return json.load(urllib.request.urlopen("https://example.com/thresholds")) + return dict(_DEFAULT_THRESHOLDS) diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/constants.py b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/constants.py new file mode 100644 index 000000000..67efb1a4b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/constants.py @@ -0,0 +1,5 @@ +"""Restock thresholds per warehouse.""" + +from python.inventory.restock._helpers import thresholds + +RESTOCK_THRESHOLDS: dict[str, int] = thresholds.load_thresholds() diff --git a/crates/sqlbuild-pysource/src/node_units/_helpers/deferred.rs b/crates/sqlbuild-pysource/src/node_units/_helpers/deferred.rs new file mode 100644 index 000000000..ebd08cea7 --- /dev/null +++ b/crates/sqlbuild-pysource/src/node_units/_helpers/deferred.rs @@ -0,0 +1,57 @@ +//! The visitor that finds lambda bodies, and the `__main__` test. + +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{CmpOp, Expr, Stmt}; +use ruff_text_size::{Ranged, TextRange, TextSize}; + +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::models::SourceLines; +use crate::node_units::models::DeferredSpan; + +/// Whether `test` is `__name__ == "__main__"`, either way round. +pub(crate) fn runs_as_main(test: &Expr) -> bool { + let Expr::Compare(compare) = test else { + return false; + }; + let ([CmpOp::Eq], [right]) = (&*compare.ops, &*compare.comparators) else { + return false; + }; + let name = |expr: &Expr| matches!(expr, Expr::Name(name) if name.id.as_str() == "__name__"); + let main = |expr: &Expr| matches!(expr, Expr::StringLiteral(text) if text.value.to_str() == "__main__"); + (name(&compare.left) && main(right)) || (main(&compare.left) && name(right)) +} + +/// Collects lambda bodies without descending into them. +pub(crate) struct Spans<'a> { + pub(crate) lines: &'a SourceLines, + pub(crate) spans: Vec, +} + +impl Spans<'_> { + /// Record `range` as deferred. + pub(crate) fn push(&mut self, range: TextRange) { + let position = |offset: TextSize| source_position(self.lines, offset); + let span = DeferredSpan { + start: position(range.start()), + end: position(range.end()), + }; + self.spans.push(span); + } +} + +impl<'a> Visitor<'a> for Spans<'_> { + fn visit_stmt(&mut self, statement: &'a Stmt) { + walk_stmt(self, statement); + } + + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Lambda(lambda) = expr { + if let Some(parameters) = &lambda.parameters { + self.visit_parameters(parameters); + } + self.push(lambda.body.range()); + return; + } + walk_expr(self, expr); + } +} diff --git a/crates/sqlbuild-pysource/src/node_units/_helpers/mod.rs b/crates/sqlbuild-pysource/src/node_units/_helpers/mod.rs index da1b2da49..a194766b3 100644 --- a/crates/sqlbuild-pysource/src/node_units/_helpers/mod.rs +++ b/crates/sqlbuild-pysource/src/node_units/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod deferred; pub(crate) mod functions; pub(crate) mod imports; pub(crate) mod positions; diff --git a/crates/sqlbuild-pysource/src/node_units/main/deferred_spans.rs b/crates/sqlbuild-pysource/src/node_units/main/deferred_spans.rs new file mode 100644 index 000000000..5c1e4c241 --- /dev/null +++ b/crates/sqlbuild-pysource/src/node_units/main/deferred_spans.rs @@ -0,0 +1,34 @@ +//! The code of a module that importing it never runs: lambda bodies and the `__main__` block. + +use ruff_python_ast::Stmt; +use ruff_python_ast::visitor::Visitor; +use ruff_text_size::{Ranged, TextRange}; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::models::ParsedSource; +use crate::node_units::_helpers::deferred::{Spans, runs_as_main}; +use crate::node_units::models::DeferredSpan; + +/// Each lambda body and the body of a top-level `if __name__ == "__main__":`, or `None` when `contents` does not parse. +#[must_use] +pub fn deferred_spans(contents: &str, python_version: (u8, u8)) -> Option> { + let parsed: ParsedSource = parse_cpython_module(contents, python_version)?; + let mut spans = Spans { + lines: &parsed.lines, + spans: Vec::new(), + }; + for statement in &parsed.module.body { + if let Stmt::If(block) = statement + && runs_as_main(&block.test) + && let (Some(first), Some(last)) = (block.body.first(), block.body.last()) + { + spans.push(TextRange::new(first.start(), last.end())); + for clause in &block.elif_else_clauses { + spans.visit_elif_else_clause(clause); + } + continue; + } + spans.visit_stmt(statement); + } + Some(spans.spans) +} diff --git a/crates/sqlbuild-pysource/src/node_units/main/mod.rs b/crates/sqlbuild-pysource/src/node_units/main/mod.rs index 77bf4695f..b803b4c23 100644 --- a/crates/sqlbuild-pysource/src/node_units/main/mod.rs +++ b/crates/sqlbuild-pysource/src/node_units/main/mod.rs @@ -1 +1,2 @@ +pub mod deferred_spans; pub mod node_unit_module; diff --git a/crates/sqlbuild-pysource/src/node_units/models.rs b/crates/sqlbuild-pysource/src/node_units/models.rs index e1587cd6d..d2abc1394 100644 --- a/crates/sqlbuild-pysource/src/node_units/models.rs +++ b/crates/sqlbuild-pysource/src/node_units/models.rs @@ -102,3 +102,18 @@ pub struct UnitParameter { pub name: String, pub annotation: Option, } + +/// Source a module's import never runs, from `start` up to `end`, each a 1-based line and 0-based UTF-8 byte column. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DeferredSpan { + pub start: (usize, usize), + pub end: (usize, usize), +} + +impl DeferredSpan { + /// Whether the 1-based `line` and 0-based byte `column` fall inside the span. + #[must_use] + pub fn contains(&self, line: usize, column: usize) -> bool { + self.start <= (line, column) && (line, column) < self.end + } +} diff --git a/crates/sqlbuild-pysource/src/node_units/tests.rs b/crates/sqlbuild-pysource/src/node_units/tests.rs index 60fba9106..f42ff87fb 100644 --- a/crates/sqlbuild-pysource/src/node_units/tests.rs +++ b/crates/sqlbuild-pysource/src/node_units/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_deferred_spans.rs"] +mod deferred_spans; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_node_unit_module.rs"] diff --git a/crates/sqlbuild-pysource/src/node_units/tests/helpers.rs b/crates/sqlbuild-pysource/src/node_units/tests/helpers.rs index 1f767fd75..8a4f29378 100644 --- a/crates/sqlbuild-pysource/src/node_units/tests/helpers.rs +++ b/crates/sqlbuild-pysource/src/node_units/tests/helpers.rs @@ -1,6 +1,6 @@ use crate::node_units::main::node_unit_module::node_unit_module; use crate::node_units::models::{ - UnitDecorator, UnitImport, UnitParameter, UnitPosition, UnitStatementKind, + DeferredSpan, UnitDecorator, UnitImport, UnitParameter, UnitPosition, UnitStatementKind, }; const PYTHON_VERSION: (u8, u8) = (3, 12); @@ -80,3 +80,8 @@ pub(super) fn import( only_in_depends_on: false, } } + +/// The span from `start` to `end`, each a 1-based line and 0-based byte column. +pub(super) fn span(start: (usize, usize), end: (usize, usize)) -> DeferredSpan { + DeferredSpan { start, end } +} diff --git a/crates/sqlbuild-pysource/src/node_units/tests/test_deferred_spans.rs b/crates/sqlbuild-pysource/src/node_units/tests/test_deferred_spans.rs new file mode 100644 index 000000000..d214bf542 --- /dev/null +++ b/crates/sqlbuild-pysource/src/node_units/tests/test_deferred_spans.rs @@ -0,0 +1,52 @@ +use crate::node_units::main::deferred_spans::deferred_spans; +use crate::node_units::tests::helpers::span; +use crate::node_units::tests::test_types::DeferredSpansTestCase; + +#[test] +fn given_module_when_finding_deferred_code_then_lambda_bodies_and_the_main_block_are_spans() { + let test_cases = [ + DeferredSpansTestCase { + description: "a lambda body, but not its default, which runs at definition", + contents: "H = {'x': lambda a=f(): run(a)}\n", + expected_spans: Some(vec![span((1, 24), (1, 30))]), + }, + DeferredSpansTestCase { + description: "a nested lambda is inside the outer span only", + contents: "H = lambda: (lambda: g())\n", + expected_spans: Some(vec![span((1, 13), (1, 24))]), + }, + DeferredSpansTestCase { + description: "a lambda inside a function body", + contents: "def f():\n return sorted(r, key=lambda x: k(x))\n", + expected_spans: Some(vec![span((2, 35), (2, 39))]), + }, + DeferredSpansTestCase { + description: "the main block body, either way round, but not its else clause", + contents: "if '__main__' == __name__:\n run()\n stop()\nelse:\n other(lambda: z())\n", + expected_spans: Some(vec![span((2, 4), (3, 10)), span((5, 18), (5, 21))]), + }, + DeferredSpansTestCase { + description: "a main check inside a function is not the module's main block", + contents: "def f():\n if __name__ == '__main__':\n run()\n", + expected_spans: Some(Vec::new()), + }, + DeferredSpansTestCase { + description: "another comparison with __name__ runs on import", + contents: "if __name__ != '__main__':\n run()\n", + expected_spans: Some(Vec::new()), + }, + DeferredSpansTestCase { + description: "a source that does not parse", + contents: "def (:\n", + expected_spans: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + deferred_spans(test_case.contents, (3, 12)), + test_case.expected_spans, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/node_units/tests/test_types.rs b/crates/sqlbuild-pysource/src/node_units/tests/test_types.rs index 66490f98e..401a6b646 100644 --- a/crates/sqlbuild-pysource/src/node_units/tests/test_types.rs +++ b/crates/sqlbuild-pysource/src/node_units/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::node_units::models::{UnitFunction, UnitImport}; +use crate::node_units::models::{DeferredSpan, UnitFunction, UnitImport}; pub(super) struct StatementKindsTestCase { pub(super) description: &'static str, @@ -17,3 +17,9 @@ pub(super) struct FunctionTestCase { pub(super) contents: &'static str, pub(super) expected_function: UnitFunction, } + +pub(super) struct DeferredSpansTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) expected_spans: Option>, +} diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index 501756281..64b59420c 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -6,6 +6,7 @@ use crate::constants::{ RELATION_EXCEPTION_RULES, }; use crate::models::RulesConfig; +use crate::python_rules::constants::{APPROVED_LOADER_BOUNDARIES_OPTION, COMPILE_TIME_PURITY_CODE}; use fensu_policy::policy::types::RuleCodeGrammar; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_sqltext::header_strings::main::fold_prose::fold_prose; @@ -248,6 +249,7 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { } } validate_published_layer_options(config)?; + validate_approved_loader_boundaries(config)?; for exception in &config.rule_exceptions { if exception.rule.trim().is_empty() || exception.path.trim().is_empty() @@ -309,6 +311,40 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { Ok(()) } +/// Approved loader boundaries are dotted function names, such as `python.orders._helpers.specs.load_specs`. +fn validate_approved_loader_boundaries(config: &RulesConfig) -> Result<(), String> { + let Some(values) = config + .rule_options + .get(COMPILE_TIME_PURITY_CODE) + .and_then(|options| options.get(APPROVED_LOADER_BOUNDARIES_OPTION)) + .and_then(serde_json::Value::as_array) + else { + return Ok(()); + }; + for value in values { + let name: &str = value.as_str().unwrap_or_default(); + let parts: Vec<&str> = name.split('.').collect(); + if !name.contains('.') || !parts.iter().all(|part| python_identifier(part)) { + return Err(format!( + "rules.rule_options.{COMPILE_TIME_PURITY_CODE}.{APPROVED_LOADER_BOUNDARIES_OPTION} \ + entries must be fully qualified project functions such as \ + python.orders._helpers.specs.load_specs; globs are not accepted: {value}" + )); + } + } + Ok(()) +} + +/// Whether `part` is a Python identifier. +fn python_identifier(part: &str) -> bool { + part.chars() + .next() + .is_some_and(|first| first == '_' || first.is_alphabetic()) + && part + .chars() + .all(|character| character == '_' || character.is_alphanumeric()) +} + /// Published layers name model layers; allowed qualifiers are literal and carry a reason. fn validate_published_layer_options(config: &RulesConfig) -> Result<(), String> { for code in [PUBLISHED_READ_CODE, PUBLISHED_SHAPE_CODE] { diff --git a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs index 7fd6a6967..f8521c390 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs @@ -68,6 +68,7 @@ const REVIEWED_SCOPES: &[(&str, RuleScope)] = &[ ("SQBRPYTHON019", RuleScope::Python), ("SQBRPYTHON020", RuleScope::Python), ("SQBRPYTHON021", RuleScope::Python), + ("SQBRPYTHON022", RuleScope::Python), ("SQBRPYTHON101", RuleScope::Python), ("SQBRPYTHON102", RuleScope::Python), ("SQBRPYTHON103", RuleScope::Python), diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/faults.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/faults.rs index 5b31b6d6c..e13f67e7e 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/faults.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/faults.rs @@ -45,7 +45,7 @@ pub(crate) fn python_fault( } /// The 1-based character column of 0-based UTF-8 byte offset `column` on 1-based `line`. -fn character_column(text: &str, line: u32, column: u32) -> u64 { +pub(crate) fn character_column(text: &str, line: u32, column: u32) -> u64 { let Some(start): Option = line_start(text, line) else { return 1; }; diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/compile_purity.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/compile_purity.rs new file mode 100644 index 000000000..167537863 --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/compile_purity.rs @@ -0,0 +1,339 @@ +//! Compile-time purity: the Python a compile runs reaches no subprocess, network, database or remote client. + +use std::collections::hash_map::Entry; +use std::collections::{BTreeSet, HashMap}; + +use fensu_facts::ProgramHandle; +use fensu_facts::facts::models::{ImportRow, RuleNamedCallRow}; +use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; +use sqlbuild_pysource::node_units::main::deferred_spans::deferred_spans; +use sqlbuild_pysource::node_units::main::node_unit_module::node_unit_module; +use sqlbuild_pysource::node_units::models::{DeferredSpan, UnitStatementKind}; + +use crate::models::{EvaluateRequest, Fault, PythonSource}; +use crate::python_rules::_helpers::engine::faults::{catalogue_fault, character_column}; +use crate::python_rules::_helpers::engine::programs::Programs; +use crate::python_rules::constants::{ + APPROVED_LOADER_BOUNDARIES_OPTION, COMPILE_TIME_PURITY_CODE, CONNECTING_FUNCTIONS, + CONNECTING_PACKAGES, DEFERRED_CODE_MARKERS, EXTERNAL_CALL_PREFIXES, EXTERNAL_CALLS, + FACTORY_DECORATORS, PACKAGE_INIT_MODULE, +}; + +/// The external call a followed call first reaches, with its file and line. +#[derive(Clone, Debug)] +struct Sink { + call: String, + path: String, + line: u32, +} + +/// A walk over project functions that remembers the external call each one reaches. +struct Walk<'a> { + programs: &'a Programs<'a>, + approved: &'a BTreeSet, + /// Per source, the code its import never runs, whose calls the walk skips. + deferred: &'a [Vec], + reached: HashMap>, + bodies: HashMap>>, +} + +/// Faults of compile-time purity when `codes` selects it: each root call that reaches an external system. +pub(crate) fn compile_purity_faults( + request: &EvaluateRequest, + codes: &BTreeSet<&str>, +) -> Result, String> { + if !codes.contains(COMPILE_TIME_PURITY_CODE) || request.python_sources.is_empty() { + return Ok(Vec::new()); + } + let approved: BTreeSet = request + .config + .rule_option_strings(COMPILE_TIME_PURITY_CODE, APPROVED_LOADER_BOUNDARIES_OPTION) + .into_iter() + .collect(); + let programs = Programs::new(&request.python_sources, request.python_version); + if let Some(module) = approved + .iter() + .find(|name| programs.module_index(name).is_some()) + { + return Err(format!( + "rules.rule_options.{COMPILE_TIME_PURITY_CODE}.{APPROVED_LOADER_BOUNDARIES_OPTION} names \ + the module {module}; list the function that loads the data instead" + )); + } + (0..request.python_sources.len()) + .into_par_iter() + .try_for_each(|index| programs.program(index).and(Ok(())))?; + let deferred: Vec> = request + .python_sources + .par_iter() + .map(|source| source_deferred_spans(source, request.python_version)) + .collect(); + let mut walk = Walk { + programs: &programs, + approved: &approved, + deferred: &deferred, + reached: HashMap::new(), + bodies: HashMap::new(), + }; + let mut faults: Vec = Vec::new(); + for (index, source) in request.python_sources.iter().enumerate() { + let program: &ProgramHandle = programs.program(index)?; + for call in root_calls( + (program, &deferred[index]), + source, + (&approved, request.python_version), + ) { + if let Some(sink) = walk.reach(index, call)? { + faults.extend(purity_fault(source, call, &sink)); + } + } + } + faults.sort_by(|left, right| { + (&left.path, left.line, left.column).cmp(&(&right.path, right.line, right.column)) + }); + faults.dedup_by(|left, right| { + left.path == right.path && left.line == right.line && left.column == right.column + }); + Ok(faults) +} + +impl<'a> Walk<'a> { + /// The external call `call` in source `index` reaches, directly or through project functions. + fn reach(&mut self, index: usize, call: &RuleNamedCallRow) -> Result, String> { + let programs: &'a Programs<'a> = self.programs; + let source: &PythonSource = &programs.sources[index]; + let module: Vec = module_parts(&source.relative_path); + let parts: Vec = absolute_parts( + call, + &programs.program(index)?.reference_rows().imports, + &package_parts(&source.relative_path, &module), + ); + if let Some(external) = external_call(&parts) { + return Ok(Some(Sink { + call: external, + path: source.relative_path.clone(), + line: call.line, + })); + } + let (target, function): (String, String) = match parts.as_slice() { + [] => return Ok(None), + [function] => (module.join("."), function.clone()), + [owner @ .., function] => (owner.join("."), function.clone()), + }; + let Some(target_index) = programs.module_index(&target) else { + return Ok(None); + }; + let identity: String = format!("{target}.{function}"); + if self.approved.contains(&identity) { + return Ok(None); + } + if let Some(known) = self.reached.get(&identity) { + return Ok(known.clone()); + } + self.reached.insert(identity.clone(), None); + let mut sink: Option = None; + for body_call in self.body(target_index, &function)? { + sink = self.reach(target_index, body_call)?; + if sink.is_some() { + break; + } + } + self.reached.insert(identity, sink.clone()); + Ok(sink) + } + + /// The calls written directly in top-level function `function` of source `index`. + fn body(&mut self, index: usize, function: &str) -> Result, String> { + let programs: &'a Programs<'a> = self.programs; + if let Entry::Vacant(entry) = self.bodies.entry(index) { + let mut bodies: HashMap> = HashMap::new(); + for call in programs.program(index)?.named_call_rows() { + if let ([owner], []) = ( + call.enclosing_functions.as_slice(), + call.enclosing_classes.as_slice(), + ) && !deferred(&self.deferred[index], call) + { + bodies.entry(owner.name.clone()).or_default().push(call); + } + } + entry.insert(bodies); + } + Ok(self + .bodies + .get(&index) + .and_then(|bodies| bodies.get(function)) + .cloned() + .unwrap_or_default()) + } +} + +/// The lambda bodies and `__main__` block of `source`, which its import never runs. +fn source_deferred_spans(source: &PythonSource, python_version: (u8, u8)) -> Vec { + if !DEFERRED_CODE_MARKERS + .iter() + .any(|marker| source.text.contains(marker)) + { + return Vec::new(); + } + deferred_spans(&source.text, python_version).unwrap_or_default() +} + +/// Whether `call` sits in code its module's import never runs. +fn deferred(spans: &[DeferredSpan], call: &RuleNamedCallRow) -> bool { + spans + .iter() + .any(|span| span.contains(call.line as usize, call.column as usize)) +} + +/// The calls `source`'s import runs: module and class scope and unapproved `@factory` bodies, outside `spans`. +fn root_calls<'p>( + (program, spans): (&'p ProgramHandle, &[DeferredSpan]), + source: &PythonSource, + (approved, python_version): (&BTreeSet, (u8, u8)), +) -> Vec<&'p RuleNamedCallRow> { + let module: String = module_parts(&source.relative_path).join("."); + let factories: BTreeSet = factory_names(source, python_version) + .into_iter() + .filter(|name| !approved.contains(&format!("{module}.{name}"))) + .collect(); + let mut calls: Vec<&RuleNamedCallRow> = Vec::new(); + for call in program.named_call_rows() { + let root: bool = match call.enclosing_functions.as_slice() { + [] => true, + [owner] => call.enclosing_classes.is_empty() && factories.contains(&owner.name), + _ => false, + }; + if root && !deferred(spans, call) { + calls.push(call); + } + } + calls +} + +/// The top-level functions of `source` decorated as factories. +fn factory_names(source: &PythonSource, python_version: (u8, u8)) -> BTreeSet { + let mut names: BTreeSet = BTreeSet::new(); + if !FACTORY_DECORATORS + .iter() + .any(|decorator| source.text.contains(decorator)) + { + return names; + } + let Some(module) = node_unit_module(&source.text, python_version) else { + return names; + }; + for statement in module.statements { + if let UnitStatementKind::Function(function) = statement.kind + && function + .decorators + .iter() + .any(|decorator| FACTORY_DECORATORS.contains(&decorator.name.as_str())) + { + names.insert(function.name); + } + } + names +} + +/// The dotted module of a project file, such as `python.orders.export` for `python/orders/export/__init__.py`. +fn module_parts(relative_path: &str) -> Vec { + let mut parts: Vec = relative_path + .trim_end_matches(".py") + .split('/') + .map(str::to_owned) + .collect(); + if parts.last().is_some_and(|last| last == PACKAGE_INIT_MODULE) { + parts.pop(); + } + parts +} + +/// The package a relative import in the file at `relative_path` starts from. +fn package_parts(relative_path: &str, module: &[String]) -> Vec { + if relative_path.ends_with(&format!("{PACKAGE_INIT_MODULE}.py")) { + module.to_vec() + } else { + module[..module.len().saturating_sub(1)].to_vec() + } +} + +/// The dotted name `call` targets once its first name is resolved through `imports`; empty when unresolvable. +fn absolute_parts( + call: &RuleNamedCallRow, + imports: &[ImportRow], + package: &[String], +) -> Vec { + let parts: &[String] = &call.reference.parts; + let Some(first) = parts.first() else { + return Vec::new(); + }; + for row in imports { + let Some(alias) = row.aliases.iter().find(|alias| &alias.bound_name == first) else { + continue; + }; + let imported: Vec = alias.imported_name.split('.').map(str::to_owned).collect(); + if !row.from_import && imported.first() == Some(first) { + return parts.to_vec(); + } + let mut resolved: Vec = if row.from_import { + let Some(base) = import_base(row, package) else { + return Vec::new(); + }; + base + } else { + Vec::new() + }; + resolved.extend(imported); + resolved.extend(parts.iter().skip(1).cloned()); + return resolved; + } + parts.to_vec() +} + +/// The module a `from ... import` reads from, resolving leading dots against `package`. +fn import_base(row: &ImportRow, package: &[String]) -> Option> { + let mut base: Vec = if row.relative_level == 0 { + Vec::new() + } else { + package.to_vec() + }; + for _ in 1..row.relative_level { + base.pop()?; + } + base.extend(row.module_parts.iter().cloned()); + Some(base) +} + +/// The dotted call when `parts` names a call that reaches an external system. +fn external_call(parts: &[String]) -> Option { + let joined: String = parts.join("."); + let connecting: bool = parts.len() > 1 + && CONNECTING_PACKAGES.contains(&parts[0].as_str()) + && parts + .last() + .is_some_and(|last| CONNECTING_FUNCTIONS.contains(&last.as_str())); + (connecting + || EXTERNAL_CALLS.contains(&joined.as_str()) + || EXTERNAL_CALL_PREFIXES + .iter() + .any(|prefix| joined.starts_with(prefix))) + .then_some(joined) +} + +/// The fault on root `call` in `source`, naming the external call it reaches and where. +fn purity_fault(source: &PythonSource, call: &RuleNamedCallRow, sink: &Sink) -> Option { + let detail: String = if sink.path == source.relative_path && sink.line == call.line { + format!("calls {}", sink.call) + } else { + format!("reaches {} at {}:{}", sink.call, sink.path, sink.line) + }; + catalogue_fault( + COMPILE_TIME_PURITY_CODE, + &source.relative_path, + ( + u64::from(call.line.max(1)), + character_column(&source.text, call.line, call.column), + ), + &detail, + ) +} diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs index 3a3e3300c..838ad1748 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod compile_purity; pub(crate) mod imports; pub(crate) mod layout; pub(crate) mod modules; diff --git a/crates/sqlbuild-rules/src/python_rules/constants.rs b/crates/sqlbuild-rules/src/python_rules/constants.rs index 175edd738..a2225bbbf 100644 --- a/crates/sqlbuild-rules/src/python_rules/constants.rs +++ b/crates/sqlbuild-rules/src/python_rules/constants.rs @@ -5,7 +5,7 @@ use fensu_native::rules::constants as fensu; use crate::python_rules::models::{PythonFiles, PythonRuleSpec}; /// Every `SQBRPYTHON` rule; numbers are never reused, and new rules append to their band. -pub(crate) const PYTHON_RULES: [PythonRuleSpec; 107] = [ +pub(crate) const PYTHON_RULES: [PythonRuleSpec; 108] = [ PythonRuleSpec { code: "SQBRPYTHON001", fensu_code: None, @@ -258,6 +258,18 @@ pub(crate) const PYTHON_RULES: [PythonRuleSpec; 107] = [ message: "Python tests must mirror an existing module under python/ or providers/", remediation: "Move the test to tests/python/ or tests/providers/ at the path of the module it tests: tests/python/orders/storefront_import/test_main.py tests python/orders/storefront_import/main.py.", }, + PythonRuleSpec { + code: "SQBRPYTHON022", + fensu_code: None, + slug: "compile-time-purity", + family: "purity", + owner: "file", + applies: PythonFiles::All, + enabled_by_default: true, + options: &[], + message: "Python that runs while the project compiles must not reach a subprocess, the network, a database or a remote client", + remediation: "Keep module scope and @factory bodies to static data such as specs in constants.py; move the call into a node body, main() or a provider, or list a deliberate discovery function in approved_loader_boundaries.", + }, PythonRuleSpec { code: "SQBRPYTHON101", fensu_code: Some(fensu::PARAMETER_ANNOTATION_CODE), @@ -1463,6 +1475,89 @@ pub(crate) const ORCHESTRATION_MODULE_SURFACE_CODE: &str = "SQBRPYTHON018"; pub(crate) const ORCHESTRATION_IMPORT_DIRECTION_CODE: &str = "SQBRPYTHON019"; pub(crate) const ORCHESTRATION_NO_DOCSTRING_CODE: &str = "SQBRPYTHON020"; pub(crate) const PYTHON_TEST_PLACEMENT_CODE: &str = "SQBRPYTHON021"; +pub(crate) const COMPILE_TIME_PURITY_CODE: &str = "SQBRPYTHON022"; +/// The option listing project functions whose calls compile-time purity does not follow. +pub(crate) const APPROVED_LOADER_BOUNDARIES_OPTION: &str = "approved_loader_boundaries"; +/// Calls that start a subprocess, send a request, open a socket or a database, or build a remote client. +pub(crate) const EXTERNAL_CALLS: [&str; 51] = [ + "subprocess.run", + "subprocess.Popen", + "subprocess.call", + "subprocess.check_call", + "subprocess.check_output", + "subprocess.getoutput", + "subprocess.getstatusoutput", + "os.system", + "os.popen", + "os.posix_spawn", + "asyncio.create_subprocess_exec", + "asyncio.create_subprocess_shell", + "requests.get", + "requests.post", + "requests.put", + "requests.patch", + "requests.delete", + "requests.head", + "requests.options", + "requests.request", + "httpx.get", + "httpx.post", + "httpx.put", + "httpx.patch", + "httpx.delete", + "httpx.head", + "httpx.options", + "httpx.request", + "httpx.stream", + "aiohttp.request", + "urllib.request.urlopen", + "urllib3.request", + "http.client.HTTPConnection", + "http.client.HTTPSConnection", + "socket.socket", + "socket.create_connection", + "socket.getaddrinfo", + "socket.gethostbyname", + "sqlalchemy.create_engine", + "sqlalchemy.ext.asyncio.create_async_engine", + "pymongo.MongoClient", + "redis.Redis", + "redis.StrictRedis", + "redis.from_url", + "clickhouse_connect.get_client", + "google.cloud.bigquery.Client", + "boto3.client", + "boto3.resource", + "paramiko.SSHClient", + "paramiko.Transport", + "fabric.Connection", +]; +/// Call prefixes that replace the process or spawn another, such as `os.execv` and `os.spawnl`. +pub(crate) const EXTERNAL_CALL_PREFIXES: [&str; 2] = ["os.exec", "os.spawn"]; +/// Driver packages whose `connect` or `create_pool` opens a database or remote session. +pub(crate) const CONNECTING_PACKAGES: [&str; 17] = [ + "asyncpg", + "asyncssh", + "clickhouse_driver", + "cx_Oracle", + "databricks", + "duckdb", + "oracledb", + "psycopg", + "psycopg2", + "pymssql", + "pymysql", + "pyodbc", + "redshift_connector", + "snowflake", + "sqlite3", + "teradatasql", + "trino", +]; +/// The functions of a connecting package that open a connection. +pub(crate) const CONNECTING_FUNCTIONS: [&str; 2] = ["connect", "create_pool"]; +/// The module file of a package. +pub(crate) const PACKAGE_INIT_MODULE: &str = "__init__"; /// Fensu's scope for test files, the only scope its test rules check. pub(crate) const TEST_SCOPE: &str = "test"; /// The prefix of a test module's file name; other files in a mirrored test folder need only the folder. @@ -1537,6 +1632,9 @@ pub(crate) const NODE_KINDS: [(&str, &str); 3] = [ pub(crate) const CHECK_KIND: (&str, &str) = ("check", "CheckContext"); /// Decorators that generate nodes or checks. pub(crate) const FACTORY_DECORATORS: [&str; 1] = ["factory"]; + +/// Text a source must contain to have code its import never runs: a lambda or a `__main__` block. +pub(crate) const DEFERRED_CODE_MARKERS: [&str; 2] = ["lambda", "__main__"]; /// The loader decorator, which takes no description because its managed source has one. pub(crate) const LOADER_DECORATOR: &str = "loader"; /// Return annotations `main()` may use, by node decorator. diff --git a/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs b/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs index f5ddcd959..dadb4480a 100644 --- a/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs +++ b/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs @@ -6,6 +6,7 @@ use crate::models::{EvaluateRequest, RuleMetadata}; use crate::python_rules::_helpers::engine::execution::selected_rules; use crate::python_rules::_helpers::engine::layout::layout_faults; use crate::python_rules::_helpers::engine::passes::evaluated_pass; +use crate::python_rules::_helpers::units::compile_purity::compile_purity_faults; use crate::python_rules::_helpers::units::orchestration::orchestration_faults; use crate::python_rules::_helpers::units::project::unit_faults; use crate::python_rules::_helpers::units::providers::provider_faults; @@ -51,6 +52,9 @@ pub(crate) fn evaluate_python_rules( &request.python_test_sources, &codes, )); + evaluation + .faults + .extend(compile_purity_faults(request, &codes)?); evaluation.faults.sort_by(|left, right| { (&left.path, left.line, left.column, &left.code).cmp(&( &right.path, diff --git a/crates/sqlbuild-rules/src/python_rules/tests.rs b/crates/sqlbuild-rules/src/python_rules/tests.rs index a28c2f837..a90737194 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_compile_purity_rules.rs"] +mod compile_purity_rules; #[path = "tests/test_evaluate_python_rules.rs"] mod evaluate_python_rules; #[path = "tests/helpers.rs"] diff --git a/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs b/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs index 32e134882..90f58d176 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs @@ -248,6 +248,43 @@ fn unit_files(unit: usize, upstream: &str, arguments: &str) -> [(String, String) ] } +/// Root modules calling into helper chains that each end in a request. +pub(super) fn purity_chain_project( + roots: usize, + chains: usize, + depth: usize, +) -> Vec<(String, String)> { + let mut files: Vec<(String, String)> = Vec::new(); + for chain in 0..chains { + for step in 1..depth { + files.push(( + format!("python/utils/chain_{chain}/step_{previous}.py", previous = step - 1), + format!( + "from python.utils.chain_{chain} import step_{step}\n\n\ndef run():\n return step_{step}.run()\n" + ), + )); + } + files.push(( + format!( + "python/utils/chain_{chain}/step_{last}.py", + last = depth - 1 + ), + "import requests\n\n\ndef run():\n return requests.get(\"https://example.com\")\n" + .to_owned(), + )); + } + for root in 0..roots { + files.push(( + format!("python/orders/export_{root}/constants.py"), + format!( + "from python.utils.chain_{chain} import step_0\n\nREGIONS = step_0.run()\n", + chain = root % chains + ), + )); + } + files +} + /// The codes of the Python faults `files` produce under `config` with the cache off, and how long evaluation took. pub(super) fn timed_python_codes( files: &[(String, String)], diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_compile_purity_rules.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_compile_purity_rules.rs new file mode 100644 index 000000000..ec5d6273b --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_compile_purity_rules.rs @@ -0,0 +1,281 @@ +use serde_json::json; + +use crate::python_rules::tests::helpers::{owned_faults, python_rule_faults}; +use crate::python_rules::tests::test_types::{PythonRunFailureTestCase, PythonRunTestCase}; + +const PURE_FACTORY: &str = r#"from sqlbuild.tasks import factory, task + +from python.inventory.snapshots.constants import WAREHOUSE_SPECS + + +@factory +def warehouse_snapshots(): + nodes = [] + for spec in WAREHOUSE_SPECS: + + @task(name=spec) + def snapshot(ctx): + return requests.get(spec) + + nodes.append(snapshot) + return nodes +"#; +const RUNTIME_ONLY: &str = r#"import psycopg +import requests + +from sqlbuild.tasks import task + + +def main(ctx): + return psycopg.connect("dsn").execute("SELECT 1") + + +@task +def export(ctx): + return requests.post("https://example.com/orders") + + +class CarrierApi: + def setup(self): + self.session = requests.get("https://example.com/token") +"#; +const QUERYING_FACTORY: &str = r#"import psycopg + +from sqlbuild.tasks import factory + +from ._helpers.spec_loading import load_specs + + +@factory +def warehouse_snapshots(): + codes = psycopg.connect("dsn").execute("SELECT code FROM warehouses").fetchall() + return load_specs(codes) +"#; +const SPEC_LOADING: &str = r#"import subprocess + + +def load_specs(codes): + return list_specs(codes) + + +def list_specs(codes): + return subprocess.run(["ls"], check=True) +"#; +const REGIONS: &str = r#"import requests + +STOREFRONT_REGIONS = requests.get("https://example.com/regions").json() +"#; +const SENSOR: &str = r#"from orchestration.sensors.inventory._helpers import feed + +LATEST = feed.latest() +"#; +const FEED: &str = r#"from python.utils.network import ping + + +def latest(): + return ping() + + +def again(): + return latest() +"#; +const NETWORK: &str = r#"import socket + +from python.utils import network + + +def ping(): + network.pong() + return socket.create_connection(("example.com", 443)) + + +def pong(): + return ping() +"#; +const CLIENTS: &str = r#"import boto3 +import snowflake.connector as connector +import urllib.request +from paramiko import SSHClient + +BUCKET = boto3.client("s3") +SESSION = connector.connect(account="example") +PAGE = urllib.request.urlopen("https://example.com") +SHELL = SSHClient() +"#; +const DEFERRED: &str = r#"import subprocess + +from python.utils.shell import deferred_runner + +HANDLERS = {"ls": lambda: subprocess.run(["ls"])} +RUNNER = deferred_runner() +DEFAULT = (lambda code=subprocess.run(["id"]): code) + +if __name__ == "__main__": + subprocess.run(["ls"]) +else: + subprocess.run(["id"]) +"#; +const SHELL: &str = r#"import subprocess + + +def deferred_runner(): + return lambda: subprocess.run(["ls"]) +"#; + +#[test] +fn given_compile_time_python_when_evaluating_compile_time_purity_then_reports_calls_reaching_external_systems() + { + let test_cases = [ + PythonRunTestCase { + description: "static specs, node bodies, main() and provider methods report nothing", + files: &[ + ("python/inventory/snapshots/node.py", PURE_FACTORY), + ( + "python/inventory/snapshots/constants.py", + "WAREHOUSE_SPECS = (\"north\", \"south\")\n", + ), + ("python/orders/export/main.py", RUNTIME_ONLY), + ], + config: json!({ "select": ["SQBRPYTHON022"] }), + expected_faults: vec![], + }, + PythonRunTestCase { + description: "module scope, factory bodies and the project functions they call are followed to each external call", + files: &[ + ("python/inventory/snapshots/node.py", QUERYING_FACTORY), + ( + "python/inventory/snapshots/_helpers/spec_loading.py", + SPEC_LOADING, + ), + ("python/orders/storefront_import/constants.py", REGIONS), + ("orchestration/sensors/inventory/stock_feed.py", SENSOR), + ("orchestration/sensors/inventory/_helpers/feed.py", FEED), + ("python/utils/network.py", NETWORK), + ("providers/carrier_api/constants.py", CLIENTS), + ], + config: json!({ "select": ["SQBRPYTHON022"] }), + expected_faults: vec![ + ( + "SQBRPYTHON022", + "orchestration/sensors/inventory/stock_feed.py", + 3, + 10, + ), + ("SQBRPYTHON022", "providers/carrier_api/constants.py", 6, 10), + ("SQBRPYTHON022", "providers/carrier_api/constants.py", 7, 11), + ("SQBRPYTHON022", "providers/carrier_api/constants.py", 8, 8), + ("SQBRPYTHON022", "providers/carrier_api/constants.py", 9, 9), + ( + "SQBRPYTHON022", + "python/inventory/snapshots/node.py", + 10, + 13, + ), + ( + "SQBRPYTHON022", + "python/inventory/snapshots/node.py", + 11, + 12, + ), + ( + "SQBRPYTHON022", + "python/orders/storefront_import/constants.py", + 3, + 22, + ), + ], + }, + PythonRunTestCase { + description: "lambda bodies and the __main__ block never run on import, but a lambda default and an else branch do", + files: &[ + ("python/orders/export/constants.py", DEFERRED), + ("python/utils/shell.py", SHELL), + ], + config: json!({ "select": ["SQBRPYTHON022"] }), + expected_faults: vec![ + ("SQBRPYTHON022", "python/orders/export/constants.py", 7, 24), + ("SQBRPYTHON022", "python/orders/export/constants.py", 12, 5), + ], + }, + PythonRunTestCase { + description: "an approved loader boundary stops the walk at exactly that function", + files: &[ + ("python/inventory/snapshots/node.py", QUERYING_FACTORY), + ( + "python/inventory/snapshots/_helpers/spec_loading.py", + SPEC_LOADING, + ), + ], + config: json!({ + "select": ["SQBRPYTHON022"], + "rule_options": { "SQBRPYTHON022": { "approved_loader_boundaries": [ + "python.inventory.snapshots._helpers.spec_loading.load_specs", + ] } }, + }), + expected_faults: vec![( + "SQBRPYTHON022", + "python/inventory/snapshots/node.py", + 10, + 13, + )], + }, + PythonRunTestCase { + description: "an approved factory runs its body unchecked", + files: &[ + ("python/inventory/snapshots/node.py", QUERYING_FACTORY), + ( + "python/inventory/snapshots/_helpers/spec_loading.py", + SPEC_LOADING, + ), + ], + config: json!({ + "select": ["SQBRPYTHON022"], + "rule_options": { "SQBRPYTHON022": { "approved_loader_boundaries": [ + "python.inventory.snapshots.node.warehouse_snapshots", + ] } }, + }), + expected_faults: vec![], + }, + ]; + for test_case in test_cases { + let faults = python_rule_faults(test_case.files, test_case.config).expect("rules run"); + assert_eq!( + faults, + owned_faults(&test_case.expected_faults), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_boundary_naming_a_module_when_evaluating_compile_time_purity_then_the_run_fails() { + let test_cases = [PythonRunFailureTestCase { + description: "a module is not a loader boundary", + files: &[ + ("python/inventory/snapshots/node.py", QUERYING_FACTORY), + ( + "python/inventory/snapshots/_helpers/spec_loading.py", + SPEC_LOADING, + ), + ], + config: json!({ + "select": ["SQBRPYTHON022"], + "rule_options": { "SQBRPYTHON022": { "approved_loader_boundaries": [ + "python.inventory.snapshots._helpers.spec_loading", + ] } }, + }), + expected_error: "names the module python.inventory.snapshots._helpers.spec_loading", + }]; + for test_case in test_cases { + let faults = python_rule_faults(test_case.files, test_case.config); + + assert!( + faults + .as_ref() + .is_err_and(|error| error.contains(test_case.expected_error)), + "{}: {faults:?}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs index d677ae2b8..8faa13bcf 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs @@ -16,12 +16,12 @@ fn given_the_catalogue_when_listing_python_rules_then_each_has_its_own_project_w CatalogueTestCase { description: "21 SQLBuild layout, unit, provider, orchestration and test placement rules and 86 Fensu core and test rules have distinct codes", count: distinct_python_codes, - expected_count: 107, + expected_count: 108, }, CatalogueTestCase { description: "every Python rule reports against the project in the python family", count: project_wide_python_rules, - expected_count: 107, + expected_count: 108, }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_python_scaling.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_python_scaling.rs index 2c17e9de5..9a8561239 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_python_scaling.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_python_scaling.rs @@ -4,10 +4,11 @@ use std::time::Duration; use serde_json::json; use crate::python_rules::tests::helpers::{ - median_growth, scaling_project, serial_timing, timed_python_codes, unit_scaling_project, + median_growth, purity_chain_project, scaling_project, serial_timing, timed_python_codes, + unit_scaling_project, }; use crate::python_rules::tests::test_types::{ - PythonGrowthTestCase, PythonScalingTestCase, UnitScalingTestCase, + PurityScalingTestCase, PythonGrowthTestCase, PythonScalingTestCase, UnitScalingTestCase, }; #[test] @@ -126,3 +127,33 @@ fn given_many_linked_node_units_when_evaluating_unit_rules_then_finishes_within_ ); } } + +#[test] +fn given_many_roots_sharing_deep_call_chains_when_checking_compile_time_purity_then_finishes_within_budget() + { + let _serial = serial_timing(); + let test_cases = [PurityScalingTestCase { + description: "5000 module-scope roots into 10 helper chains of 400 calls each", + roots: 5000, + chains: 10, + depth: 400, + expected_faults: 5000, + expected_max_duration: Duration::from_secs(3), + }]; + for test_case in test_cases { + let files = purity_chain_project(test_case.roots, test_case.chains, test_case.depth); + let (codes, elapsed) = + timed_python_codes(&files, json!({ "select": ["SQBRPYTHON022"] })).expect("rules run"); + assert_eq!( + codes.len(), + test_case.expected_faults, + "{}", + test_case.description + ); + assert!( + elapsed < test_case.expected_max_duration, + "{}: took {elapsed:?}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_types.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_types.rs index 41b7f36ce..f600b2fd7 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_types.rs @@ -41,6 +41,13 @@ pub(super) struct PythonRunTestCase { pub(super) expected_faults: Vec<(&'static str, &'static str, u64, u64)>, } +pub(super) struct PythonRunFailureTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [(&'static str, &'static str)], + pub(super) config: serde_json::Value, + pub(super) expected_error: &'static str, +} + pub(super) struct PythonParseFailureTestCase { pub(super) description: &'static str, pub(super) files: &'static [(&'static str, &'static str)], @@ -74,6 +81,15 @@ pub(super) struct PythonScalingTestCase { pub(super) expected_max_duration: std::time::Duration, } +pub(super) struct PurityScalingTestCase { + pub(super) description: &'static str, + pub(super) roots: usize, + pub(super) chains: usize, + pub(super) depth: usize, + pub(super) expected_faults: usize, + pub(super) expected_max_duration: std::time::Duration, +} + pub(super) struct UnitScalingTestCase { pub(super) description: &'static str, pub(super) units: usize, diff --git a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs index 55bbd8805..62aeb655f 100644 --- a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs +++ b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs @@ -81,6 +81,13 @@ pub fn builtin_options(code: &str) -> Vec { description: "Reviewed SQL test names that may assert only that empty inputs produce \ no rows; listed tests count toward min_tests_per_model.", }], + "SQBRPYTHON022" => vec![BuiltinOption { + kind: BuiltinOptionKind::StringList(&[]), + name: "approved_loader_boundaries", + description: "Fully qualified project functions, such as \ + python.inventory.snapshots._helpers.specs.load_specs, whose calls \ + compile-time purity does not follow.", + }], _ => python_rule_options(code), } } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md index 6b1be9af6..eb800afc8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md @@ -15,6 +15,7 @@ Online: https://sqlbuild.com/docs/concepts/rules/python-rules/ - Laying out providers - Laying out sensors and calendars - Testing Python +- Keeping compile-time Python pure - Main modules and entry modules - Unreachable definitions - Rule options @@ -282,6 +283,47 @@ def test_given_a_status_when_mapping_then_returns_its_label(test_case: RowMappin | `SQBRPYTHON701` to `SQBRPYTHON711` | A test folder or module whose layout, imports, helpers, conditionals or test-case dataclasses break the pattern above | Follow the pattern above | | `SQBRPYTHON712` to `SQBRPYTHON724` | A test function or `pytest.mark.parametrize` that is not named `test_given__when__then_`, or does not take dataclass cases through `test_case` with ids from each `description` | Follow the pattern above | +## Keeping compile-time Python pure + +`sqb compile` imports project Python with no warehouse connection, so the code that runs then must +not reach outside the machine. `SQBRPYTHON022` checks every call made at module scope, in a class +body, or directly in a `@factory` body, in every file it reads, and follows the project functions +those calls reach. A call that reaches a subprocess, an HTTP request, a socket, a database or +warehouse driver, or a cloud or SSH client is reported at the call where the walk started: + +```python +# python/orders/storefront_import/constants.py +import requests + +STOREFRONT_REGIONS = requests.get("https://example.com/regions").json() # reported +``` + +```text +error[SQBRPYTHON022]: Python that runs while the project compiles must not reach a subprocess, the network, a database or a remote client: calls requests.get + --> python/orders/storefront_import/constants.py:3:22 +``` + +When the external call sits in a project function, the message names it and its line instead, such +as `reaches requests.get at python/orders/storefront_import/_helpers/regions.py:8`. + +Calls inside node bodies, `main()`, the nodes a factory generates, and provider methods run when +the node runs, so they are not checked. Only calls whose target is known from the imports are +followed, so a call through a variable or a method of an object is never reported. + +Keep the data a factory loops over in its unit's `constants.py` or `models.py`. When a factory must +discover what to generate, put that discovery behind one function and approve it. The walk does not +enter an approved function, and an approved factory's body is not checked: + +```toml +[rules.rule_options.SQBRPYTHON022] +approved_loader_boundaries = [ + "python.inventory.warehouse_snapshots._helpers.spec_loading.load_warehouse_specs", +] +``` + +Each entry is the fully qualified name of a project function. A glob fails configuration with +`C000`, and an entry that names a module fails the run. + ## Main modules and entry modules Some Rules apply only to a domain's main modules: files under a `main/` folder, and a unit's @@ -312,7 +354,7 @@ reason = "Carriers are looked up by name from the shipment's carrier code." ## Rule options -Configure the size limits by exact code: +Configure the size limits and approved boundaries by exact code: ```toml [rules.rule_options.SQBRPYTHON504] @@ -332,8 +374,10 @@ max_arguments = 6 | `SQBRPYTHON611` | `max_role_depth` | 1 | How many package levels may sit below a role package. | | `SQBRPYTHON615` | `min_shared_domain_prefix_packages` | 2 | Fewest sibling domains sharing a name prefix that are reported; 0 turns the rule off. | | `SQBRPYTHON627` | `max_file_lines` | 2000 | Most lines one Python file may hold. | +| `SQBRPYTHON022` | `approved_loader_boundaries` | none | Project functions the [compile-time purity](#keeping-compile-time-python-pure) walk does not enter. | -Each option takes a whole number of 0 or more. Unknown option names, text values and negative +`approved_loader_boundaries` takes a list of fully qualified function names; every other option +takes a whole number of 0 or more. Unknown option names, values of the wrong type and negative numbers fail configuration with `C000`. ## Rule reference @@ -381,6 +425,12 @@ remediation. | `SQBRPYTHON019` | orchestration-import-direction | All | yes | Sensors and calendars import only providers, `python/utils/` and their own support; nothing outside `orchestration/` imports them | | `SQBRPYTHON020` | orchestration-no-docstring | All | yes | Sensor and calendar functions have no docstring | +### Compile-time purity + +| Code | Name | Files | Default | Checks | +|---|---|---|---|---| +| `SQBRPYTHON022` | compile-time-purity | All | yes | Module scope, class bodies and `@factory` bodies reach no subprocess, network, database or remote client | + ### Python tests | Code | Name | Files | Default | Checks | diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 07d2a5d75..d243810b9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -3425,6 +3425,28 @@ def test_given_star_in_set_operation_branches_when_compiling_then_each_branch_is "stale rule exception suppresses no finding: SQBRPYTHON101 at python/report.py" ), ), + PythonRulesIntegrationTestCase( + description="an approved loader boundary stops compile-time purity at that function", + fixture="violations", + rules_toml=( + 'select = ["SQBRPYTHON022"]\n\n[rules.rule_options.SQBRPYTHON022]\n' + "approved_loader_boundaries = " + '["python.inventory.restock._helpers.thresholds.load_thresholds"]\n' + ), + expected_exit_code=0, + expected_findings=(), + ), + PythonRulesIntegrationTestCase( + description="a loader boundary glob fails configuration", + fixture="violations", + rules_toml=( + 'select = ["SQBRPYTHON022"]\n\n[rules.rule_options.SQBRPYTHON022]\n' + 'approved_loader_boundaries = ["python.inventory.*"]\n' + ), + expected_exit_code=1, + expected_findings=(("C000", None, None, None),), + expected_text="globs are not accepted", + ), PythonRulesIntegrationTestCase( description="a rule option tightens a Python threshold", fixture="violations", @@ -3520,7 +3542,7 @@ def test_given_project_python_when_running_python_rules_then_findings_match_comp _PYTHON_ANNOTATED_ALERT, ), ), - expected_hits_and_misses=((0, 1), (0, 18), (17, 1), (0, 18), (19, 0)), + expected_hits_and_misses=((0, 1), (0, 21), (20, 1), (0, 21), (22, 0)), expected_python_cache=(False, True, True, True, True), ) ], diff --git a/website/src/content/docs/docs/concepts/rules/python-rules.mdx b/website/src/content/docs/docs/concepts/rules/python-rules.mdx index a7bf6e3a8..319fc4de9 100644 --- a/website/src/content/docs/docs/concepts/rules/python-rules.mdx +++ b/website/src/content/docs/docs/concepts/rules/python-rules.mdx @@ -267,6 +267,47 @@ def test_given_a_status_when_mapping_then_returns_its_label(test_case: RowMappin | `SQBRPYTHON701` to `SQBRPYTHON711` | A test folder or module whose layout, imports, helpers, conditionals or test-case dataclasses break the pattern above | Follow the pattern above | | `SQBRPYTHON712` to `SQBRPYTHON724` | A test function or `pytest.mark.parametrize` that is not named `test_given__when__then_`, or does not take dataclass cases through `test_case` with ids from each `description` | Follow the pattern above | +## Keeping compile-time Python pure + +`sqb compile` imports project Python with no warehouse connection, so the code that runs then must +not reach outside the machine. `SQBRPYTHON022` checks every call made at module scope, in a class +body, or directly in a `@factory` body, in every file it reads, and follows the project functions +those calls reach. A call that reaches a subprocess, an HTTP request, a socket, a database or +warehouse driver, or a cloud or SSH client is reported at the call where the walk started: + +```python +# python/orders/storefront_import/constants.py +import requests + +STOREFRONT_REGIONS = requests.get("https://example.com/regions").json() # reported +``` + +```text +error[SQBRPYTHON022]: Python that runs while the project compiles must not reach a subprocess, the network, a database or a remote client: calls requests.get + --> python/orders/storefront_import/constants.py:3:22 +``` + +When the external call sits in a project function, the message names it and its line instead, such +as `reaches requests.get at python/orders/storefront_import/_helpers/regions.py:8`. + +Calls inside node bodies, `main()`, the nodes a factory generates, and provider methods run when +the node runs, so they are not checked. Only calls whose target is known from the imports are +followed, so a call through a variable or a method of an object is never reported. + +Keep the data a factory loops over in its unit's `constants.py` or `models.py`. When a factory must +discover what to generate, put that discovery behind one function and approve it. The walk does not +enter an approved function, and an approved factory's body is not checked: + +```toml +[rules.rule_options.SQBRPYTHON022] +approved_loader_boundaries = [ + "python.inventory.warehouse_snapshots._helpers.spec_loading.load_warehouse_specs", +] +``` + +Each entry is the fully qualified name of a project function. A glob fails configuration with +`C000`, and an entry that names a module fails the run. + ## Main modules and entry modules Some Rules apply only to a domain's main modules: files under a `main/` folder, and a unit's @@ -297,7 +338,7 @@ reason = "Carriers are looked up by name from the shipment's carrier code." ## Rule options -Configure the size limits by exact code: +Configure the size limits and approved boundaries by exact code: ```toml [rules.rule_options.SQBRPYTHON504] @@ -317,8 +358,10 @@ max_arguments = 6 | `SQBRPYTHON611` | `max_role_depth` | 1 | How many package levels may sit below a role package. | | `SQBRPYTHON615` | `min_shared_domain_prefix_packages` | 2 | Fewest sibling domains sharing a name prefix that are reported; 0 turns the rule off. | | `SQBRPYTHON627` | `max_file_lines` | 2000 | Most lines one Python file may hold. | +| `SQBRPYTHON022` | `approved_loader_boundaries` | none | Project functions the [compile-time purity](#keeping-compile-time-python-pure) walk does not enter. | -Each option takes a whole number of 0 or more. Unknown option names, text values and negative +`approved_loader_boundaries` takes a list of fully qualified function names; every other option +takes a whole number of 0 or more. Unknown option names, values of the wrong type and negative numbers fail configuration with `C000`. ## Rule reference @@ -366,6 +409,12 @@ remediation. | `SQBRPYTHON019` | orchestration-import-direction | All | yes | Sensors and calendars import only providers, `python/utils/` and their own support; nothing outside `orchestration/` imports them | | `SQBRPYTHON020` | orchestration-no-docstring | All | yes | Sensor and calendar functions have no docstring | +### Compile-time purity + +| Code | Name | Files | Default | Checks | +|---|---|---|---|---| +| `SQBRPYTHON022` | compile-time-purity | All | yes | Module scope, class bodies and `@factory` bodies reach no subprocess, network, database or remote client | + ### Python tests | Code | Name | Files | Default | Checks | From 9f2fbe00803c49783854ba5df75580651e80707c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 15 Nov 2026 11:21:50 +0000 Subject: [PATCH 5/6] test: list allow_missing_secrets in the unknown local key golden --- tests/goldens/compiler/failure/local-config-unknown-key.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/goldens/compiler/failure/local-config-unknown-key.json b/tests/goldens/compiler/failure/local-config-unknown-key.json index 5b94d037c..60b80858d 100644 --- a/tests/goldens/compiler/failure/local-config-unknown-key.json +++ b/tests/goldens/compiler/failure/local-config-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_local.toml settings contains unknown key(s): sql_analysiss. Allowed keys: auto_load_sources, check_declared_columns, column_contract_mode, concurrency, default_audit_run_scope, default_audit_severity, delta_compile, max_concurrency, microbatch_concurrency, microbatch_unaccounted_partition_policy, query_change_tracking, require_sql_analysis, scenario_timeout, sql_analysis, sql_validation, table_promotion_mode, test_timeout, unit_test_max_real_models", + "message": "/sqlbuild_local.toml settings contains unknown key(s): sql_analysiss. Allowed keys: allow_missing_secrets, auto_load_sources, check_declared_columns, column_contract_mode, concurrency, default_audit_run_scope, default_audit_severity, delta_compile, max_concurrency, microbatch_concurrency, microbatch_unaccounted_partition_policy, query_change_tracking, require_sql_analysis, scenario_timeout, sql_analysis, sql_validation, table_promotion_mode, test_timeout, unit_test_max_real_models", "help": "did you mean 'sql_analysis'?" } ] From 2b7b33ad3110f01ac5d9e80297fb58270b8ca4b0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 15 Nov 2026 11:26:48 +0000 Subject: [PATCH 6/6] fix: send pytest output to stderr portably for sqb test --json --- .../src/bindings/_helpers/sql_test_command/python_tests.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs index 8c1aab682..07e0f7645 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs @@ -1,7 +1,6 @@ //! The project's Python tests, which a local `sqb test` runs with pytest after its SQL tests. use std::collections::BTreeSet; -use std::os::fd::AsFd; use std::path::{Path, PathBuf}; use std::process::{Command, ExitStatus, Stdio}; use std::time::Instant; @@ -166,7 +165,7 @@ pub(crate) fn run_python_tests( .env("PYTHONPATH", python_path(&project_dir)?) .stdin(Stdio::null()); if native_field(request, "json_output")?.is_truthy()? { - command.stdout(Stdio::from(std::io::stderr().as_fd().try_clone_to_owned()?)); + command.stdout(Stdio::from(std::io::stderr())); } let started: Instant = Instant::now(); let status: ExitStatus = py