diff --git a/benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/jobs/inventory_job.toml b/benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/jobs/inventory_job.toml new file mode 100644 index 000000000..6d273e69a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/jobs/inventory_job.toml @@ -0,0 +1,3 @@ +name = "inventory_job" +description = "Destination inventory." +select = "stg_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/sensors/inventory/_helpers/on_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/sensors/inventory/_helpers/on_orders_sensor.py new file mode 100644 index 000000000..b0eae0644 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/private_folder_sensor/sensors/inventory/_helpers/on_orders_sensor.py @@ -0,0 +1,10 @@ +"""A sensor declared inside a private support folder fails compile instead of never running.""" + +from sqlbuild.orchestration import NodeEventKind, node_event, sensor + +on_orders = sensor( + "on_orders_sensor", + description="Build inventory for every genuine orders rebuild.", + when=node_event(select="orders orders*", kinds=[NodeEventKind.SUCCESS]), + run="inventory_job", +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/jobs/inventory_job.toml b/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/jobs/inventory_job.toml new file mode 100644 index 000000000..6d273e69a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/jobs/inventory_job.toml @@ -0,0 +1,3 @@ +name = "inventory_job" +description = "Destination inventory." +select = "stg_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/_helpers/selection.py b/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/_helpers/selection.py new file mode 100644 index 000000000..d7e21a644 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/_helpers/selection.py @@ -0,0 +1,3 @@ +"""Private support: modules in a _-prefixed folder declare nothing.""" + +ORDERS_SELECTION: str = "orders orders*" diff --git a/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/on_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/on_orders_sensor.py new file mode 100644 index 000000000..9dcfcbb4e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/private_support_orders/sensors/inventory/on_orders_sensor.py @@ -0,0 +1,12 @@ +"""A sensor in its domain folder reads its selection from a private support folder.""" + +from orchestration.sensors.inventory._helpers.selection import ORDERS_SELECTION + +from sqlbuild.orchestration import NodeEventKind, node_event, sensor + +on_orders = sensor( + "on_orders_sensor", + description="Build inventory for every genuine orders rebuild.", + when=node_event(select=ORDERS_SELECTION, kinds=[NodeEventKind.SUCCESS]), + run="inventory_job", +) diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/jobs/restock_shipments_job.toml b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/jobs/restock_shipments_job.toml new file mode 100644 index 000000000..6b22853d5 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/jobs/restock_shipments_job.toml @@ -0,0 +1,3 @@ +name = "restock_shipments_job" +description = "Rebuild shipments." +select = "shipments" diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_constants.py b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_constants.py new file mode 100644 index 000000000..e3b657540 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_constants.py @@ -0,0 +1,3 @@ +"""Shipment backlog settings.""" + +BACKLOG_REASON: str = "No shipment backlog" diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_helpers/reasons.py b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_helpers/reasons.py new file mode 100644 index 000000000..b518cf30d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/_helpers/reasons.py @@ -0,0 +1,6 @@ +"""Skip reasons for the fulfillment sensors.""" + + +def prefixed(reason: str) -> str: + """Prefix a skip reason with its domain.""" + return f"fulfillment: {reason}" diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/shipment_backlog.py b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/shipment_backlog.py new file mode 100644 index 000000000..0a9c1bdbe --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/orchestration/sensors/fulfillment/shipment_backlog.py @@ -0,0 +1,18 @@ +"""Skip restocking while the shipment backlog is empty.""" + +from orchestration.sensors.fulfillment._constants import BACKLOG_REASON +from orchestration.sensors.fulfillment._helpers.reasons import prefixed + +from sqlbuild.orchestration import SensorContext, SkipReason, sensor + +_REASON_SUFFIX: str = "." + + +@sensor( + "shipment_backlog_sensor", + description="Rebuild shipments when the backlog grows.", + run="restock_shipments_job", + minimum_interval_seconds=60, +) +def shipment_backlog(ctx: SensorContext) -> SkipReason: + return SkipReason(prefixed(BACKLOG_REASON) + _REASON_SUFFIX) diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/_test_types.py b/benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/_test_types.py new file mode 100644 index 000000000..6448f05b0 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/_test_types.py @@ -0,0 +1,12 @@ +"""Test cases for carrier money formatting.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CentsTextTestCase: + """One amount in cents and its carrier text.""" + + description: str + cents: int + expected_text: str diff --git a/benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/test_amounts.py b/benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/test_amounts.py new file mode 100644 index 000000000..ce9ebf710 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/conforming/tests/python/utils/currency/test_amounts.py @@ -0,0 +1,22 @@ +"""Tests for carrier money formatting.""" + +import pytest +from python.utils.currency import amounts +from tests.python.utils.currency._test_types import CentsTextTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + CentsTextTestCase( + description="whole amounts keep two decimals", cents=1200, expected_text="12.00" + ), + CentsTextTestCase(description="single cents are padded", cents=1205, expected_text="12.05"), + ], + ids=lambda case: case.description, +) +def test_given_cents_when_formatting_then_returns_carrier_text( + test_case: CentsTextTestCase, +) -> None: + """Format cents as carrier text.""" + assert amounts.cents_text(test_case.cents) == test_case.expected_text diff --git a/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json b/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json index f7ae9dfa5..6b7fa5c2c 100644 --- a/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json +++ b/benchmarks/duckshop/edge_cases/python_rules/expected_findings.json @@ -2,6 +2,13 @@ "select": ["SQBRPYTHON", "SQBRPYTHON306"], "findings": [ ["SQBRPYTHON101", "hooks/python/alert.py", 4, 11], + ["SQBRPYTHON301", "orchestration/sensors/inventory/_helpers/feed_window.py", 3, 1], + ["SQBRPYTHON017", "orchestration/sensors/inventory/_helpers/feed_window.py", 3, 43], + ["SQBRPYTHON017", "orchestration/sensors/inventory/_lib/age.py", 1, 1], + ["SQBRPYTHON016", "orchestration/sensors/stock_feed.py", 1, 1], + ["SQBRPYTHON019", "orchestration/sensors/stock_feed.py", 3, 8], + ["SQBRPYTHON018", "orchestration/sensors/stock_feed.py", 7, 1], + ["SQBRPYTHON020", "orchestration/sensors/stock_feed.py", 16, 5], ["SQBRPYTHON015", "providers/carrier_api/provider.py", 3, 8], ["SQBRPYTHON014", "providers/carrier_api/provider.py", 8, 5], ["SQBRPYTHON013", "providers/stock_feed.py", 1, 1], @@ -12,16 +19,32 @@ ["SQBRPYTHON306", "python/inventory/main/publish.py", 4, 1], ["SQBRPYTHON614", "python/inventory/replenishment.py", 1, 1], ["SQBRPYTHON012", "python/inventory/replenishment.py", 7, 5], + ["SQBRPYTHON306", "python/inventory/restock/_helpers/thresholds.py", 6, 1], + ["SQBRPYTHON306", "python/inventory/restock/_helpers/thresholds.py", 9, 1], + ["SQBRPYTHON306", "python/inventory/restock/constants.py", 5, 1], + ["SQBRPYTHON022", "python/inventory/restock/constants.py", 5, 38], ["SQBRPYTHON617", "python/inventory/restock/main.py", 1, 1], ["SQBRPYTHON306", "python/inventory/restock/main.py", 4, 1], ["SQBRPYTHON306", "python/inventory/restock/main.py", 9, 1], ["SQBRPYTHON003", "python/inventory/stock_count/node.py", 1, 1], - ["SQBRPYTHON004", "python/inventory/stock_count/node.py", 5, 1], - ["SQBRPYTHON605", "python/inventory/stock_count/node.py", 5, 1], - ["SQBRPYTHON005", "python/inventory/stock_count/node.py", 9, 5], - ["SQBRPYTHON007", "python/inventory/stock_count/node.py", 9, 5], - ["SQBRPYTHON011", "python/inventory/stock_count/node.py", 9, 5], + ["SQBRPYTHON008", "python/inventory/stock_count/node.py", 3, 62], + ["SQBRPYTHON004", "python/inventory/stock_count/node.py", 7, 1], + ["SQBRPYTHON605", "python/inventory/stock_count/node.py", 7, 1], + ["SQBRPYTHON005", "python/inventory/stock_count/node.py", 11, 5], + ["SQBRPYTHON007", "python/inventory/stock_count/node.py", 11, 5], + ["SQBRPYTHON011", "python/inventory/stock_count/node.py", 11, 5], ["SQBRPYTHON001", "python/report.py", 1, 1], - ["SQBRPYTHON614", "python/report.py", 1, 1] + ["SQBRPYTHON614", "python/report.py", 1, 1], + ["SQBRPYTHON021", "tests/python/inventory/test_restock_window.py", 1, 1], + ["SQBRPYTHON710", "tests/python/inventory/test_restock_window.py", 1, 1], + ["SQBRPYTHON722", "tests/python/inventory/test_restock_window.py", 6, 35], + ["SQBRPYTHON712", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON714", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON715", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON716", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON718", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON719", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON724", "tests/python/inventory/test_restock_window.py", 7, 1], + ["SQBRPYTHON704", "tests/python/inventory/test_restock_window.py", 9, 5] ] } diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/jobs/stock_count_job.toml b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/jobs/stock_count_job.toml new file mode 100644 index 000000000..7d91d3f88 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/jobs/stock_count_job.toml @@ -0,0 +1,3 @@ +name = "stock_count_job" +description = "Count stock." +select = "stock_count" diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/fulfillment/_helpers/hours.py b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/fulfillment/_helpers/hours.py new file mode 100644 index 000000000..528014737 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/fulfillment/_helpers/hours.py @@ -0,0 +1,3 @@ +"""Shipment cut-off hours, private to the fulfillment sensors.""" + +CUTOFF_HOURS: int = 4 diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_helpers/feed_window.py b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_helpers/feed_window.py new file mode 100644 index 000000000..d1841f8ca --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_helpers/feed_window.py @@ -0,0 +1,5 @@ +"""Feed window that reads support private to another domain.""" + +from ...fulfillment._helpers.hours import CUTOFF_HOURS + +FEED_WINDOW_HOURS: int = CUTOFF_HOURS * 2 diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_lib/age.py b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_lib/age.py new file mode 100644 index 000000000..320f377af --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/inventory/_lib/age.py @@ -0,0 +1,3 @@ +"""Private support in a folder other than _helpers/.""" + +FEED_AGE_HOURS: int = 6 diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/stock_feed.py b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/stock_feed.py new file mode 100644 index 000000000..b6998d066 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/orchestration/sensors/stock_feed.py @@ -0,0 +1,18 @@ +"""A sensor outside a domain folder that documents itself twice and reads a node module.""" + +import python.report + +from sqlbuild.orchestration import SensorContext, SkipReason, sensor + +STALE_REASON: str = "Stock feed is fresh" + + +@sensor( + "stock_feed_stale_sensor", + description="Count stock when the supplier feed goes stale.", + run="stock_count_job", + minimum_interval_seconds=60, +) +def stock_feed_stale(ctx: SensorContext) -> SkipReason: + """Count stock when the supplier feed goes stale.""" + return SkipReason(f"{STALE_REASON} {python.report.__name__}") diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/_helpers/__init__.py b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/_helpers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/_helpers/thresholds.py b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/_helpers/thresholds.py new file mode 100644 index 000000000..da04d662d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/_helpers/thresholds.py @@ -0,0 +1,13 @@ +"""Read restock thresholds, from the stock service when asked to refresh.""" + +import json +import urllib.request + +_DEFAULT_THRESHOLDS: dict[str, int] = {"north": 10, "south": 5} + + +def load_thresholds(refresh: bool = False) -> dict[str, int]: + """Return each warehouse's restock threshold.""" + if refresh: + return json.load(urllib.request.urlopen("https://example.com/thresholds")) + return dict(_DEFAULT_THRESHOLDS) diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/constants.py b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/constants.py new file mode 100644 index 000000000..67efb1a4b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/restock/constants.py @@ -0,0 +1,5 @@ +"""Restock thresholds per warehouse.""" + +from python.inventory.restock._helpers import thresholds + +RESTOCK_THRESHOLDS: dict[str, int] = thresholds.load_thresholds() diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/stock_count/node.py b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/stock_count/node.py index 1e0c4bf07..4d0159b67 100644 --- a/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/stock_count/node.py +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/python/inventory/stock_count/node.py @@ -1,5 +1,7 @@ """Count stock in every warehouse.""" +from orchestration.sensors.fulfillment._helpers.hours import CUTOFF_HOURS + from sqlbuild.tasks import TaskContext, task WAREHOUSES: tuple[str, ...] = ("north", "south") @@ -8,4 +10,4 @@ @task def stock_count(context: TaskContext) -> int: """Count stock in every warehouse.""" - return len(WAREHOUSES) + return len(WAREHOUSES) + CUTOFF_HOURS diff --git a/benchmarks/duckshop/edge_cases/python_rules/violations/tests/python/inventory/test_restock_window.py b/benchmarks/duckshop/edge_cases/python_rules/violations/tests/python/inventory/test_restock_window.py new file mode 100644 index 000000000..b1257ee1a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/python_rules/violations/tests/python/inventory/test_restock_window.py @@ -0,0 +1,10 @@ +"""Tests for a restock window that has no module.""" + +import pytest + + +@pytest.mark.parametrize("case", [{"days": 3, "expected": True}]) +def test_window(case: dict[str, object]) -> None: + """Check the restock window.""" + if case["days"]: + assert case["expected"] diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index be0db5ce1..15e3989cf 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1929,6 +1929,13 @@ export interface components { healthy: boolean; /** @enum {string} */ kind: "connection"; + } | { + /** Format: int64 */ + generation: number; + /** @enum {string} */ + kind: "provider_secrets"; + /** Format: int64 */ + missing: number; } | { /** Format: int64 */ elapsed_millis: number; @@ -2571,6 +2578,8 @@ export interface components { description?: string | null; /** @description The environment variable the setting is read from. */ env: string; + /** @description Whether this required secret is unset in the location's environment. */ + missing?: boolean; name: string; required: boolean; /** @description Whether the annotation is a pydantic secret type. */ diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index fdf96c060..cc1c12d0d 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1500,6 +1500,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--allow-missing-secrets" + ], + "dest": "allow_missing_secrets", + "nargs": 0, + "const": true, + "default": false, + "help": "Compile without provider secrets: skip the check for missing secret settings, still validating the others", + "group": 1 + }, { "kind": "store_true", "options": [ @@ -1639,6 +1651,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--allow-missing-secrets" + ], + "dest": "allow_missing_secrets", + "nargs": 0, + "const": true, + "default": false, + "help": "Compile without provider secrets: skip the check for missing secret settings, still validating the others", + "group": 1 + }, { "kind": "store_true", "options": [ @@ -2861,6 +2885,9 @@ "title": "options" } ], + "mutex_groups": [ + false + ], "actions": [ { "kind": "help", @@ -2970,6 +2997,32 @@ "help": "with --since, show why each test was selected", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--sql" + ], + "dest": "sql_only", + "nargs": 0, + "const": true, + "default": false, + "help": "Run only SQL tests", + "mutex_group": 0, + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--python" + ], + "dest": "python_only", + "nargs": 0, + "const": true, + "default": false, + "help": "Run only Python tests under tests/python/ and tests/providers/", + "mutex_group": 0, + "group": 1 + }, { "kind": "store", "options": [ @@ -3688,6 +3741,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--allow-missing-secrets" + ], + "dest": "allow_missing_secrets", + "nargs": 0, + "const": true, + "default": false, + "help": "Compile without provider secrets: skip the check for missing secret settings, still validating the others", + "group": 1 + }, { "kind": "store_true", "options": [ @@ -5243,6 +5308,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--allow-missing-secrets" + ], + "dest": "allow_missing_secrets", + "nargs": 0, + "const": true, + "default": false, + "help": "Compile without provider secrets: skip the check for missing secret settings, still validating the others", + "group": 1 + }, { "kind": "store_true", "options": [ @@ -5353,6 +5430,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--verbose" + ], + "dest": "verbose", + "nargs": 0, + "const": true, + "default": false, + "help": "List every provider and secret setting, not only the missing ones", + "group": 1 + }, { "kind": "store_true", "options": [ @@ -5900,6 +5989,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--allow-missing-secrets" + ], + "dest": "allow_missing_secrets", + "nargs": 0, + "const": true, + "default": false, + "help": "Compile without provider secrets: skip the check for missing secret settings, still validating the others", + "group": 1 + }, { "kind": "store_true", "options": [ @@ -8158,6 +8259,18 @@ "help": "SQLBuild project directory (default: the current directory)", "group": 1 }, + { + "kind": "store_true", + "options": [ + "--allow-missing-secrets" + ], + "dest": "allow_missing_secrets", + "nargs": 0, + "const": true, + "default": false, + "help": "Compile without provider secrets: skip the check for missing secret settings, still validating the others", + "group": 1 + }, { "kind": "store_true", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 5d238abd5..fc1ffb0ad 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -73,7 +73,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 187] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 190] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -86,6 +86,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 187] = [ ("skip_dbt_debug", NamespaceDefault::Bool(false)), ("no_sql_validation", NamespaceDefault::Bool(false)), ("no_cache", NamespaceDefault::Bool(false)), + ("allow_missing_secrets", NamespaceDefault::Bool(false)), ("defer_to", NamespaceDefault::None), ("defer_clone_from", NamespaceDefault::None), ("defer_sources_to", NamespaceDefault::None), @@ -95,6 +96,8 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 187] = [ ("inspect", NamespaceDefault::Bool(false)), ("since", NamespaceDefault::None), ("explain", NamespaceDefault::Bool(false)), + ("sql_only", NamespaceDefault::Bool(false)), + ("python_only", NamespaceDefault::Bool(false)), ("review_against", NamespaceDefault::None), ("review_downstream", NamespaceDefault::None), ("review_no_clone", NamespaceDefault::Bool(false)), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 43487a369..237838934 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -213,7 +213,7 @@ pub(crate) const FRESHNESS_ROUTE: Route = routed( REQUEST_KEYWORD_SHAPE, ); -const TEST_FIELDS: [Field; 15] = [ +const TEST_FIELDS: [Field; 17] = [ PROJECT_DIR, NO_SQL_VALIDATION, NO_COLOR, @@ -229,6 +229,8 @@ const TEST_FIELDS: [Field; 15] = [ same("test_timeout"), same("since"), same("explain"), + same("sql_only"), + same("python_only"), ]; const TEST_ARGUMENTS: [Field; 1] = [request(class("TestCommandRequest"), &TEST_FIELDS)]; pub(crate) const TEST_ROUTE: Route = routed( @@ -504,12 +506,13 @@ pub(crate) const COST_ROUTE: Route = routed( REQUEST_SHAPE, ); -const DEBUG_ARGUMENTS: [Field; 5] = [ +const DEBUG_ARGUMENTS: [Field; 6] = [ PROJECT_DIR, NO_COLOR, same("no_connection"), SELECTED_TARGET, JSON_OUTPUT, + same("verbose"), ]; pub(crate) const DEBUG_ROUTE: Route = routed( "run_debug", diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 9fd9f3af5..035882d3e 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -812,7 +812,7 @@ pub(crate) async fn check_published_location( "Checking location connection with sqb debug...\n", )])); tokio::pin!(stopped); - let Some((connected, files)) = + let Some((connected, missing, files)) = check_location_connection(options, script, stopped.as_mut()).await? else { return Ok(None); @@ -821,6 +821,18 @@ pub(crate) async fn check_published_location( result = &mut stopped => { result?; return Ok(None); }, () = report_location_health(options, bearer, session_id, LocationHealth::Connection { generation, healthy: connected }) => {}, } + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = report_location_health(options, bearer, session_id, LocationHealth::ProviderSecrets { generation, missing }) => {}, + } + if missing > 0 { + drop(write_output(&[( + Stream::Stderr, + &format!( + "{missing} provider secret(s) are unset in this location's environment; runs that use them fail. Run sqb debug for the list.\n" + ), + )])); + } drop(write_output(&[( Stream::Stderr, if connected { @@ -832,12 +844,12 @@ pub(crate) async fn check_published_location( Ok(Some(files)) } -/// Capture the ordinary debug check's exact local file paths without contaminating publication stdout. +/// Capture the ordinary debug check's exact local file paths and missing provider secret count without contaminating publication stdout. pub(crate) async fn check_location_connection( options: &LocationServe, script: &Path, stopped: impl Future>, -) -> Result)>, ControlError> { +) -> Result)>, ControlError> { let mut child = location_child(options, script) .args(["debug", "--target", &options.target, "--json"]) .stdout(Stdio::piped()) @@ -856,20 +868,27 @@ pub(crate) async fn check_location_connection( }, result = async { tokio::try_join!(child.wait(), stdout.read_to_end(&mut output)) } => result?.0.success(), }; - let files: Vec = match serde_json::from_slice::(&output) - .and_then(|report| serde_json::from_value(report["local_files"].clone())) - { - Ok(files) => files, + let (files, missing): (Vec, u64) = match serde_json::from_slice::( + &output, + ) + .and_then(|report| { + Ok(( + serde_json::from_value(report["local_files"].clone())?, + report["missing_secrets"].as_u64().unwrap_or_default(), + )) + }) { + Ok(report) => report, Err(_) => { drop(write_output(&[( Stream::Stderr, "Location connection check did not produce a valid report; retaining the published snapshot.\n", )])); - return Ok(Some((false, Vec::new()))); + return Ok(Some((false, 0, Vec::new()))); } }; Ok(Some(( connected, + missing, location_database_files(&options.project_dir, files), ))) } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index 8b40bcfe0..67ebe7e7d 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -139,7 +139,7 @@ async fn perform_work( LocationWork::SyncWarehouse {} => Some((LocationWorkOutcome::SyncStarted {}, Vec::new())), LocationWork::CheckConnection {} => check_location_connection(options, script, expires) .await? - .map(|(healthy, files)| (LocationWorkOutcome::Connection { healthy }, files)), + .map(|(healthy, _, files)| (LocationWorkOutcome::Connection { healthy }, files)), LocationWork::EvaluateSensor { evaluation } => { evaluate_location_sensor(options, script, &evaluation, expires) .await? diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs index 49ca0c930..ed7947b94 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_location_invocation.rs @@ -91,7 +91,7 @@ fn location_health(namespace: &Namespace) -> i32 { ])); i32::from(matches!( state, - LocationState::Broken | LocationState::FallingBack + LocationState::Broken | LocationState::FallingBack | LocationState::MissingSecrets )) } Err(error) => { diff --git a/crates/sqlbuild-cli/tests/native_commands/test_location_health.rs b/crates/sqlbuild-cli/tests/native_commands/test_location_health.rs index 4d32b485a..92b4c5e03 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_location_health.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_location_health.rs @@ -13,7 +13,7 @@ use crate::helpers::{api_ready, start_control}; use crate::test_types::LocationHealthTestCase; #[test] -fn given_location_health_reports_when_checking_health_then_down_or_falling_back_locations_exit_nonzero() +fn given_location_health_reports_when_checking_health_then_down_falling_back_or_missing_secret_locations_exit_nonzero() { let test_cases = [ LocationHealthTestCase { @@ -60,6 +60,26 @@ fn given_location_health_reports_when_checking_health_then_down_or_falling_back_ expected_state: "falling_back", expected_exit: 1, }, + LocationHealthTestCase { + description: "a provider secret is unset, even with a failed connection check", + reports: &[ + r#"{"kind":"connection","generation":2,"healthy":false}"#, + r#"{"kind":"provider_secrets","generation":2,"missing":1}"#, + ], + other_target_connections: 0, + expected_state: "missing_secrets", + expected_exit: 1, + }, + LocationHealthTestCase { + description: "a later load found every provider secret set", + reports: &[ + r#"{"kind":"provider_secrets","generation":2,"missing":2}"#, + r#"{"kind":"provider_secrets","generation":3,"missing":0}"#, + ], + other_target_connections: 0, + expected_state: "healthy", + expected_exit: 0, + }, LocationHealthTestCase { description: "the warehouse connection check failed", reports: &[r#"{"kind":"connection","generation":2,"healthy":false}"#], diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 09e0abbf4..fe516eaeb 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -24,8 +24,9 @@ use crate::project::_helpers::targets::{ }; use crate::project::_helpers::values::is_truthy; use crate::project::constants::{ - COLUMN_CONTRACT_MODES, LEGACY_COLUMN_CONTRACT_MODE_KEY, LEGACY_EVENT_EXPORTERS_KEY, - LOCAL_CONFIG_FILENAME, QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY, + ALLOW_MISSING_SECRETS_KEY, COLUMN_CONTRACT_MODES, LEGACY_COLUMN_CONTRACT_MODE_KEY, + LEGACY_EVENT_EXPORTERS_KEY, LOCAL_CONFIG_FILENAME, QUERY_CHANGE_TRACKING_KEY, + REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY, }; use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; use crate::project::models::{ @@ -184,6 +185,21 @@ fn identity(payload: &Entries) -> Loaded { } fn sections(payload: &Entries) -> Loaded { + if get(payload, "settings") + .and_then(ConfigValue::as_map) + .is_some_and(|settings| has(settings, ALLOW_MISSING_SECRETS_KEY)) + { + return Err(ConfigFailure::at( + PROJECT_FILE, + format!( + "cannot set [settings] {ALLOW_MISSING_SECRETS_KEY}; it applies to one machine, \ + so only {LOCAL_CONFIG_FILENAME} can set it" + ), + ) + .with_help(format!( + "move it to {LOCAL_CONFIG_FILENAME}, or set SQLBUILD_ALLOW_MISSING_SECRETS=1 in CI" + ))); + } Ok(vec![ ( "settings", diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 32947c975..5d966d192 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -10,6 +10,11 @@ use crate::project::_helpers::fields::{ use crate::project::_helpers::keys::{Place, reject_removed_virtual_keys, validate_allowed_keys}; use crate::project::_helpers::targets::seconds_value; use crate::project::_helpers::values::{int_below, python_equal}; +use crate::project::constants::{ + ALLOW_MISSING_SECRETS_KEY, DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, + DELTA_COMPILE_KEY, DISABLED_VALUE, SCENARIO_TIMEOUT_KEY, TEST_TIMEOUT_KEY, + UNIT_TEST_MAX_REAL_MODELS_KEY, +}; use crate::project::constants::{ AUDIT_SEVERITIES, CHECK_DECLARED_COLUMNS_KEY, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES, CONCURRENCY_KEY, DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, @@ -26,10 +31,6 @@ use crate::project::constants::{ CHANGE_COUNTS, CHANGE_REACHES, CHANGES_KEYS, CONTROL_KEYS, MIN_SYNC_INTERVAL_SECONDS, SYNC_INTERVAL_HELP, SYNC_INTERVAL_OFF, }; -use crate::project::constants::{ - DEFAULT_TEST_TIMEOUT, DEFAULT_UNIT_TEST_MAX_REAL_MODELS, DELTA_COMPILE_KEY, DISABLED_VALUE, - SCENARIO_TIMEOUT_KEY, TEST_TIMEOUT_KEY, UNIT_TEST_MAX_REAL_MODELS_KEY, -}; use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ChangesConfig, ConfigFile, ControlConfig, ReviewConfig}; use crate::project::types::{Entries, Fields, Loaded}; @@ -276,6 +277,14 @@ pub(crate) fn load_settings(payload: Option<&ConfigValue>, file: ConfigFile) -> true, )?)), ), + ( + ALLOW_MISSING_SECRETS_KEY, + Built::Value(ConfigValue::Bool(optional_bool( + mapping, + ALLOW_MISSING_SECRETS_KEY, + false, + )?)), + ), ]; let mut fields = fields; for (key, seconds_key, default) in [ diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 5bc122f08..8c7ec909f 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -47,9 +47,10 @@ pub(crate) const UNIT_TEST_MAX_REAL_MODELS_KEY: &str = "unit_test_max_real_model pub(crate) const DEFAULT_UNIT_TEST_MAX_REAL_MODELS: i64 = 10; pub(crate) const TEST_TIMEOUT_KEY: &str = "test_timeout"; pub(crate) const DELTA_COMPILE_KEY: &str = "delta_compile"; +pub(crate) const ALLOW_MISSING_SECRETS_KEY: &str = "allow_missing_secrets"; pub(crate) const SCENARIO_TIMEOUT_KEY: &str = "scenario_timeout"; pub(crate) const DEFAULT_TEST_TIMEOUT: &str = "30s"; -pub(crate) const SETTINGS_KEYS: [&str; 15] = [ +pub(crate) const SETTINGS_KEYS: [&str; 16] = [ "sql_analysis", "require_sql_analysis", "query_change_tracking", @@ -65,6 +66,7 @@ pub(crate) const SETTINGS_KEYS: [&str; 15] = [ "test_timeout", "scenario_timeout", "delta_compile", + "allow_missing_secrets", ]; pub(crate) const DEFAULTS_KEYS: [&str; 33] = [ "append_cursor_inclusive", diff --git a/crates/sqlbuild-core/src/debug_report/main/debug_provider_lines.rs b/crates/sqlbuild-core/src/debug_report/main/debug_provider_lines.rs index 0e411d654..aad527755 100644 --- a/crates/sqlbuild-core/src/debug_report/main/debug_provider_lines.rs +++ b/crates/sqlbuild-core/src/debug_report/main/debug_provider_lines.rs @@ -2,9 +2,9 @@ use crate::debug_report::models::{DebugLine, DebugProvider, DebugStatus}; -/// The provider count followed by one line per provider. +/// How many providers have every secret set, then the missing secrets, or with `verbose` every provider's secrets. #[must_use] -pub fn debug_provider_lines(providers: &[DebugProvider]) -> Vec { +pub fn debug_provider_lines(providers: &[DebugProvider], verbose: bool) -> Vec { if providers.is_empty() { return vec![DebugLine::checked( "providers", @@ -12,17 +12,53 @@ pub fn debug_provider_lines(providers: &[DebugProvider]) -> Vec { (DebugStatus::Ok, None), )]; } + let ready: usize = providers + .iter() + .filter(|provider| has_every_secret(provider)) + .count(); + let all_ready: bool = ready == providers.len(); let mut lines: Vec = vec![DebugLine::checked( "providers", - &providers.len().to_string(), - (DebugStatus::Ok, Some("discovered")), + &format!("{ready} of {} ready", providers.len()), + if all_ready { + (DebugStatus::Ok, Some("secrets set")) + } else { + (DebugStatus::Warn, Some("missing secrets")) + }, )]; - lines.extend(providers.iter().map(|provider| { - DebugLine::checked( - &provider.name, - &format!("{}:{}", provider.relative_path, provider.class_name), - (DebugStatus::Ok, Some("valid settings")), - ) - })); + for provider in providers { + if verbose && provider.secrets.is_empty() { + lines.push(DebugLine::checked( + &provider.name, + "(no secrets)", + (DebugStatus::Ok, Some("ok")), + )); + } + for (variable, set) in &provider.secrets { + if *set && !verbose { + continue; + } + lines.push(DebugLine::checked( + &provider.name, + variable, + if *set { + (DebugStatus::Ok, Some("set")) + } else { + (DebugStatus::Warn, Some("missing")) + }, + )); + } + } + if !verbose && !all_ready { + lines.push(DebugLine::checked( + "more", + "sqb debug --verbose lists every provider", + (DebugStatus::Skip, None), + )); + } lines } + +fn has_every_secret(provider: &DebugProvider) -> bool { + provider.secrets.iter().all(|(_, set)| *set) +} diff --git a/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs index 9eb5e8408..d4a6aece7 100644 --- a/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs +++ b/crates/sqlbuild-core/src/debug_report/main/format_debug_json.rs @@ -2,7 +2,7 @@ use crate::debug_report::models::{DebugLine, DebugReport}; use crate::json::main::dumps::dumps; -use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; +use crate::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; fn line_value(line: &DebugLine) -> JsonValue { let mut fields: Vec<(String, JsonValue)> = vec![ @@ -42,6 +42,10 @@ pub fn format_debug_json(report: &DebugReport) -> String { lines_value(&report.configuration), ), ("providers".to_owned(), lines_value(&report.providers)), + ( + "missing_secrets".to_owned(), + JsonValue::Integer(JsonInteger::from(report.missing_secrets as u64)), + ), ("connection".to_owned(), lines_value(&report.connection)), ( "local_files".to_owned(), diff --git a/crates/sqlbuild-core/src/debug_report/models.rs b/crates/sqlbuild-core/src/debug_report/models.rs index 144079f7d..71b592f3f 100644 --- a/crates/sqlbuild-core/src/debug_report/models.rs +++ b/crates/sqlbuild-core/src/debug_report/models.rs @@ -6,6 +6,8 @@ use std::path::PathBuf; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DebugStatus { Ok, + /// Worth fixing, but the report still succeeds, such as a provider secret this machine lacks. + Warn, Error, Skip, } @@ -16,6 +18,7 @@ impl DebugStatus { pub fn label(self) -> &'static str { match self { Self::Ok => "OK", + Self::Warn => "WARN", Self::Error => "ERROR", Self::Skip => "SKIP", } @@ -38,6 +41,8 @@ pub struct DebugReport { pub configuration: Vec, pub providers: Vec, pub connection: Vec, + /// How many provider secret settings the environment leaves unset. + pub missing_secrets: usize, /// Exact local database paths, not the shortened display values in connection lines. pub local_files: Vec, } @@ -118,6 +123,8 @@ pub struct DebugProvider { pub name: String, pub relative_path: String, pub class_name: String, + /// Each secret setting's environment variable and whether the environment sets it. + pub secrets: Vec<(String, bool)>, } /// One connection setting value, as Python would print it. diff --git a/crates/sqlbuild-core/src/debug_report/tests/helpers.rs b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs index 240efc55d..f1582adba 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/helpers.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/helpers.rs @@ -1,8 +1,11 @@ -use crate::debug_report::models::{ConnectionValue, DebugLine, DebugReport, DebugStatus}; +use crate::debug_report::models::{ + ConnectionValue, DebugLine, DebugProvider, DebugReport, DebugStatus, +}; pub(super) fn sample_report() -> DebugReport { DebugReport { local_files: vec!["/repo/warehouse.db".into()], + missing_secrets: 0, runtime: vec![DebugLine::plain("sqlbuild version", "0.2.1")], configuration: vec![DebugLine::checked( "project file", @@ -40,3 +43,15 @@ pub(super) fn text(value: &str) -> ConnectionValue { pub(super) fn line_summary(line: &DebugLine) -> (&str, &str, Option<&str>) { (&line.label, &line.message, line.status_message.as_deref()) } + +pub(super) fn provider(name: &str, secrets: &[(&str, bool)]) -> DebugProvider { + DebugProvider { + name: name.to_owned(), + relative_path: format!("providers/{name}/provider.py"), + class_name: "Provider".to_owned(), + secrets: secrets + .iter() + .map(|(variable, set)| ((*variable).to_owned(), *set)) + .collect(), + } +} diff --git a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs index 5fb74ee02..c90dd3f04 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/test_debug_report.rs @@ -1,14 +1,15 @@ use crate::cli_text::models::CliStyle; use crate::debug_report::main::debug_connection_lines::debug_connection_lines; use crate::debug_report::main::debug_failure::debug_failure; +use crate::debug_report::main::debug_provider_lines::debug_provider_lines; use crate::debug_report::main::format_debug_json::format_debug_json; use crate::debug_report::main::format_debug_text::format_debug_text; use crate::debug_report::models::{ ConnectionCheck, ConnectionValue, DebugFailure, DebugLine, DebugReport, DebugStatus, }; -use crate::debug_report::tests::helpers::{line_summary, sample_report, text}; +use crate::debug_report::tests::helpers::{line_summary, provider, sample_report, text}; use crate::debug_report::tests::test_types::{ - ConnectionLinesTestCase, DebugFailureTestCase, FormatTestCase, + ConnectionLinesTestCase, DebugFailureTestCase, FormatTestCase, ProviderLinesTestCase, }; /// Expectations recorded from the retired Python formatter. @@ -18,7 +19,7 @@ fn given_debug_report_when_formatting_then_matches_python_outputs() { description: "renders diagnostics sections and json sections", report: sample_report(), expected_text: "\nSQLBuild Diagnostics\n\nRuntime:\n sqlbuild version: 0.2.1\n\nConfiguration:\n project file: /repo/sqlbuild_project.toml [OK found and valid]\n\nProviders:\n providers: 1 [OK discovered]\n marker_provider: providers/marker.py:MarkerProvider [OK valid settings]\n\nConnection:\n token: ****\n connection test: [ERROR authentication failed]\n query test: [SKIP connection failed]\n\n", - expected_json: r#"{"configuration": [{"label": "project file", "message": "/repo/sqlbuild_project.toml", "status": "OK", "status_message": "found and valid"}], "connection": [{"label": "token", "message": "****"}, {"label": "connection test", "message": "", "status": "ERROR", "status_message": "authentication failed"}, {"label": "query test", "message": "", "status": "SKIP", "status_message": "connection failed"}], "local_files": ["/repo/warehouse.db"], "providers": [{"label": "providers", "message": "1", "status": "OK", "status_message": "discovered"}, {"label": "marker_provider", "message": "providers/marker.py:MarkerProvider", "status": "OK", "status_message": "valid settings"}], "runtime": [{"label": "sqlbuild version", "message": "0.2.1"}], "success": false}"#, + expected_json: r#"{"configuration": [{"label": "project file", "message": "/repo/sqlbuild_project.toml", "status": "OK", "status_message": "found and valid"}], "connection": [{"label": "token", "message": "****"}, {"label": "connection test", "message": "", "status": "ERROR", "status_message": "authentication failed"}, {"label": "query test", "message": "", "status": "SKIP", "status_message": "connection failed"}], "local_files": ["/repo/warehouse.db"], "missing_secrets": 0, "providers": [{"label": "providers", "message": "1", "status": "OK", "status_message": "discovered"}, {"label": "marker_provider", "message": "providers/marker.py:MarkerProvider", "status": "OK", "status_message": "valid settings"}], "runtime": [{"label": "sqlbuild version", "message": "0.2.1"}], "success": false}"#, expected_color_fragments: &[ "\x1b[34m\x1b[1mSQLBuild Diagnostics\x1b[0m", "\x1b[1mRuntime:\x1b[0m", @@ -156,3 +157,57 @@ fn given_a_debug_report_when_its_failure_is_read_then_the_first_failed_check_is_ ); } } + +#[test] +fn given_providers_with_and_without_secrets_when_listing_then_missing_secrets_lead_and_verbose_lists_all() + { + let providers = vec![ + provider("slack_alerts", &[("SLACK_WEBHOOK_URL", false)]), + provider( + "partner_sftp", + &[("PARTNER_SFTP_PASSWORD", true), ("PARTNER_SFTP_KEY", false)], + ), + provider("storefront_api", &[]), + ]; + let test_cases = [ + ProviderLinesTestCase { + description: "default lists only the missing secrets, then a hint", + providers: providers.clone(), + verbose: false, + expected_lines: vec![ + ("providers", "1 of 3 ready", Some("missing secrets")), + ("slack_alerts", "SLACK_WEBHOOK_URL", Some("missing")), + ("partner_sftp", "PARTNER_SFTP_KEY", Some("missing")), + ("more", "sqb debug --verbose lists every provider", None), + ], + }, + ProviderLinesTestCase { + description: "verbose lists every provider and secret", + providers, + verbose: true, + expected_lines: vec![ + ("providers", "1 of 3 ready", Some("missing secrets")), + ("slack_alerts", "SLACK_WEBHOOK_URL", Some("missing")), + ("partner_sftp", "PARTNER_SFTP_PASSWORD", Some("set")), + ("partner_sftp", "PARTNER_SFTP_KEY", Some("missing")), + ("storefront_api", "(no secrets)", Some("ok")), + ], + }, + ProviderLinesTestCase { + description: "every secret set prints one line", + providers: vec![provider("partner_sftp", &[("PARTNER_SFTP_PASSWORD", true)])], + verbose: false, + expected_lines: vec![("providers", "1 of 1 ready", Some("secrets set"))], + }, + ]; + for test_case in test_cases { + let lines = debug_provider_lines(&test_case.providers, test_case.verbose); + + assert_eq!( + lines.iter().map(line_summary).collect::>(), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/debug_report/tests/test_types.rs b/crates/sqlbuild-core/src/debug_report/tests/test_types.rs index 685417e0c..2e539a46e 100644 --- a/crates/sqlbuild-core/src/debug_report/tests/test_types.rs +++ b/crates/sqlbuild-core/src/debug_report/tests/test_types.rs @@ -1,4 +1,6 @@ -use crate::debug_report::models::{ConnectionCheck, ConnectionValue, DebugFailure, DebugReport}; +use crate::debug_report::models::{ + ConnectionCheck, ConnectionValue, DebugFailure, DebugProvider, DebugReport, +}; pub(super) struct FormatTestCase { pub(super) description: &'static str, @@ -20,3 +22,10 @@ pub(super) struct DebugFailureTestCase { pub(super) report: DebugReport, pub(super) expected_failure: Option, } + +pub(super) struct ProviderLinesTestCase { + pub(super) description: &'static str, + pub(super) providers: Vec, + pub(super) verbose: bool, + pub(super) expected_lines: Vec<(&'static str, &'static str, Option<&'static str>)>, +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs index 57c463b88..5d8ba7447 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs @@ -157,13 +157,8 @@ pub(crate) fn read_python_modules( (SENSORS_DIRECTORY, PythonDeclarationKind::Sensor), ] { for path in role_files(tree, role, PYTHON_FILE_SUFFIX)? { - let stem: &str = path - .rsplit('/') - .next() - .and_then(|name| name.strip_suffix(PYTHON_FILE_SUFFIX)) - .unwrap_or_default(); let declared: Option = - module_declaration(&read(tree, &path)?, stem, kind, python_version) + module_declaration(&read(tree, &path)?, &path, kind, python_version) .map_err(|error| failure(&path, error))?; if let Some(mut module) = declared { module.path = path.clone(); diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs index d299717f2..18129c54a 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs @@ -5,8 +5,9 @@ use sqlbuild_pysource::orchestration_modules::models::{ LiteralArgument, ModuleDeclarations, PythonDeclaration, PythonDeclarationKind, SourcePosition, }; +use crate::constants::PYTHON_FILE_SUFFIX; use crate::orchestration_files::constants::{ - CALENDAR_KIND, PRIVATE_PREFIX, SENSOR_KIND, TOML_JOBS_HELP, + CALENDAR_KIND, PRIVATE_PREFIX, ROLE_FOLDER_DEPTH, SENSOR_KIND, TOML_JOBS_HELP, }; use crate::orchestration_files::errors::FileError; use crate::orchestration_files::models::{FilePosition, NamedModule, Positioned}; @@ -55,13 +56,21 @@ fn literal(argument: LiteralArgument, what: &str) -> Result, } } -/// The one declaration a public module in a `kind` folder makes; `None` for a private support module. +/// The one declaration a public module at `path` in a `kind` folder makes; `None` for a private support module. pub(crate) fn module_declaration( text: &str, - stem: &str, + path: &str, kind: PythonDeclarationKind, python_version: (u8, u8), ) -> Result, FileError> { + let (folders, file_name) = path.rsplit_once('/').unwrap_or(("", path)); + let stem: &str = file_name + .strip_suffix(PYTHON_FILE_SUFFIX) + .unwrap_or(file_name); + let private_folder: Option<&str> = folders + .split('/') + .skip(ROLE_FOLDER_DEPTH) + .find(|folder| folder.starts_with(PRIVATE_PREFIX)); let start = SourcePosition { line: 1, column: 1 }; let Some(ModuleDeclarations { declarations, @@ -98,7 +107,7 @@ pub(crate) fn module_declaration( None, )); } - let private: bool = stem.starts_with(PRIVATE_PREFIX); + let private: bool = stem.starts_with(PRIVATE_PREFIX) || private_folder.is_some(); let mut declarations = declarations.into_iter(); let first: Option = declarations.next(); if let Some(second) = declarations.next() { @@ -120,6 +129,17 @@ pub(crate) fn module_declaration( )), )); }; + if let Some(folder) = private_folder { + return Err(error( + declaration.position, + format!( + "declares a {noun} inside the private folder {folder}/; declarations can't live in a private folder" + ), + Some(format!( + "move the module out of {folder}/ into its domain folder, or move the {noun} into a module there" + )), + )); + } if private { return Err(error( declaration.position, diff --git a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs index c722549fb..304139ac9 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs @@ -54,6 +54,8 @@ pub(crate) const BYTECODE_DIRECTORY: &str = "__pycache__"; /// The prefix of hidden files and folders, and of private support modules. pub(crate) const HIDDEN_PREFIX: char = '.'; pub(crate) const PRIVATE_PREFIX: char = '_'; +/// The folders of a module path before the ones below its role folder: `orchestration` and the role. +pub(crate) const ROLE_FOLDER_DEPTH: usize = 2; /// Every folder `orchestration/` may hold. pub const ROLE_DIRECTORIES: [&str; 4] = [ JOBS_DIRECTORY, diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs index 0e63db921..7ee11f7aa 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs @@ -39,6 +39,46 @@ fn given_projects_when_inventorying_then_reads_each_kind_and_names_them() { ], expected_empty: false, }, + InventoryTestCase { + description: "a module inside a private folder is support and declares nothing", + files: [ + valid(), + vec![( + "orchestration/sensors/inventory/_helpers/feed_age.py", + "def feed_age() -> int:\n return 0\n", + )], + ] + .concat(), + expected_names: vec![ + named( + "orders_export_job", + "job", + "orchestration/jobs/orders/export.toml", + ), + named( + "orders_nightly_job_group", + "job group", + "orchestration/job_groups/orders/nightly.toml", + ), + named( + "trading_days_calendar", + "calendar", + "orchestration/calendars/trading_days.py", + ), + named( + "stock_stale_sensor", + "sensor", + "orchestration/sensors/inventory/stock.py", + ), + ], + expected_modules: vec![ + "orchestration/calendars/_holidays.py", + "orchestration/calendars/trading_days.py", + "orchestration/sensors/inventory/_helpers/feed_age.py", + "orchestration/sensors/inventory/stock.py", + ], + expected_empty: false, + }, InventoryTestCase { description: "no orchestration folder", files: vec![("models/orders.sql", "select 1")], @@ -177,6 +217,14 @@ fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_ "declare one sensor, or start the file name with `_` to make it a private support module", ), }, + InventoryErrorTestCase { + description: "a sensor declared inside a private folder", + files: with_file("orchestration/sensors/inventory/_helpers/stock.py", SENSOR), + expected_message: "orchestration/sensors/inventory/_helpers/stock.py:3:9: declares a sensor inside the private folder _helpers/; declarations can't live in a private folder", + expected_help: Some( + "move the module out of _helpers/ into its domain folder, or move the sensor into a module there", + ), + }, InventoryErrorTestCase { description: "removed Python job API", files: with_file( diff --git a/crates/sqlbuild-discovery/src/python_paths/_helpers/python_files.rs b/crates/sqlbuild-discovery/src/python_paths/_helpers/python_files.rs index 1c700e44e..9c59eda73 100644 --- a/crates/sqlbuild-discovery/src/python_paths/_helpers/python_files.rs +++ b/crates/sqlbuild-discovery/src/python_paths/_helpers/python_files.rs @@ -6,10 +6,10 @@ use std::path::Path; use crate::constants::PYTHON_FILE_SUFFIX; use crate::python_paths::constants::{HIDDEN_PREFIX, IGNORED_ROOTS, PYTHON_CACHE_DIRECTORY}; -/// The relative parts of every Python file below `project_dir`, sorted part by part. -pub(crate) fn python_files(project_dir: &Path) -> Vec> { +/// The relative parts of every Python file in the `start` folder below `project_dir`, sorted part by part. +pub(crate) fn python_files(project_dir: &Path, start: &[&str]) -> Vec> { let mut files: Vec> = Vec::new(); - let mut pending: Vec> = vec![Vec::new()]; + let mut pending: Vec> = vec![start.iter().map(|part| (*part).to_owned()).collect()]; while let Some(parts) = pending.pop() { let directory = parts .iter() diff --git a/crates/sqlbuild-discovery/src/python_paths/constants.rs b/crates/sqlbuild-discovery/src/python_paths/constants.rs index 4483a9f78..6069594a4 100644 --- a/crates/sqlbuild-discovery/src/python_paths/constants.rs +++ b/crates/sqlbuild-discovery/src/python_paths/constants.rs @@ -32,3 +32,5 @@ pub(crate) const GROUPED_PYTHON_HOOK_ROLES: [[&str; 3]; 2] = [ ]; /// The prefix of a hidden directory the walk skips. pub(crate) const HIDDEN_PREFIX: &str = "."; +/// The folders holding project Python tests, each mirroring the project folder its second part names. +pub(crate) const PYTHON_TEST_ROOTS: [[&str; 2]; 2] = [["tests", "providers"], ["tests", "python"]]; diff --git a/crates/sqlbuild-discovery/src/python_paths/main/mod.rs b/crates/sqlbuild-discovery/src/python_paths/main/mod.rs index 07d5f14de..a8412c27a 100644 --- a/crates/sqlbuild-discovery/src/python_paths/main/mod.rs +++ b/crates/sqlbuild-discovery/src/python_paths/main/mod.rs @@ -1,2 +1,3 @@ +pub mod python_test_paths; pub mod supported_python_paths; pub mod unsupported_python_paths; diff --git a/crates/sqlbuild-discovery/src/python_paths/main/python_test_paths.rs b/crates/sqlbuild-discovery/src/python_paths/main/python_test_paths.rs new file mode 100644 index 000000000..55f3ce38f --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/main/python_test_paths.rs @@ -0,0 +1,14 @@ +//! Project Python tests, which `sqb test` runs with pytest and discovery never imports. + +use std::path::Path; + +use crate::python_paths::_helpers::python_files::python_files; +use crate::python_paths::constants::PYTHON_TEST_ROOTS; + +/// Each Python file under `tests/providers/` or `tests/python/` as relative parts, sorted part by part. +pub fn python_test_paths(project_dir: &Path) -> Vec> { + PYTHON_TEST_ROOTS + .iter() + .flat_map(|root| python_files(project_dir, root)) + .collect() +} diff --git a/crates/sqlbuild-discovery/src/python_paths/main/supported_python_paths.rs b/crates/sqlbuild-discovery/src/python_paths/main/supported_python_paths.rs index 3c4a75691..e76d6e610 100644 --- a/crates/sqlbuild-discovery/src/python_paths/main/supported_python_paths.rs +++ b/crates/sqlbuild-discovery/src/python_paths/main/supported_python_paths.rs @@ -7,7 +7,7 @@ use crate::python_paths::_helpers::supported::is_supported; /// Each supported Python file below `project_dir` as relative parts, sorted part by part. pub fn supported_python_paths(project_dir: &Path) -> Vec> { - python_files(project_dir) + python_files(project_dir, &[]) .into_iter() .filter(|parts| is_supported(parts)) .collect() diff --git a/crates/sqlbuild-discovery/src/python_paths/main/unsupported_python_paths.rs b/crates/sqlbuild-discovery/src/python_paths/main/unsupported_python_paths.rs index 337529281..cf4b4dc7d 100644 --- a/crates/sqlbuild-discovery/src/python_paths/main/unsupported_python_paths.rs +++ b/crates/sqlbuild-discovery/src/python_paths/main/unsupported_python_paths.rs @@ -7,7 +7,7 @@ use crate::python_paths::_helpers::supported::is_supported; /// Each unsupported Python file below `project_dir` as relative parts, sorted part by part. pub fn unsupported_python_paths(project_dir: &Path) -> Vec> { - python_files(project_dir) + python_files(project_dir, &[]) .into_iter() .filter(|parts| !is_supported(parts)) .collect() diff --git a/crates/sqlbuild-discovery/src/python_paths/tests.rs b/crates/sqlbuild-discovery/src/python_paths/tests.rs index cf987b755..c7677845e 100644 --- a/crates/sqlbuild-discovery/src/python_paths/tests.rs +++ b/crates/sqlbuild-discovery/src/python_paths/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_python_test_paths.rs"] +mod python_test_paths; #[path = "tests/test_supported_python_paths.rs"] mod supported_python_paths; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-discovery/src/python_paths/tests/test_python_test_paths.rs b/crates/sqlbuild-discovery/src/python_paths/tests/test_python_test_paths.rs new file mode 100644 index 000000000..7922ba8e1 --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/tests/test_python_test_paths.rs @@ -0,0 +1,51 @@ +use crate::python_paths::main::python_test_paths::python_test_paths; +use crate::python_paths::main::supported_python_paths::supported_python_paths; +use crate::python_paths::tests::test_types::TestPathsTestCase; + +#[test] +fn given_project_tests_when_listing_python_test_paths_then_only_mirrored_test_roots_remain() { + let test_cases = [ + TestPathsTestCase { + description: "tests/python and tests/providers, skipping SQL test folders, caches and hidden folders", + files: &[ + "tests/python/orders/import/test_main.py", + "tests/providers/storefront/test_provider.py", + "tests/python/orders/_test_types.py", + "tests/rules/test_rule.py", + "tests/unit/orders.py", + "tests/python/__pycache__/test_main.py", + "tests/python/.hidden/test_main.py", + "python/orders/import/main.py", + ], + expected_tests: &[ + "tests/providers/storefront/test_provider.py", + "tests/python/orders/_test_types.py", + "tests/python/orders/import/test_main.py", + ], + }, + TestPathsTestCase { + description: "a project without Python tests", + files: &["python/orders/import/main.py"], + expected_tests: &[], + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("temporary project"); + for file in test_case.files { + let path = directory.path().join(file); + std::fs::create_dir_all(path.parent().expect("parent")).expect("folders"); + std::fs::write(&path, "").expect("file"); + } + let tests: Vec = python_test_paths(directory.path()) + .into_iter() + .map(|parts| parts.join("/")) + .collect(); + let supported: Vec> = supported_python_paths(directory.path()); + assert_eq!(tests, test_case.expected_tests, "{}", test_case.description); + assert!( + supported.iter().all(|parts| parts[0] != "tests"), + "{}: discovery never reads tests/", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/python_paths/tests/test_types.rs b/crates/sqlbuild-discovery/src/python_paths/tests/test_types.rs index 640c9b2ae..3679d3b8e 100644 --- a/crates/sqlbuild-discovery/src/python_paths/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/python_paths/tests/test_types.rs @@ -11,3 +11,10 @@ pub(super) struct SupportedPathsTestCase { pub(super) files: &'static [&'static str], pub(super) expected_supported: &'static [&'static str], } + +/// One project layout and its Python test files. +pub(super) struct TestPathsTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [&'static str], + pub(super) expected_tests: &'static [&'static str], +} diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 8af0a3282..56f9c1218 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -391,6 +391,9 @@ pub struct ProviderSetting { /// The environment variable the setting is read from. pub env: String, pub description: Option, + /// Whether this required secret is unset in the location's environment. + #[serde(default)] + pub missing: bool, } /// Code that injects a provider: a node (`loader`, `task`, `asset`, `check`, `hook` or `materialization`, keyed `kind:name`), an `event_exporter` or a `sink`. diff --git a/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs index 7d2dad4bc..06b6761a5 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/report_location_health.rs @@ -26,6 +26,7 @@ pub async fn report_location_health( LocationHealth::Heartbeat {} => None, LocationHealth::CompileFailed { generation } | LocationHealth::Connection { generation, .. } + | LocationHealth::ProviderSecrets { generation, .. } | LocationHealth::WarehouseSync { generation, .. } => Some(generation), }; if generation.is_some_and(|generation| generation < 1) { diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index efcacecef..626b9b821 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -43,6 +43,11 @@ pub enum LocationHealth { generation: i64, healthy: bool, }, + /// How many provider secret settings the location's environment leaves unset. + ProviderSecrets { + generation: i64, + missing: u64, + }, WarehouseSync { generation: i64, location: WarehouseStateLocation, @@ -96,12 +101,14 @@ pub enum LocationState { Degraded, /// The last load failed, so runs use the previous snapshot. FallingBack, + /// A provider secret is unset in the location's environment, so runs that use the provider fail. + MissingSecrets, /// No instance has checked in for 90 seconds. Broken, } impl LocationState { - /// `target`'s state at `now` from observations that may cover several targets: down without a recent check-in, then a failed load, then a failed connection. + /// `target`'s state at `now` from observations that may cover several targets: down without a recent check-in, then a failed load, then missing provider secrets, then a failed connection. #[must_use] pub fn of( observations: &[LocationHealthObservation], @@ -121,6 +128,7 @@ impl LocationState { } let mut failed: Option<(DateTime, i64)> = None; let mut connection: Option<(DateTime, i64, bool)> = None; + let mut secrets: Option<(DateTime, u64)> = None; for observation in own { let seen = observation.received_at; match observation.report.health { @@ -135,15 +143,21 @@ impl LocationState { } if connection.is_none_or(|(at, _, _)| seen > at) => { connection = Some((seen, generation, healthy)); } + LocationHealth::ProviderSecrets { missing, .. } + if secrets.is_none_or(|(at, _)| seen > at) => + { + secrets = Some((seen, missing)); + } _ => {} } } - match (failed, connection) { - (Some((_, failed)), Some((_, connected, _))) if failed >= connected => { + match (failed, connection, secrets) { + (Some((_, failed)), Some((_, connected, _)), _) if failed >= connected => { Self::FallingBack } - (Some(_), None) => Self::FallingBack, - (_, Some((_, _, false))) => Self::Degraded, + (Some(_), None, _) => Self::FallingBack, + (_, _, Some((_, missing))) if missing > 0 => Self::MissingSecrets, + (_, Some((_, _, false)), _) => Self::Degraded, _ => Self::Healthy, } } diff --git a/crates/sqlbuild-orchestration/tests/api/test_providers.rs b/crates/sqlbuild-orchestration/tests/api/test_providers.rs index 01e151856..72a747f78 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_providers.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_providers.rs @@ -91,7 +91,7 @@ fn given_published_providers_when_people_read_them_then_jobs_using_their_nodes_a "description": "Client for the fulfillment service.", "path": "providers/fulfillment.py", "settings": [ - {"name": "api_key", "type": "SecretStr", "required": true, "secret": true, "env": "API_KEY", "description": null}, + {"name": "api_key", "type": "SecretStr", "required": true, "secret": true, "env": "API_KEY", "description": null, "missing": true}, {"name": "endpoint", "type": "str", "required": false, "secret": false, "env": "ENDPOINT", "description": "Base URL"}, ], "used_by": [ @@ -113,8 +113,8 @@ fn given_published_providers_when_people_read_them_then_jobs_using_their_nodes_a "description": "Client for the fulfillment service.", "path": "providers/fulfillment.py", "settings": [ - {"name": "api_key", "type": "SecretStr", "required": true, "secret": true, "env": "API_KEY", "description": null}, - {"name": "endpoint", "type": "str", "required": false, "secret": false, "env": "ENDPOINT", "description": "Base URL"}, + {"name": "api_key", "type": "SecretStr", "required": true, "secret": true, "env": "API_KEY", "description": null, "missing": true}, + {"name": "endpoint", "type": "str", "required": false, "secret": false, "env": "ENDPOINT", "description": "Base URL", "missing": false}, ], "used_by": [ {"kind": "hook", "name": "notify_fulfillment"}, diff --git a/crates/sqlbuild-pysource/src/node_units/_helpers/deferred.rs b/crates/sqlbuild-pysource/src/node_units/_helpers/deferred.rs new file mode 100644 index 000000000..ebd08cea7 --- /dev/null +++ b/crates/sqlbuild-pysource/src/node_units/_helpers/deferred.rs @@ -0,0 +1,57 @@ +//! The visitor that finds lambda bodies, and the `__main__` test. + +use ruff_python_ast::visitor::{Visitor, walk_expr, walk_stmt}; +use ruff_python_ast::{CmpOp, Expr, Stmt}; +use ruff_text_size::{Ranged, TextRange, TextSize}; + +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::models::SourceLines; +use crate::node_units::models::DeferredSpan; + +/// Whether `test` is `__name__ == "__main__"`, either way round. +pub(crate) fn runs_as_main(test: &Expr) -> bool { + let Expr::Compare(compare) = test else { + return false; + }; + let ([CmpOp::Eq], [right]) = (&*compare.ops, &*compare.comparators) else { + return false; + }; + let name = |expr: &Expr| matches!(expr, Expr::Name(name) if name.id.as_str() == "__name__"); + let main = |expr: &Expr| matches!(expr, Expr::StringLiteral(text) if text.value.to_str() == "__main__"); + (name(&compare.left) && main(right)) || (main(&compare.left) && name(right)) +} + +/// Collects lambda bodies without descending into them. +pub(crate) struct Spans<'a> { + pub(crate) lines: &'a SourceLines, + pub(crate) spans: Vec, +} + +impl Spans<'_> { + /// Record `range` as deferred. + pub(crate) fn push(&mut self, range: TextRange) { + let position = |offset: TextSize| source_position(self.lines, offset); + let span = DeferredSpan { + start: position(range.start()), + end: position(range.end()), + }; + self.spans.push(span); + } +} + +impl<'a> Visitor<'a> for Spans<'_> { + fn visit_stmt(&mut self, statement: &'a Stmt) { + walk_stmt(self, statement); + } + + fn visit_expr(&mut self, expr: &'a Expr) { + if let Expr::Lambda(lambda) = expr { + if let Some(parameters) = &lambda.parameters { + self.visit_parameters(parameters); + } + self.push(lambda.body.range()); + return; + } + walk_expr(self, expr); + } +} diff --git a/crates/sqlbuild-pysource/src/node_units/_helpers/mod.rs b/crates/sqlbuild-pysource/src/node_units/_helpers/mod.rs index da1b2da49..a194766b3 100644 --- a/crates/sqlbuild-pysource/src/node_units/_helpers/mod.rs +++ b/crates/sqlbuild-pysource/src/node_units/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod deferred; pub(crate) mod functions; pub(crate) mod imports; pub(crate) mod positions; diff --git a/crates/sqlbuild-pysource/src/node_units/main/deferred_spans.rs b/crates/sqlbuild-pysource/src/node_units/main/deferred_spans.rs new file mode 100644 index 000000000..5c1e4c241 --- /dev/null +++ b/crates/sqlbuild-pysource/src/node_units/main/deferred_spans.rs @@ -0,0 +1,34 @@ +//! The code of a module that importing it never runs: lambda bodies and the `__main__` block. + +use ruff_python_ast::Stmt; +use ruff_python_ast::visitor::Visitor; +use ruff_text_size::{Ranged, TextRange}; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::models::ParsedSource; +use crate::node_units::_helpers::deferred::{Spans, runs_as_main}; +use crate::node_units::models::DeferredSpan; + +/// Each lambda body and the body of a top-level `if __name__ == "__main__":`, or `None` when `contents` does not parse. +#[must_use] +pub fn deferred_spans(contents: &str, python_version: (u8, u8)) -> Option> { + let parsed: ParsedSource = parse_cpython_module(contents, python_version)?; + let mut spans = Spans { + lines: &parsed.lines, + spans: Vec::new(), + }; + for statement in &parsed.module.body { + if let Stmt::If(block) = statement + && runs_as_main(&block.test) + && let (Some(first), Some(last)) = (block.body.first(), block.body.last()) + { + spans.push(TextRange::new(first.start(), last.end())); + for clause in &block.elif_else_clauses { + spans.visit_elif_else_clause(clause); + } + continue; + } + spans.visit_stmt(statement); + } + Some(spans.spans) +} diff --git a/crates/sqlbuild-pysource/src/node_units/main/mod.rs b/crates/sqlbuild-pysource/src/node_units/main/mod.rs index 77bf4695f..b803b4c23 100644 --- a/crates/sqlbuild-pysource/src/node_units/main/mod.rs +++ b/crates/sqlbuild-pysource/src/node_units/main/mod.rs @@ -1 +1,2 @@ +pub mod deferred_spans; pub mod node_unit_module; diff --git a/crates/sqlbuild-pysource/src/node_units/models.rs b/crates/sqlbuild-pysource/src/node_units/models.rs index e1587cd6d..d2abc1394 100644 --- a/crates/sqlbuild-pysource/src/node_units/models.rs +++ b/crates/sqlbuild-pysource/src/node_units/models.rs @@ -102,3 +102,18 @@ pub struct UnitParameter { pub name: String, pub annotation: Option, } + +/// Source a module's import never runs, from `start` up to `end`, each a 1-based line and 0-based UTF-8 byte column. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DeferredSpan { + pub start: (usize, usize), + pub end: (usize, usize), +} + +impl DeferredSpan { + /// Whether the 1-based `line` and 0-based byte `column` fall inside the span. + #[must_use] + pub fn contains(&self, line: usize, column: usize) -> bool { + self.start <= (line, column) && (line, column) < self.end + } +} diff --git a/crates/sqlbuild-pysource/src/node_units/tests.rs b/crates/sqlbuild-pysource/src/node_units/tests.rs index 60fba9106..f42ff87fb 100644 --- a/crates/sqlbuild-pysource/src/node_units/tests.rs +++ b/crates/sqlbuild-pysource/src/node_units/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_deferred_spans.rs"] +mod deferred_spans; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_node_unit_module.rs"] diff --git a/crates/sqlbuild-pysource/src/node_units/tests/helpers.rs b/crates/sqlbuild-pysource/src/node_units/tests/helpers.rs index 1f767fd75..8a4f29378 100644 --- a/crates/sqlbuild-pysource/src/node_units/tests/helpers.rs +++ b/crates/sqlbuild-pysource/src/node_units/tests/helpers.rs @@ -1,6 +1,6 @@ use crate::node_units::main::node_unit_module::node_unit_module; use crate::node_units::models::{ - UnitDecorator, UnitImport, UnitParameter, UnitPosition, UnitStatementKind, + DeferredSpan, UnitDecorator, UnitImport, UnitParameter, UnitPosition, UnitStatementKind, }; const PYTHON_VERSION: (u8, u8) = (3, 12); @@ -80,3 +80,8 @@ pub(super) fn import( only_in_depends_on: false, } } + +/// The span from `start` to `end`, each a 1-based line and 0-based byte column. +pub(super) fn span(start: (usize, usize), end: (usize, usize)) -> DeferredSpan { + DeferredSpan { start, end } +} diff --git a/crates/sqlbuild-pysource/src/node_units/tests/test_deferred_spans.rs b/crates/sqlbuild-pysource/src/node_units/tests/test_deferred_spans.rs new file mode 100644 index 000000000..d214bf542 --- /dev/null +++ b/crates/sqlbuild-pysource/src/node_units/tests/test_deferred_spans.rs @@ -0,0 +1,52 @@ +use crate::node_units::main::deferred_spans::deferred_spans; +use crate::node_units::tests::helpers::span; +use crate::node_units::tests::test_types::DeferredSpansTestCase; + +#[test] +fn given_module_when_finding_deferred_code_then_lambda_bodies_and_the_main_block_are_spans() { + let test_cases = [ + DeferredSpansTestCase { + description: "a lambda body, but not its default, which runs at definition", + contents: "H = {'x': lambda a=f(): run(a)}\n", + expected_spans: Some(vec![span((1, 24), (1, 30))]), + }, + DeferredSpansTestCase { + description: "a nested lambda is inside the outer span only", + contents: "H = lambda: (lambda: g())\n", + expected_spans: Some(vec![span((1, 13), (1, 24))]), + }, + DeferredSpansTestCase { + description: "a lambda inside a function body", + contents: "def f():\n return sorted(r, key=lambda x: k(x))\n", + expected_spans: Some(vec![span((2, 35), (2, 39))]), + }, + DeferredSpansTestCase { + description: "the main block body, either way round, but not its else clause", + contents: "if '__main__' == __name__:\n run()\n stop()\nelse:\n other(lambda: z())\n", + expected_spans: Some(vec![span((2, 4), (3, 10)), span((5, 18), (5, 21))]), + }, + DeferredSpansTestCase { + description: "a main check inside a function is not the module's main block", + contents: "def f():\n if __name__ == '__main__':\n run()\n", + expected_spans: Some(Vec::new()), + }, + DeferredSpansTestCase { + description: "another comparison with __name__ runs on import", + contents: "if __name__ != '__main__':\n run()\n", + expected_spans: Some(Vec::new()), + }, + DeferredSpansTestCase { + description: "a source that does not parse", + contents: "def (:\n", + expected_spans: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + deferred_spans(test_case.contents, (3, 12)), + test_case.expected_spans, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/node_units/tests/test_types.rs b/crates/sqlbuild-pysource/src/node_units/tests/test_types.rs index 66490f98e..401a6b646 100644 --- a/crates/sqlbuild-pysource/src/node_units/tests/test_types.rs +++ b/crates/sqlbuild-pysource/src/node_units/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::node_units::models::{UnitFunction, UnitImport}; +use crate::node_units::models::{DeferredSpan, UnitFunction, UnitImport}; pub(super) struct StatementKindsTestCase { pub(super) description: &'static str, @@ -17,3 +17,9 @@ pub(super) struct FunctionTestCase { pub(super) contents: &'static str, pub(super) expected_function: UnitFunction, } + +pub(super) struct DeferredSpansTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) expected_spans: Option>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 98cff67d7..b42d11672 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -19,8 +19,14 @@ use crate::bindings::_helpers::build_command::planning::{ compile_build_plan, enforce_build_plan_policies, run_defer_clone_prephase, write_compile_target, write_plan_text, }; +use crate::bindings::_helpers::build_command::support::{frozenset, tuple}; +use crate::bindings::_helpers::check::check_core::relevant_check_functions; use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild; use crate::bindings::_helpers::diagnostics::trackers::BuildPhaseTimingTracker; +use crate::bindings::_helpers::discovery::provider_secrets::{ + consumer_uses, provider_uses, require_used_secrets, sink_uses, +}; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::observations::source_capture::{ arm_source_capture, take_source_capture, @@ -69,9 +75,61 @@ fn run_cached_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> built } +/// The post-build check functions that run if every planned node succeeds. +fn planned_check_functions<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, +) -> PyResult>> { + if !native_field(&context.request, "run_audits")?.extract::()? { + return Ok(Vec::new()); + } + let py: Python<'py> = context.py(); + let plan: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?; + let mut dependencies: Vec> = Vec::new(); + for entry in native_field(pipeline, "python_plan_entries")?.try_iter()? { + dependencies.push(native_field(&entry?, "name")?); + } + let mut completed: Vec> = Vec::new(); + for entries in ["model_entries", "seed_entries", "source_load_entries"] { + for entry in native_field(&plan, entries)?.try_iter()? { + completed.push(native_field(&entry?, "name")?); + } + } + for load in native_field(&plan, "source_load_entries")?.try_iter()? { + dependencies.push(native_field(&load?, "loader")?); + } + let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; + relevant_check_functions( + &discovered, + &python_node_graph(&discovered)?, + native_field(&context.request, "exclude")?, + (&frozenset(py, &dependencies)?, &frozenset(py, &completed)?), + ) +} + +/// Fail before anything runs when planned nodes, their post-build checks or sinks lack provider secrets. +fn require_plan_secrets(context: &BuildContext<'_>, pipeline: &Bound<'_, PyAny>) -> PyResult<()> { + let discovered: Bound<'_, PyAny> = context.get("discovered_inputs")?; + let mut uses: Vec<(String, String)> = provider_uses( + &native_field(&native_field(pipeline, "plan_output")?, "provider_usages")?, + None, + )?; + uses.extend(consumer_uses(&native_field( + pipeline, + "python_plan_entries", + )?)?); + uses.extend(consumer_uses(&tuple( + context.py(), + &planned_check_functions(context, pipeline)?, + )?)?); + uses.extend(sink_uses(&discovered)?); + require_used_secrets(&discovered, &uses) +} + /// Plan and run the build, sharing one relation metadata cache from planning to the run's end. fn run_captured_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { let pipeline: Bound<'_, PyAny> = compile_build_plan(context)?; + require_plan_secrets(context, &pipeline)?; if let Some(worker) = &context.worker { worker.enforce_plan_writers(&pipeline)?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index a2a5d45df..2f6d753cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -24,6 +24,7 @@ use crate::bindings::_helpers::compiled_project::snapshot_execution::WorkerBuild use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::{new_record, new_record_with}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -124,6 +125,10 @@ pub(crate) fn resolve_build_context<'py>( &[ ("project_dir", project_dir.clone()), ("extract_output_column_locations", boolean(py, false)), + ( + "secret_policy", + pyo3::types::PyString::new(py, SecretPolicy::Defer.name()).into_any(), + ), ], )?), ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index f9b8a4d5c..ce9feea89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -10,6 +10,9 @@ use crate::bindings::_helpers::compile_driver::relation_targets::build_project_r use crate::bindings::_helpers::compile_driver::relation_targets::build_python_relation_targets; use crate::bindings::_helpers::compile_driver::static_context::compile_static_command_context; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; +use crate::bindings::_helpers::discovery::provider_secrets::{ + consumer_uses, require_used_secrets, sink_uses, +}; use crate::bindings::_helpers::graph::algorithms::transitive_closure_many; use crate::bindings::_helpers::graph::execution_gates::sql_ref_key; use crate::bindings::_helpers::graph::python_lifecycle::build_python_sql_run_lifecycle; @@ -661,6 +664,10 @@ pub(crate) fn run_check_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> P header.set_item("use_color", native_field(&invocation, "use_color")?)?; write_execution_header(py, Some(header))?; let prepared: Prepared<'_> = prepare(request, &invocation)?; + let discovered: Bound<'_, PyAny> = native_field(&invocation, "discovered_inputs")?; + let mut uses: Vec<(String, String)> = consumer_uses(&prepared.check_functions)?; + uses.extend(sink_uses(&discovered)?); + require_used_secrets(&discovered, &uses)?; let session: Bound<'_, PyAny> = call_with( &host::entry(py, UserCodeHost::BuildProviderSession)?, &[( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 5f6e49b0b..234f01bd6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -18,6 +18,7 @@ use crate::bindings::_helpers::dbt_command::init::run_dbt_init_command; use crate::bindings::_helpers::debug_command::run_debug::run_debug_command; use crate::bindings::_helpers::diff_report::command::run_diff_command; use crate::bindings::_helpers::diff_report::support::cli_error; +use crate::bindings::_helpers::discovery::provider_secrets::MissingSecretsScope; use crate::bindings::_helpers::format_command::command::run_format_command; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::init_command::command::run_init_command; @@ -76,6 +77,9 @@ const RAW_QUERY_ATTRIBUTES: [&str; 4] = [ "right_query_file", ]; +/// The namespace attribute of `--allow-missing-secrets`. +const ALLOW_MISSING_SECRETS_DEST: &str = "allow_missing_secrets"; + /// One invocation being dispatched: its namespace and the values derived from it once. pub(crate) struct Invocation<'py> { args: Bound<'py, PyAny>, @@ -448,6 +452,10 @@ fn routed_command<'py>( ) -> PyResult> { let py: Python<'py> = args.py(); let invocation = Invocation::new(args)?; + let _secrets = MissingSecretsScope::enter( + args.getattr_opt(ALLOW_MISSING_SECRETS_DEST)? + .is_some_and(|allowed| allowed.is_truthy().unwrap_or(false)), + ); match command_dispatch(&command_facts(args)?) { Dispatch::Nothing => 0.into_bound_py_any(py), Dispatch::Fail(failure) => Err(user_error(py, failure)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index 45f161496..0f0db48fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; @@ -81,7 +82,16 @@ fn invocation_context<'py>( Some(discovered) => discovered.clone(), None => discover_project_inputs( py, - Some(keywords(py, &[("project_dir", project_dir.clone())])?), + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ( + "secret_policy", + pyo3::types::PyString::new(py, SecretPolicy::Defer.name()).into_any(), + ), + ], + )?), )?, }; let adapter_context: Bound<'py, PyAny> = resolve_adapter_connection_context( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs index 0f9299991..64245f373 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/runtime_identity.rs @@ -8,6 +8,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::python_calls::keywords; +use crate::bindings::_helpers::discovery::provider_secrets::invocation_allows_missing_secrets; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; @@ -86,6 +87,7 @@ pub(crate) fn invocation_identity<'py>( for field in REQUEST_CHOICES { payload.set_item(field, native_field(request, field)?)?; } + payload.set_item("allow_missing_secrets", invocation_allows_missing_secrets())?; payload.set_item("cwd", py.import("os")?.getattr("getcwd")?.call0()?)?; payload.set_item("use_color", use_color)?; payload.set_item("select", listed(native_field(request, "select")?)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index c565a917a..f78bbff89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -14,6 +14,9 @@ use crate::bindings::_helpers::compile_driver::config_validation::{ }; use crate::bindings::_helpers::compile_driver::integration_loaders::integration_loader_functions; use crate::bindings::_helpers::compile_driver::stage_work::{PROJECT_DISCOVERIES, count_stage}; +use crate::bindings::_helpers::discovery::constants::{ + ALLOW_MISSING_SECRETS_ENV, ALLOW_MISSING_SECRETS_SETTING, TRUE_ENV_VALUES, +}; use crate::bindings::_helpers::discovery::declaration_records::{ discover_adapter_file, discover_audit_files, discover_constant_files, discover_enum_files, discover_macro_files, discover_model_schema_files, discover_seed_files, @@ -31,6 +34,9 @@ use crate::bindings::_helpers::discovery::model_records::{ use crate::bindings::_helpers::discovery::project_config::{ load_local_config_file, load_project_config_file, warn_deprecated_settings, }; +use crate::bindings::_helpers::discovery::provider_secrets::{ + SecretPolicy, invocation_allows_missing_secrets, +}; use crate::bindings::_helpers::discovery::python_function_files::discover_python_function_files; use crate::bindings::_helpers::discovery::python_paths::reject_unsupported_python_paths; use crate::bindings::_helpers::discovery::snapshot::in_snapshot; @@ -186,6 +192,27 @@ struct DiscoveryRequest { project_dir: Py, sql_analysis_enabled_override: Py, extract_output_column_locations: bool, + secret_policy: SecretPolicy, +} + +/// Whether `--allow-missing-secrets`, `SQLBUILD_ALLOW_MISSING_SECRETS` or the local `allow_missing_secrets` setting lets a static command run without provider secrets. +fn missing_secrets_allowed(local_config: &Bound<'_, PyAny>) -> PyResult { + if invocation_allows_missing_secrets() { + return Ok(true); + } + if std::env::var(ALLOW_MISSING_SECRETS_ENV) + .is_ok_and(|value| TRUE_ENV_VALUES.contains(&value.trim().to_ascii_lowercase().as_str())) + { + return Ok(true); + } + Ok( + native_field(local_config, "setting_overrides")?.contains(ALLOW_MISSING_SECRETS_SETTING)? + && native_field( + &native_field(local_config, "settings")?, + ALLOW_MISSING_SECRETS_SETTING, + )? + .is_truthy()?, + ) } /// Load all raw project inputs from disk before semantic resolution. @@ -204,15 +231,22 @@ pub(crate) fn discover_project_inputs<'py>( &[ "sql_analysis_enabled_override", "extract_output_column_locations", + "secret_policy", ], ), )?; count_stage(py, PROJECT_DISCOVERIES, 1)?; + let secret_policy: SecretPolicy = if arguments.passed("secret_policy")? { + SecretPolicy::parse(&arguments.value("secret_policy")?.extract::()?)? + } else { + SecretPolicy::Require + }; let request: DiscoveryRequest = DiscoveryRequest { project_dir: arguments.value("project_dir")?.unbind(), sql_analysis_enabled_override: arguments.value("sql_analysis_enabled_override")?.unbind(), extract_output_column_locations: !arguments.passed("extract_output_column_locations")? || arguments.flag("extract_output_column_locations")?, + secret_policy, }; within(&lifecycle(py, "project_discovery")?, |_| { let assembled: Bound<'py, PyAny> = @@ -263,6 +297,13 @@ fn assembled_inputs<'py>( } else { native_field(&project_config, "settings")?.getattr("sql_analysis")? }; + let secret_policy: SecretPolicy = if request.secret_policy == SecretPolicy::Require + && missing_secrets_allowed(&local_config)? + { + SecretPolicy::Allow + } else { + request.secret_policy + }; let discovered: Bound<'py, PyAny> = discovered_inputs( py, (project_dir, &project_config, &local_config), @@ -270,6 +311,7 @@ fn assembled_inputs<'py>( sql_analysis_enabled.is_truthy()?, request.extract_output_column_locations, ), + secret_policy, )?; host::entry(py, UserCodeHost::ValidateDiscoveredInputs)?.call1((&discovered,))?; configure_discovered_event_exporters(&discovered)?; @@ -309,6 +351,7 @@ fn build_discovered_project_inputs<'py>( !arguments.passed("extract_output_column_locations")? || arguments.flag("extract_output_column_locations")?, ), + SecretPolicy::Require, ) } @@ -321,6 +364,7 @@ fn discovered_inputs<'py>( &Bound<'py, PyAny>, ), flags: (bool, bool), + secret_policy: SecretPolicy, ) -> PyResult> { let declarations: Bound<'py, PyDict> = within( &lifecycle(py, "discovery_declaration_parse")?, @@ -338,7 +382,7 @@ fn discovered_inputs<'py>( let python: Bound<'py, PyDict> = within( &lifecycle(py, "discovery_python_import")?, |python_lifecycle| { - let python: Bound<'py, PyDict> = imported_python(project_dir)?; + let python: Bound<'py, PyDict> = imported_python(project_dir, secret_policy)?; completed(python_lifecycle, imported_python_paths(&python)?)?; Ok(python) }, @@ -392,10 +436,14 @@ fn declaration_files<'py>( } /// Providers, then every kind of imported project Python that may use them. -fn imported_python<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { +fn imported_python<'py>( + project_dir: &Bound<'py, PyAny>, + secret_policy: SecretPolicy, +) -> PyResult> { let py: Python<'py> = project_dir.py(); - let providers: Bound<'py, PyAny> = discover_provider_classes(project_dir.clone()) - .and_then(|value| value.into_bound_py_any(py))?; + let providers: Bound<'py, PyAny> = + discover_provider_classes(project_dir.clone(), secret_policy.name()) + .and_then(|value| value.into_bound_py_any(py))?; let python: Bound<'py, PyDict> = PyDict::new(py); python.set_item("providers", &providers)?; for (field, discoverer) in PYTHON_DISCOVERERS { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs index 3ab1dc70e..396f3a9ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/static_context.rs @@ -10,7 +10,9 @@ use crate::bindings::_helpers::compile_driver::deferral::{ build_deferred_locations, resolve_deferred_target_config, }; use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; +use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; use crate::bindings::_helpers::compile_driver::selection_phases::unit_test_selectors_applied; +use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; use crate::bindings::_helpers::planner_entries::command_plans::{ resolve_static_command_scope, resolve_static_relation_context, }; @@ -104,6 +106,26 @@ pub(crate) fn compile_static_command_context<'py>( )?), )?; let project: Bound<'py, PyAny> = native_field(&compiled, "project")?; + let selective: bool = native_field(&options, "select")?.is_truthy()? + || native_field(&options, "exclude")?.is_truthy()?; + if selected_keys.is_none() + && selective + && native_field(&options, "resolve_python_run_selectors")?.is_truthy()? + { + let graph: Bound<'py, PyAny> = build_project_graph_from_compiled_project( + py, + Some(keywords(py, &[("project", project.clone())])?), + )?; + let selection: Bound<'py, PyAny> = run_selection_from_inputs( + ( + native_field(&options, "select")?.extract()?, + native_field(&options, "exclude")?.extract()?, + ), + &graph, + &discovered, + )?; + selected_keys = native_field(&selection, "sql_keys")?; + } let deferred: Bound<'py, PyAny> = deferred_locations((&discovered, &adapter), &options, &project)?; let scope: Bound<'py, PyAny> = resolve_static_command_scope( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs index eadc8f712..e6224c7dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -214,6 +214,7 @@ const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("scenario_timeout", FieldDefault::None), FieldSpec::new("scenario_timeout_seconds", FieldDefault::None), FieldSpec::new("delta_compile", FieldDefault::True), + FieldSpec::new("allow_missing_secrets", FieldDefault::False), ], }; const R_SCOPES_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 1ebb3f2c6..95983bfa4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -42,7 +42,11 @@ use crate::bindings::_helpers::compile_attachment::target::build_effective_targe use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::diff_report::query::NAMESPACELESS_ADAPTERS; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::native_host::host::redirect_stdout_to_stderr; +use crate::bindings::_helpers::native_host::provider_settings::{ + missing_secrets, secret_variables, +}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::fields::field as native_field; @@ -56,6 +60,7 @@ const DEBUG_OPTIONS: (&[&str], &[&str]) = ( "no_connection", "selected_target", "json_output", + "verbose", ], ); @@ -167,7 +172,16 @@ fn providers(discovered_inputs: &Bound<'_, PyAny>) -> PyResult = Vec::new(); for provider in native_field(discovered_inputs, "providers")?.try_iter()? { let provider: Bound<'_, PyAny> = provider?; + let provider_class: Bound<'_, PyAny> = native_field(&provider, "provider_class")?; + let missing: Vec = missing_secrets(&provider_class)?; providers.push(DebugProvider { + secrets: secret_variables(&provider_class)? + .into_iter() + .map(|variable| { + let set: bool = !missing.contains(&variable); + (variable, set) + }) + .collect(), name: native_field(&provider, "name")?.str()?.extract()?, relative_path: native_field(&provider, "relative_path")?.str()?.extract()?, class_name: native_field(&provider, "provider_class")? @@ -262,7 +276,7 @@ fn effective_connection<'py>( fn build_report( py: Python<'_>, project_dir: &PathBuf, - (selected_target, check_connection): (Bound<'_, PyAny>, bool), + (selected_target, check_connection, verbose): (Bound<'_, PyAny>, bool, bool), ) -> PyResult { let project_dir_object: Bound<'_, PyAny> = imported(py, "pathlib", "Path")?.call1((project_dir,))?; @@ -270,7 +284,13 @@ fn build_report( py, Some(keywords( py, - &[("project_dir", project_dir_object.clone())], + &[ + ("project_dir", project_dir_object.clone()), + ( + "secret_policy", + PyString::new(py, SecretPolicy::Defer.name()).into_any(), + ), + ], )?), )?; let files: DebugConfigFiles = debug_config_files(project_dir); @@ -359,6 +379,7 @@ fn build_report( }; let warehouses: Vec = warehouse_checks(&context, (&command_scope, &base_connection))?; + let providers: Vec = providers(&context.discovered_inputs)?; let check: ConnectionCheck = if check_connection { connection_check(&context.adapter, &connection_config)? } else { @@ -368,7 +389,12 @@ fn build_report( local_files: local_database_files(&facts.adapter_name, &connection_config)?, runtime: debug_runtime_lines(&runtime(py)?), configuration: debug_configuration_lines(&facts, &warehouses), - providers: debug_provider_lines(&providers(&context.discovered_inputs)?), + missing_secrets: providers + .iter() + .flat_map(|provider| &provider.secrets) + .filter(|(_, set)| !set) + .count(), + providers: debug_provider_lines(&providers, verbose), connection: debug_connection_lines(&connection_settings(&connection_config)?, &check), }) } @@ -398,6 +424,7 @@ pub(crate) fn run_debug_command( ( arguments.value("selected_target")?, !arguments.flag("no_connection")?, + arguments.passed("verbose")? && arguments.flag("verbose")?, ), ) }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs index ef9a10eba..9e24509e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/constants.rs @@ -9,3 +9,13 @@ pub(crate) const SQL_HOOK_OUTPUT_FIELDS: [&str; 6] = [ "kwargs", "description", ]; + +/// The flag of the static commands that lets them compile without provider secrets. +pub(crate) const ALLOW_MISSING_SECRETS_FLAG: &str = "--allow-missing-secrets"; +/// The environment variable that sets `--allow-missing-secrets`. +pub(crate) const ALLOW_MISSING_SECRETS_ENV: &str = "SQLBUILD_ALLOW_MISSING_SECRETS"; +/// The `[settings]` key of `sqlbuild_local.toml` that sets `--allow-missing-secrets`. +pub(crate) const ALLOW_MISSING_SECRETS_SETTING: &str = "allow_missing_secrets"; + +/// Environment values that switch a boolean `SQLBUILD_*` variable on. +pub(crate) const TRUE_ENV_VALUES: [&str; 4] = ["1", "on", "true", "yes"]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs index d3eefb90c..702b84ed2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs @@ -3,7 +3,7 @@ use std::collections::HashSet; use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyType}; @@ -12,11 +12,14 @@ use sqlbuild_host::entries::models::UserCodeHost; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; use crate::bindings::_helpers::compile_context::input_reads::record_settings_class_read; use crate::bindings::_helpers::contracts::resource_identities::validate_resource_identity; +use crate::bindings::_helpers::discovery::constants::ALLOW_MISSING_SECRETS_FLAG; use crate::bindings::_helpers::discovery::imported_modules::{ ProjectDir, discovered, discovery_error, discovery_error_from, module_functions, provider_by_name, provider_usages, shim, tuple_of, }; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::provider_settings::{ProviderState, provider_state}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get_or; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; @@ -26,7 +29,6 @@ const SINK_ERROR: ErrorId = ErrorId::EventExporterDiscovery; /// The provider name kept free so a sink's `RecallWriter` parameter can never be mistaken for a provider. const RESERVED_PROVIDER_NAME: &str = "recall_writer"; const PROVIDER_INPUT_ERROR: UserCodeHost = UserCodeHost::ProviderInputError; -const VALIDATION_ERROR: (&str, &str) = ("pydantic", "ValidationError"); const SINK_DEFINITION: UserCodeHost = UserCodeHost::GetLifecycleEventSinkDefinition; const ROOT_LOCATION: &str = ""; const DEFAULT_MESSAGE: &str = "invalid value"; @@ -108,41 +110,82 @@ fn validation_details(error: &Bound<'_, PyAny>) -> PyResult { Ok(details.join("\n")) } -/// An instance of the provider class, reporting invalid settings with each field's error. +/// An instance of the provider class, with missing secrets an error unless `policy` defers or allows them. fn provider_instance<'py>( provider_class: &Bound<'py, PyAny>, (provider_name, relative_path): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { + policy: SecretPolicy, +) -> PyResult<(Bound<'py, PyAny>, Vec)> { let py: Python<'py> = provider_class.py(); - provider_class.call0().or_else(|error| { - let validation_error: Bound<'py, PyAny> = - imported(py, VALIDATION_ERROR.0, VALIDATION_ERROR.1)?; - if !error.value(py).is_instance(&validation_error)? { - return Err(error); + let missing: Vec = match provider_state(provider_class)? { + ProviderState::Ready(instance) => return Ok((instance, Vec::new())), + ProviderState::MissingSecrets(missing) => missing, + ProviderState::Invalid(error) => { + let message: String = format!( + "Provider '{}' in {} has invalid settings:\n{}", + provider_name.str()?, + relative_path.str()?, + validation_details(error.value(py))? + ); + return Err(discovery_error_from(py, (PROVIDER_ERROR, message), error)); } - let message: String = format!( - "Provider '{}' in {} has invalid settings:\n{}", - provider_name.str()?, - relative_path.str()?, - validation_details(error.value(py))? + }; + if policy == SecretPolicy::Require { + let variables: String = missing.join(", "); + let noun: &str = if missing.len() == 1 { + "secret" + } else { + "secrets" + }; + let error: PyErr = discovery_error( + py, + PROVIDER_ERROR, + format!( + "provider '{}' ({}) is missing {noun} {variables}", + provider_name.str()?, + relative_path.str()? + ), ); - Err(discovery_error_from(py, (PROVIDER_ERROR, message), error)) - }) + error.value(py).setattr( + "help", + format!("set {variables}, or pass {ALLOW_MISSING_SECRETS_FLAG} where secrets aren't available"), + )?; + return Err(error); + } + Ok((provider_class.call_method0("model_construct")?, missing)) } /// Provider classes under providers/. #[pyfunction] -#[pyo3(signature = (*, project_dir))] +#[pyo3(signature = (*, project_dir, secret_policy = SecretPolicy::Require.name()))] pub(crate) fn discover_provider_classes<'py>( project_dir: Bound<'py, PyAny>, + secret_policy: &str, ) -> PyResult> { let py: Python<'py> = project_dir.py(); + let policy: SecretPolicy = SecretPolicy::parse(secret_policy)?; + let mut allowed: Vec = Vec::new(); let project = ProjectDir { path: project_dir }; if let Some(cached) = project.cached_extensions()? { let providers: Bound<'py, PyAny> = cached.get_item(0)?; for provider in providers.try_iter()? { - record_settings_class_read(&native_field(&provider?, "provider_class")?)?; + let provider: Bound<'py, PyAny> = provider?; + let provider_class: Bound<'py, PyAny> = native_field(&provider, "provider_class")?; + record_settings_class_read(&provider_class)?; + if policy == SecretPolicy::Defer { + continue; + } + let name: Bound<'py, PyAny> = native_field(&provider, "name")?; + let (_, missing) = provider_instance( + &provider_class, + (&name, &native_field(&provider, "relative_path")?), + policy, + )?; + if !missing.is_empty() { + allowed.push(name.extract()?); + } } + report_allowed(py, policy, &allowed)?; return Ok(providers); } if !project @@ -188,8 +231,11 @@ pub(crate) fn discover_provider_classes<'py>( } seen.push((name_text, file_path.clone())); record_settings_class_read(&provider_class)?; - let settings: Bound<'py, PyAny> = - provider_instance(&provider_class, (&name, &relative_path))?; + let (settings, missing) = + provider_instance(&provider_class, (&name, &relative_path), policy)?; + if !missing.is_empty() { + allowed.push(name.extract()?); + } providers.push(discovered( py, RecordId::DiscoveredProvider, @@ -203,9 +249,31 @@ pub(crate) fn discover_provider_classes<'py>( )?); } } + report_allowed(py, policy, &allowed)?; tuple_of(py, providers) } +/// Note on stderr which providers run with missing secrets because the command allows them. +fn report_allowed(py: Python<'_>, policy: SecretPolicy, allowed: &[String]) -> PyResult<()> { + if policy != SecretPolicy::Allow || allowed.is_empty() { + return Ok(()); + } + let noun: &str = if allowed.len() == 1 { + "provider has" + } else { + "providers have" + }; + host::write_console( + py, + "stderr", + &format!( + "{} {noun} missing secrets (allowed): {}\n", + allowed.len(), + allowed.join(", ") + ), + ) +} + /// Sink modules imported and their lifecycle declarations collected, without providers. #[pyfunction] #[pyo3(signature = (*, project_dir))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 4fe23096f..a83fee866 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -23,6 +23,7 @@ pub(crate) mod project_config; pub(crate) mod project_facts; pub(crate) mod project_files; pub(crate) mod project_validation; +pub(crate) mod provider_secrets; pub(crate) mod python_function_files; pub(crate) mod python_hook_files; pub(crate) mod python_module_files; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs new file mode 100644 index 000000000..64d1f0feb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/provider_secrets.rs @@ -0,0 +1,182 @@ +//! A provider class's setting fields and which of its secret settings the environment leaves unset. + +use std::cell::Cell; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; + +use crate::bindings::_helpers::native_host::provider_settings::missing_secrets; +use crate::bindings::_helpers::records::construct::new_annotated_error; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::ids::ErrorId; + +/// Discovered collections of sinks that run with every command. +const SINK_COLLECTIONS: [&str; 2] = ["event_exporters", "command_output_sinks"]; + +thread_local! { + /// Whether the invocation running on this thread passed `--allow-missing-secrets`. + static INVOCATION_ALLOWS_MISSING_SECRETS: Cell = const { Cell::new(false) }; +} + +/// `--allow-missing-secrets` for one invocation, restored when the invocation ends. +pub(crate) struct MissingSecretsScope { + previous: bool, +} + +impl MissingSecretsScope { + pub(crate) fn enter(allowed: bool) -> Self { + Self { + previous: INVOCATION_ALLOWS_MISSING_SECRETS.replace(allowed), + } + } +} + +impl Drop for MissingSecretsScope { + fn drop(&mut self) { + INVOCATION_ALLOWS_MISSING_SECRETS.set(self.previous); + } +} + +/// Whether the running invocation passed `--allow-missing-secrets`. +pub(crate) fn invocation_allows_missing_secrets() -> bool { + INVOCATION_ALLOWS_MISSING_SECRETS.get() +} + +/// What discovery does when a provider's required secrets are unset. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SecretPolicy { + /// Fail with D012: static commands by default. + Require, + /// Continue and list the providers on stderr: static commands that opted out. + Allow, + /// Continue silently; the command checks the providers it uses before running anything. + Defer, +} + +impl SecretPolicy { + /// The keyword value that selects the policy. + pub(crate) const fn name(self) -> &'static str { + match self { + Self::Require => "require", + Self::Allow => "allow", + Self::Defer => "defer", + } + } + + pub(crate) fn parse(value: &str) -> PyResult { + [Self::Require, Self::Allow, Self::Defer] + .into_iter() + .find(|policy| policy.name() == value) + .ok_or_else(|| PyValueError::new_err(format!("unknown secret policy '{value}'"))) + } +} + +/// `(provider, consumer)` for each usage, named by the usage's own `consumer_name` or else by `consumer`. +pub(crate) fn provider_uses( + usages: &Bound<'_, PyAny>, + consumer: Option<&str>, +) -> PyResult> { + usages + .try_iter()? + .map(|usage| usage_pair(&usage?, consumer)) + .collect() +} + +fn usage_pair(usage: &Bound<'_, PyAny>, consumer: Option<&str>) -> PyResult<(String, String)> { + let provider: String = field(usage, "provider_name")?.extract()?; + let name: String = match consumer { + Some(name) => name.to_owned(), + None => field(usage, "consumer_name")?.extract()?, + }; + Ok((provider, name)) +} + +/// `(provider, consumer)` for each provider the named consumers, such as checks or loaders, inject. +pub(crate) fn consumer_uses(consumers: &Bound<'_, PyAny>) -> PyResult> { + let mut uses: Vec<(String, String)> = Vec::new(); + for consumer in consumers.try_iter()? { + let consumer: Bound<'_, PyAny> = consumer?; + let name: String = field(&consumer, "name")?.extract()?; + uses.extend(provider_uses( + &field(&consumer, "provider_usages")?, + Some(&name), + )?); + } + Ok(uses) +} + +/// The providers the project's lifecycle and command-output sinks use, which run with every command. +pub(crate) fn sink_uses(discovered: &Bound<'_, PyAny>) -> PyResult> { + let mut uses: Vec<(String, String)> = Vec::new(); + for collection in SINK_COLLECTIONS { + if let Some(sinks) = discovered.getattr_opt(collection)? { + uses.extend(consumer_uses(&sinks)?); + } + } + Ok(uses) +} + +/// Fail before anything runs when a provider in `uses` lacks required secrets, listing each with what needs it. +pub(crate) fn require_used_secrets( + discovered: &Bound<'_, PyAny>, + uses: &[(String, String)], +) -> PyResult<()> { + let py: Python<'_> = discovered.py(); + let mut consumers: Vec<(String, Vec)> = Vec::new(); + for (provider, consumer) in uses { + match consumers.iter_mut().find(|(name, _)| name == provider) { + Some((_, names)) if !names.contains(consumer) => names.push(consumer.clone()), + Some(_) => {} + None => consumers.push((provider.clone(), vec![consumer.clone()])), + } + } + let mut rows: Vec<[String; 3]> = Vec::new(); + for provider in field(discovered, "providers")?.try_iter()? { + let provider: Bound<'_, PyAny> = provider?; + let name: String = field(&provider, "name")?.extract()?; + let Some((_, names)) = consumers.iter().find(|(used, _)| *used == name) else { + continue; + }; + let needed_by: String = match names.as_slice() { + [only] => only.clone(), + [first, rest @ ..] => format!("{first}, +{}", rest.len()), + [] => String::new(), + }; + for variable in missing_secrets(&field(&provider, "provider_class")?)? { + rows.push([name.clone(), variable, format!("({needed_by})")]); + } + } + if rows.is_empty() { + return Ok(()); + } + let widths: [usize; 2] = [0, 1].map(|column| column_width(&rows, column)); + let lines: Vec = rows + .iter() + .map(|[provider, variable, needed_by]| { + format!( + " {provider: usize { + rows.iter() + .map(|row| row[column].len()) + .max() + .unwrap_or_default() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs index 3c86c9b39..5379e7b86 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs @@ -1,5 +1,6 @@ //! `EventExporterCommandScope`: typed sink binding, its provider session and the active command scope. +use std::collections::HashMap; use std::sync::{Mutex, MutexGuard}; use pyo3::exceptions::PyRuntimeError; @@ -17,6 +18,7 @@ use crate::bindings::_helpers::boundary::python_calls::{ use crate::bindings::_helpers::event_exporting::constants::SCOPE_CONTEXT_VARIABLE; use crate::bindings::_helpers::event_exporting::models::BoundEventExporter; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::provider_settings::{ProviderState, provider_state}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; @@ -77,15 +79,58 @@ fn provider_arguments<'py>( Ok(arguments) } -/// Bound lifecycle exporters and command-output sinks over `session`'s provider instances. +/// Each discovered provider whose required secret settings are unset, with their variables. +fn providers_missing_secrets( + providers: &Bound<'_, PyAny>, +) -> PyResult>> { + let mut missing: HashMap> = HashMap::new(); + for provider in providers.try_iter()? { + let provider: Bound<'_, PyAny> = provider?; + if let ProviderState::MissingSecrets(variables) = + provider_state(&native_field(&provider, "provider_class")?)? + { + missing.insert(native_field(&provider, "name")?.extract()?, variables); + } + } + Ok(missing) +} + +/// Whether `declaration` can bind; when a provider it takes is missing secrets, a notice says it is left unbound. +fn bindable( + declaration: &Bound<'_, PyAny>, + missing: &HashMap>, +) -> PyResult { + for usage in native_field(declaration, "provider_usages")?.try_iter()? { + let provider: String = native_field(&usage?, "provider_name")?.extract()?; + if let Some(variables) = missing.get(&provider) { + host::write_console( + declaration.py(), + "stderr", + &format!( + "sink '{}' is not bound: provider '{provider}' is missing {}\n", + native_field(declaration, "name")?.str()?, + variables.join(", ") + ), + )?; + return Ok(false); + } + } + Ok(true) +} + +/// Bound lifecycle exporters and command-output sinks over `session`, except those using `missing` providers. fn bindings<'py>( session: &Bound<'py, PyAny>, (event_exporters, command_output_sinks): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + missing: &HashMap>, ) -> PyResult<(Bound<'py, PyTuple>, Bound<'py, PyTuple>)> { let py: Python<'py> = session.py(); let mut exporters: Vec> = Vec::new(); for exporter in event_exporters.try_iter()? { let exporter: Bound<'py, PyAny> = exporter?; + if !bindable(&exporter, missing)? { + continue; + } let arguments: Bound<'py, PyDict> = provider_arguments(session, &exporter)?; exporters.push( Bound::new( @@ -109,6 +154,9 @@ fn bindings<'py>( let mut sinks: Vec> = Vec::new(); for sink in command_output_sinks.try_iter()? { let sink: Bound<'py, PyAny> = sink?; + if !bindable(&sink, missing)? { + continue; + } let arguments: Bound<'py, PyDict> = provider_arguments(session, &sink)?; sinks.push(call_with( &sink_class, @@ -162,6 +210,7 @@ impl EventExporterCommandScope { let empty = PyTuple::empty(py).into_any(); return self.bind_dispatcher(py, (empty.clone(), empty)); } + let missing: HashMap> = providers_missing_secrets(&providers)?; let session: Bound<'_, PyAny> = call_with( &host::entry(py, UserCodeHost::BuildProviderSession)?, &[ @@ -170,8 +219,11 @@ impl EventExporterCommandScope { ], )?; session.call_method0("hold_open")?; - let (exporters, sinks) = match bindings(&session, (&event_exporters, &command_output_sinks)) - { + let (exporters, sinks) = match bindings( + &session, + (&event_exporters, &command_output_sinks), + &missing, + ) { Ok(bound) => bound, Err(error) => { session.call_method0("release")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index beae4031d..8fc5d16e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -29,6 +29,9 @@ use crate::bindings::_helpers::compile_attachment::effective_config::build_effec use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; +use crate::bindings::_helpers::discovery::provider_secrets::{ + SecretPolicy, consumer_uses, require_used_secrets, sink_uses, +}; use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, }; @@ -85,6 +88,27 @@ fn write(stream: &Bound<'_, PyAny>, value: &str) -> PyResult<()> { Ok(()) } +/// Fail before loading anything when the selected sources' loaders or the sinks need provider secrets that are unset. +fn require_loader_secrets( + discovered: &Bound<'_, PyAny>, + selected: &[Bound<'_, PyAny>], +) -> PyResult<()> { + let mut loaders: Vec = Vec::new(); + for entry in selected { + let loader: Bound<'_, PyAny> = native_field(entry, "loader")?; + if !loader.is_none() { + loaders.push(loader.extract()?); + } + } + let mut uses: Vec<(String, String)> = + consumer_uses(&native_field(discovered, "loader_functions")?)? + .into_iter() + .filter(|(_, loader)| loaders.contains(loader)) + .collect(); + uses.extend(sink_uses(discovered)?); + require_used_secrets(discovered, &uses) +} + fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult> { let project_dir: Bound<'py, PyAny> = match native_field(request, "project_dir")? { directory if !directory.is_none() => directory, @@ -92,7 +116,16 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = discover_project_inputs( py, - Some(keywords(py, &[("project_dir", project_dir.clone())])?), + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ( + "secret_policy", + PyString::new(py, SecretPolicy::Defer.name()).into_any(), + ), + ], + )?), )?; let select: Bound<'py, PyAny> = native_field(request, "select")?; let exclude: Bound<'py, PyAny> = native_field(request, "exclude")?; @@ -122,6 +155,7 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult( let project = project_adapter( py, Some(arguments.value("project_dir")?), - ProjectInputs::Discovered, + ProjectInputs::DiscoveredDeferringSecrets, )?; status(py, "Location project discovered. Compiling snapshot...")?; let (snapshot, blocked) = compile_snapshot(py, arguments, (input, &project), None)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index fb1c70fdf..27ea6aed4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -19,10 +19,6 @@ pub(crate) const COMMAND_IMPLEMENTATIONS: [(RunCommand, &str); 5] = [ pub(crate) const MODEL_NODE_KIND: &str = "model"; pub(crate) const SOURCE_NODE_KIND: &str = "source"; -/// `env_prefix_target` values under which pydantic-settings prefixes alias names too. -pub(crate) const ALIAS_PREFIX_TARGET: &str = "alias"; -pub(crate) const ALL_PREFIX_TARGET: &str = "all"; - /// DuckDB's message when another process, such as a running build, holds the database file. pub(crate) const DUCKDB_LOCK_CONFLICT: &str = "Conflicting lock is held"; @@ -36,3 +32,5 @@ pub(crate) const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; /// A SQL unit test's published kind; a frozen run selects it back with `test:`. pub(crate) const SQL_TEST_NODE_KIND: &str = "sql_test"; +/// The setting-row field the published providers document leaves out: secret matching reads it, the UI does not. +pub(crate) const SNAPSHOT_HIDDEN_SETTING: &str = "alias"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs index 55d170d62..9b5f318e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot_providers.rs @@ -4,15 +4,14 @@ use std::collections::{BTreeMap, BTreeSet}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use serde_json::{Value, json}; -use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::snapshots::models::SnapshotNode; -use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::compiled_project::snapshot_boundary::relative_path; -use crate::bindings::_helpers::compiled_project::values::plain_value; -use crate::bindings::_helpers::discovery::imported_modules::shim; use crate::bindings::_helpers::location_command::constants::{ - ALIAS_PREFIX_TARGET, ALL_PREFIX_TARGET, MODEL_NODE_KIND, SOURCE_NODE_KIND, + MODEL_NODE_KIND, SNAPSHOT_HIDDEN_SETTING, SOURCE_NODE_KIND, +}; +use crate::bindings::_helpers::native_host::provider_settings::{ + missing_secrets, provider_class_settings, setting_rows, }; use crate::bindings::_helpers::records::fields::field; @@ -46,7 +45,6 @@ pub(crate) fn providers_document( discovered_inputs: &Bound<'_, PyAny>, nodes: &BTreeMap, ) -> PyResult> { - let py = discovered_inputs.py(); let providers = field(discovered_inputs, "providers")?; if providers.len()? == 0 { return Ok(None); @@ -68,21 +66,29 @@ pub(crate) fn providers_document( } } } - let read_class = shim(py, UserCodeHost::ProviderSettings)?; let mut rows: Vec = Vec::new(); for provider in providers.try_iter()? { let provider = provider?; let name: String = field(&provider, "name")?.extract()?; - let class = plain_value(&call_with( - &read_class, - &[("provider_class", field(&provider, "provider_class")?)], - )?)?; + let provider_class = field(&provider, "provider_class")?; + let class = provider_class_settings(&provider_class)?; + let missing: Vec = missing_secrets(&provider_class)?; + let mut settings: Vec = setting_rows(&class); + for setting in &mut settings { + let unset = setting["env"] + .as_str() + .is_some_and(|env| missing.iter().any(|name| name == env)); + setting["missing"] = Value::Bool(unset); + if let Some(row) = setting.as_object_mut() { + row.remove(SNAPSHOT_HIDDEN_SETTING); + } + } rows.push(json!({ "name": name, "class": class["class"], "description": class["description"], "path": relative_path(&provider)?, - "settings": settings(&class), + "settings": settings, "nodes": running.remove(&name).unwrap_or_default(), "used_by": used_by.remove(&name).unwrap_or_default(), })); @@ -134,36 +140,3 @@ fn calls_hook(definition: &Value, hook: &str) -> bool { } false } - -/// Declared fields with the environment variable each is read from, which pydantic-settings derives from the prefix unless an alias names it. -fn settings(class: &Value) -> Vec { - let prefix = class["env_prefix"].as_str().unwrap_or_default(); - let case_sensitive = class["case_sensitive"].as_bool().unwrap_or_default(); - let target = class["env_prefix_target"].as_str(); - let alias_prefix = match target { - Some(ALIAS_PREFIX_TARGET | ALL_PREFIX_TARGET) => prefix, - _ => "", - }; - let name_prefix = match target { - Some(ALIAS_PREFIX_TARGET) => "", - _ => prefix, - }; - let mut rows: Vec = Vec::new(); - for field in class["fields"].as_array().into_iter().flatten() { - let name = field["name"].as_str().unwrap_or_default(); - let type_name = field["type"].as_str().unwrap_or_default(); - let env = field["alias"].as_str().map_or_else( - || format!("{name_prefix}{name}"), - |alias| format!("{alias_prefix}{alias}"), - ); - rows.push(json!({ - "name": name, - "type": type_name, - "required": field["required"], - "secret": type_name.contains("Secret"), - "env": if case_sensitive { env } else { env.to_uppercase() }, - "description": field["description"], - })); - } - rows -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index 2f12728a8..9de61aa2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -19,3 +19,7 @@ pub(crate) const SENSOR_CLASS: &str = "Sensor"; /// Pages one bounded state-history SELECT streams before the scan continues from its cursor. pub(crate) const HISTORY_STREAM_PAGES: usize = 50; + +/// `env_prefix_target` values under which pydantic-settings prefixes alias names too. +pub(crate) const ALIAS_PREFIX_TARGET: &str = "alias"; +pub(crate) const ALL_PREFIX_TARGET: &str = "all"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs index 84d87dcdc..c372767b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs @@ -1,5 +1,7 @@ //! Resolve host entries: targets read once from `sqlbuild._native_host`, attributes read live. +use std::path::PathBuf; + use pyo3::PyErr; use pyo3::exceptions::PyImportError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -73,6 +75,25 @@ pub(crate) fn write_console(py: Python<'_>, stream: &str, text: &str) -> PyResul Ok(()) } +/// Flush the interpreter's live console `stream` so text written before a child process appears first. +pub(crate) fn flush_console(py: Python<'_>, stream: &str) -> PyResult<()> { + py.import("sys")?.getattr(stream)?.call_method0("flush")?; + Ok(()) +} + +/// The interpreter running SQLBuild, which child processes reuse. +pub(crate) fn python_executable(py: Python<'_>) -> PyResult { + py.import("sys")?.getattr("executable")?.extract() +} + +/// Whether this interpreter can import the top-level module `name`. +pub(crate) fn module_installed(py: Python<'_>, name: &str) -> PyResult { + Ok(!py + .import("importlib.util")? + .call_method1("find_spec", (name,))? + .is_none()) +} + /// Keep user prints off machine-readable stdout while running project code. pub(crate) fn redirect_stdout_to_stderr(py: Python<'_>) -> PyResult> { py.import("contextlib")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 5a6e327e6..48803fe01 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -13,6 +13,7 @@ pub(crate) mod orchestration_declarations; pub(crate) mod orchestration_environment; pub(crate) mod orchestration_sensors; pub(crate) mod orchestration_triggers; +pub(crate) mod provider_settings; pub(crate) mod registration; pub(crate) mod resource_costs_state; pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index edb486b35..640f5d6be 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::compiled_project::snapshot_declarations::python_d use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::native_host::constants::SENSOR_CONTEXT_PARAMETER; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_environment::{ @@ -70,8 +71,10 @@ pub(crate) fn evaluate_sensor( .import("sqlbuild.orchestration")? .getattr("SensorContext")? .call((), Some(&fields))?; - let providers = - discover_provider_classes(imported(py, "pathlib", "Path")?.call1((project_dir,))?)?; + let providers = discover_provider_classes( + imported(py, "pathlib", "Path")?.call1((project_dir,))?, + SecretPolicy::Defer.name(), + )?; let by_name = provider_by_name(&providers)?; let session = call_with( &entry(py, UserCodeHost::BuildProviderSession)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index a76334202..ab265c388 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -20,6 +20,7 @@ use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::constants::{ CALENDAR_WINDOW_PARAMETER, SOURCE_NEW_DATA_CONDITION, SOURCE_STALE_CONDITION, @@ -156,8 +157,10 @@ pub(crate) fn compile_calendars( if requests.is_empty() { return Ok(definitions); } - let providers = - discover_provider_classes(imported(py, "pathlib", "Path")?.call1((project_dir,))?)?; + let providers = discover_provider_classes( + imported(py, "pathlib", "Path")?.call1((project_dir,))?, + SecretPolicy::Defer.name(), + )?; let by_name = provider_by_name(&providers)?; let session = call_with( &entry(py, UserCodeHost::BuildProviderSession)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/provider_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/provider_settings.rs new file mode 100644 index 000000000..b0d1905c5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/provider_settings.rs @@ -0,0 +1,150 @@ +//! A provider class's declared settings and whether the environment sets its required secrets. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use serde_json::{Value, json}; +use sqlbuild_host::entries::models::UserCodeHost; + +use crate::bindings::_helpers::boundary::python_calls::{call_with, imported}; +use crate::bindings::_helpers::native_host::constants::{ALIAS_PREFIX_TARGET, ALL_PREFIX_TARGET}; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::records::fields::mapping_get_or; + +const VALIDATION_ERROR: (&str, &str) = ("pydantic", "ValidationError"); +const MISSING_ERROR_TYPE: &str = "missing"; + +/// How a provider class instantiates from the environment. +pub(crate) enum ProviderState<'py> { + Ready(Bound<'py, PyAny>), + /// Only required secret settings are unset: their environment variables, sorted. + MissingSecrets(Vec), + /// Another setting is invalid: pydantic's validation error. + Invalid(PyErr), +} + +/// The class's description and declared setting fields, read without any setting value. +pub(crate) fn provider_class_settings(provider_class: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = provider_class.py(); + let settings: Bound<'_, PyAny> = call_with( + &host::entry(py, UserCodeHost::ProviderSettings)?, + &[("provider_class", provider_class.clone())], + )?; + let text: String = imported(py, "json", "dumps")? + .call1((settings,))? + .extract()?; + serde_json::from_str(&text).map_err(|error| PyValueError::new_err(error.to_string())) +} + +/// Declared fields with the environment variable each is read from, which pydantic-settings derives from the prefix unless an alias names it. +pub(crate) fn setting_rows(class: &Value) -> Vec { + let prefix = class["env_prefix"].as_str().unwrap_or_default(); + let case_sensitive = class["case_sensitive"].as_bool().unwrap_or_default(); + let target = class["env_prefix_target"].as_str(); + let alias_prefix = match target { + Some(ALIAS_PREFIX_TARGET | ALL_PREFIX_TARGET) => prefix, + _ => "", + }; + let name_prefix = match target { + Some(ALIAS_PREFIX_TARGET) => "", + _ => prefix, + }; + let mut rows: Vec = Vec::new(); + for field in class["fields"].as_array().into_iter().flatten() { + let name = field["name"].as_str().unwrap_or_default(); + let type_name = field["type"].as_str().unwrap_or_default(); + let env = field["alias"].as_str().map_or_else( + || format!("{name_prefix}{name}"), + |alias| format!("{alias_prefix}{alias}"), + ); + rows.push(json!({ + "name": name, + "alias": field["alias"], + "type": type_name, + "required": field["required"], + "secret": type_name.contains("Secret"), + "env": if case_sensitive { env } else { env.to_uppercase() }, + "description": field["description"], + })); + } + rows +} + +/// The environment variables of the class's secret settings. +pub(crate) fn secret_variables(provider_class: &Bound<'_, PyAny>) -> PyResult> { + Ok(setting_rows(&provider_class_settings(provider_class)?) + .iter() + .filter(|row| row["secret"].as_bool().unwrap_or_default()) + .filter_map(|row| row["env"].as_str().map(str::to_owned)) + .collect()) +} + +/// Instantiate the class, telling unset required secrets apart from other invalid settings. +pub(crate) fn provider_state<'py>( + provider_class: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = provider_class.py(); + let error: PyErr = match provider_class.call0() { + Ok(instance) => return Ok(ProviderState::Ready(instance)), + Err(error) => error, + }; + if !error + .value(py) + .is_instance(&imported(py, VALIDATION_ERROR.0, VALIDATION_ERROR.1)?)? + { + return Err(error); + } + let rows: Vec = setting_rows(&provider_class_settings(provider_class)?); + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("include_input", false)?; + let mut missing: Vec = Vec::new(); + for item in error + .value(py) + .call_method("errors", (), Some(&options))? + .try_iter()? + { + let item: Bound<'py, PyAny> = item?; + let kind: String = mapping_get_or(&item, "type", "")?.str()?.extract()?; + let location: Vec = mapping_get_or(&item, "loc", ())? + .try_iter()? + .map(|part| part?.str()?.extract::()) + .collect::>>()?; + let secret: Option<&str> = match location.as_slice() { + [field] if kind == MISSING_ERROR_TYPE => secret_env(&rows, field), + _ => None, + }; + let Some(env) = secret else { + return Ok(ProviderState::Invalid(error)); + }; + missing.push(env.to_owned()); + } + if missing.is_empty() { + return Ok(ProviderState::Invalid(error)); + } + missing.sort(); + missing.dedup(); + Ok(ProviderState::MissingSecrets(missing)) +} + +/// The environment variables of the class's required secrets the environment leaves unset; an invalid setting is an error. +pub(crate) fn missing_secrets(provider_class: &Bound<'_, PyAny>) -> PyResult> { + match provider_state(provider_class)? { + ProviderState::Ready(_) => Ok(Vec::new()), + ProviderState::MissingSecrets(missing) => Ok(missing), + ProviderState::Invalid(error) => Err(error), + } +} + +/// The environment variable of the secret setting pydantic reports as `field`, by its name or alias. +fn secret_env<'a>(rows: &'a [Value], field: &str) -> Option<&'a str> { + rows.iter() + .filter(|row| row["secret"].as_bool().unwrap_or_default()) + .find(|row| { + row["name"].as_str() == Some(field) + || row["alias"].as_str() == Some(field) + || row["env"] + .as_str() + .is_some_and(|env| env.eq_ignore_ascii_case(field)) + }) + .and_then(|row| row["env"].as_str()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs index fbbeb3b01..1d20d2386 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -8,6 +8,7 @@ use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::discovery::configuration::discover_connection_configuration; +use crate::bindings::_helpers::discovery::provider_secrets::SecretPolicy; use crate::bindings::_helpers::records::fields::field as native_field; /// The project directory, defaulting to the working directory, with its inputs and adapter. @@ -22,6 +23,8 @@ pub(crate) struct ProjectAdapter<'py> { pub(crate) enum ProjectInputs { /// Every declaration file and the imported project Python. Discovered, + /// As `Discovered`, leaving missing provider secrets to the runs that use the providers. + DiscoveredDeferringSecrets, /// Only the project and local configuration, with its connection settings validated. Configuration, } @@ -40,6 +43,19 @@ pub(crate) fn project_adapter<'py>( py, Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?, + ProjectInputs::DiscoveredDeferringSecrets => discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ( + "secret_policy", + pyo3::types::PyString::new(py, SecretPolicy::Defer.name()).into_any(), + ), + ], + )?), + )?, ProjectInputs::Configuration => discover_connection_configuration(py, &project_dir)?, }; let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index caae4c98c..8647dd03c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -698,6 +698,8 @@ const TEST_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("test_timeout", FieldDefault::None), FieldSpec::new("since", FieldDefault::None), FieldSpec::new("explain", FieldDefault::False), + FieldSpec::new("sql_only", FieldDefault::False), + FieldSpec::new("python_only", FieldDefault::False), ], }; const REFACTOR_COMMAND_REQUEST: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs index c590e518e..53eba661c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/execution.rs @@ -5,6 +5,7 @@ use std::path::{Path, PathBuf}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyBool; use serde_json::Value; use sqlbuild_analysis::review_command::constants::{ CLONE_START, FULL_REFRESH_START, SEED_KIND, UNIT_TEST_LABEL, @@ -289,6 +290,7 @@ pub(crate) fn run_changed_tests( ("json_output_path", path_object(py, &output)?), ("concurrency", native_field(inputs.request, "concurrency")?), ("since", text(py, &settings.since)), + ("sql_only", PyBool::new(py, true).to_owned().into_any()), ], )?; let ran: PyResult = quietly(inputs.output, || run_test_command(py, &request)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs index 5db811030..1751c40d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod python_tests; pub(crate) mod registration; pub(crate) mod test_command; pub(crate) mod timeouts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs new file mode 100644 index 000000000..07e0f7645 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/python_tests.rs @@ -0,0 +1,271 @@ +//! The project's Python tests, which a local `sqb test` runs with pytest after its SQL tests. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitStatus, Stdio}; +use std::time::Instant; + +use pyo3::exceptions::{PyOSError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyFrozenSetMethods; +use serde_json::{Value, json}; +use sqlbuild_core::cli_text::main::format_phase_line::format_phase_line; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_discovery::python_paths::main::python_test_paths::python_test_paths; + +use crate::bindings::_helpers::build_progress::terminal::{flush, write}; +use crate::bindings::_helpers::compile_driver::pipeline_entries::project_graph; +use crate::bindings::_helpers::graph::python_node_graph::{ + python_node_graph, run_selection_from_inputs, +}; +use crate::bindings::_helpers::graph::python_selection::python_node_names; +use crate::bindings::_helpers::native_host::host::{ + flush_console, module_installed, python_executable, +}; +use crate::bindings::_helpers::planner_selection::unit_tests::split_project_unit_test_selectors; +use crate::bindings::_helpers::records::construct::new_helped_error; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::ErrorId; + +/// The prefix of a Python test module's file name. +const TEST_MODULE_PREFIX: &str = "test_"; +const PYTEST_MODULE: &str = "pytest"; +/// The diagnostic code of a Python test run that cannot start. +pub(crate) const PYTHON_TESTS_INPUT_CODE: &str = "S000"; +/// The project folder whose `python/` and `providers/` folders mirror the project's. +const TESTS_FOLDER: &str = "tests"; +/// The discovered Python node collections whose folders a selection's tests mirror. +const NODE_COLLECTIONS: [&str; 4] = [ + "loader_functions", + "task_functions", + "asset_functions", + "check_functions", +]; +/// Lets tests import project and `tests.*` modules, and mirrored folders reuse module names. +const IMPORT_MODE: &str = "--import-mode=importlib"; +/// The JSON section reporting the Python tests a `sqb test` ran. +const PYTHON_TESTS_SECTION: &str = "python_tests"; +const PASSED_STATUS: &str = "success"; +const FAILED_STATUS: &str = "failed"; + +/// Whether the request selects or excludes anything. +pub(crate) fn selective(request: &Bound<'_, PyAny>) -> PyResult { + Ok(native_field(request, "select")?.is_truthy()? + || native_field(request, "exclude")?.is_truthy()?) +} + +/// Every `test_*.py` module under the project's `tests/python/` and `tests/providers/`. +pub(crate) fn python_test_modules(invocation: &Bound<'_, PyAny>) -> PyResult> { + let project_dir: PathBuf = native_field(invocation, "effective_project_dir")?.extract()?; + Ok(python_test_paths(&project_dir) + .into_iter() + .filter(|parts| { + parts + .last() + .is_some_and(|name| name.starts_with(TEST_MODULE_PREFIX)) + }) + .map(|parts| parts.join("/")) + .collect()) +} + +/// Whether a selector of the request reaches a discovered Python node, such as `+storefront_import` or `tag:ingest`. +pub(crate) fn names_python_node( + request: &Bound<'_, PyAny>, + discovered: &Bound<'_, PyAny>, +) -> PyResult { + let graph: Bound<'_, PyAny> = python_node_graph(discovered)?; + let mut selectors: Vec = native_field(request, "select")?.extract()?; + selectors.extend(native_field(request, "exclude")?.extract::>()?); + for selector in selectors { + if python_node_names(&graph, (vec![selector], Vec::new())) + .is_ok_and(|names| !PyFrozenSetMethods::is_empty(&names)) + { + return Ok(true); + } + } + Ok(false) +} + +/// The Python test `modules` a local `sqb test` runs: all, or those mirroring the selected Python nodes. +pub(crate) fn selected_python_tests( + request: &Bound<'_, PyAny>, + invocation: &Bound<'_, PyAny>, + (pipeline, modules): (Option<&Bound<'_, PyAny>>, Vec), +) -> PyResult> { + let py: Python<'_> = request.py(); + let selected: Vec = match pipeline { + Some(pipeline) if !modules.is_empty() && selective(request)? => { + let folders: BTreeSet = mirrored_folders( + request, + pipeline, + &native_field(invocation, "discovered_inputs")?, + )?; + let mut mirrored: Vec = Vec::new(); + for module in modules { + if folders.iter().any(|folder| module.starts_with(folder)) { + mirrored.push(module); + } + } + mirrored + } + _ => modules, + }; + if native_field(request, "python_only")?.is_truthy()? && selected.is_empty() { + return Err(new_helped_error( + py, + ErrorId::PlannerInput, + ("no Python tests found".to_owned(), PYTHON_TESTS_INPUT_CODE), + Some( + "add test_*.py modules under tests/python/ or tests/providers/, mirroring the code they test", + ), + )); + } + if !selected.is_empty() && !module_installed(py, PYTEST_MODULE)? { + return Err(new_helped_error( + py, + ErrorId::PlannerInput, + ( + "Python tests found but pytest isn't installed".to_owned(), + PYTHON_TESTS_INPUT_CODE, + ), + Some("install the dev dependencies, or run `sqb test --sql`"), + )); + } + Ok(selected) +} + +/// Run `modules` with this interpreter's pytest from the project folder; pytest's exit code. +pub(crate) fn run_python_tests( + request: &Bound<'_, PyAny>, + invocation: &Bound<'_, PyAny>, + modules: &[String], +) -> PyResult { + let py: Python<'_> = request.py(); + let stream: Bound<'_, PyAny> = native_field(invocation, "progress_stream")?; + let style: CliStyle = CliStyle { + use_color: native_field(invocation, "use_color")?.is_truthy()?, + }; + let project_dir: PathBuf = native_field(invocation, "effective_project_dir")?.extract()?; + let interpreter: PathBuf = python_executable(py)?; + let files: String = match modules.len() { + 1 => "1 file".to_owned(), + count => format!("{count} files"), + }; + write( + &stream, + &format!("\nRunning Python tests with pytest ({files})\n"), + )?; + flush(&stream)?; + flush_console(py, "stdout")?; + let mut command: Command = Command::new(&interpreter); + command + .args(["-m", PYTEST_MODULE, IMPORT_MODE]) + .args(modules) + .current_dir(&project_dir) + .env("PYTHONPATH", python_path(&project_dir)?) + .stdin(Stdio::null()); + if native_field(request, "json_output")?.is_truthy()? { + command.stdout(Stdio::from(std::io::stderr())); + } + let started: Instant = Instant::now(); + let status: ExitStatus = py + .detach(|| command.status()) + .map_err(|error| PyOSError::new_err(format!("cannot run pytest: {error}")))?; + let elapsed: f64 = started.elapsed().as_secs_f64(); + let code: i32 = status.code().unwrap_or(1); + let line: String = if code == 0 { + format_phase_line( + style, + true, + ("Python tests passed", &format!("{files} ({elapsed:.2}s)")), + ) + } else { + format_phase_line( + style, + false, + ( + "Python tests failed", + &format!("pytest exit code {code} ({elapsed:.2}s)"), + ), + ) + }; + write(&stream, &format!("{line}\n"))?; + flush(&stream)?; + Ok(code) +} + +/// The `tests/` folders mirroring the Python nodes the request's resource selectors resolve to. +fn mirrored_folders( + request: &Bound<'_, PyAny>, + pipeline: &Bound<'_, PyAny>, + discovered: &Bound<'_, PyAny>, +) -> PyResult> { + let split: Bound<'_, PyAny> = split_project_unit_test_selectors( + &native_field(request, "select")?, + &native_field(request, "exclude")?, + discovered, + true, + )?; + let select: Bound<'_, PyAny> = native_field(&split, "select")?; + let exclude: Bound<'_, PyAny> = native_field(&split, "exclude")?; + if !select.is_truthy()? && native_field(request, "select")?.is_truthy()? { + return Ok(BTreeSet::new()); + } + let selection: Bound<'_, PyAny> = run_selection_from_inputs( + (select.extract()?, exclude.extract()?), + &project_graph(&native_field(pipeline, "project")?)?, + discovered, + )?; + let names: Bound<'_, PyAny> = native_field(&selection, "python_node_names")?; + let mut folders: BTreeSet = BTreeSet::new(); + for collection in NODE_COLLECTIONS { + for node in native_field(discovered, collection)?.try_iter()? { + let node: Bound<'_, PyAny> = node?; + if !names.contains(native_field(&node, "name")?)? { + continue; + } + let path: PathBuf = native_field(&node, "relative_path")?.extract()?; + if let Some(folder) = path.parent() { + folders.insert(format!("{TESTS_FOLDER}/{}/", folder.to_string_lossy())); + } + } + } + Ok(folders) +} + +/// The project folder ahead of any inherited `PYTHONPATH`. +fn python_path(project_dir: &Path) -> PyResult { + let inherited: Vec = std::env::var_os("PYTHONPATH") + .map(|value| std::env::split_paths(&value).collect()) + .unwrap_or_default(); + std::env::join_paths(std::iter::once(project_dir.to_path_buf()).chain(inherited)) + .map_err(|error| PyOSError::new_err(format!("cannot build PYTHONPATH: {error}"))) +} + +/// `payload` with a `python_tests` section for `modules` and pytest's exit `code`; a failure fails it. +pub(crate) fn with_python_section( + payload: &str, + modules: &[String], + code: i32, +) -> PyResult { + let invalid = |error: serde_json::Error| PyValueError::new_err(error.to_string()); + let mut document: Value = serde_json::from_str(payload).map_err(invalid)?; + let status: &str = if code == 0 { + PASSED_STATUS + } else { + FAILED_STATUS + }; + if let Value::Object(fields) = &mut document { + if code != 0 { + fields.insert("status".to_owned(), json!(FAILED_STATUS)); + } + fields.insert( + PYTHON_TESTS_SECTION.to_owned(), + json!({ "status": status, "exit_code": code, "files": modules }), + ); + } + Ok(format!( + "{}\n", + serde_json::to_string_pretty(&document).map_err(invalid)? + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 35accda3c..744efeed5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -1,4 +1,4 @@ -//! `sqb test`: compile the selected SQL unit tests, inspect or run them, and report. +//! `sqb test`: compile the selected SQL unit tests, inspect or run them, and report; locally, then run the Python tests. use std::collections::{BTreeSet, HashMap}; use std::path::PathBuf; @@ -50,7 +50,7 @@ use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::planner_entries::command_plans::build_test_command_plan; -use crate::bindings::_helpers::records::construct::error_class; +use crate::bindings::_helpers::records::construct::{error_class, new_helped_error}; use crate::bindings::_helpers::records::construct::{new_record_with, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced; @@ -59,6 +59,10 @@ use crate::bindings::_helpers::result_output::documents::test_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; use crate::bindings::_helpers::result_output::resource_ids::sql_test_identity; +use crate::bindings::_helpers::sql_test_command::python_tests::{ + PYTHON_TESTS_INPUT_CODE, names_python_node, python_test_modules, run_python_tests, + selected_python_tests, selective, with_python_section, +}; use crate::bindings::_helpers::sql_test_command::timeouts::{ RunTimeout, TimeoutKind, resolve_run_timeout, }; @@ -304,6 +308,7 @@ fn unknown_case(py: Python<'_>, message: String) -> PyErr { fn compile_tests<'py>( request: &Bound<'py, PyAny>, invocation: &Bound<'py, PyAny>, + python_selectors: bool, ) -> PyResult> { let py: Python<'py> = request.py(); let discovered: Bound<'py, PyAny> = invocation.getattr("discovered_inputs")?; @@ -328,6 +333,10 @@ fn compile_tests<'py>( "accepts_unit_test_selectors", PyBool::new(py, true).to_owned().into_any(), ), + ( + "resolve_python_run_selectors", + PyBool::new(py, python_selectors).to_owned().into_any(), + ), ( "connection_config", invocation.getattr("connection_config")?, @@ -639,13 +648,12 @@ pub(crate) fn timeout_record<'py>( host::entry(py, ExecutorHost::RunItemTimeouts)?.call((), Some(&timeout.keywords(py)?)) } -/// Write the summary, failures and JSON output; returns the exit code. +/// Write the summary and failures; returns the exit code and the `--json` document. fn complete( - request: &Bound<'_, PyAny>, invocation: &Bound<'_, PyAny>, (results, since_json): (&[SqlTestExecutionResult>], Option<&str>), -) -> PyResult { - let py: Python<'_> = request.py(); +) -> PyResult<(i32, String)> { + let py: Python<'_> = invocation.py(); let mut failures: Vec = Vec::new(); for result in results { if result.outcome == SqlTestOutcome::Pass { @@ -672,18 +680,10 @@ fn complete( ),), )?; stream.call_method0("flush")?; - let payload: Bound<'_, PyAny> = PyString::new( - py, - &with_since_section(test_execution_json(py, results)?, since_json)?, - ) - .into_any(); - write_execution_json_output( - py, - payload, - request.getattr("json_output")?.extract()?, - request.getattr("json_output_path")?, - )?; - Ok(test_exit_code(passed, results.len())) + Ok(( + test_exit_code(passed, results.len()), + with_since_section(test_execution_json(py, results)?, since_json)?, + )) } /// Run one `sqb test` request. @@ -700,14 +700,105 @@ pub(crate) fn run_test<'py>( verified: Option>, ) -> PyResult { let (worker, discovered) = verified.unzip(); - let inspect_only: bool = request.getattr("inspect")?.is_truthy()?; + let inspect_only: bool = native_field(request, "inspect")?.is_truthy()?; + if native_field(request, "python_only")?.is_truthy()? && worker.is_some() { + return Err(new_helped_error( + py, + ErrorId::PlannerInput, + ( + "--python cannot run on a server location".to_owned(), + PYTHON_TESTS_INPUT_CODE, + ), + Some("runs on a server location run SQL tests only; run `sqb test --python` locally"), + )); + } + if native_field(request, "python_only")?.is_truthy()? + && (inspect_only || !native_field(request, "since")?.is_none()) + { + return Err(new_helped_error( + py, + ErrorId::PlannerInput, + ( + "--python cannot be combined with --inspect or --since".to_owned(), + PYTHON_TESTS_INPUT_CODE, + ), + Some( + "--inspect and --since apply to SQL tests only; run `sqb test --python` without them", + ), + )); + } let invocation: Bound<'_, PyAny> = resolve_reported_command_invocation( request, &record_class(py, RecordId::TestInvocation)?, (!inspect_only, discovered), )?; + let python: bool = worker.is_none() + && !inspect_only + && !native_field(request, "sql_only")?.is_truthy()? + && native_field(request, "since")?.is_none(); + let modules: Option> = if python { + Some(python_test_modules(&invocation)?) + } else { + None + }; + let resolve: bool = match &modules { + Some(modules) if !modules.is_empty() => { + names_python_node(request, &native_field(&invocation, "discovered_inputs")?)? + } + _ => false, + }; + let (sql_code, python_tests, payload): (i32, Vec, Option) = match modules { + Some(modules) + if native_field(request, "python_only")?.is_truthy()? && !selective(request)? => + { + ( + 0, + selected_python_tests(request, &invocation, (None, modules))?, + Some(test_execution_json(py, &[])?), + ) + } + modules => run_sql_tests(request, &invocation, worker, (modules, resolve))?, + }; + if python_tests.is_empty() { + write_json(request, payload)?; + return Ok(sql_code); + } + let python_code: i32 = run_python_tests(request, &invocation, &python_tests)?; + write_json( + request, + payload + .map(|payload| with_python_section(&payload, &python_tests, python_code)) + .transpose()?, + )?; + Ok(i32::from(sql_code != 0 || python_code != 0)) +} + +/// Write a `--json` document to stdout or `--json-output`, when there is one. +fn write_json(request: &Bound<'_, PyAny>, payload: Option) -> PyResult<()> { + let Some(payload) = payload else { + return Ok(()); + }; + let py: Python<'_> = request.py(); + write_execution_json_output( + py, + PyString::new(py, &payload).into_any(), + native_field(request, "json_output")?.extract()?, + native_field(request, "json_output_path")?, + )?; + Ok(()) +} + +/// Compile, inspect or run the selected SQL tests and pick the Python test `modules` to run after. +fn run_sql_tests<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, + worker: Option>, + (modules, resolve): (Option>, bool), +) -> PyResult<(i32, Vec, Option)> { + let py: Python<'py> = request.py(); + let inspect_only: bool = native_field(request, "inspect")?.is_truthy()?; let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; - let project_dir: Bound<'_, PyAny> = native_field(&invocation, "effective_project_dir")?; + let project_dir: Bound<'_, PyAny> = native_field(invocation, "effective_project_dir")?; let since: Option = native_field(request, "since")?.extract()?; let base: Option = match since.as_deref() { Some(since) => Some(base_comparison( @@ -720,7 +811,14 @@ pub(crate) fn run_test<'py>( )?), None => None, }; - let mut pipeline: Bound<'_, PyAny> = compile_tests(request, &invocation)?; + let mut pipeline: Bound<'_, PyAny> = compile_tests(request, invocation, resolve)?; + let python_tests: Vec = match modules { + Some(modules) => selected_python_tests(request, invocation, (Some(&pipeline), modules))?, + None => Vec::new(), + }; + if native_field(request, "python_only")?.is_truthy()? { + return Ok((0, python_tests, Some(test_execution_json(py, &[])?))); + } let mut reasons: HashMap> = HashMap::new(); let mut since_json: Option = None; if let Some(base) = base { @@ -729,7 +827,7 @@ pub(crate) fn run_test<'py>( ( &pipeline, &stream, - &native_field(&invocation, "discovered_inputs")?, + &native_field(invocation, "discovered_inputs")?, ), &project_dir.extract::()?, base, @@ -738,11 +836,30 @@ pub(crate) fn run_test<'py>( since_json = Some(json); } if inspect_only { - return inspect(&stream, &pipeline.getattr("plan_output")?, &reasons); + return Ok(( + inspect(&stream, &pipeline.getattr("plan_output")?, &reasons)?, + python_tests, + None, + )); + } + if !python_tests.is_empty() + && !pipeline + .getattr("plan_output")? + .getattr("test_entries")? + .is_truthy()? + { + return Ok(( + 0, + python_tests, + Some(with_since_section( + test_execution_json(py, &[])?, + since_json.as_deref(), + )?), + )); } let concurrency: i64 = resolve_concurrency( &request.getattr("concurrency")?, - &native_field(&invocation, "discovered_inputs")?, + &native_field(invocation, "discovered_inputs")?, &request.getattr("selected_target")?, )? .limit; @@ -763,17 +880,18 @@ pub(crate) fn run_test<'py>( TimeoutKind::Test, )?; let results: Vec>> = - run_tests(&invocation, &pipeline, (concurrency, &timeout, worker))?; + run_tests(invocation, &pipeline, (concurrency, &timeout, worker))?; write_test_runtime_target( ( - &native_field(&invocation, "effective_project_dir")? + &native_field(invocation, "effective_project_dir")? .call_method1("__truediv__", ("target",))?, &invocation.getattr("adapter")?, ), &pipeline.getattr("plan_output")?, &results, )?; - complete(request, &invocation, (&results, since_json.as_deref())) + let (code, payload): (i32, String) = complete(invocation, (&results, since_json.as_deref()))?; + Ok((code, python_tests, Some(payload))) } /// Add source and safe typed values only for parameterized test rows. diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index 501756281..64b59420c 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -6,6 +6,7 @@ use crate::constants::{ RELATION_EXCEPTION_RULES, }; use crate::models::RulesConfig; +use crate::python_rules::constants::{APPROVED_LOADER_BOUNDARIES_OPTION, COMPILE_TIME_PURITY_CODE}; use fensu_policy::policy::types::RuleCodeGrammar; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_sqltext::header_strings::main::fold_prose::fold_prose; @@ -248,6 +249,7 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { } } validate_published_layer_options(config)?; + validate_approved_loader_boundaries(config)?; for exception in &config.rule_exceptions { if exception.rule.trim().is_empty() || exception.path.trim().is_empty() @@ -309,6 +311,40 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { Ok(()) } +/// Approved loader boundaries are dotted function names, such as `python.orders._helpers.specs.load_specs`. +fn validate_approved_loader_boundaries(config: &RulesConfig) -> Result<(), String> { + let Some(values) = config + .rule_options + .get(COMPILE_TIME_PURITY_CODE) + .and_then(|options| options.get(APPROVED_LOADER_BOUNDARIES_OPTION)) + .and_then(serde_json::Value::as_array) + else { + return Ok(()); + }; + for value in values { + let name: &str = value.as_str().unwrap_or_default(); + let parts: Vec<&str> = name.split('.').collect(); + if !name.contains('.') || !parts.iter().all(|part| python_identifier(part)) { + return Err(format!( + "rules.rule_options.{COMPILE_TIME_PURITY_CODE}.{APPROVED_LOADER_BOUNDARIES_OPTION} \ + entries must be fully qualified project functions such as \ + python.orders._helpers.specs.load_specs; globs are not accepted: {value}" + )); + } + } + Ok(()) +} + +/// Whether `part` is a Python identifier. +fn python_identifier(part: &str) -> bool { + part.chars() + .next() + .is_some_and(|first| first == '_' || first.is_alphabetic()) + && part + .chars() + .all(|character| character == '_' || character.is_alphanumeric()) +} + /// Published layers name model layers; allowed qualifiers are literal and carry a reason. fn validate_published_layer_options(config: &RulesConfig) -> Result<(), String> { for code in [PUBLISHED_READ_CODE, PUBLISHED_SHAPE_CODE] { diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs index cb8c5ca23..7a101e305 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -14,6 +14,7 @@ use crate::errors::RowsError; use crate::models::{EvaluateRequest, Model, RetainedRows, RowsEvaluation, RowsRequest}; use crate::orchestration_rules::main::orchestration_facts::orchestration_facts; use crate::python_rules::main::python_sources::python_sources; +use crate::python_rules::main::python_test_sources::python_test_sources; /// The shared key prefix of fields a request serializes but evaluation derives again. const DERIVED_FIELD_PREFIX: &str = ",\"empty_input_only"; @@ -60,6 +61,7 @@ pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result>()?; request.python_sources = python_sources(&request).map_err(RowsError::Rules)?; + request.python_test_sources = python_test_sources(&request).map_err(RowsError::Rules)?; request.orchestration = orchestration_facts(&request).map_err(RowsError::Rules)?; let mut request_json: Option> = None; let memo: Option<(String, PathBuf)> = match memo { diff --git a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs index e6eee93ec..f8521c390 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs @@ -62,6 +62,13 @@ const REVIEWED_SCOPES: &[(&str, RuleScope)] = &[ ("SQBRPYTHON013", RuleScope::Python), ("SQBRPYTHON014", RuleScope::Python), ("SQBRPYTHON015", RuleScope::Python), + ("SQBRPYTHON016", RuleScope::Python), + ("SQBRPYTHON017", RuleScope::Python), + ("SQBRPYTHON018", RuleScope::Python), + ("SQBRPYTHON019", RuleScope::Python), + ("SQBRPYTHON020", RuleScope::Python), + ("SQBRPYTHON021", RuleScope::Python), + ("SQBRPYTHON022", RuleScope::Python), ("SQBRPYTHON101", RuleScope::Python), ("SQBRPYTHON102", RuleScope::Python), ("SQBRPYTHON103", RuleScope::Python), @@ -124,6 +131,30 @@ const REVIEWED_SCOPES: &[(&str, RuleScope)] = &[ ("SQBRPYTHON625", RuleScope::Python), ("SQBRPYTHON626", RuleScope::Python), ("SQBRPYTHON627", RuleScope::Python), + ("SQBRPYTHON701", RuleScope::Python), + ("SQBRPYTHON702", RuleScope::Python), + ("SQBRPYTHON703", RuleScope::Python), + ("SQBRPYTHON704", RuleScope::Python), + ("SQBRPYTHON705", RuleScope::Python), + ("SQBRPYTHON706", RuleScope::Python), + ("SQBRPYTHON707", RuleScope::Python), + ("SQBRPYTHON708", RuleScope::Python), + ("SQBRPYTHON709", RuleScope::Python), + ("SQBRPYTHON710", RuleScope::Python), + ("SQBRPYTHON711", RuleScope::Python), + ("SQBRPYTHON712", RuleScope::Python), + ("SQBRPYTHON713", RuleScope::Python), + ("SQBRPYTHON714", RuleScope::Python), + ("SQBRPYTHON715", RuleScope::Python), + ("SQBRPYTHON716", RuleScope::Python), + ("SQBRPYTHON717", RuleScope::Python), + ("SQBRPYTHON718", RuleScope::Python), + ("SQBRPYTHON719", RuleScope::Python), + ("SQBRPYTHON720", RuleScope::Python), + ("SQBRPYTHON721", RuleScope::Python), + ("SQBRPYTHON722", RuleScope::Python), + ("SQBRPYTHON723", RuleScope::Python), + ("SQBRPYTHON724", RuleScope::Python), ("SQBRTEST101", RuleScope::Project), ("SQBRTEST102", RuleScope::Project), ("SQBRTEST103", RuleScope::Project), diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index e50fe328a..dfcdb43bf 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -773,6 +773,8 @@ pub struct EvaluateRequest { pub python_version: (u8, u8), /// The supported project Python files, filled from disk only when a Python rule is selected. pub python_sources: Vec, + /// The project's Python tests, filled from disk only when a Python test rule is selected. + pub python_test_sources: Vec, /// The job and sensor selections, filled from disk only when an orchestration rule is selected. pub orchestration: OrchestrationFacts, } @@ -825,6 +827,7 @@ impl Default for EvaluateRequest { rules_cache_path: String::new(), python_version: (3, 12), python_sources: vec![], + python_test_sources: vec![], orchestration: OrchestrationFacts::default(), } } diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/contexts.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/contexts.rs index 23e2d3a0e..8e646e4eb 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/contexts.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/contexts.rs @@ -11,7 +11,7 @@ use std::path::Path; use crate::models::{EvaluateRequest, RulesConfig}; use crate::python_rules::constants::{ - NAMING_CONTRACTS, NAMING_FAMILY, PACKAGE_ROOT, PROJECT_CONFIG_FILE, ROOT_SCOPE, + NAMING_CONTRACTS, NAMING_FAMILY, PACKAGE_ROOT, PROJECT_CONFIG_FILE, }; use crate::python_rules::models::{PythonRuleSpec, PythonTarget}; @@ -68,12 +68,12 @@ pub(crate) fn context_settings( /// The context of `target`, without thresholds, observations or the dead-code context. pub(crate) fn rule_context(target: &PythonTarget, settings: &ContextSettings) -> NativeRuleContext { let shared = NativeRuleContext { - scope: ROOT_SCOPE.to_owned(), + scope: target.scope.to_owned(), repository_path: target.repository_path.clone(), relative_parts: target.relative_parts.clone(), contracts: settings.contracts.clone(), repo_root: settings.repo_root.clone(), - scope_roots: vec![(ROOT_SCOPE.to_owned(), target.root.clone())], + scope_roots: vec![(target.scope.to_owned(), target.root.clone())], ..NativeRuleContext::default() }; if !target.package { diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/execution.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/execution.rs index 9ef5c21bd..8a8c81249 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/execution.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/execution.rs @@ -19,7 +19,7 @@ use crate::python_rules::_helpers::engine::file_cache::{answers_digest, file_ide use crate::python_rules::_helpers::engine::observations::{ observations, relative_queries, repository_index, }; -use crate::python_rules::constants::{PACKAGE_ROOT, PYTHON_RULES, ROOT_SCOPE}; +use crate::python_rules::constants::{PACKAGE_ROOT, PYTHON_RULES}; use crate::python_rules::models::{ CacheSplit, CallInputs, FilePlan, PythonCacheEntry, PythonEvaluation, PythonFiles, PythonRuleSpec, PythonTarget, RuleCall, @@ -258,7 +258,7 @@ fn file_rules<'a>( let applicable: Vec<&PythonRuleSpec> = rules .iter() .copied() - .filter(|rule| package || rule.applies == PythonFiles::All) + .filter(|rule| package || rule.applies != PythonFiles::Package) .collect(); let execution_targets: Vec = indexes .iter() @@ -290,7 +290,7 @@ fn execution_rule(rule: &PythonRuleSpec) -> NativeExecutionRule { fn execution_target(target: &PythonTarget) -> NativeExecutionTarget { NativeExecutionTarget { repository_path: target.repository_path.clone(), - scope: ROOT_SCOPE.to_owned(), + scope: target.scope.to_owned(), root: target.root.clone(), relative_parts: target.relative_parts.clone(), direct: true, diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/faults.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/faults.rs index 5b31b6d6c..e13f67e7e 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/faults.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/faults.rs @@ -45,7 +45,7 @@ pub(crate) fn python_fault( } /// The 1-based character column of 0-based UTF-8 byte offset `column` on 1-based `line`. -fn character_column(text: &str, line: u32, column: u32) -> u64 { +pub(crate) fn character_column(text: &str, line: u32, column: u32) -> u64 { let Some(start): Option = line_start(text, line) else { return 1; }; diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/mod.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/mod.rs index 6685ab9f6..2d83f7e21 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/mod.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/mod.rs @@ -4,6 +4,8 @@ pub(crate) mod faults; pub(crate) mod file_cache; pub(crate) mod layout; pub(crate) mod observations; +pub(crate) mod passes; pub(crate) mod placement; pub(crate) mod programs; pub(crate) mod project_plane; +pub(crate) mod source_files; diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/observations.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/observations.rs index d34f67a6c..201e96f43 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/observations.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/observations.rs @@ -98,6 +98,15 @@ fn answer( Some(module) => programs.program(module)?.is_public_facade(&query.argument), None => false, })], + "dataclasses" => match programs.index_of(path) { + Some(module) => programs + .program(module)? + .dataclass_rows() + .iter() + .map(|row| row.name.clone()) + .collect(), + None => Vec::new(), + }, "package_anchor" => vec![bool_text( package_anchor(programs.sources, path).is_some_and(|anchor| anchor == query.argument), )], diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/passes.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/passes.rs new file mode 100644 index 000000000..7db5fe0ca --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/passes.rs @@ -0,0 +1,58 @@ +//! One Python rule pass: the selected Fensu-backed rules over project Python, or over its tests. + +use std::collections::BTreeMap; + +use fensu_native::NativeProjectPlane; + +use crate::models::{EvaluateRequest, PythonSource}; +use crate::python_rules::_helpers::engine::contexts::context_settings; +use crate::python_rules::_helpers::engine::execution::{evaluated_files, file_plans}; +use crate::python_rules::_helpers::engine::file_cache::project_digest; +use crate::python_rules::_helpers::engine::placement::{python_targets, test_targets}; +use crate::python_rules::_helpers::engine::programs::Programs; +use crate::python_rules::_helpers::engine::project_plane::loaded_project; +use crate::python_rules::models::{ + CallInputs, PythonCacheEntry, PythonEvaluation, PythonFiles, PythonRuleSpec, PythonTarget, +}; + +/// Cache-aware findings of the `selected` rules over project Python tests, or the rest of project Python. +pub(crate) fn evaluated_pass( + request: &EvaluateRequest, + (selected, tests): (&[&'static PythonRuleSpec], bool), + cached: &BTreeMap, + fingerprint: &str, +) -> Result { + let rules: Vec<&PythonRuleSpec> = selected + .iter() + .copied() + .filter(|rule| (rule.applies == PythonFiles::Tests) == tests) + .collect(); + let sources: &[PythonSource] = if tests { + &request.python_test_sources + } else { + &request.python_sources + }; + if rules.is_empty() || sources.is_empty() { + return Ok(PythonEvaluation::default()); + } + let programs = Programs::new(sources, request.python_version); + let targets: Vec = if tests { + test_targets(sources) + } else { + python_targets(sources) + }; + let (plane, entrypoints) = if tests { + (NativeProjectPlane::default(), Vec::new()) + } else { + loaded_project(&rules, &targets, &programs, request)? + }; + let inputs = CallInputs { + request, + settings: context_settings(request, &rules, entrypoints), + programs: &programs, + cached, + fingerprint, + project_digest: project_digest(sources), + }; + evaluated_files(file_plans(&targets, &rules, &inputs)?, &inputs, &plane) +} diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/placement.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/placement.rs index 4239ce8a5..750b2305c 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/placement.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/placement.rs @@ -2,7 +2,8 @@ use crate::models::PythonSource; use crate::python_rules::constants::{ - INIT_FILE, MAIN_ROLE, PACKAGE_ROOT, ROLE_DIRECTORIES, ROLE_FILES, UNIT_MAIN_FILE, + INIT_FILE, MAIN_ROLE, PACKAGE_ROOT, ROLE_DIRECTORIES, ROLE_FILES, ROOT_SCOPE, TEST_SCOPE, + UNIT_MAIN_FILE, }; use crate::python_rules::models::PythonTarget; @@ -11,6 +12,18 @@ pub(crate) fn python_targets(sources: &[PythonSource]) -> Vec { sources.iter().enumerate().map(placed).collect() } +/// Place every Python test under its own folder in Fensu's test scope. +pub(crate) fn test_targets(sources: &[PythonSource]) -> Vec { + sources + .iter() + .enumerate() + .map(|source| PythonTarget { + scope: TEST_SCOPE, + ..placed(source) + }) + .collect() +} + fn placed((source, python): (usize, &PythonSource)) -> PythonTarget { let parts: Vec = python.relative_path.split('/').map(str::to_owned).collect(); if parts.len() > 1 && parts[0] == PACKAGE_ROOT { @@ -32,6 +45,7 @@ fn placed((source, python): (usize, &PythonSource)) -> PythonTarget { is_entry_module: is_main_module && relative_parts.last().is_some_and(|name| name != INIT_FILE), relative_parts, + scope: ROOT_SCOPE, }; } let (root, file) = python @@ -47,6 +61,7 @@ fn placed((source, python): (usize, &PythonSource)) -> PythonTarget { role: None, is_main_module: false, is_entry_module: false, + scope: ROOT_SCOPE, } } diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/engine/source_files.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/source_files.rs new file mode 100644 index 000000000..a118570ea --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/engine/source_files.rs @@ -0,0 +1,22 @@ +//! Read one project Python file as a rule source, digested so the memo sees every edit. + +use std::path::Path; + +use crate::models::PythonSource; + +/// The project file at `relative_path`, with its digest. +pub(crate) fn read_source( + project_dir: &Path, + relative_path: String, +) -> Result { + let bytes: Vec = std::fs::read(project_dir.join(&relative_path)) + .map_err(|error| format!("{relative_path}: cannot read Python source: {error}"))?; + let digest: String = blake3::hash(&bytes).to_hex().to_string(); + let text: String = String::from_utf8(bytes) + .map_err(|_| format!("{relative_path}: Python source is not valid UTF-8"))?; + Ok(PythonSource { + relative_path, + digest, + text, + }) +} diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/compile_purity.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/compile_purity.rs new file mode 100644 index 000000000..167537863 --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/compile_purity.rs @@ -0,0 +1,339 @@ +//! Compile-time purity: the Python a compile runs reaches no subprocess, network, database or remote client. + +use std::collections::hash_map::Entry; +use std::collections::{BTreeSet, HashMap}; + +use fensu_facts::ProgramHandle; +use fensu_facts::facts::models::{ImportRow, RuleNamedCallRow}; +use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; +use sqlbuild_pysource::node_units::main::deferred_spans::deferred_spans; +use sqlbuild_pysource::node_units::main::node_unit_module::node_unit_module; +use sqlbuild_pysource::node_units::models::{DeferredSpan, UnitStatementKind}; + +use crate::models::{EvaluateRequest, Fault, PythonSource}; +use crate::python_rules::_helpers::engine::faults::{catalogue_fault, character_column}; +use crate::python_rules::_helpers::engine::programs::Programs; +use crate::python_rules::constants::{ + APPROVED_LOADER_BOUNDARIES_OPTION, COMPILE_TIME_PURITY_CODE, CONNECTING_FUNCTIONS, + CONNECTING_PACKAGES, DEFERRED_CODE_MARKERS, EXTERNAL_CALL_PREFIXES, EXTERNAL_CALLS, + FACTORY_DECORATORS, PACKAGE_INIT_MODULE, +}; + +/// The external call a followed call first reaches, with its file and line. +#[derive(Clone, Debug)] +struct Sink { + call: String, + path: String, + line: u32, +} + +/// A walk over project functions that remembers the external call each one reaches. +struct Walk<'a> { + programs: &'a Programs<'a>, + approved: &'a BTreeSet, + /// Per source, the code its import never runs, whose calls the walk skips. + deferred: &'a [Vec], + reached: HashMap>, + bodies: HashMap>>, +} + +/// Faults of compile-time purity when `codes` selects it: each root call that reaches an external system. +pub(crate) fn compile_purity_faults( + request: &EvaluateRequest, + codes: &BTreeSet<&str>, +) -> Result, String> { + if !codes.contains(COMPILE_TIME_PURITY_CODE) || request.python_sources.is_empty() { + return Ok(Vec::new()); + } + let approved: BTreeSet = request + .config + .rule_option_strings(COMPILE_TIME_PURITY_CODE, APPROVED_LOADER_BOUNDARIES_OPTION) + .into_iter() + .collect(); + let programs = Programs::new(&request.python_sources, request.python_version); + if let Some(module) = approved + .iter() + .find(|name| programs.module_index(name).is_some()) + { + return Err(format!( + "rules.rule_options.{COMPILE_TIME_PURITY_CODE}.{APPROVED_LOADER_BOUNDARIES_OPTION} names \ + the module {module}; list the function that loads the data instead" + )); + } + (0..request.python_sources.len()) + .into_par_iter() + .try_for_each(|index| programs.program(index).and(Ok(())))?; + let deferred: Vec> = request + .python_sources + .par_iter() + .map(|source| source_deferred_spans(source, request.python_version)) + .collect(); + let mut walk = Walk { + programs: &programs, + approved: &approved, + deferred: &deferred, + reached: HashMap::new(), + bodies: HashMap::new(), + }; + let mut faults: Vec = Vec::new(); + for (index, source) in request.python_sources.iter().enumerate() { + let program: &ProgramHandle = programs.program(index)?; + for call in root_calls( + (program, &deferred[index]), + source, + (&approved, request.python_version), + ) { + if let Some(sink) = walk.reach(index, call)? { + faults.extend(purity_fault(source, call, &sink)); + } + } + } + faults.sort_by(|left, right| { + (&left.path, left.line, left.column).cmp(&(&right.path, right.line, right.column)) + }); + faults.dedup_by(|left, right| { + left.path == right.path && left.line == right.line && left.column == right.column + }); + Ok(faults) +} + +impl<'a> Walk<'a> { + /// The external call `call` in source `index` reaches, directly or through project functions. + fn reach(&mut self, index: usize, call: &RuleNamedCallRow) -> Result, String> { + let programs: &'a Programs<'a> = self.programs; + let source: &PythonSource = &programs.sources[index]; + let module: Vec = module_parts(&source.relative_path); + let parts: Vec = absolute_parts( + call, + &programs.program(index)?.reference_rows().imports, + &package_parts(&source.relative_path, &module), + ); + if let Some(external) = external_call(&parts) { + return Ok(Some(Sink { + call: external, + path: source.relative_path.clone(), + line: call.line, + })); + } + let (target, function): (String, String) = match parts.as_slice() { + [] => return Ok(None), + [function] => (module.join("."), function.clone()), + [owner @ .., function] => (owner.join("."), function.clone()), + }; + let Some(target_index) = programs.module_index(&target) else { + return Ok(None); + }; + let identity: String = format!("{target}.{function}"); + if self.approved.contains(&identity) { + return Ok(None); + } + if let Some(known) = self.reached.get(&identity) { + return Ok(known.clone()); + } + self.reached.insert(identity.clone(), None); + let mut sink: Option = None; + for body_call in self.body(target_index, &function)? { + sink = self.reach(target_index, body_call)?; + if sink.is_some() { + break; + } + } + self.reached.insert(identity, sink.clone()); + Ok(sink) + } + + /// The calls written directly in top-level function `function` of source `index`. + fn body(&mut self, index: usize, function: &str) -> Result, String> { + let programs: &'a Programs<'a> = self.programs; + if let Entry::Vacant(entry) = self.bodies.entry(index) { + let mut bodies: HashMap> = HashMap::new(); + for call in programs.program(index)?.named_call_rows() { + if let ([owner], []) = ( + call.enclosing_functions.as_slice(), + call.enclosing_classes.as_slice(), + ) && !deferred(&self.deferred[index], call) + { + bodies.entry(owner.name.clone()).or_default().push(call); + } + } + entry.insert(bodies); + } + Ok(self + .bodies + .get(&index) + .and_then(|bodies| bodies.get(function)) + .cloned() + .unwrap_or_default()) + } +} + +/// The lambda bodies and `__main__` block of `source`, which its import never runs. +fn source_deferred_spans(source: &PythonSource, python_version: (u8, u8)) -> Vec { + if !DEFERRED_CODE_MARKERS + .iter() + .any(|marker| source.text.contains(marker)) + { + return Vec::new(); + } + deferred_spans(&source.text, python_version).unwrap_or_default() +} + +/// Whether `call` sits in code its module's import never runs. +fn deferred(spans: &[DeferredSpan], call: &RuleNamedCallRow) -> bool { + spans + .iter() + .any(|span| span.contains(call.line as usize, call.column as usize)) +} + +/// The calls `source`'s import runs: module and class scope and unapproved `@factory` bodies, outside `spans`. +fn root_calls<'p>( + (program, spans): (&'p ProgramHandle, &[DeferredSpan]), + source: &PythonSource, + (approved, python_version): (&BTreeSet, (u8, u8)), +) -> Vec<&'p RuleNamedCallRow> { + let module: String = module_parts(&source.relative_path).join("."); + let factories: BTreeSet = factory_names(source, python_version) + .into_iter() + .filter(|name| !approved.contains(&format!("{module}.{name}"))) + .collect(); + let mut calls: Vec<&RuleNamedCallRow> = Vec::new(); + for call in program.named_call_rows() { + let root: bool = match call.enclosing_functions.as_slice() { + [] => true, + [owner] => call.enclosing_classes.is_empty() && factories.contains(&owner.name), + _ => false, + }; + if root && !deferred(spans, call) { + calls.push(call); + } + } + calls +} + +/// The top-level functions of `source` decorated as factories. +fn factory_names(source: &PythonSource, python_version: (u8, u8)) -> BTreeSet { + let mut names: BTreeSet = BTreeSet::new(); + if !FACTORY_DECORATORS + .iter() + .any(|decorator| source.text.contains(decorator)) + { + return names; + } + let Some(module) = node_unit_module(&source.text, python_version) else { + return names; + }; + for statement in module.statements { + if let UnitStatementKind::Function(function) = statement.kind + && function + .decorators + .iter() + .any(|decorator| FACTORY_DECORATORS.contains(&decorator.name.as_str())) + { + names.insert(function.name); + } + } + names +} + +/// The dotted module of a project file, such as `python.orders.export` for `python/orders/export/__init__.py`. +fn module_parts(relative_path: &str) -> Vec { + let mut parts: Vec = relative_path + .trim_end_matches(".py") + .split('/') + .map(str::to_owned) + .collect(); + if parts.last().is_some_and(|last| last == PACKAGE_INIT_MODULE) { + parts.pop(); + } + parts +} + +/// The package a relative import in the file at `relative_path` starts from. +fn package_parts(relative_path: &str, module: &[String]) -> Vec { + if relative_path.ends_with(&format!("{PACKAGE_INIT_MODULE}.py")) { + module.to_vec() + } else { + module[..module.len().saturating_sub(1)].to_vec() + } +} + +/// The dotted name `call` targets once its first name is resolved through `imports`; empty when unresolvable. +fn absolute_parts( + call: &RuleNamedCallRow, + imports: &[ImportRow], + package: &[String], +) -> Vec { + let parts: &[String] = &call.reference.parts; + let Some(first) = parts.first() else { + return Vec::new(); + }; + for row in imports { + let Some(alias) = row.aliases.iter().find(|alias| &alias.bound_name == first) else { + continue; + }; + let imported: Vec = alias.imported_name.split('.').map(str::to_owned).collect(); + if !row.from_import && imported.first() == Some(first) { + return parts.to_vec(); + } + let mut resolved: Vec = if row.from_import { + let Some(base) = import_base(row, package) else { + return Vec::new(); + }; + base + } else { + Vec::new() + }; + resolved.extend(imported); + resolved.extend(parts.iter().skip(1).cloned()); + return resolved; + } + parts.to_vec() +} + +/// The module a `from ... import` reads from, resolving leading dots against `package`. +fn import_base(row: &ImportRow, package: &[String]) -> Option> { + let mut base: Vec = if row.relative_level == 0 { + Vec::new() + } else { + package.to_vec() + }; + for _ in 1..row.relative_level { + base.pop()?; + } + base.extend(row.module_parts.iter().cloned()); + Some(base) +} + +/// The dotted call when `parts` names a call that reaches an external system. +fn external_call(parts: &[String]) -> Option { + let joined: String = parts.join("."); + let connecting: bool = parts.len() > 1 + && CONNECTING_PACKAGES.contains(&parts[0].as_str()) + && parts + .last() + .is_some_and(|last| CONNECTING_FUNCTIONS.contains(&last.as_str())); + (connecting + || EXTERNAL_CALLS.contains(&joined.as_str()) + || EXTERNAL_CALL_PREFIXES + .iter() + .any(|prefix| joined.starts_with(prefix))) + .then_some(joined) +} + +/// The fault on root `call` in `source`, naming the external call it reaches and where. +fn purity_fault(source: &PythonSource, call: &RuleNamedCallRow, sink: &Sink) -> Option { + let detail: String = if sink.path == source.relative_path && sink.line == call.line { + format!("calls {}", sink.call) + } else { + format!("reaches {} at {}:{}", sink.call, sink.path, sink.line) + }; + catalogue_fault( + COMPILE_TIME_PURITY_CODE, + &source.relative_path, + ( + u64::from(call.line.max(1)), + character_column(&source.text, call.line, call.column), + ), + &detail, + ) +} diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs index 4be254067..838ad1748 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/mod.rs @@ -1,6 +1,9 @@ +pub(crate) mod compile_purity; pub(crate) mod imports; pub(crate) mod layout; pub(crate) mod modules; pub(crate) mod nodes; +pub(crate) mod orchestration; pub(crate) mod project; pub(crate) mod providers; +pub(crate) mod python_tests; diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs new file mode 100644 index 000000000..6a1f75d1a --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs @@ -0,0 +1,243 @@ +//! Sensor and calendar modules under orchestration/: where they sit, what they hold and what they import. + +use std::collections::BTreeSet; + +use sqlbuild_pysource::node_units::models::{UnitPosition, UnitStatementKind}; +use sqlbuild_pysource::orchestration_modules::main::orchestration_module_declarations::orchestration_module_declarations; + +use crate::models::{Fault, PythonSource}; +use crate::python_rules::_helpers::engine::faults::catalogue_fault; +use crate::python_rules::_helpers::units::project::{ + files_naming, nearest_unit, project_modules, rejected_import_faults, unit_fault, unit_folders, +}; +use crate::python_rules::constants::{ + INIT_FILE, ORCHESTRATION_GENERIC_MODULES, ORCHESTRATION_IMPORT_DIRECTION_CODE, + ORCHESTRATION_MODULE_PATHS_CODE, ORCHESTRATION_MODULE_SURFACE_CODE, + ORCHESTRATION_NO_DOCSTRING_CODE, ORCHESTRATION_PRIVATE_SUPPORT_CODE, + ORCHESTRATION_PYTHON_ROLES, ORCHESTRATION_ROOT, ORCHESTRATION_RULE_CODES, + ORCHESTRATION_SUPPORT_FILES, ORCHESTRATION_SUPPORT_FOLDER, PACKAGE_ROOT, PRIVATE_PREFIX, + PROVIDERS_ROOT, SHARED_UTILITIES_PACKAGE, +}; +use crate::python_rules::models::ParsedFile; + +const OTHER_STATEMENT: &str = + "a statement that is not an import, a private constant or the declaration"; + +/// Faults of the selected orchestration rules among `codes`. +pub(crate) fn orchestration_faults( + sources: &[PythonSource], + codes: &BTreeSet<&str>, + python_version: (u8, u8), +) -> Vec { + if !ORCHESTRATION_RULE_CODES + .iter() + .any(|code| codes.contains(code)) + { + return Vec::new(); + } + let mut faults: Vec = sources.iter().filter_map(layout_fault).collect(); + let files: Vec> = files_naming(sources, ORCHESTRATION_ROOT, python_version); + let modules: BTreeSet = project_modules(sources); + let units: BTreeSet = unit_folders(sources); + for file in &files { + if role_folders(&file.parts).is_some_and(|rest| !is_private(rest)) { + faults.extend(surface_faults(file, python_version)); + } + faults.extend(import_faults(file, &modules, &units)); + } + faults.retain(|fault| codes.contains(fault.code.as_str())); + faults +} + +/// The path parts below `orchestration//` for a sensor or calendar file; `None` for any other file. +fn role_folders<'p>(parts: &'p [&'p str]) -> Option<&'p [&'p str]> { + match parts { + [root, role, rest @ ..] + if *root == ORCHESTRATION_ROOT && ORCHESTRATION_PYTHON_ROLES.contains(role) => + { + Some(rest) + } + _ => None, + } +} + +/// Whether a file below a role folder is private support: its name or a folder above it starts with `_`. +fn is_private(rest: &[&str]) -> bool { + rest.iter().any(|part| part.starts_with(PRIVATE_PREFIX)) +} + +/// Where a sensor or calendar file sits wrongly: outside a domain folder, nested, generic, or odd support. +fn layout_fault(source: &PythonSource) -> Option { + let path: &str = source.relative_path.as_str(); + let parts: Vec<&str> = path.split('/').collect(); + let rest: &[&str] = role_folders(&parts)?; + let role: String = format!("{ORCHESTRATION_ROOT}/{}", parts[1]); + let (code, detail): (&str, String) = match rest { + [file] if *file == INIT_FILE => return None, + [file] if file.starts_with(PRIVATE_PREFIX) => ( + ORCHESTRATION_PRIVATE_SUPPORT_CODE, + format!("{path} is private support outside a domain folder of {role}/"), + ), + [_] => ( + ORCHESTRATION_MODULE_PATHS_CODE, + format!("{path} sits directly in {role}/, not in a domain folder"), + ), + [domain, ..] if domain.starts_with(PRIVATE_PREFIX) => ( + ORCHESTRATION_PRIVATE_SUPPORT_CODE, + format!("{path} is in private folder {domain}/ outside a domain folder of {role}/"), + ), + [_, file] if *file == INIT_FILE => return None, + [_, file] if file.starts_with(PRIVATE_PREFIX) => { + if ORCHESTRATION_SUPPORT_FILES.contains(file) { + return None; + } + ( + ORCHESTRATION_PRIVATE_SUPPORT_CODE, + format!("{path} is not _models.py, _types.py, _constants.py or _exceptions.py"), + ) + } + [_, file] => { + if !ORCHESTRATION_GENERIC_MODULES.contains(file) { + return None; + } + ( + ORCHESTRATION_MODULE_PATHS_CODE, + format!("{file} is too generic; name the module for its capability"), + ) + } + [_, folder, ..] if *folder == ORCHESTRATION_SUPPORT_FOLDER => return None, + [domain, folder, ..] if folder.starts_with(PRIVATE_PREFIX) => ( + ORCHESTRATION_PRIVATE_SUPPORT_CODE, + format!("{path} is in {folder}/, but {domain}/'s only private folder is _helpers/"), + ), + [domain, ..] => ( + ORCHESTRATION_MODULE_PATHS_CODE, + format!("{path} is nested below its domain folder {domain}/"), + ), + [] => return None, + }; + catalogue_fault(code, path, (1, 1), &detail) +} + +/// A declaration module's statements beyond imports, private constants and the declaration, and its docstring. +fn surface_faults(file: &ParsedFile<'_>, python_version: (u8, u8)) -> Vec { + let declaration_lines: Vec = + orchestration_module_declarations(file.text, python_version) + .map(|module| module.declarations) + .unwrap_or_default() + .iter() + .map(|declaration| declaration.position.line) + .collect(); + let statements = &file.module.statements; + let mut faults: Vec = Vec::new(); + for (index, statement) in statements.iter().enumerate() { + let end: usize = statements + .get(index + 1) + .map_or(usize::MAX, |next| next.position.line); + let declares: bool = declaration_lines + .iter() + .any(|line| (statement.position.line..end).contains(line)); + let (code, detail, position): (&str, String, UnitPosition) = match &statement.kind { + UnitStatementKind::Docstring | UnitStatementKind::Import => continue, + UnitStatementKind::Function(function) if declares && function.docstring => ( + ORCHESTRATION_NO_DOCSTRING_CODE, + format!( + "'{}' has a docstring; its description is description=", + function.name + ), + function.position, + ), + _ if declares => continue, + UnitStatementKind::Assignment(names) => match public_name(names) { + Some(detail) => ( + ORCHESTRATION_MODULE_SURFACE_CODE, + detail, + statement.position, + ), + None => continue, + }, + UnitStatementKind::Function(function) => ( + ORCHESTRATION_MODULE_SURFACE_CODE, + format!("function '{}' belongs in _helpers/", function.name), + function.position, + ), + UnitStatementKind::Class(class) => ( + ORCHESTRATION_MODULE_SURFACE_CODE, + format!("class '{}' belongs in _models.py or _helpers/", class.name), + class.position, + ), + UnitStatementKind::Other => ( + ORCHESTRATION_MODULE_SURFACE_CODE, + OTHER_STATEMENT.to_owned(), + statement.position, + ), + }; + faults.extend(unit_fault(code, file.path, position, &detail)); + } + faults +} + +/// Why an assignment does not belong in a declaration module; `None` for a private constant. +fn public_name(names: &[String]) -> Option { + if names.is_empty() { + return Some(OTHER_STATEMENT.to_owned()); + } + let name: &String = names + .iter() + .find(|name| !name.starts_with(PRIVATE_PREFIX))?; + Some(format!("public constant '{name}' belongs in _constants.py")) +} + +/// Imports of another domain's private support, and imports across the orchestration boundary. +fn import_faults( + file: &ParsedFile<'_>, + modules: &BTreeSet, + units: &BTreeSet, +) -> Vec { + rejected_import_faults(file, modules, |module| { + boundary_detail(&file.parts, module, units) + }) +} + +/// Why a file at `parts` may not import `module`; node units and providers are left to 008 and 015. +fn boundary_detail( + parts: &[&str], + module: &str, + units: &BTreeSet, +) -> Option<(&'static str, String)> { + let imported: Vec<&str> = module.split('.').collect(); + let Some(rest) = role_folders(parts) else { + let in_unit: bool = + parts.first() == Some(&PACKAGE_ROOT) && nearest_unit(&parts[1..], units).is_some(); + let outside: bool = parts.first() != Some(&ORCHESTRATION_ROOT) + && parts.first() != Some(&PROVIDERS_ROOT) + && !in_unit + && imported.first() == Some(&ORCHESTRATION_ROOT); + return outside.then(|| { + ( + ORCHESTRATION_IMPORT_DIRECTION_CODE, + format!("but only {ORCHESTRATION_ROOT}/ imports from {ORCHESTRATION_ROOT}/"), + ) + }); + }; + match imported.as_slice() { + [root, utils, ..] if *root == PACKAGE_ROOT && *utils == SHARED_UTILITIES_PACKAGE => None, + [root, ..] if *root == PACKAGE_ROOT => Some(( + ORCHESTRATION_IMPORT_DIRECTION_CODE, + format!( + "but sensors and calendars import only {PACKAGE_ROOT}/{SHARED_UTILITIES_PACKAGE}/ from {PACKAGE_ROOT}/" + ), + )), + [root, role, domain, support @ ..] + if *root == ORCHESTRATION_ROOT + && is_private(support) + && (Some(role) != parts.get(1) || Some(domain) != rest.first()) => + { + Some(( + ORCHESTRATION_PRIVATE_SUPPORT_CODE, + format!("which is private to {ORCHESTRATION_ROOT}/{role}/{domain}/"), + )) + } + _ => None, + } +} diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/project.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/project.rs index 8299d9621..26a085421 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/units/project.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/project.rs @@ -22,7 +22,7 @@ use crate::python_rules::constants::{ NODE_MODULE, PACKAGE_ROOT, ROLE_DIRECTORIES, SHARED_UTILITIES_PACKAGE, UNIT_NODE_FILE, UNIT_RULE_CODES, }; -use crate::python_rules::models::{UnitFile, UnitProject}; +use crate::python_rules::models::{ParsedFile, UnitFile, UnitProject}; /// Faults of the selected unit rules among `codes`. pub(crate) fn unit_faults( @@ -60,10 +60,7 @@ fn unit_project(sources: &[PythonSource], python_version: (u8, u8)) -> UnitProje (root == PACKAGE_ROOT).then(|| (source, rest.split('/').collect())) }) .collect(); - let units: BTreeSet = package - .iter() - .filter_map(|(_, parts)| unit_folder(parts)) - .collect(); + let units: BTreeSet = unit_folders(sources); let files: Vec> = package .par_iter() .filter_map(|(source, parts)| { @@ -91,15 +88,82 @@ fn unit_project(sources: &[PythonSource], python_version: (u8, u8)) -> UnitProje .iter() .map(|source| source.relative_path.as_str()) .collect(), - modules: sources - .iter() - .flat_map(|source| dotted_names(&source.relative_path)) - .collect(), + modules: project_modules(sources), } } +/// Every dotted module the project's Python files define, packages included. +pub(crate) fn project_modules(sources: &[PythonSource]) -> BTreeSet { + sources + .iter() + .flat_map(|source| dotted_names(&source.relative_path)) + .collect() +} + +/// The files under `root/` or whose text names it, parsed; a file that does not parse is left out. +pub(crate) fn files_naming<'a>( + sources: &'a [PythonSource], + root: &str, + python_version: (u8, u8), +) -> Vec> { + sources + .par_iter() + .filter(|source| { + source.relative_path.split('/').next() == Some(root) || source.text.contains(root) + }) + .filter_map(|source| { + Some(ParsedFile { + path: source.relative_path.as_str(), + parts: source.relative_path.split('/').collect(), + text: source.text.as_str(), + module: node_unit_module(&source.text, python_version)?, + }) + }) + .collect() +} + +/// Each import in `file` of a project module that `reason` rejects, as a fault of its rule and detail. +pub(crate) fn rejected_import_faults( + file: &ParsedFile<'_>, + modules: &BTreeSet, + reason: impl Fn(&str) -> Option<(&'static str, String)>, +) -> Vec { + let folder: &[&str] = &file.parts[..file.parts.len() - 1]; + file.module + .imports + .iter() + .filter_map(|import| { + let module: String = absolute_module(import, folder, modules)?; + if !modules.contains(&module) { + return None; + } + let (code, detail): (&str, String) = reason(&module)?; + unit_fault( + code, + file.path, + import.position, + &format!("{} imports {module}, {detail}", file.path), + ) + }) + .collect() +} + +/// The unit folders under `python/`, each relative to it. +pub(crate) fn unit_folders(sources: &[PythonSource]) -> BTreeSet { + sources + .iter() + .filter_map(|source| { + let (root, rest) = source.relative_path.split_once('/')?; + let parts: Vec<&str> = rest.split('/').collect(); + (root == PACKAGE_ROOT) + .then(|| unit_folder(&parts)) + .flatten() + }) + .collect() +} + /// The deepest unit folder among the file's folders. -fn nearest_unit(parts: &[&str], units: &BTreeSet) -> Option { +pub(crate) fn nearest_unit(parts: &[&str], units: &BTreeSet) -> Option { (1..parts.len()) .rev() .map(|depth| parts[..depth].join("/")) diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs index 971166a87..b88f4e2db 100644 --- a/crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs @@ -2,37 +2,28 @@ use std::collections::{BTreeMap, BTreeSet}; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; - -use sqlbuild_pysource::node_units::main::node_unit_module::node_unit_module; -use sqlbuild_pysource::node_units::models::{ - UnitClass, UnitModule, UnitPosition, UnitStatementKind, -}; +use sqlbuild_pysource::node_units::models::{UnitClass, UnitPosition, UnitStatementKind}; use crate::models::{Fault, PythonSource}; use crate::python_rules::_helpers::engine::faults::catalogue_fault; use crate::python_rules::_helpers::engine::placement::package_anchor; -use crate::python_rules::_helpers::units::project::{absolute_module, dotted_names, unit_fault}; +use crate::python_rules::_helpers::units::project::{ + absolute_module, dotted_names, files_naming, project_modules, rejected_import_faults, + unit_fault, +}; use crate::python_rules::constants::{ INIT_FILE, ORCHESTRATION_ROOT, PACKAGE_ROOT, PRIVATE_PREFIX, PROVIDER_BASE, PROVIDER_FILE, PROVIDER_IMPORT_BOUNDARIES_CODE, PROVIDER_LAYOUT_CODE, PROVIDER_MODULE_SHAPE_CODE, PROVIDER_PRIVATE_FOLDERS, PROVIDER_ROLE_FILES, PROVIDER_RULE_CODES, PROVIDERS_ROOT, SHARED_UTILITIES_PACKAGE, STAR_IMPORT, }; +use crate::python_rules::models::ParsedFile; const FILE_START: UnitPosition = UnitPosition { line: 1, column: 1 }; const OTHER_STATEMENT: &str = "a statement that is not an import, a constant or the provider class"; /// The package Provider is imported from. const SQLBUILD_PACKAGE: &str = "sqlbuild"; -/// One parsed project file and its path parts. -#[derive(Debug)] -struct ParsedFile<'a> { - path: &'a str, - parts: Vec<&'a str>, - module: UnitModule, -} - /// Faults of the selected provider rules among `codes`. pub(crate) fn provider_faults( sources: &[PythonSource], @@ -43,24 +34,8 @@ pub(crate) fn provider_faults( return Vec::new(); } let mut faults: Vec = provider_layout_faults(sources); - let files: Vec> = sources - .par_iter() - .filter(|source| { - source.relative_path.split('/').next() == Some(PROVIDERS_ROOT) - || source.text.contains(PROVIDERS_ROOT) - }) - .filter_map(|source| { - Some(ParsedFile { - path: source.relative_path.as_str(), - parts: source.relative_path.split('/').collect(), - module: node_unit_module(&source.text, python_version)?, - }) - }) - .collect(); - let modules: BTreeSet = sources - .iter() - .flat_map(|source| dotted_names(&source.relative_path)) - .collect(); + let files: Vec> = files_naming(sources, PROVIDERS_ROOT, python_version); + let modules: BTreeSet = project_modules(sources); let classes: ProviderClasses = provider_classes(&files, &modules); for file in &files { if matches!(file.parts.as_slice(), [root, _, file_name] if *root == PROVIDERS_ROOT && *file_name == PROVIDER_FILE) @@ -357,24 +332,9 @@ fn import_faults(file: &ParsedFile<'_>, modules: &BTreeSet) -> Vec Some(capability), _ => None, }; - let folder: &[&str] = &file.parts[..file.parts.len() - 1]; - file.module - .imports - .iter() - .filter_map(|import| { - let module: String = absolute_module(import, folder, modules)?; - if !modules.contains(&module) { - return None; - } - let detail: String = boundary_detail(&module, capability)?; - unit_fault( - PROVIDER_IMPORT_BOUNDARIES_CODE, - file.path, - import.position, - &format!("{} imports {module}, {detail}", file.path), - ) - }) - .collect() + rejected_import_faults(file, modules, |module| { + boundary_detail(module, capability).map(|detail| (PROVIDER_IMPORT_BOUNDARIES_CODE, detail)) + }) } /// Why a file in provider `capability` (or outside providers/ when `None`) may not import `module`. diff --git a/crates/sqlbuild-rules/src/python_rules/_helpers/units/python_tests.rs b/crates/sqlbuild-rules/src/python_rules/_helpers/units/python_tests.rs new file mode 100644 index 000000000..ed2447812 --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/_helpers/units/python_tests.rs @@ -0,0 +1,65 @@ +//! Where project Python tests sit: each mirrors an existing folder, and each test module a module, under `python/` or `providers/`. + +use std::collections::BTreeSet; + +use crate::models::{Fault, PythonSource}; +use crate::python_rules::_helpers::engine::faults::catalogue_fault; +use crate::python_rules::constants::{PYTHON_TEST_PLACEMENT_CODE, TEST_FILE_PREFIX}; + +/// A fault on each Python test whose mirrored folder, or for a test module whose module, is not in the project. +pub(crate) fn python_test_faults( + sources: &[PythonSource], + tests: &[PythonSource], + codes: &BTreeSet<&str>, +) -> Vec { + if !codes.contains(PYTHON_TEST_PLACEMENT_CODE) { + return Vec::new(); + } + let paths: BTreeSet<&str> = sources + .iter() + .map(|source| source.relative_path.as_str()) + .collect(); + tests + .iter() + .filter_map(|test| { + let detail: String = missing_counterpart(&test.relative_path, &paths)?; + catalogue_fault( + PYTHON_TEST_PLACEMENT_CODE, + &test.relative_path, + (1, 1), + &detail, + ) + }) + .collect() +} + +/// What the test at `path`, such as `tests/python/orders/test_export.py`, mirrors that the project lacks. +fn missing_counterpart(path: &str, paths: &BTreeSet<&str>) -> Option { + let parts: Vec<&str> = path.split('/').collect(); + let [_, root, folders @ .., file] = parts.as_slice() else { + return None; + }; + if folders.is_empty() { + return Some(format!( + "it sits directly in tests/{root}/, not in a folder mirroring one under {root}/" + )); + } + let folder: String = format!("{root}/{}", folders.join("/")); + if !holds_files(paths, &folder) { + return Some(format!("{folder}/ does not exist")); + } + let module: &str = file.strip_prefix(TEST_FILE_PREFIX)?; + let tested: String = format!("{folder}/{module}"); + let package: &str = tested.trim_end_matches(".py"); + (!paths.contains(tested.as_str()) && !holds_files(paths, package)) + .then(|| format!("{tested} does not exist")) +} + +/// Whether any project Python file sits below `folder`. +fn holds_files(paths: &BTreeSet<&str>, folder: &str) -> bool { + let prefix: String = format!("{folder}/"); + paths + .range(prefix.as_str()..) + .next() + .is_some_and(|path| path.starts_with(&prefix)) +} diff --git a/crates/sqlbuild-rules/src/python_rules/constants.rs b/crates/sqlbuild-rules/src/python_rules/constants.rs index 77649efd2..a2225bbbf 100644 --- a/crates/sqlbuild-rules/src/python_rules/constants.rs +++ b/crates/sqlbuild-rules/src/python_rules/constants.rs @@ -1,11 +1,11 @@ -//! The Python rule codes: two SQLBuild layout rules, then Fensu's core rules in bands by Fensu family. +//! The Python rule codes: SQLBuild's own project rules, then Fensu's core rules in bands by Fensu family. use fensu_native::rules::constants as fensu; use crate::python_rules::models::{PythonFiles, PythonRuleSpec}; /// Every `SQBRPYTHON` rule; numbers are never reused, and new rules append to their band. -pub(crate) const PYTHON_RULES: [PythonRuleSpec; 77] = [ +pub(crate) const PYTHON_RULES: [PythonRuleSpec; 108] = [ PythonRuleSpec { code: "SQBRPYTHON001", fensu_code: None, @@ -186,6 +186,90 @@ pub(crate) const PYTHON_RULES: [PythonRuleSpec; 77] = [ message: "a provider's _helpers/ and _classes/ are private to it, and providers import only python/utils/ from the runtime package and nothing from orchestration/", remediation: "Import a provider through provider.py or its role files; move code a provider shares with nodes into python/utils//.", }, + PythonRuleSpec { + code: "SQBRPYTHON016", + fensu_code: None, + slug: "orchestration-module-paths", + family: "orchestration", + owner: "file", + applies: PythonFiles::All, + enabled_by_default: true, + options: &[], + message: "sensor and calendar modules sit at orchestration///.py and are named for their capability", + remediation: "Move the module into a domain folder directly under orchestration/sensors/ or orchestration/calendars/, and name it for what it checks instead of main.py, definitions.py, sensors.py, calendars.py or processing.py.", + }, + PythonRuleSpec { + code: "SQBRPYTHON017", + fensu_code: None, + slug: "orchestration-private-support", + family: "orchestration", + owner: "file", + applies: PythonFiles::All, + enabled_by_default: true, + options: &[], + message: "a sensor or calendar domain folder's private support is _models.py, _types.py, _constants.py, _exceptions.py and _helpers/, imported only from that folder", + remediation: "Keep support beside its declarations in one of the support files or _helpers/; move code another domain needs into python/utils// or a provider.", + }, + PythonRuleSpec { + code: "SQBRPYTHON018", + fensu_code: None, + slug: "orchestration-module-surface", + family: "orchestration", + owner: "file", + applies: PythonFiles::All, + enabled_by_default: true, + options: &[], + message: "a sensor or calendar module holds imports, private constants and its one declaration", + remediation: "Move functions and classes into the folder's _helpers/ or _models.py, and public constants into _constants.py.", + }, + PythonRuleSpec { + code: "SQBRPYTHON019", + fensu_code: None, + slug: "orchestration-import-direction", + family: "orchestration", + owner: "file", + applies: PythonFiles::All, + enabled_by_default: true, + options: &[], + message: "sensor and calendar code imports providers, python/utils/ and its own folder's support, and nothing outside orchestration/ imports from it", + remediation: "Read node results through the sensor's context instead of importing node units, share code through python/utils// or a provider, and keep other code from importing orchestration/.", + }, + PythonRuleSpec { + code: "SQBRPYTHON020", + fensu_code: None, + slug: "orchestration-no-docstring", + family: "orchestration", + owner: "file", + applies: PythonFiles::All, + enabled_by_default: true, + options: &[], + message: "sensor and calendar functions have no docstring; their description is the declaration's description=", + remediation: "Remove the docstring and keep the text in description=.", + }, + PythonRuleSpec { + code: "SQBRPYTHON021", + fensu_code: None, + slug: "python-test-placement", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "Python tests must mirror an existing module under python/ or providers/", + remediation: "Move the test to tests/python/ or tests/providers/ at the path of the module it tests: tests/python/orders/storefront_import/test_main.py tests python/orders/storefront_import/main.py.", + }, + PythonRuleSpec { + code: "SQBRPYTHON022", + fensu_code: None, + slug: "compile-time-purity", + family: "purity", + owner: "file", + applies: PythonFiles::All, + enabled_by_default: true, + options: &[], + message: "Python that runs while the project compiles must not reach a subprocess, the network, a database or a remote client", + remediation: "Keep module scope and @factory bodies to static data such as specs in constants.py; move the call into a node body, main() or a provider, or list a deliberate discovery function in approved_loader_boundaries.", + }, PythonRuleSpec { code: "SQBRPYTHON101", fensu_code: Some(fensu::PARAMETER_ANNOTATION_CODE), @@ -980,6 +1064,294 @@ pub(crate) const PYTHON_RULES: [PythonRuleSpec; 77] = [ message: "source files must stay below the configured line limit", remediation: "Split the file by a cohesive role or concern instead of extracting arbitrary numbered fragments.", }, + PythonRuleSpec { + code: "SQBRPYTHON701", + fensu_code: Some(fensu::TEST_INIT_MODULE_EMPTY_CODE), + slug: "test-init-module-empty", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test package __init__.py files must be empty or docstring-only", + remediation: "Remove declarations from __init__.py and import them from their owning module.", + }, + PythonRuleSpec { + code: "SQBRPYTHON702", + fensu_code: Some(fensu::TEST_ABSOLUTE_IMPORTS_CODE), + slug: "test-absolute-imports", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "tests must use absolute imports", + remediation: "Replace the relative import with the full tests or project package path.", + }, + PythonRuleSpec { + code: "SQBRPYTHON703", + fensu_code: Some(fensu::TEST_NO_TOP_LEVEL_HELPERS_CODE), + slug: "test-no-top-level-helpers", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test modules may contain only tests, imports, and declarations", + remediation: "Move reusable functions into the local helpers.py module.", + }, + PythonRuleSpec { + code: "SQBRPYTHON704", + fensu_code: Some(fensu::TEST_NO_IF_IN_TESTS_CODE), + slug: "test-no-conditionals", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "tests and local test helpers must not contain conditional control flow", + remediation: "Use parametrized cases when setup and assertions remain branch-free; otherwise split the behavior into separate test functions.", + }, + PythonRuleSpec { + code: "SQBRPYTHON705", + fensu_code: Some(fensu::TEST_PRIVATE_CONSTANT_ORDER_CODE), + slug: "test-private-constant-order", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "private test constants must appear before test functions", + remediation: "Move the private constant above the first test so module setup is visible before behavior.", + }, + PythonRuleSpec { + code: "SQBRPYTHON706", + fensu_code: Some(fensu::TEST_NO_COMPLEX_COMPREHENSIONS_CODE), + slug: "test-no-complex-comprehensions", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "nested or multi-generator comprehensions hide control flow and data shapes", + remediation: "Use simple statements with named intermediate values, or a named helper in helpers.py.", + }, + PythonRuleSpec { + code: "SQBRPYTHON707", + fensu_code: Some(fensu::TEST_TYPES_DESCRIPTION_CODE), + slug: "test-case-description", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test-case dataclasses must define a description field", + remediation: "Add description: str so parametrized cases explain the behavior they represent.", + }, + PythonRuleSpec { + code: "SQBRPYTHON708", + fensu_code: Some(fensu::TEST_TYPES_EXPECTED_FIELD_CODE), + slug: "test-case-expected-field", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test-case dataclasses must define at least one expected_ field", + remediation: "Name expected outcomes with an expected_ prefix and assert against them in the test.", + }, + PythonRuleSpec { + code: "SQBRPYTHON709", + fensu_code: Some(fensu::TEST_LOCAL_TEST_TYPES_IMPORT_CODE), + slug: "test-local-types-import", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "tests must import test-case types from their local _test_types.py", + remediation: "Move the dataclass beside the test and import it through its absolute tests path.", + }, + PythonRuleSpec { + code: "SQBRPYTHON710", + fensu_code: Some(fensu::TEST_LOCAL_TEST_TYPES_FILE_CODE), + slug: "test-local-types-file", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test directories must provide a local _test_types.py", + remediation: "Create _test_types.py beside the test and define its test-case dataclasses there.", + }, + PythonRuleSpec { + code: "SQBRPYTHON711", + fensu_code: Some(fensu::TEST_FILE_NAME_CODE), + slug: "test-file-name", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test modules must use a test_ filename", + remediation: "Rename the module to test_.py after the module it tests.", + }, + PythonRuleSpec { + code: "SQBRPYTHON712", + fensu_code: Some(fensu::TEST_FUNCTION_NAME_CODE), + slug: "test-function-name", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test functions must use test_given__when__then_", + remediation: "Rename the test so its precondition, action, and expected behavior are explicit.", + }, + PythonRuleSpec { + code: "SQBRPYTHON713", + fensu_code: Some(fensu::TEST_DATACLASS_PARAMETRIZE_CODE), + slug: "test-dataclass-parametrize", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "tests must use dataclass-backed pytest parameterization", + remediation: "Add @pytest.mark.parametrize with local test-case dataclass instances.", + }, + PythonRuleSpec { + code: "SQBRPYTHON714", + fensu_code: Some(fensu::TEST_ACCEPTS_TEST_CASE_CODE), + slug: "test-accepts-test-case", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "parametrized tests must accept a test_case argument", + remediation: "Name the parameter test_case and read inputs and expectations from it.", + }, + PythonRuleSpec { + code: "SQBRPYTHON715", + fensu_code: Some(fensu::TEST_CASE_ANNOTATION_CODE), + slug: "test-case-annotation", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "test_case parameters must use a local test-case dataclass annotation", + remediation: "Annotate test_case with a dataclass imported from the local _test_types.py.", + }, + PythonRuleSpec { + code: "SQBRPYTHON716", + fensu_code: Some(fensu::TEST_EXPECTED_FIELD_ASSERTION_CODE), + slug: "test-expected-field-assertion", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "tests must assert against an expected_ field from test_case", + remediation: "Store the expected outcome on test_case and reference it in an assertion.", + }, + PythonRuleSpec { + code: "SQBRPYTHON717", + fensu_code: Some(fensu::TEST_PARAMETRIZE_ARGUMENTS_CODE), + slug: "test-parametrize-arguments", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest parametrize decorators must define parameter names and values", + remediation: "Pass both the parameter-name string and the case sequence to parametrize.", + }, + PythonRuleSpec { + code: "SQBRPYTHON718", + fensu_code: Some(fensu::TEST_PARAMETRIZE_TEST_CASE_CODE), + slug: "test-parametrize-test-case", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest parametrize must expose cases through the test_case parameter", + remediation: "Use \"test_case\" as the parametrize parameter name.", + }, + PythonRuleSpec { + code: "SQBRPYTHON719", + fensu_code: Some(fensu::TEST_PARAMETRIZE_IDS_CODE), + slug: "test-parametrize-ids", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest parametrize decorators must define readable case ids", + remediation: "Set ids=lambda case: case.description.", + }, + PythonRuleSpec { + code: "SQBRPYTHON720", + fensu_code: Some(fensu::TEST_INLINE_PARAMETRIZE_VALUES_CODE), + slug: "test-inline-parametrize-values", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest parametrize values must be a visible list, tuple, or local comprehension", + remediation: "Inline the case sequence in @pytest.mark.parametrize so its cases are visible beside the test.", + }, + PythonRuleSpec { + code: "SQBRPYTHON721", + fensu_code: Some(fensu::TEST_NONEMPTY_PARAMETRIZE_VALUES_CODE), + slug: "test-nonempty-parametrize-values", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest parametrize case sequences must not be empty", + remediation: "Add at least one behavior case or remove the test until a real case exists.", + }, + PythonRuleSpec { + code: "SQBRPYTHON722", + fensu_code: Some(fensu::TEST_NO_DICT_TEST_CASES_CODE), + slug: "test-no-dict-cases", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest cases must use typed dataclasses instead of dictionaries", + remediation: "Define a local frozen test-case dataclass and construct one instance per case.", + }, + PythonRuleSpec { + code: "SQBRPYTHON723", + fensu_code: Some(fensu::TEST_LOCAL_TEST_CASE_CONSTRUCTORS_CODE), + slug: "test-local-case-constructors", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest cases must construct dataclasses from the local _test_types.py", + remediation: "Parametrize with dataclasses imported from the local _test_types.py.", + }, + PythonRuleSpec { + code: "SQBRPYTHON724", + fensu_code: Some(fensu::TEST_DESCRIPTION_LAMBDA_IDS_CODE), + slug: "test-description-ids", + family: "tests", + owner: "file", + applies: PythonFiles::Tests, + enabled_by_default: true, + options: &[], + message: "pytest case ids must come from each test case description", + remediation: "Use ids=lambda case: case.description so failures name the behavior.", + }, ]; /// The project folder that holds the runtime Python package, imported as `python`. @@ -1097,6 +1469,122 @@ pub(crate) const PROVIDER_PRIVATE_FOLDERS: [&str; 2] = ["_helpers", "_classes"]; pub(crate) const PROVIDER_BASE: &str = "Provider"; /// The folder holding orchestration declarations, imported as `orchestration`. pub(crate) const ORCHESTRATION_ROOT: &str = "orchestration"; +pub(crate) const ORCHESTRATION_MODULE_PATHS_CODE: &str = "SQBRPYTHON016"; +pub(crate) const ORCHESTRATION_PRIVATE_SUPPORT_CODE: &str = "SQBRPYTHON017"; +pub(crate) const ORCHESTRATION_MODULE_SURFACE_CODE: &str = "SQBRPYTHON018"; +pub(crate) const ORCHESTRATION_IMPORT_DIRECTION_CODE: &str = "SQBRPYTHON019"; +pub(crate) const ORCHESTRATION_NO_DOCSTRING_CODE: &str = "SQBRPYTHON020"; +pub(crate) const PYTHON_TEST_PLACEMENT_CODE: &str = "SQBRPYTHON021"; +pub(crate) const COMPILE_TIME_PURITY_CODE: &str = "SQBRPYTHON022"; +/// The option listing project functions whose calls compile-time purity does not follow. +pub(crate) const APPROVED_LOADER_BOUNDARIES_OPTION: &str = "approved_loader_boundaries"; +/// Calls that start a subprocess, send a request, open a socket or a database, or build a remote client. +pub(crate) const EXTERNAL_CALLS: [&str; 51] = [ + "subprocess.run", + "subprocess.Popen", + "subprocess.call", + "subprocess.check_call", + "subprocess.check_output", + "subprocess.getoutput", + "subprocess.getstatusoutput", + "os.system", + "os.popen", + "os.posix_spawn", + "asyncio.create_subprocess_exec", + "asyncio.create_subprocess_shell", + "requests.get", + "requests.post", + "requests.put", + "requests.patch", + "requests.delete", + "requests.head", + "requests.options", + "requests.request", + "httpx.get", + "httpx.post", + "httpx.put", + "httpx.patch", + "httpx.delete", + "httpx.head", + "httpx.options", + "httpx.request", + "httpx.stream", + "aiohttp.request", + "urllib.request.urlopen", + "urllib3.request", + "http.client.HTTPConnection", + "http.client.HTTPSConnection", + "socket.socket", + "socket.create_connection", + "socket.getaddrinfo", + "socket.gethostbyname", + "sqlalchemy.create_engine", + "sqlalchemy.ext.asyncio.create_async_engine", + "pymongo.MongoClient", + "redis.Redis", + "redis.StrictRedis", + "redis.from_url", + "clickhouse_connect.get_client", + "google.cloud.bigquery.Client", + "boto3.client", + "boto3.resource", + "paramiko.SSHClient", + "paramiko.Transport", + "fabric.Connection", +]; +/// Call prefixes that replace the process or spawn another, such as `os.execv` and `os.spawnl`. +pub(crate) const EXTERNAL_CALL_PREFIXES: [&str; 2] = ["os.exec", "os.spawn"]; +/// Driver packages whose `connect` or `create_pool` opens a database or remote session. +pub(crate) const CONNECTING_PACKAGES: [&str; 17] = [ + "asyncpg", + "asyncssh", + "clickhouse_driver", + "cx_Oracle", + "databricks", + "duckdb", + "oracledb", + "psycopg", + "psycopg2", + "pymssql", + "pymysql", + "pyodbc", + "redshift_connector", + "snowflake", + "sqlite3", + "teradatasql", + "trino", +]; +/// The functions of a connecting package that open a connection. +pub(crate) const CONNECTING_FUNCTIONS: [&str; 2] = ["connect", "create_pool"]; +/// The module file of a package. +pub(crate) const PACKAGE_INIT_MODULE: &str = "__init__"; +/// Fensu's scope for test files, the only scope its test rules check. +pub(crate) const TEST_SCOPE: &str = "test"; +/// The prefix of a test module's file name; other files in a mirrored test folder need only the folder. +pub(crate) const TEST_FILE_PREFIX: &str = "test_"; +/// The orchestration rules, which read sensor and calendar modules and their importers. +pub(crate) const ORCHESTRATION_RULE_CODES: [&str; 5] = [ + ORCHESTRATION_MODULE_PATHS_CODE, + ORCHESTRATION_PRIVATE_SUPPORT_CODE, + ORCHESTRATION_MODULE_SURFACE_CODE, + ORCHESTRATION_IMPORT_DIRECTION_CODE, + ORCHESTRATION_NO_DOCSTRING_CODE, +]; +/// The orchestration folders that hold Python sensor and calendar modules. +pub(crate) const ORCHESTRATION_PYTHON_ROLES: [&str; 2] = ["sensors", "calendars"]; +/// Module names too generic to say which capability a sensor or calendar checks. +pub(crate) const ORCHESTRATION_GENERIC_MODULES: [&str; 5] = [ + "main.py", + "definitions.py", + "sensors.py", + "calendars.py", + "processing.py", +]; +/// The private support files a sensor or calendar domain folder may hold. +pub(crate) const ORCHESTRATION_SUPPORT_FILES: [&str; 4] = + ["_models.py", "_types.py", "_constants.py", "_exceptions.py"]; +/// The private support folder a sensor or calendar domain folder may hold. +pub(crate) const ORCHESTRATION_SUPPORT_FOLDER: &str = "_helpers"; /// The prefix of a private name. pub(crate) const PRIVATE_PREFIX: &str = "_"; /// The name a `from module import *` binds. @@ -1144,6 +1632,9 @@ pub(crate) const NODE_KINDS: [(&str, &str); 3] = [ pub(crate) const CHECK_KIND: (&str, &str) = ("check", "CheckContext"); /// Decorators that generate nodes or checks. pub(crate) const FACTORY_DECORATORS: [&str; 1] = ["factory"]; + +/// Text a source must contain to have code its import never runs: a lambda or a `__main__` block. +pub(crate) const DEFERRED_CODE_MARKERS: [&str; 2] = ["lambda", "__main__"]; /// The loader decorator, which takes no description because its managed source has one. pub(crate) const LOADER_DECORATOR: &str = "loader"; /// Return annotations `main()` may use, by node decorator. diff --git a/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs b/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs index 103cc7fe3..dadb4480a 100644 --- a/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs +++ b/crates/sqlbuild-rules/src/python_rules/main/evaluate_python_rules.rs @@ -3,20 +3,15 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::models::{EvaluateRequest, RuleMetadata}; -use crate::python_rules::_helpers::engine::contexts::context_settings; -use crate::python_rules::_helpers::engine::execution::{ - evaluated_files, file_plans, selected_rules, -}; -use crate::python_rules::_helpers::engine::file_cache::project_digest; +use crate::python_rules::_helpers::engine::execution::selected_rules; use crate::python_rules::_helpers::engine::layout::layout_faults; -use crate::python_rules::_helpers::engine::placement::python_targets; -use crate::python_rules::_helpers::engine::programs::Programs; -use crate::python_rules::_helpers::engine::project_plane::loaded_project; +use crate::python_rules::_helpers::engine::passes::evaluated_pass; +use crate::python_rules::_helpers::units::compile_purity::compile_purity_faults; +use crate::python_rules::_helpers::units::orchestration::orchestration_faults; use crate::python_rules::_helpers::units::project::unit_faults; use crate::python_rules::_helpers::units::providers::provider_faults; -use crate::python_rules::models::{ - CallInputs, PythonCacheEntry, PythonEvaluation, PythonRuleSpec, PythonTarget, -}; +use crate::python_rules::_helpers::units::python_tests::python_test_faults; +use crate::python_rules::models::{PythonCacheEntry, PythonEvaluation, PythonRuleSpec}; /// Sorted findings of the selected Python rules, reusing `cached` ones made under `fingerprint` that still hold. pub(crate) fn evaluate_python_rules( @@ -26,22 +21,14 @@ pub(crate) fn evaluate_python_rules( fingerprint: &str, ) -> Result { let codes: BTreeSet<&str> = selected.iter().map(|rule| rule.code.as_str()).collect(); - let rules: Vec<&PythonRuleSpec> = selected_rules(selected); - let mut evaluation = PythonEvaluation::default(); - if !rules.is_empty() && !request.python_sources.is_empty() { - let programs = Programs::new(&request.python_sources, request.python_version); - let targets: Vec = python_targets(&request.python_sources); - let (plane, entrypoints) = loaded_project(&rules, &targets, &programs, request)?; - let inputs = CallInputs { - request, - settings: context_settings(request, &rules, entrypoints), - programs: &programs, - cached, - fingerprint, - project_digest: project_digest(&request.python_sources), - }; - evaluation = evaluated_files(file_plans(&targets, &rules, &inputs)?, &inputs, &plane)?; - } + let rules: Vec<&'static PythonRuleSpec> = selected_rules(selected); + let mut evaluation: PythonEvaluation = + evaluated_pass(request, (&rules, false), cached, fingerprint)?; + let tests: PythonEvaluation = evaluated_pass(request, (&rules, true), cached, fingerprint)?; + evaluation.faults.extend(tests.faults); + evaluation.entries.extend(tests.entries); + evaluation.hits += tests.hits; + evaluation.misses += tests.misses; evaluation .faults .extend(layout_faults(&request.python_sources, &codes)); @@ -55,6 +42,19 @@ pub(crate) fn evaluate_python_rules( &codes, request.python_version, )); + evaluation.faults.extend(orchestration_faults( + &request.python_sources, + &codes, + request.python_version, + )); + evaluation.faults.extend(python_test_faults( + &request.python_sources, + &request.python_test_sources, + &codes, + )); + evaluation + .faults + .extend(compile_purity_faults(request, &codes)?); evaluation.faults.sort_by(|left, right| { (&left.path, left.line, left.column, &left.code).cmp(&( &right.path, diff --git a/crates/sqlbuild-rules/src/python_rules/main/mod.rs b/crates/sqlbuild-rules/src/python_rules/main/mod.rs index b735fb0d5..17a3a7ea9 100644 --- a/crates/sqlbuild-rules/src/python_rules/main/mod.rs +++ b/crates/sqlbuild-rules/src/python_rules/main/mod.rs @@ -2,3 +2,4 @@ pub mod evaluate_python_rules; pub mod python_catalogue; pub mod python_rule_options; pub mod python_sources; +pub mod python_test_sources; diff --git a/crates/sqlbuild-rules/src/python_rules/main/python_sources.rs b/crates/sqlbuild-rules/src/python_rules/main/python_sources.rs index 1ec2fdbaa..807758102 100644 --- a/crates/sqlbuild-rules/src/python_rules/main/python_sources.rs +++ b/crates/sqlbuild-rules/src/python_rules/main/python_sources.rs @@ -5,6 +5,7 @@ use std::path::Path; use sqlbuild_discovery::python_paths::main::supported_python_paths::supported_python_paths; use crate::models::{EvaluateRequest, PythonSource}; +use crate::python_rules::_helpers::engine::source_files::read_source; use crate::rules::main::request_selects::request_selects; /// Every supported project Python file when the request selects a Python rule, otherwise none. @@ -18,16 +19,3 @@ pub(crate) fn python_sources(request: &EvaluateRequest) -> Result Result { - let bytes: Vec = std::fs::read(project_dir.join(&relative_path)) - .map_err(|error| format!("{relative_path}: cannot read Python source: {error}"))?; - let digest: String = blake3::hash(&bytes).to_hex().to_string(); - let text: String = String::from_utf8(bytes) - .map_err(|_| format!("{relative_path}: Python source is not valid UTF-8"))?; - Ok(PythonSource { - relative_path, - digest, - text, - }) -} diff --git a/crates/sqlbuild-rules/src/python_rules/main/python_test_sources.rs b/crates/sqlbuild-rules/src/python_rules/main/python_test_sources.rs new file mode 100644 index 000000000..1b411c927 --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/main/python_test_sources.rs @@ -0,0 +1,28 @@ +//! The project Python test files the selected Python test rules read. + +use std::path::Path; + +use sqlbuild_discovery::python_paths::main::python_test_paths::python_test_paths; + +use crate::models::{EvaluateRequest, PythonSource}; +use crate::python_rules::_helpers::engine::source_files::read_source; +use crate::python_rules::constants::PYTHON_RULES; +use crate::python_rules::models::PythonFiles; +use crate::rules::main::request_selects::request_selects; + +/// Every project Python test file when the request selects a Python test rule, otherwise none. +pub(crate) fn python_test_sources(request: &EvaluateRequest) -> Result, String> { + let test_rule = |code: &str| { + PYTHON_RULES + .iter() + .any(|rule| rule.code == code && rule.applies == PythonFiles::Tests) + }; + if !request_selects(request, |rule| test_rule(&rule.code))? { + return Ok(Vec::new()); + } + let project_dir: &Path = Path::new(&request.project_dir); + python_test_paths(project_dir) + .into_iter() + .map(|parts| read_source(project_dir, parts.join("/"))) + .collect() +} diff --git a/crates/sqlbuild-rules/src/python_rules/models.rs b/crates/sqlbuild-rules/src/python_rules/models.rs index 80628f54f..d228ee9ad 100644 --- a/crates/sqlbuild-rules/src/python_rules/models.rs +++ b/crates/sqlbuild-rules/src/python_rules/models.rs @@ -7,6 +7,8 @@ pub enum PythonFiles { Package, /// Every supported project Python file, including hooks, UDFs and orchestration modules. All, + /// Only the project's Python tests under `tests/python/` and `tests/providers/`. + Tests, } /// One `SQBRPYTHON` rule and the Fensu core rule it runs, if any. @@ -42,6 +44,8 @@ pub(crate) struct PythonTarget { pub role: Option, pub is_main_module: bool, pub is_entry_module: bool, + /// Fensu's scope for the file: `test` for project Python tests, else the root scope. + pub scope: &'static str, } /// One `evaluate_core_rules` call on a file: the Fensu codes it runs and the context they read. @@ -102,6 +106,15 @@ pub(crate) struct CacheSplit<'a> { pub stale: Vec>, } +/// One parsed project file, its path parts from the project root, and its text. +#[derive(Debug)] +pub(crate) struct ParsedFile<'a> { + pub path: &'a str, + pub parts: Vec<&'a str>, + pub text: &'a str, + pub module: sqlbuild_pysource::node_units::models::UnitModule, +} + /// One parsed module under `python/` and the unit folder it sits in. #[derive(Debug)] pub(crate) struct UnitFile<'a> { diff --git a/crates/sqlbuild-rules/src/python_rules/tests.rs b/crates/sqlbuild-rules/src/python_rules/tests.rs index ae23fb200..a90737194 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests.rs @@ -1,9 +1,13 @@ +#[path = "tests/test_compile_purity_rules.rs"] +mod compile_purity_rules; #[path = "tests/test_evaluate_python_rules.rs"] mod evaluate_python_rules; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_node_units.rs"] mod node_units; +#[path = "tests/test_orchestration_rules.rs"] +mod orchestration_rules; #[path = "tests/test_provider_rules.rs"] mod provider_rules; #[path = "tests/test_python_cache.rs"] @@ -12,5 +16,7 @@ mod python_cache; mod python_catalogue; #[path = "tests/test_python_scaling.rs"] mod python_scaling; +#[path = "tests/test_python_test_rules.rs"] +mod python_test_rules; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs b/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs index 32e134882..90f58d176 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/helpers.rs @@ -248,6 +248,43 @@ fn unit_files(unit: usize, upstream: &str, arguments: &str) -> [(String, String) ] } +/// Root modules calling into helper chains that each end in a request. +pub(super) fn purity_chain_project( + roots: usize, + chains: usize, + depth: usize, +) -> Vec<(String, String)> { + let mut files: Vec<(String, String)> = Vec::new(); + for chain in 0..chains { + for step in 1..depth { + files.push(( + format!("python/utils/chain_{chain}/step_{previous}.py", previous = step - 1), + format!( + "from python.utils.chain_{chain} import step_{step}\n\n\ndef run():\n return step_{step}.run()\n" + ), + )); + } + files.push(( + format!( + "python/utils/chain_{chain}/step_{last}.py", + last = depth - 1 + ), + "import requests\n\n\ndef run():\n return requests.get(\"https://example.com\")\n" + .to_owned(), + )); + } + for root in 0..roots { + files.push(( + format!("python/orders/export_{root}/constants.py"), + format!( + "from python.utils.chain_{chain} import step_0\n\nREGIONS = step_0.run()\n", + chain = root % chains + ), + )); + } + files +} + /// The codes of the Python faults `files` produce under `config` with the cache off, and how long evaluation took. pub(super) fn timed_python_codes( files: &[(String, String)], diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_compile_purity_rules.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_compile_purity_rules.rs new file mode 100644 index 000000000..ec5d6273b --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_compile_purity_rules.rs @@ -0,0 +1,281 @@ +use serde_json::json; + +use crate::python_rules::tests::helpers::{owned_faults, python_rule_faults}; +use crate::python_rules::tests::test_types::{PythonRunFailureTestCase, PythonRunTestCase}; + +const PURE_FACTORY: &str = r#"from sqlbuild.tasks import factory, task + +from python.inventory.snapshots.constants import WAREHOUSE_SPECS + + +@factory +def warehouse_snapshots(): + nodes = [] + for spec in WAREHOUSE_SPECS: + + @task(name=spec) + def snapshot(ctx): + return requests.get(spec) + + nodes.append(snapshot) + return nodes +"#; +const RUNTIME_ONLY: &str = r#"import psycopg +import requests + +from sqlbuild.tasks import task + + +def main(ctx): + return psycopg.connect("dsn").execute("SELECT 1") + + +@task +def export(ctx): + return requests.post("https://example.com/orders") + + +class CarrierApi: + def setup(self): + self.session = requests.get("https://example.com/token") +"#; +const QUERYING_FACTORY: &str = r#"import psycopg + +from sqlbuild.tasks import factory + +from ._helpers.spec_loading import load_specs + + +@factory +def warehouse_snapshots(): + codes = psycopg.connect("dsn").execute("SELECT code FROM warehouses").fetchall() + return load_specs(codes) +"#; +const SPEC_LOADING: &str = r#"import subprocess + + +def load_specs(codes): + return list_specs(codes) + + +def list_specs(codes): + return subprocess.run(["ls"], check=True) +"#; +const REGIONS: &str = r#"import requests + +STOREFRONT_REGIONS = requests.get("https://example.com/regions").json() +"#; +const SENSOR: &str = r#"from orchestration.sensors.inventory._helpers import feed + +LATEST = feed.latest() +"#; +const FEED: &str = r#"from python.utils.network import ping + + +def latest(): + return ping() + + +def again(): + return latest() +"#; +const NETWORK: &str = r#"import socket + +from python.utils import network + + +def ping(): + network.pong() + return socket.create_connection(("example.com", 443)) + + +def pong(): + return ping() +"#; +const CLIENTS: &str = r#"import boto3 +import snowflake.connector as connector +import urllib.request +from paramiko import SSHClient + +BUCKET = boto3.client("s3") +SESSION = connector.connect(account="example") +PAGE = urllib.request.urlopen("https://example.com") +SHELL = SSHClient() +"#; +const DEFERRED: &str = r#"import subprocess + +from python.utils.shell import deferred_runner + +HANDLERS = {"ls": lambda: subprocess.run(["ls"])} +RUNNER = deferred_runner() +DEFAULT = (lambda code=subprocess.run(["id"]): code) + +if __name__ == "__main__": + subprocess.run(["ls"]) +else: + subprocess.run(["id"]) +"#; +const SHELL: &str = r#"import subprocess + + +def deferred_runner(): + return lambda: subprocess.run(["ls"]) +"#; + +#[test] +fn given_compile_time_python_when_evaluating_compile_time_purity_then_reports_calls_reaching_external_systems() + { + let test_cases = [ + PythonRunTestCase { + description: "static specs, node bodies, main() and provider methods report nothing", + files: &[ + ("python/inventory/snapshots/node.py", PURE_FACTORY), + ( + "python/inventory/snapshots/constants.py", + "WAREHOUSE_SPECS = (\"north\", \"south\")\n", + ), + ("python/orders/export/main.py", RUNTIME_ONLY), + ], + config: json!({ "select": ["SQBRPYTHON022"] }), + expected_faults: vec![], + }, + PythonRunTestCase { + description: "module scope, factory bodies and the project functions they call are followed to each external call", + files: &[ + ("python/inventory/snapshots/node.py", QUERYING_FACTORY), + ( + "python/inventory/snapshots/_helpers/spec_loading.py", + SPEC_LOADING, + ), + ("python/orders/storefront_import/constants.py", REGIONS), + ("orchestration/sensors/inventory/stock_feed.py", SENSOR), + ("orchestration/sensors/inventory/_helpers/feed.py", FEED), + ("python/utils/network.py", NETWORK), + ("providers/carrier_api/constants.py", CLIENTS), + ], + config: json!({ "select": ["SQBRPYTHON022"] }), + expected_faults: vec![ + ( + "SQBRPYTHON022", + "orchestration/sensors/inventory/stock_feed.py", + 3, + 10, + ), + ("SQBRPYTHON022", "providers/carrier_api/constants.py", 6, 10), + ("SQBRPYTHON022", "providers/carrier_api/constants.py", 7, 11), + ("SQBRPYTHON022", "providers/carrier_api/constants.py", 8, 8), + ("SQBRPYTHON022", "providers/carrier_api/constants.py", 9, 9), + ( + "SQBRPYTHON022", + "python/inventory/snapshots/node.py", + 10, + 13, + ), + ( + "SQBRPYTHON022", + "python/inventory/snapshots/node.py", + 11, + 12, + ), + ( + "SQBRPYTHON022", + "python/orders/storefront_import/constants.py", + 3, + 22, + ), + ], + }, + PythonRunTestCase { + description: "lambda bodies and the __main__ block never run on import, but a lambda default and an else branch do", + files: &[ + ("python/orders/export/constants.py", DEFERRED), + ("python/utils/shell.py", SHELL), + ], + config: json!({ "select": ["SQBRPYTHON022"] }), + expected_faults: vec![ + ("SQBRPYTHON022", "python/orders/export/constants.py", 7, 24), + ("SQBRPYTHON022", "python/orders/export/constants.py", 12, 5), + ], + }, + PythonRunTestCase { + description: "an approved loader boundary stops the walk at exactly that function", + files: &[ + ("python/inventory/snapshots/node.py", QUERYING_FACTORY), + ( + "python/inventory/snapshots/_helpers/spec_loading.py", + SPEC_LOADING, + ), + ], + config: json!({ + "select": ["SQBRPYTHON022"], + "rule_options": { "SQBRPYTHON022": { "approved_loader_boundaries": [ + "python.inventory.snapshots._helpers.spec_loading.load_specs", + ] } }, + }), + expected_faults: vec![( + "SQBRPYTHON022", + "python/inventory/snapshots/node.py", + 10, + 13, + )], + }, + PythonRunTestCase { + description: "an approved factory runs its body unchecked", + files: &[ + ("python/inventory/snapshots/node.py", QUERYING_FACTORY), + ( + "python/inventory/snapshots/_helpers/spec_loading.py", + SPEC_LOADING, + ), + ], + config: json!({ + "select": ["SQBRPYTHON022"], + "rule_options": { "SQBRPYTHON022": { "approved_loader_boundaries": [ + "python.inventory.snapshots.node.warehouse_snapshots", + ] } }, + }), + expected_faults: vec![], + }, + ]; + for test_case in test_cases { + let faults = python_rule_faults(test_case.files, test_case.config).expect("rules run"); + assert_eq!( + faults, + owned_faults(&test_case.expected_faults), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_boundary_naming_a_module_when_evaluating_compile_time_purity_then_the_run_fails() { + let test_cases = [PythonRunFailureTestCase { + description: "a module is not a loader boundary", + files: &[ + ("python/inventory/snapshots/node.py", QUERYING_FACTORY), + ( + "python/inventory/snapshots/_helpers/spec_loading.py", + SPEC_LOADING, + ), + ], + config: json!({ + "select": ["SQBRPYTHON022"], + "rule_options": { "SQBRPYTHON022": { "approved_loader_boundaries": [ + "python.inventory.snapshots._helpers.spec_loading", + ] } }, + }), + expected_error: "names the module python.inventory.snapshots._helpers.spec_loading", + }]; + for test_case in test_cases { + let faults = python_rule_faults(test_case.files, test_case.config); + + assert!( + faults + .as_ref() + .is_err_and(|error| error.contains(test_case.expected_error)), + "{}: {faults:?}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_orchestration_rules.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_orchestration_rules.rs new file mode 100644 index 000000000..d2dc267d4 --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_orchestration_rules.rs @@ -0,0 +1,261 @@ +use serde_json::json; + +use crate::python_rules::tests::helpers::{owned_faults, python_rule_faults}; +use crate::python_rules::tests::test_types::PythonRunTestCase; + +const ORCHESTRATION_RULES: [&str; 5] = [ + "SQBRPYTHON016", + "SQBRPYTHON017", + "SQBRPYTHON018", + "SQBRPYTHON019", + "SQBRPYTHON020", +]; + +#[test] +fn given_sensor_and_calendar_modules_when_evaluating_orchestration_rules_then_reports_each_broken_convention() + { + let test_cases = [ + PythonRunTestCase { + description: "domain folders with one declaration per module, private support and allowed imports report nothing", + files: &[ + ("orchestration/sensors/__init__.py", ""), + ( + "orchestration/sensors/inventory/stock_feed.py", + r#"from sqlbuild.orchestration import sensor + +from orchestration.sensors.inventory._constants import STALE_AFTER +from python.utils.retries import ATTEMPTS +from providers.carrier_api.provider import CarrierApi + +from ._helpers.feed_age import feed_age + +_LIMIT: int = 3 + + +@sensor("stock_feed_stale_sensor", description="Stale feed.", run="restock_job") +def stock_feed_stale(ctx, carrier_api: CarrierApi): + return feed_age(ctx) > STALE_AFTER * ATTEMPTS * _LIMIT +"#, + ), + ( + "orchestration/sensors/inventory/_constants.py", + "STALE_AFTER = 3\n", + ), + ( + "orchestration/sensors/inventory/_helpers/feed_age.py", + "def feed_age(ctx):\n return 0\n", + ), + ( + "orchestration/calendars/finance/month_end.py", + r#"from sqlbuild.orchestration import calendar + +month_end = calendar("month_end_calendar", description="Month end.") +"#, + ), + ("python/utils/retries.py", "ATTEMPTS = 2\n"), + ("providers/carrier_api/provider.py", ""), + ], + config: json!({ "select": ORCHESTRATION_RULES }), + expected_faults: vec![], + }, + PythonRunTestCase { + description: "layout: no domain folder, a generic name, nesting, and support outside a domain or not in the allowed files", + files: &[ + ("orchestration/sensors/stock_feed.py", ""), + ("orchestration/sensors/inventory/sensors.py", ""), + ("orchestration/sensors/inventory/sub/feed.py", ""), + ("orchestration/sensors/inventory/_constants.py", ""), + ("orchestration/sensors/inventory/_helpers/feed_age.py", ""), + ("orchestration/sensors/inventory/_utils.py", ""), + ("orchestration/sensors/inventory/_lib/age.py", ""), + ("orchestration/sensors/_shared.py", ""), + ("orchestration/sensors/_shared/age.py", ""), + ("orchestration/calendars/__init__.py", ""), + ], + config: json!({ "select": ["SQBRPYTHON016", "SQBRPYTHON017"] }), + expected_faults: vec![ + ("SQBRPYTHON017", "orchestration/sensors/_shared.py", 1, 1), + ( + "SQBRPYTHON017", + "orchestration/sensors/_shared/age.py", + 1, + 1, + ), + ( + "SQBRPYTHON017", + "orchestration/sensors/inventory/_lib/age.py", + 1, + 1, + ), + ( + "SQBRPYTHON017", + "orchestration/sensors/inventory/_utils.py", + 1, + 1, + ), + ( + "SQBRPYTHON016", + "orchestration/sensors/inventory/sensors.py", + 1, + 1, + ), + ( + "SQBRPYTHON016", + "orchestration/sensors/inventory/sub/feed.py", + 1, + 1, + ), + ("SQBRPYTHON016", "orchestration/sensors/stock_feed.py", 1, 1), + ], + }, + PythonRunTestCase { + description: "surface: a public constant, a mixed assignment, a function, a class, a loop and a docstring on the sensor", + files: &[( + "orchestration/sensors/inventory/stock_feed.py", + r#""""Stock feed sensor.""" + +from sqlbuild.orchestration import sensor + +_STALE_AFTER = 3 +STALE = 4 +_A, B.x = 1, 2 + + +def helper(): + pass + + +class Feed: + pass + + +@sensor("stock_feed_stale_sensor", description="Stale feed.", run="restock_job") +def stock_feed_stale(ctx): + """Doc.""" + return None + + +for row in []: + pass +"#, + )], + config: json!({ "select": ["SQBRPYTHON018", "SQBRPYTHON020"] }), + expected_faults: vec![ + ( + "SQBRPYTHON018", + "orchestration/sensors/inventory/stock_feed.py", + 6, + 1, + ), + ( + "SQBRPYTHON018", + "orchestration/sensors/inventory/stock_feed.py", + 7, + 1, + ), + ( + "SQBRPYTHON018", + "orchestration/sensors/inventory/stock_feed.py", + 10, + 5, + ), + ( + "SQBRPYTHON018", + "orchestration/sensors/inventory/stock_feed.py", + 14, + 7, + ), + ( + "SQBRPYTHON020", + "orchestration/sensors/inventory/stock_feed.py", + 19, + 5, + ), + ( + "SQBRPYTHON018", + "orchestration/sensors/inventory/stock_feed.py", + 24, + 1, + ), + ], + }, + PythonRunTestCase { + description: "imports: another domain's support, a node unit from a sensor or its helpers, and orchestration/ from outside it except providers and node units", + files: &[ + ( + "orchestration/sensors/orders/refund_spike.py", + r#"from orchestration.sensors.inventory._helpers.feed_age import feed_age +from python.orders.daily.main import main +from python.utils.retries import ATTEMPTS +from providers.carrier_api.provider import CarrierApi +from ._helpers.window import window +"#, + ), + ("orchestration/sensors/orders/_helpers/window.py", ""), + ("orchestration/sensors/inventory/_helpers/feed_age.py", ""), + ( + "orchestration/sensors/inventory/_helpers/age.py", + "from ...orders._helpers.window import window\nfrom python.orders.daily.main import main\n", + ), + ("python/orders/weekly/node.py", ""), + ( + "python/orders/weekly/main.py", + "from orchestration.sensors.orders._helpers.window import window\n", + ), + ( + "python/orders/daily/main.py", + "from orchestration.sensors.orders._helpers.window import window\n", + ), + ("python/utils/retries.py", ""), + ( + "providers/carrier_api/provider.py", + "import orchestration.sensors.orders.refund_spike\n", + ), + ( + "hooks/python/notify.py", + "import orchestration.sensors.orders.refund_spike\n", + ), + ], + config: json!({ "select": ["SQBRPYTHON008", "SQBRPYTHON017", "SQBRPYTHON019"] }), + expected_faults: vec![ + ("SQBRPYTHON019", "hooks/python/notify.py", 1, 8), + ( + "SQBRPYTHON017", + "orchestration/sensors/inventory/_helpers/age.py", + 1, + 39, + ), + ( + "SQBRPYTHON019", + "orchestration/sensors/inventory/_helpers/age.py", + 2, + 38, + ), + ( + "SQBRPYTHON017", + "orchestration/sensors/orders/refund_spike.py", + 1, + 63, + ), + ( + "SQBRPYTHON019", + "orchestration/sensors/orders/refund_spike.py", + 2, + 38, + ), + ("SQBRPYTHON019", "python/orders/daily/main.py", 1, 58), + ("SQBRPYTHON008", "python/orders/weekly/main.py", 1, 58), + ], + }, + ]; + for test_case in test_cases { + let faults = python_rule_faults(test_case.files, test_case.config.clone()); + + assert_eq!( + faults, + Ok(owned_faults(&test_case.expected_faults)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs index 95ce1c8d8..8faa13bcf 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_python_catalogue.rs @@ -14,14 +14,14 @@ use crate::rules::models::BuiltinOptionKind; fn given_the_catalogue_when_listing_python_rules_then_each_has_its_own_project_wide_code() { let test_cases = [ CatalogueTestCase { - description: "15 SQLBuild layout, unit and provider rules and 62 Fensu core rules have distinct codes", + description: "21 SQLBuild layout, unit, provider, orchestration and test placement rules and 86 Fensu core and test rules have distinct codes", count: distinct_python_codes, - expected_count: 77, + expected_count: 108, }, CatalogueTestCase { description: "every Python rule reports against the project in the python family", count: project_wide_python_rules, - expected_count: 77, + expected_count: 108, }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_python_scaling.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_python_scaling.rs index 2c17e9de5..9a8561239 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_python_scaling.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_python_scaling.rs @@ -4,10 +4,11 @@ use std::time::Duration; use serde_json::json; use crate::python_rules::tests::helpers::{ - median_growth, scaling_project, serial_timing, timed_python_codes, unit_scaling_project, + median_growth, purity_chain_project, scaling_project, serial_timing, timed_python_codes, + unit_scaling_project, }; use crate::python_rules::tests::test_types::{ - PythonGrowthTestCase, PythonScalingTestCase, UnitScalingTestCase, + PurityScalingTestCase, PythonGrowthTestCase, PythonScalingTestCase, UnitScalingTestCase, }; #[test] @@ -126,3 +127,33 @@ fn given_many_linked_node_units_when_evaluating_unit_rules_then_finishes_within_ ); } } + +#[test] +fn given_many_roots_sharing_deep_call_chains_when_checking_compile_time_purity_then_finishes_within_budget() + { + let _serial = serial_timing(); + let test_cases = [PurityScalingTestCase { + description: "5000 module-scope roots into 10 helper chains of 400 calls each", + roots: 5000, + chains: 10, + depth: 400, + expected_faults: 5000, + expected_max_duration: Duration::from_secs(3), + }]; + for test_case in test_cases { + let files = purity_chain_project(test_case.roots, test_case.chains, test_case.depth); + let (codes, elapsed) = + timed_python_codes(&files, json!({ "select": ["SQBRPYTHON022"] })).expect("rules run"); + assert_eq!( + codes.len(), + test_case.expected_faults, + "{}", + test_case.description + ); + assert!( + elapsed < test_case.expected_max_duration, + "{}: took {elapsed:?}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs new file mode 100644 index 000000000..009100532 --- /dev/null +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_python_test_rules.rs @@ -0,0 +1,237 @@ +use serde_json::json; + +use crate::python_rules::tests::helpers::{owned_faults, python_rule_faults}; +use crate::python_rules::tests::test_types::PythonRunTestCase; + +const RUNTIME_FILES: [(&str, &str); 5] = [ + ( + "python/orders/storefront_import/node.py", + r#"from sqlbuild.tasks import TaskContext, task + +from python.orders.storefront_import.main import main + + +@task(description="Import storefront orders.") +def storefront_import(ctx: TaskContext) -> None: + return main(ctx) +"#, + ), + ( + "python/orders/storefront_import/main.py", + r#"from sqlbuild.tasks import PythonNodeResult, TaskContext + + +def main(ctx: TaskContext) -> PythonNodeResult: + return PythonNodeResult() +"#, + ), + ( + "python/orders/storefront_import/_helpers/row_mapping.py", + r#"def map_row(raw: dict[str, object]) -> dict[str, object]: + return {"status": str(raw["state"]).lower()} +"#, + ), + ( + "providers/storefront_api/provider.py", + r#"from sqlbuild.providers import Provider + + +class StorefrontApi(Provider): + region: str = "eu" +"#, + ), + ("providers/storefront_api/__init__.py", ""), +]; + +const CASE_TYPES: &str = r#"from dataclasses import dataclass + + +@dataclass(frozen=True) +class RowMappingCase: + description: str + raw: dict[str, object] + expected_status: str +"#; + +const ROW_MAPPING_TEST: &str = r#"import pytest + +from python.orders.storefront_import._helpers.row_mapping import map_row +from tests.python.orders.storefront_import._helpers._test_types import RowMappingCase + + +@pytest.mark.parametrize( + "test_case", + [ + RowMappingCase(description="paid order", raw={"state": "PAID"}, expected_status="paid"), + ], + ids=lambda case: case.description, +) +def test_given_raw_order_when_mapping_then_status_is_normalised(test_case: RowMappingCase) -> None: + assert map_row(test_case.raw)["status"] == test_case.expected_status +"#; + +const PROVIDER_CASE_TYPES: &str = r#"from dataclasses import dataclass + + +@dataclass(frozen=True) +class RegionCase: + description: str + region: str + expected_region: str +"#; + +const PROVIDER_TEST: &str = r#"import pytest + +from providers.storefront_api.provider import StorefrontApi +from tests.providers.storefront_api._test_types import RegionCase + + +@pytest.mark.parametrize( + "test_case", + [RegionCase(description="explicit region", region="us", expected_region="us")], + ids=lambda case: case.description, +) +def test_given_region_when_constructing_then_provider_keeps_it(test_case: RegionCase) -> None: + assert StorefrontApi(region=test_case.region).region == test_case.expected_region +"#; + +const BRANCHING_TEST: &str = r#"import pytest + +from .._test_types import RowMappingCase + + +def build_raw(state: str) -> dict[str, object]: + return {"state": state} + + +@pytest.mark.parametrize( + "test_case", + [RowMappingCase(description="paid order", raw={"state": "PAID"}, expected_status="paid")], + ids=lambda case: case.description, +) +def test_given_raw_order_when_mapping_then_status_is_normalised(test_case: RowMappingCase) -> None: + if test_case.raw: + assert build_raw("PAID") == test_case.raw + assert test_case.expected_status == "paid" +"#; + +#[test] +fn given_project_python_tests_when_evaluating_test_rules_then_reports_misplaced_and_unconventional_tests() + { + let test_cases = [ + PythonRunTestCase { + description: "tests mirroring a unit, its helpers and a provider, with dataclass cases, report nothing", + files: &[ + RUNTIME_FILES[0], + RUNTIME_FILES[1], + RUNTIME_FILES[2], + RUNTIME_FILES[3], + RUNTIME_FILES[4], + ( + "tests/python/orders/storefront_import/_helpers/_test_types.py", + CASE_TYPES, + ), + ( + "tests/python/orders/storefront_import/_helpers/test_row_mapping.py", + ROW_MAPPING_TEST, + ), + ( + "tests/providers/storefront_api/_test_types.py", + PROVIDER_CASE_TYPES, + ), + ( + "tests/providers/storefront_api/test_provider.py", + PROVIDER_TEST, + ), + ], + config: json!({"select": ["SQBRPYTHON021", "SQBRPYTHON701", "SQBRPYTHON702", "SQBRPYTHON703", "SQBRPYTHON704", "SQBRPYTHON705", "SQBRPYTHON706", "SQBRPYTHON707", "SQBRPYTHON708", "SQBRPYTHON709", "SQBRPYTHON710", "SQBRPYTHON711", "SQBRPYTHON712", "SQBRPYTHON713", "SQBRPYTHON714", "SQBRPYTHON715", "SQBRPYTHON716", "SQBRPYTHON717", "SQBRPYTHON718", "SQBRPYTHON719", "SQBRPYTHON720", "SQBRPYTHON721", "SQBRPYTHON722", "SQBRPYTHON723", "SQBRPYTHON724"]}), + expected_faults: vec![], + }, + PythonRunTestCase { + description: "tests for missing modules or folders, or outside any folder, fail placement", + files: &[ + RUNTIME_FILES[0], + RUNTIME_FILES[1], + ("tests/python/orders/_test_types.py", CASE_TYPES), + ("tests/python/orders/test_storefront.py", ROW_MAPPING_TEST), + ("tests/python/inventory/_test_types.py", CASE_TYPES), + ("tests/python/inventory/test_stock.py", ROW_MAPPING_TEST), + ("tests/python/_test_types.py", CASE_TYPES), + ("tests/python/test_everything.py", ROW_MAPPING_TEST), + ], + config: json!({"select": ["SQBRPYTHON021"]}), + expected_faults: vec![ + ("SQBRPYTHON021", "tests/python/_test_types.py", 1, 1), + ( + "SQBRPYTHON021", + "tests/python/inventory/_test_types.py", + 1, + 1, + ), + ( + "SQBRPYTHON021", + "tests/python/inventory/test_stock.py", + 1, + 1, + ), + ( + "SQBRPYTHON021", + "tests/python/orders/test_storefront.py", + 1, + 1, + ), + ("SQBRPYTHON021", "tests/python/test_everything.py", 1, 1), + ], + }, + PythonRunTestCase { + description: "a relative import, a top-level helper and a branch in a test break the test conventions", + files: &[ + RUNTIME_FILES[0], + RUNTIME_FILES[1], + RUNTIME_FILES[2], + ( + "tests/python/orders/storefront_import/_test_types.py", + CASE_TYPES, + ), + ( + "tests/python/orders/storefront_import/_helpers/_test_types.py", + CASE_TYPES, + ), + ( + "tests/python/orders/storefront_import/_helpers/test_row_mapping.py", + BRANCHING_TEST, + ), + ], + config: json!({"select": ["SQBRPYTHON702", "SQBRPYTHON703", "SQBRPYTHON704"]}), + expected_faults: vec![ + ( + "SQBRPYTHON702", + "tests/python/orders/storefront_import/_helpers/test_row_mapping.py", + 3, + 1, + ), + ( + "SQBRPYTHON703", + "tests/python/orders/storefront_import/_helpers/test_row_mapping.py", + 6, + 1, + ), + ( + "SQBRPYTHON704", + "tests/python/orders/storefront_import/_helpers/test_row_mapping.py", + 16, + 5, + ), + ], + }, + ]; + for test_case in test_cases { + let faults = python_rule_faults(test_case.files, test_case.config).expect("rules run"); + assert_eq!( + faults, + owned_faults(&test_case.expected_faults), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/python_rules/tests/test_types.rs b/crates/sqlbuild-rules/src/python_rules/tests/test_types.rs index 41b7f36ce..f600b2fd7 100644 --- a/crates/sqlbuild-rules/src/python_rules/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/python_rules/tests/test_types.rs @@ -41,6 +41,13 @@ pub(super) struct PythonRunTestCase { pub(super) expected_faults: Vec<(&'static str, &'static str, u64, u64)>, } +pub(super) struct PythonRunFailureTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [(&'static str, &'static str)], + pub(super) config: serde_json::Value, + pub(super) expected_error: &'static str, +} + pub(super) struct PythonParseFailureTestCase { pub(super) description: &'static str, pub(super) files: &'static [(&'static str, &'static str)], @@ -74,6 +81,15 @@ pub(super) struct PythonScalingTestCase { pub(super) expected_max_duration: std::time::Duration, } +pub(super) struct PurityScalingTestCase { + pub(super) description: &'static str, + pub(super) roots: usize, + pub(super) chains: usize, + pub(super) depth: usize, + pub(super) expected_faults: usize, + pub(super) expected_max_duration: std::time::Duration, +} + pub(super) struct UnitScalingTestCase { pub(super) description: &'static str, pub(super) units: usize, diff --git a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs index 55bbd8805..62aeb655f 100644 --- a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs +++ b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs @@ -81,6 +81,13 @@ pub fn builtin_options(code: &str) -> Vec { description: "Reviewed SQL test names that may assert only that empty inputs produce \ no rows; listed tests count toward min_tests_per_model.", }], + "SQBRPYTHON022" => vec![BuiltinOption { + kind: BuiltinOptionKind::StringList(&[]), + name: "approved_loader_boundaries", + description: "Fully qualified project functions, such as \ + python.inventory.snapshots._helpers.specs.load_specs, whose calls \ + compile-time purity does not follow.", + }], _ => python_rule_options(code), } } diff --git a/fensu.toml b/fensu.toml index 3f1d83277..f6a70bb98 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2471,7 +2471,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=100", + "crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs=99", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", @@ -3016,3 +3016,7 @@ reason = "Each is a short stack walk over a JSON syntax tree with its own stop a [[dupes.allowlist]] paths = ["crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs"] reason = "Field lookups on unrelated value types in different crates: project configuration takes the first matching field of a built object, while rule headers take the last entry of an authored map so a repeated key overrides earlier ones." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-rules/src/python_rules/_helpers/units/orchestration.rs", "crates/sqlbuild-rules/src/python_rules/_helpers/units/providers.rs"] +reason = "Each is one rule family's entry point: return early when no rule is selected, then run its own layout, module and import checks. Parsing, project modules and import resolution share files_naming, project_modules and rejected_import_faults." diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 1f393a71a..6073b2a6e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -140,7 +140,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [sqb debug](cli/debug.md) (`cli/debug`) - Validate project configuration and test the warehouse connection. - [sqb compile](cli/compile.md) (`cli/compile`) - Compile models into resolved SQL, validate contracts, and write target artifacts - fully offline. - [sqb format](cli/format.md) (`cli/format`) - Apply canonical SQLBuild SQL formatting. -- [sqb test](cli/test.md) (`cli/test`) - Run SQL unit tests and multi-model tests in isolation. +- [sqb test](cli/test.md) (`cli/test`) - Run SQL unit tests, multi-model tests and the project's Python tests in isolation. - [sqb scenario](cli/scenario.md) (`cli/scenario`) - Run end-to-end scenario tests against the warehouse or locally with DuckDB. - [sqb lineage](cli/lineage.md) (`cli/lineage`) - Explore model and column-level dependency graphs from the command line. - [sqb scope](cli/scope.md) (`cli/scope`) - Inspect declaration visibility, usage, placement, and move impact offline. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index a716fb092..e7b91404d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -51,7 +51,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | Flag | Description | |------|-------------| | `--target` | Build against a configured target instead of the active/default target | -| `--no-tests` | Skip SQL unit tests | +| `--no-tests` | Skip SQL unit tests; a build never runs [Python tests](test.md#python-tests) | | `--no-audits` | Skip audits | | `--no-python` | Skip read-side Python tasks and assets (loader-side Python still runs for selected sources) | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 48b1f1d84..e7b3dc61e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -66,6 +66,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--exclude` | Remove resources from the selected scope | | `--no-sql-analysis` | Turn off SQL syntax, binding, type inference and semantic checks (`--no-sql-validation` is an alias) | | `--no-cache` | Compile in full, without reusing earlier compiles or the Rules cache; see [Forcing a fresh compile](#forcing-a-fresh-compile) | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](../concepts/python-nodes/providers.md#secrets) | | `--defer-to` | Resolve unselected model references against another target | | `--json` | Print the full compile report as JSON | | `--manifest` | Also write `target/manifest.json` with project metadata | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md index 7e126a40c..dd9ae1bcf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md @@ -30,6 +30,7 @@ sqb --project-dir dag [flags] |------|-------------| | `--json` | Print the full DAG artifact as JSON to stdout | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](../concepts/python-nodes/providers.md#secrets) | | `--vars` | JSON object of project variable overrides | Without `--json`, the command prints a summary: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md index 4f53bb48a..95697cdcf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/debug.md @@ -24,7 +24,14 @@ The command reports four groups of checks: **Configuration** - Finds and validates `sqlbuild_project.toml`, loads `sqlbuild_local.toml` if present, resolves the adapter, active target, connection, database, and schema. For Snowflake it also shows the warehouse each [command group](../concepts/adapters/snowflake.md#command-group-warehouses) connects with and where that value came from. -**Providers** - Lists the [providers](../concepts/python-nodes/providers.md) the project defines. +**Providers** - Counts the [providers](../concepts/python-nodes/providers.md) whose [secrets](../concepts/python-nodes/providers.md#secrets) are all set, and lists each missing secret as `WARN`. When every secret is set it prints one line. `--verbose` lists every provider and secret. + +```text +Providers: + providers: 2 of 3 ready [WARN missing secrets] + slack_alerts: SLACK_WEBHOOK_URL [WARN missing] + more: sqb debug --verbose lists every provider [SKIP] +``` **Connection** - Displays connection settings as `sqb debug` itself uses them, with the `query` warehouse group applied (secrets are masked), attempts to connect to the warehouse, and runs `SELECT 1` to verify query execution. @@ -33,7 +40,8 @@ The command reports four groups of checks: | Flag | Description | |------|-------------| | `--no-connection` | Skip the connection and query tests. Useful for validating config without warehouse access. | -| `--json` | Output results as JSON instead of formatted text. | +| `--json` | Output results as JSON instead of formatted text, with `missing_secrets` counting the unset provider secrets. | +| `--verbose` | List every provider and secret setting, set or missing. | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/adapters/snowflake.md#command-group-warehouses) and the connection warehouse | ## Example @@ -81,7 +89,7 @@ Connection: ## Exit codes -Returns `0` when all checks pass, `1` when any check fails (e.g. connection refused, invalid config). +Returns `0` when all checks pass, `1` when any check fails (e.g. connection refused, invalid config). A missing provider secret is a warning and doesn't fail the command. A failed run is recorded in [Recall](../concepts/observability/recall.md) with the first failed check as its cause: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md index 1c09acc5b..297fc5c28 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -27,6 +27,7 @@ sqb format [flags] | `--fix` | Also apply compiler-verified, meaning-preserving Rule fixes before layout formatting | | `--json` | Print structured results | | `--no-cache` | Format every file instead of skipping files a previous run found clean | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](../concepts/python-nodes/providers.md#secrets) | | `--select`, `-s` | Format selected models | | `--select-file` | Read selectors from a file | | `--exclude` | Exclude models from the selected scope | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md index 710838a91..8762e344d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/lineage.md @@ -44,6 +44,7 @@ Exactly one of a positional target or `--select` is required. | `--json` | Shorthand for `--format json`. | | `--mode` | Column lineage analysis mode: `rich` (default) or `fast`. | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias). | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](../concepts/python-nodes/providers.md#secrets) | | `--select`, `-s` | Select resources using the same selector syntax as `sqb compile`, `sqb plan` and `sqb build`, for example `path:models/marts`, `tag:finance` or `stg_*`. | | `--exclude` | Exclude resources from the selection. | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md index e9e56aa8f..448a5c8a4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/location.md @@ -34,6 +34,7 @@ sqb location health --server https://control.example.com --location-id "$LOCATIO | --- | --- | --- | | `healthy` | An instance checked in within 90 seconds and its last load published | `0` | | `degraded` | The warehouse connection check after the last load failed | `0` | +| `missing_secrets` | A provider [secret](../concepts/python-nodes/providers.md#secrets) is unset in the location's environment, so runs that use it fail | `1` | | `falling_back` | The last load failed, so runs use the previous snapshot | `1` | | `broken` | No instance has checked in for 90 seconds | `1` | @@ -51,3 +52,6 @@ healthcheck: A `falling_back` location still runs scheduled jobs from its last good snapshot. Restarting it does not fix the failed load; fix the project and deploy it again. + +A `missing_secrets` location publishes and runs jobs whose nodes have their secrets. Set the +missing variables in its environment and restart it; `sqb debug` in the location lists them. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md index 590199fdb..fcb239757 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rules.md @@ -34,7 +34,7 @@ Lifecycle progress is written to stderr so stdout remains valid JSON. - `run`: compile and evaluate only one exact code or family prefix. - `skills`: install guidance derived from configured Rules; `--check` verifies freshness without writing. -`run` accepts normal SQLBuild model selectors. Native built-ins execute before custom Python Rules. +`run` accepts normal SQLBuild model selectors and `--allow-missing-secrets`, which runs without provider [secrets](../concepts/python-nodes/providers.md#secrets). Native built-ins execute before custom Python Rules. ## Exit codes diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md index 7106ff1ba..98c66c22f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scope.md @@ -85,6 +85,7 @@ These identities are for inspection. SQL authored inside the owning model contin | `--kind KIND` | Keep `macro`, `enum`, or `constant` declarations. Repeat to include multiple kinds. | | `--match GLOB` | Match declaration names or qualified identities with a deterministic, case-sensitive glob. | | `--used-only` | Keep declarations used by the target. | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](../concepts/python-nodes/providers.md#secrets) | | `--include-nearby` | Include bounded nearby declarations that are unavailable to the target. | | `--nearby-depth N` | Set filesystem proximity for nearby descendant and sibling discovery (default: `1`). | | `--dependency-depth N` | Expand dependencies from declarations in the used section by `N` edges (default: `0`). | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md index 70ab7391c..59804b6c8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/test.md @@ -2,7 +2,7 @@ # sqb test -> Run SQL unit tests and multi-model tests in isolation. +> Run SQL unit tests, multi-model tests and the project's Python tests in isolation. Online: https://sqlbuild.com/docs/cli/test/ @@ -10,11 +10,13 @@ Online: https://sqlbuild.com/docs/cli/test/ - Usage - Flags +- Python tests - Run only affected tests - Examples Runs SQL unit tests, independently reported parameterized cases, and multi-model tests without -building models. Useful for validating test logic independently. +building models, then the project's [Python tests](#python-tests) with pytest. Useful for +validating test logic independently. ## Usage @@ -36,6 +38,8 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--test-timeout ` | Time limit for each test in this run, such as `30s` or `2m`, or `disabled`; overrides `TEST(timeout ...)`, the target and `[settings] test_timeout` | | `--since ` | Run only the tests affected by changes since the merge base of `` and `HEAD` (see [Run only affected tests](#run-only-affected-tests)); `@base` reads `[git] base_ref` | | `--explain` | With `--since`, print why each selected test was selected | +| `--sql` | Run only SQL tests | +| `--python` | Run only [Python tests](#python-tests); cannot be combined with `--inspect` or `--since` | Test names are globally unique, so a bare test name selects exactly that test. `test:` is the explicit form and accepts the same glob patterns as names, such as `test:orders_*`. A bare pattern @@ -60,6 +64,40 @@ counted under `TIMEOUT=` and excluded from `FAIL=` in the summary line output, `summary.fail_count` still includes timed-out tests, and `summary.timeout_count` reports how many of them timed out. A timeout still fails the run: `sqb test` exits non-zero. +## Python tests + +Python tests are pytest modules named `test_*.py` under `tests/python/` and `tests/providers/`. Each +sits in the folder that mirrors the code it tests, so the tests of +`python/orders/storefront_import/node.py` live in `tests/python/orders/storefront_import/`. + +| Command | SQL unit tests | Python tests | +|---------|----------------|--------------| +| `sqb test` | Yes | Yes, when any exist; needs pytest installed | +| `sqb test --sql` | Yes | No | +| `sqb test --python` | No | Yes | +| `sqb test --since ` | The affected tests | No | +| `sqb build` | Yes | Never | + +pytest runs from the project directory with the same Python as SQLBuild and with +`--import-mode=importlib`, so tests import project code as `python.orders...` or `providers...`. +Its output appears after the SQL test summary, followed by `Python tests passed` or +`Python tests failed`. `sqb test` exits non-zero when either kind fails. + +With `--json`, pytest writes to stderr so stdout stays one JSON document. When Python tests ran, the +document has a `python_tests` object with `status` (`success` or `failed`), pytest's `exit_code` +and the test `files`, and the top-level `status` is `failed` when either kind failed. + +Selecting a Python node, such as `sqb test storefront_import`, runs the Python tests in that node's +mirrored folder. Selecting only models or unit tests runs no Python tests. When Python tests exist +but pytest is not installed, `sqb test` fails before running anything; install it, or run +`sqb test --sql`. + +[Ad-hoc runs](server.md#run-ad-hoc-commands) on a server run SQL tests only, and reject +`--python`. + +[Python Rules](../concepts/rules/python-rules.md#testing-python) check where Python tests sit and +how they are written. + ## Run only affected tests `--since ` runs only the tests a change can affect, which keeps CI on branch pushes from diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index da2a6a9d3..0f38b9505 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -44,8 +44,12 @@ sqb build nightly_orders_job # build what the job builds, locally - Subfolders are allowed, such as `orchestration/jobs/finance/revenue_job.toml`. - A name must end with its folder's suffix. Names are unique across the whole project, with models, sources and every other resource. -- A Python module whose file name starts with `_` is a private helper and declares nothing. +- A Python module whose file name starts with `_`, or that sits in a folder starting with `_` under + `orchestration/calendars/` or `orchestration/sensors/` (such as `sensors/inventory/_helpers/`), is + private support and declares nothing. Declaring a calendar or sensor there fails compilation. - Any other file under `orchestration/`, or a folder not in the table, is a `D017` error. +- The [Python Rules](rules/python-rules.md#laying-out-sensors-and-calendars) add a + domain-folder layout for sensor and calendar modules. ## Declare a job @@ -138,7 +142,7 @@ another system. The code location evaluates it when it publishes and again befor out. ```python -# orchestration/calendars/fulfillment_calendar.py +# orchestration/calendars/fulfillment/cutoffs.py from sqlbuild.orchestration import calendar @calendar(name="fulfillment_calendar", description="Upcoming fulfillment times", horizon="1d") @@ -159,7 +163,7 @@ returns timezone-aware datetimes inside it. Other parameters name A sensor starts a job or job group when something happens, named by `run`. ```python -# orchestration/sensors/on_new_orders_sensor.py +# orchestration/sensors/orders/new_orders.py from sqlbuild.orchestration import sensor, source_new_data on_new_orders = sensor( diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index db0ae5c2e..15338d537 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -516,6 +516,7 @@ delta_compile = true | `test_timeout` | `30s` | Time limit for each SQL unit test, or `"disabled"` | | `scenario_timeout` | none | Time limit for each scenario, or `"disabled"` | | `delta_compile` | `true` | After an edit to model SQL or SQL test files only, compile just the changed files against the previous compile. See [Compiling after an edit](../cli/compile.md#compiling-after-an-edit). | +| `allow_missing_secrets` | `false` | Let `compile`, `rules run`, `format`, `dag`, `lineage` and `scope` run without provider secrets. `sqlbuild_local.toml` only. See [Secrets](python-nodes/providers.md#secrets). | ### Unit test and scenario limits @@ -769,10 +770,11 @@ Code that extends SQLBuild itself keeps its own root: Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create an unofficial project structure. -Keep repository pytest tests outside the SQLBuild project's `tests/` directory, which is reserved for -SQLBuild SQL tests and scenarios. Orchestrator code such as Dagster or Rivers definitions lives outside -the SQLBuild project directory and points at it; see [Dagster](../integrations/dagster.md) and -[Rivers](../integrations/rivers.md). +The project's `tests/` directory holds SQL tests and scenarios, and pytest tests of project Python +under `tests/python/` and `tests/providers/`, which `sqb test` runs; see +[Python tests](../cli/test.md#python-tests). Orchestrator code such as Dagster or Rivers definitions +lives outside the SQLBuild project directory and points at it; see +[Dagster](../integrations/dagster.md) and [Rivers](../integrations/rivers.md). ## sqlbuild_local.toml diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md index c9be71b78..827a3b736 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -15,6 +15,7 @@ Online: https://sqlbuild.com/docs/concepts/python-nodes/providers/ - Using providers via context - Lifecycle - Configuration from environment variables +- Secrets - Discovery rules - Plan output - Providers in the web UI @@ -150,8 +151,8 @@ Both approaches (parameter injection and `ctx.providers`) are equivalent. Parame Providers follow a lazy setup, reverse-teardown lifecycle scoped to the command invocation: -1. **Discovery** - on compile, SQLBuild discovers all `Provider` subclasses under `providers/` and validates their settings (from environment variables or field defaults). -2. **Lazy setup** - `setup(ctx)` is called the first time a provider is accessed during a build, not at startup. Providers that are never used are never set up. +1. **Discovery** - on compile, SQLBuild discovers all `Provider` subclasses under `providers/` and validates their settings (from environment variables or field defaults). Missing [secrets](#secrets) follow the command's rules. +2. **Lazy setup** - the first time a provider is accessed during a build, SQLBuild reads its settings and calls `setup(ctx)`. Providers that are never used are never set up. 3. **Teardown** - after the command completes, `teardown()` is called on all providers that were set up, in reverse setup order. Teardown runs even if the build failed. ```python @@ -183,13 +184,61 @@ class SlackNotifier(Provider): See the [pydantic-settings documentation](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) for advanced configuration like custom env prefixes, `.env` file support, and nested settings. +## Secrets + +Declare a secret as a field typed `SecretStr`. Laptops and CI jobs can then compile without production secrets, and builds check only the secrets their selection needs. + +```python +from pydantic import SecretStr + +class SlackAlerts(Provider): + """Posts alerts to Slack.""" + + channel: str = "#data-alerts" + slack_webhook_url: SecretStr # reads SLACK_WEBHOOK_URL +``` + +| Commands | When a required secret is unset | +| --- | --- | +| `compile`, `rules run`, `format`, `dag`, `lineage`, `scope` | Error `D012`, unless missing secrets are allowed | +| `build`, `load`, `check`, and runs on `sqb server` workers | Error before anything runs, only when a selected node or a sink uses the provider | +| `sqb debug` | Lists the missing secrets as warnings | +| `sqb location health` | Reports `missing_secrets` and exits `1` | + +```text +$ sqb compile +error[D012]: provider 'slack_alerts' (providers/slack_alerts/provider.py) is missing secret SLACK_WEBHOOK_URL + = help: set SLACK_WEBHOOK_URL, or pass --allow-missing-secrets where secrets aren't available +``` + +Allow missing secrets for the static commands in any of these ways: + +| Where | Setting | +| --- | --- | +| Command line | `--allow-missing-secrets` | +| Environment | `SQLBUILD_ALLOW_MISSING_SECRETS=1` | +| `sqlbuild_local.toml` | `allow_missing_secrets = true` under `[settings]` | + +The command then lists the providers it skipped on stderr, such as +`1 provider has missing secrets (allowed): slack_alerts`. Settings that aren't secrets are still +validated. Execution commands accept none of these. + +```text +$ sqb build --select python/orders +error[D012]: selected nodes need secrets that are missing: + slack_alerts SLACK_WEBHOOK_URL (partner_export_alert) +nothing was run +``` + +Run [`sqb debug`](../../cli/debug.md) to see which secrets this machine has. + ## Discovery rules - Provider classes are discovered from `.py` files under `providers/` recursively - Files named `__init__.py` or starting with `_` are skipped - Each concrete (non-abstract) subclass of `Provider` is registered - Provider names must be globally unique across project resources -- Settings are validated at discovery time. Missing required fields (without environment variables set) raise a discovery error immediately, not at runtime +- Settings are validated at discovery time. A missing or invalid setting that isn't a secret is a `D012` error; missing secrets follow [Secrets](#secrets) ## Plan output @@ -210,7 +259,7 @@ provider with its class and docstring. A provider's page shows: | Tab | Shows | | --- | --- | -| Settings | Each setting's type, environment variable and whether it is required; secret settings are marked | +| Settings | Each setting's type, environment variable and whether it is required; secret settings are marked, and flagged **missing** when the location's environment doesn't set them | | Used by | The loaders, tasks, hooks and other code that take the provider, and the jobs that run that code | Setting values, defaults included, are never published. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md index af117ddc3..81dae29d9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/custom-rules/testing-and-determinism.md @@ -60,7 +60,9 @@ passes, the case fails with `custom rule is not deterministic`. The usual module-level state, such as a set of already-seen models, that carries over from one subject to the next. Compute findings from the subject and `ctx` alone. -Repository pytest files remain outside the SQLBuild project's SQL `tests/` directory. +Keep Rule tests under the project's `tests/` directory, outside `tests/python/` and +`tests/providers/`, such as in `tests/rules/`. `sqb test` runs only those two folders, so run Rule +tests with pytest. ## Parallel evaluation diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md index 78938aa8f..eb800afc8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/python-rules.md @@ -13,6 +13,9 @@ Online: https://sqlbuild.com/docs/concepts/rules/python-rules/ - Laying out `python/` - Writing nodes as units - Laying out providers +- Laying out sensors and calendars +- Testing Python +- Keeping compile-time Python pure - Main modules and entry modules - Unreachable definitions - Rule options @@ -62,10 +65,12 @@ sqb rules run SQBRPYTHON The Python Rules read every supported project Python file: the runtime package under `python/` and the folders that extend SQLBuild, such as `hooks/python/`, `functions/python/`, `providers/`, `rules/` and `macros/`. See [Project Python](../project-configuration.md#project-python) -for the full list. Files under `tests/` are not checked. +for the full list. Under `tests/`, only the [Python tests](#testing-python) in `tests/python/` and +`tests/providers/` are checked, and only by the Rules marked `tests/`. Rules marked `python/` in the [reference](#rule-reference) check only the runtime package, because -they describe how its packages are organised. The others check every file. +they describe how its packages are organised. Rules marked `tests/` check only Python tests. The +others check every file except Python tests. ## Laying out `python/` @@ -189,6 +194,136 @@ A capability folder may also hold `types.py`, `constants.py`, `exceptions.py`, ` `Provider` through an alias or a base class in `_classes/`. A class whose base comes from another package is never reported as not being a provider. +## Laying out sensors and calendars + +`SQBRPYTHON016` to `SQBRPYTHON020` keep each sensor and calendar module in a domain folder, named +for what it checks, with its private support beside it: + +```text +orchestration/ + sensors/ + inventory/ + stock_feed.py # one sensor + _constants.py + _helpers/ + feed_age.py + calendars/ + finance/ + month_end.py # one calendar +``` + +A domain folder's private support is `_models.py`, `_types.py`, `_constants.py`, +`_exceptions.py` and `_helpers/`. These are reported: + +| Code | Finding | Fix | +|---|---|---| +| `SQBRPYTHON016` | A module directly in `sensors/` or `calendars/`, nested below its domain folder, or named `main.py`, `definitions.py`, `sensors.py`, `calendars.py` or `processing.py` | Move it to `//.py` | +| `SQBRPYTHON017` | Other private files or folders, private support outside a domain folder, or an import of another domain's private support | Use the support files or `_helpers/`; share code through `python/utils/` or a provider | +| `SQBRPYTHON018` | A sensor or calendar module holding a function, a class, a public constant or another statement besides its declaration | Move it to `_helpers/`, `_models.py` or `_constants.py` | +| `SQBRPYTHON019` | A sensor or calendar importing `python/` outside `utils/`, or code outside `orchestration/` importing it | Read node results through the sensor's context; share code through `python/utils/` or a provider | +| `SQBRPYTHON020` | A sensor or calendar function with a docstring | Keep the text in `description=` only | + +Providers importing `orchestration/` are reported by `SQBRPYTHON015` instead. + +## Testing Python + +`sqb test` runs the pytest modules under `tests/python/` and `tests/providers/`; see +[Python tests](../../cli/test.md#python-tests). `SQBRPYTHON021` keeps each test beside a mirror of the +code it tests, and `SQBRPYTHON701` to `SQBRPYTHON724` keep tests table-driven: + +```text +python/ + orders/ + storefront_import/ + node.py + _helpers/ + row_mapping.py +tests/ + python/ + orders/ + storefront_import/ + _helpers/ + _test_types.py # the test-case dataclasses + test_row_mapping.py # tests row_mapping.py +``` + +```python +# tests/python/orders/storefront_import/_helpers/_test_types.py +from dataclasses import dataclass + +@dataclass(frozen=True) +class RowMappingTestCase: + description: str + status: str + expected_label: str +``` + +```python +# tests/python/orders/storefront_import/_helpers/test_row_mapping.py +import pytest + +from python.orders.storefront_import._helpers.row_mapping import status_label +from tests.python.orders.storefront_import._helpers._test_types import RowMappingTestCase + +@pytest.mark.parametrize( + "test_case", + [ + RowMappingTestCase(description="shipped orders", status="S", expected_label="shipped"), + RowMappingTestCase(description="open orders", status="O", expected_label="open"), + ], + ids=lambda case: case.description, +) +def test_given_a_status_when_mapping_then_returns_its_label(test_case: RowMappingTestCase) -> None: + assert status_label(test_case.status) == test_case.expected_label +``` + +| Code | Finding | Fix | +|---|---|---| +| `SQBRPYTHON021` | A test module whose folder does not mirror one under `python/` or `providers/`, or a `test_.py` with no `.py` or `/` package beside the code it tests | Move the test to the mirrored folder, or rename it for the module it tests | +| `SQBRPYTHON701` to `SQBRPYTHON711` | A test folder or module whose layout, imports, helpers, conditionals or test-case dataclasses break the pattern above | Follow the pattern above | +| `SQBRPYTHON712` to `SQBRPYTHON724` | A test function or `pytest.mark.parametrize` that is not named `test_given__when__then_`, or does not take dataclass cases through `test_case` with ids from each `description` | Follow the pattern above | + +## Keeping compile-time Python pure + +`sqb compile` imports project Python with no warehouse connection, so the code that runs then must +not reach outside the machine. `SQBRPYTHON022` checks every call made at module scope, in a class +body, or directly in a `@factory` body, in every file it reads, and follows the project functions +those calls reach. A call that reaches a subprocess, an HTTP request, a socket, a database or +warehouse driver, or a cloud or SSH client is reported at the call where the walk started: + +```python +# python/orders/storefront_import/constants.py +import requests + +STOREFRONT_REGIONS = requests.get("https://example.com/regions").json() # reported +``` + +```text +error[SQBRPYTHON022]: Python that runs while the project compiles must not reach a subprocess, the network, a database or a remote client: calls requests.get + --> python/orders/storefront_import/constants.py:3:22 +``` + +When the external call sits in a project function, the message names it and its line instead, such +as `reaches requests.get at python/orders/storefront_import/_helpers/regions.py:8`. + +Calls inside node bodies, `main()`, the nodes a factory generates, and provider methods run when +the node runs, so they are not checked. Only calls whose target is known from the imports are +followed, so a call through a variable or a method of an object is never reported. + +Keep the data a factory loops over in its unit's `constants.py` or `models.py`. When a factory must +discover what to generate, put that discovery behind one function and approve it. The walk does not +enter an approved function, and an approved factory's body is not checked: + +```toml +[rules.rule_options.SQBRPYTHON022] +approved_loader_boundaries = [ + "python.inventory.warehouse_snapshots._helpers.spec_loading.load_warehouse_specs", +] +``` + +Each entry is the fully qualified name of a project function. A glob fails configuration with +`C000`, and an entry that names a module fails the run. + ## Main modules and entry modules Some Rules apply only to a domain's main modules: files under a `main/` folder, and a unit's @@ -219,7 +354,7 @@ reason = "Carriers are looked up by name from the shipment's carrier code." ## Rule options -Configure the size limits by exact code: +Configure the size limits and approved boundaries by exact code: ```toml [rules.rule_options.SQBRPYTHON504] @@ -239,8 +374,10 @@ max_arguments = 6 | `SQBRPYTHON611` | `max_role_depth` | 1 | How many package levels may sit below a role package. | | `SQBRPYTHON615` | `min_shared_domain_prefix_packages` | 2 | Fewest sibling domains sharing a name prefix that are reported; 0 turns the rule off. | | `SQBRPYTHON627` | `max_file_lines` | 2000 | Most lines one Python file may hold. | +| `SQBRPYTHON022` | `approved_loader_boundaries` | none | Project functions the [compile-time purity](#keeping-compile-time-python-pure) walk does not enter. | -Each option takes a whole number of 0 or more. Unknown option names, text values and negative +`approved_loader_boundaries` takes a list of fully qualified function names; every other option +takes a whole number of 0 or more. Unknown option names, values of the wrong type and negative numbers fail configuration with `C000`. ## Rule reference @@ -278,6 +415,52 @@ remediation. | `SQBRPYTHON014` | provider-module-shape | All | yes | `provider.py` holds imports, private constants and one public `Provider` subclass | | `SQBRPYTHON015` | provider-import-boundaries | All | yes | A provider's `_helpers/` and `_classes/` are private to it; providers import only `python/utils/` from `python/` and nothing from `orchestration/` | +### Sensors and calendars + +| Code | Name | Files | Default | Checks | +|---|---|---|---|---| +| `SQBRPYTHON016` | orchestration-module-paths | All | yes | Sensor and calendar modules sit at `orchestration///.py` with a specific name | +| `SQBRPYTHON017` | orchestration-private-support | All | yes | A domain folder's private support is its support files and `_helpers/`, imported only from that folder | +| `SQBRPYTHON018` | orchestration-module-surface | All | yes | A sensor or calendar module holds imports, private constants and its one declaration | +| `SQBRPYTHON019` | orchestration-import-direction | All | yes | Sensors and calendars import only providers, `python/utils/` and their own support; nothing outside `orchestration/` imports them | +| `SQBRPYTHON020` | orchestration-no-docstring | All | yes | Sensor and calendar functions have no docstring | + +### Compile-time purity + +| Code | Name | Files | Default | Checks | +|---|---|---|---|---| +| `SQBRPYTHON022` | compile-time-purity | All | yes | Module scope, class bodies and `@factory` bodies reach no subprocess, network, database or remote client | + +### Python tests + +| Code | Name | Files | Default | Checks | +|---|---|---|---|---| +| `SQBRPYTHON021` | python-test-placement | `tests/` | yes | Python tests must mirror an existing module under `python/` or `providers/` | +| `SQBRPYTHON701` | test-init-module-empty | `tests/` | yes | Test package `__init__.py` files must be empty or docstring-only | +| `SQBRPYTHON702` | test-absolute-imports | `tests/` | yes | Tests must use absolute imports | +| `SQBRPYTHON703` | test-no-top-level-helpers | `tests/` | yes | Test modules may contain only tests, imports, and declarations | +| `SQBRPYTHON704` | test-no-conditionals | `tests/` | yes | Tests and local test helpers must not contain conditional control flow | +| `SQBRPYTHON705` | test-private-constant-order | `tests/` | yes | Private test constants must appear before test functions | +| `SQBRPYTHON706` | test-no-complex-comprehensions | `tests/` | yes | Nested or multi-generator comprehensions hide control flow and data shapes | +| `SQBRPYTHON707` | test-case-description | `tests/` | yes | Test-case dataclasses must define a `description` field | +| `SQBRPYTHON708` | test-case-expected-field | `tests/` | yes | Test-case dataclasses must define at least one `expected_` field | +| `SQBRPYTHON709` | test-local-types-import | `tests/` | yes | Tests must import test-case types from their local `_test_types.py` | +| `SQBRPYTHON710` | test-local-types-file | `tests/` | yes | Test directories must provide a local `_test_types.py` | +| `SQBRPYTHON711` | test-file-name | `tests/` | yes | Test modules must use a `test_` filename | +| `SQBRPYTHON712` | test-function-name | `tests/` | yes | Test functions must use `test_given__when__then_` | +| `SQBRPYTHON713` | test-dataclass-parametrize | `tests/` | yes | Tests must use dataclass-backed pytest parameterization | +| `SQBRPYTHON714` | test-accepts-test-case | `tests/` | yes | Parametrized tests must accept a `test_case` argument | +| `SQBRPYTHON715` | test-case-annotation | `tests/` | yes | `test_case` parameters must use a local test-case dataclass annotation | +| `SQBRPYTHON716` | test-expected-field-assertion | `tests/` | yes | Tests must assert against an `expected_` field from `test_case` | +| `SQBRPYTHON717` | test-parametrize-arguments | `tests/` | yes | Pytest parametrize decorators must define parameter names and values | +| `SQBRPYTHON718` | test-parametrize-test-case | `tests/` | yes | Pytest parametrize must expose cases through the `test_case` parameter | +| `SQBRPYTHON719` | test-parametrize-ids | `tests/` | yes | Pytest parametrize decorators must define readable case ids | +| `SQBRPYTHON720` | test-inline-parametrize-values | `tests/` | yes | Pytest parametrize values must be a visible list, tuple, or local comprehension | +| `SQBRPYTHON721` | test-nonempty-parametrize-values | `tests/` | yes | Pytest parametrize case sequences must not be empty | +| `SQBRPYTHON722` | test-no-dict-cases | `tests/` | yes | Pytest cases must use typed dataclasses instead of dictionaries | +| `SQBRPYTHON723` | test-local-case-constructors | `tests/` | yes | Pytest cases must construct dataclasses from the local `_test_types.py` | +| `SQBRPYTHON724` | test-description-ids | `tests/` | yes | Pytest case ids must come from each test case `description` | + ### Type annotations | Code | Name | Files | Default | Checks | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index 067ba1cb3..77f54892c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -536,6 +536,9 @@ sqb test sqb test --select stg_orders ``` +`sqb test` also runs the project's pytest modules under `tests/python/` and `tests/providers/`; +`sqb build` never runs them. See [Python tests](../cli/test.md#python-tests). + ## More test features [Testing reference](testing-reference.md) covers macro, UDF and table function tests, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index cc6d6d1f4..c40b89674 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -5175,7 +5175,9 @@ def discover_hook_functions( def discover_materialization_files( *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] | None = None ) -> tuple[DiscoveredMaterializationFile, ...]: ... -def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, ...]: ... +def discover_provider_classes( + *, project_dir: Path, secret_policy: str = ... +) -> tuple[DiscoveredProvider, ...]: ... def discover_event_exporter_declarations( *, project_dir: Path ) -> tuple[DiscoveredEventExporterDeclaration, ...]: ... @@ -7100,6 +7102,8 @@ class TestCommandRequest(NativeRecord): test_timeout: str | None since: str | None explain: bool + sql_only: bool + python_only: bool def __init__( self, project_dir: Path | None = None, @@ -7117,6 +7121,8 @@ class TestCommandRequest(NativeRecord): test_timeout: str | None = None, since: str | None = None, explain: bool = False, + sql_only: bool = False, + python_only: bool = False, ) -> None: ... class RefactorCommandRequest(NativeRecord): @@ -12023,6 +12029,7 @@ class SettingsConfig(NativeRecord): scenario_timeout: str | None scenario_timeout_seconds: int | None delta_compile: bool + allow_missing_secrets: bool def __init__( self, sql_analysis: bool = ..., @@ -12042,6 +12049,7 @@ class SettingsConfig(NativeRecord): scenario_timeout: str | None = ..., scenario_timeout_seconds: int | None = ..., delta_compile: bool = ..., + allow_missing_secrets: bool = ..., ) -> None: ... @property def sql_validation(self) -> bool: ... diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py index 7813ad8e6..f8b157d1d 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py @@ -362,11 +362,17 @@ def _exec_project_module( return module +def _deepest_modules_first() -> list[tuple[str, ModuleType]]: + """Loaded modules, children first: a namespace package reads its parent's path.""" + + return sorted(sys.modules.items(), key=lambda item: item[0].count("."), reverse=True) + + def _evict_stale_project_package_modules(*, root_module: str, project_dir: Path) -> None: root_path: Path = (project_dir / root_module).resolve() module_name: str module: ModuleType - for module_name, module in tuple(sys.modules.items()): + for module_name, module in _deepest_modules_first(): if module_name != root_module and not module_name.startswith(f"{root_module}."): continue module_file: object = getattr(module, "__file__", None) diff --git a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py index 197605d5e..773d40dce 100644 --- a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py +++ b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py @@ -52,7 +52,9 @@ def discover_runtime_extensions(*, project_dir: Path) -> DiscoveredRuntimeExtens ) if not declarations and not output_declarations: return DiscoveredRuntimeExtensions() - providers: tuple[DiscoveredProvider, ...] = discover_provider_classes(project_dir=project_dir) + providers: tuple[DiscoveredProvider, ...] = discover_provider_classes( + project_dir=project_dir, secret_policy="defer" + ) exporters: tuple[DiscoveredEventExporter, ...] = bind_event_exporter_declarations( declarations=declarations, project_dir=project_dir, diff --git a/src/sqlbuild/provider/_helpers/session.py b/src/sqlbuild/provider/_helpers/session.py index 732740b55..b61f991da 100644 --- a/src/sqlbuild/provider/_helpers/session.py +++ b/src/sqlbuild/provider/_helpers/session.py @@ -9,7 +9,6 @@ if TYPE_CHECKING: from sqlbuild._native import EventExporterCommandScope - from sqlbuild.provider.classes.provider import Provider def build_provider_session( @@ -18,20 +17,23 @@ def build_provider_session( setup_context: object | None = None, allow_shared: bool = True, ) -> ProviderSession: - """Build a runtime provider session with fresh provider instances.""" + """Build a runtime provider session that reads each provider's settings on first use.""" if allow_shared: from sqlbuild._native import current_event_exporter_command_scope scope: EventExporterCommandScope | None = current_event_exporter_command_scope() if scope is not None and scope.provider_session is not None: - expected_names: tuple[str, ...] = tuple( + expected_names: frozenset[str] = frozenset( provider.name for provider in discovered_providers ) - if scope.provider_session.keys() == expected_names: + if frozenset(scope.provider_session.keys()) == expected_names: return scope.provider_session - providers: dict[str, Provider] = { - discovered_provider.name: discovered_provider.provider_class() - for discovered_provider in discovered_providers - } - return ProviderSession(providers, setup_context=setup_context) + return ProviderSession( + {}, + setup_context=setup_context, + deferred={ + discovered_provider.name: discovered_provider.provider_class + for discovered_provider in discovered_providers + }, + ) diff --git a/src/sqlbuild/provider/classes/session.py b/src/sqlbuild/provider/classes/session.py index 07e836710..81462cb14 100644 --- a/src/sqlbuild/provider/classes/session.py +++ b/src/sqlbuild/provider/classes/session.py @@ -25,8 +25,14 @@ def __init__( providers: Mapping[str, Provider] | Iterable[tuple[str, Provider]], *, setup_context: object | None = None, + deferred: Mapping[str, type[Provider]] | None = None, ) -> None: self._providers: dict[str, Provider] = dict(providers) + self._deferred: dict[str, type[Provider]] = { + name: provider_class + for name, provider_class in (deferred or {}).items() + if name not in self._providers + } self._setup_context: object | None = setup_context self._container: ProviderContainer = ProviderContainer(self) self._setup_names: list[str] = [] @@ -41,7 +47,7 @@ def providers(self) -> ProviderContainer: return self._container def __contains__(self, name: object) -> bool: - return isinstance(name, str) and name in self._providers + return isinstance(name, str) and (name in self._providers or name in self._deferred) def __enter__(self) -> ProviderSession: return self @@ -56,12 +62,14 @@ def __exit__(self, exc_type: object, exc: object, traceback: object) -> bool: return False def keys(self) -> tuple[str, ...]: - return tuple(self._providers) + return (*self._providers, *self._deferred) def get(self, name: str) -> Provider: with self._lifecycle_lock: if self._is_closed: raise ProviderLookupError(f"Provider session is closed; cannot access '{name}'") + if name in self._deferred: + self._providers[name] = self._instantiate(name=name) provider: Provider | None = self._providers.get(name) if provider is None: available: str = ", ".join(self.keys()) or "none" @@ -111,6 +119,24 @@ def close(self, *, force: bool = False) -> None: if errors: raise ProviderTeardownError("\n".join(errors)) + def _instantiate(self, *, name: str) -> Provider: + """Read a deferred provider's settings from the environment on its first use.""" + + from pydantic import ValidationError + + provider_class: type[Provider] = self._deferred.pop(name) + try: + return provider_class() + except ValidationError as error: + details: list[str] = [] + for item in error.errors(include_input=False): + location: str = ".".join(str(part) for part in item["loc"]) + details.append(f"{location}: {item['msg']}") + raise ProviderSetupError( + f"Provider '{name}' ({provider_class.__name__}) has missing or invalid " + f"settings: {'; '.join(details)}" + ) from error + def _setup_provider(self, *, name: str, provider: Provider) -> None: try: provider.setup(self._setup_context) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py index 1f508ca12..ce2573e7b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/debug/test_debug.py @@ -315,11 +315,7 @@ def test_given_local_default_and_explicit_target_when_debugging_json_then_report DebugCliTestCase( description="reports discovered providers", command=("debug", "--no-connection"), - expected_stdout_fragment=( - "Providers:\n" - " providers: 1 [OK discovered]\n" - " marker_provider: providers/marker.py:MarkerProvider [OK valid settings]\n" - ), + expected_stdout_fragment=("Providers:\n providers: 1 of 1 ready [OK secrets set]\n"), ) ], ids=lambda case: case.description, @@ -369,15 +365,9 @@ class MarkerProvider(Provider): expected_check_tail=[ { "label": "providers", - "message": "1", - "status": "OK", - "status_message": "discovered", - }, - { - "label": "marker_provider", - "message": "providers/marker.py:MarkerProvider", + "message": "1 of 1 ready", "status": "OK", - "status_message": "valid settings", + "status_message": "secrets set", }, ], ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py index a9430b328..d957f3c98 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py @@ -131,3 +131,21 @@ class SlowSinkExitE2ETestCase: description: str command: tuple[str, ...] expected_marker_entries: tuple[str, ...] + + +@dataclass(frozen=True) +class ProviderSecretE2ETestCase: + description: str + command: tuple[str, ...] + env: dict[str, str] + extra_repo_files: dict[str, str] + expected_exit_code: int + expected_output_fragments: tuple[str, ...] + expected_marker_entries: tuple[str, ...] + + +@dataclass(frozen=True) +class SharedSinkProviderE2ETestCase: + description: str + unused_provider_name: str + expected_setups: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py index 2da770b29..3dc164b6c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py @@ -15,6 +15,7 @@ FilteredSinkFailureE2ETestCase, NoExporterCommandE2ETestCase, ReadOnlyProviderE2ETestCase, + SharedSinkProviderE2ETestCase, SlowSinkExitE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -535,3 +536,98 @@ def publish_lifecycle(event: LifecycleEvent, destination: Destination) -> None: tuple(marker_path.read_text(encoding="utf-8").splitlines()) == test_case.expected_marker_entries ) + + +SHARED_PROVIDER: str = dedent( + """ + import os + from pathlib import Path + + from sqlbuild.providers import Provider + + + class {class_name}(Provider): + '''Test provider {class_name}.''' + + def setup(self, ctx): + del ctx + with Path(os.environ["SETUP_LOG_PATH"]).open("a", encoding="utf-8") as log: + log.write("{name}\\n") + """ +).strip() + + +@pytest.mark.parametrize( + "test_case", + ( + SharedSinkProviderE2ETestCase( + description="the shared provider sorts after one the sink does not use", + unused_provider_name="a_first", + expected_setups=("z_shared",), + ), + SharedSinkProviderE2ETestCase( + description="the shared provider sorts before one the sink does not use", + unused_provider_name="zz_last", + expected_setups=("z_shared",), + ), + ), + ids=lambda case: case.description, +) +def test_given_sink_and_task_sharing_a_provider_when_building_then_it_is_set_up_once( + test_case: SharedSinkProviderE2ETestCase, + tmp_path: Path, +) -> None: + setup_log: Path = tmp_path / "setups.log" + setup_log.write_text("", encoding="utf-8") + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shared_sink_provider_project", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "shared_sink_provider_project"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "shared_sink_provider_project.duckdb"\n' + ), + f"providers/{test_case.unused_provider_name}.py": SHARED_PROVIDER.format( + class_name="UnusedProvider", name=test_case.unused_provider_name + ) + + "\n", + "providers/z_shared.py": SHARED_PROVIDER.format( + class_name="SharedProvider", name="z_shared" + ) + + "\n", + "sinks/events.py": dedent( + """ + from providers.z_shared import SharedProvider + from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink + + + @lifecycle_event_sink + def export_event(event: LifecycleEvent, shared_provider: SharedProvider) -> None: + del event, shared_provider + """ + ).strip() + + "\n", + "python/tasks/uses_shared.py": dedent( + """ + from providers.z_shared import SharedProvider + from sqlbuild.tasks import task + + + @task + def uses_shared(ctx, shared_provider: SharedProvider): + '''Test task uses_shared.''' + del ctx, shared_provider + """ + ).strip() + + "\n", + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", "uses_shared"), + project_dir=project_dir, + env={"SETUP_LOG_PATH": str(setup_log)}, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert tuple(setup_log.read_text(encoding="utf-8").splitlines()) == test_case.expected_setups diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_provider_secrets.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_provider_secrets.py new file mode 100644 index 000000000..25494f864 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_provider_secrets.py @@ -0,0 +1,411 @@ +"""E2E tests for providers whose secrets are not set.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.providers._test_types import ( + ProviderCommandDiagnosticE2ETestCase, + ProviderSecretE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + run_sqb, +) + +SECRET_VARIABLE: str = "SECRETS_E2E_API_KEY" + +PROJECT_TOML: str = ( + 'name = "provider_secret_project"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "provider_secret_project.duckdb"\n' +) + + +MARKING_NODE: str = ( + dedent( + """ + import os + from pathlib import Path + + from providers.api import ApiProvider + from sqlbuild.{decorator}s import {decorator} + + + @{decorator}{arguments} + def {name}(ctx{parameter}): + '''Test node {name}.''' + with Path(os.environ["MARKER_PATH"]).open("a", encoding="utf-8") as marker: + marker.write("{name}\\n") + """ + ).strip() + + "\n{body}" +) +USES_API: str = ", api_provider: ApiProvider" + + +PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": PROJECT_TOML, + "providers/api.py": dedent( + f""" + from pydantic import Field, SecretStr + from sqlbuild.providers import Provider + + + class ApiProvider(Provider): + '''Test provider ApiProvider.''' + api_key: SecretStr = Field(validation_alias="{SECRET_VARIABLE}") + region: str = "eu" + """ + ).strip() + + "\n", + "python/tasks/uses_api.py": MARKING_NODE.format( + decorator="task", arguments="", name="uses_api", parameter=USES_API, body="" + ), + "python/tasks/no_api.py": MARKING_NODE.format( + decorator="task", arguments="", name="no_api", parameter="", body="" + ), + "python/checks/api_check.py": MARKING_NODE.format( + decorator="check", + arguments="(depends_on=())", + name="api_check", + parameter=USES_API, + body=" return True\n", + ), + "python/loaders/raw_api_events.py": MARKING_NODE.format( + decorator="loader", + arguments="", + name="raw_api_events", + parameter=USES_API, + body=' return [{"event_id": 1}]\n', + ), + "sources/raw.yml": dedent( + """ + sources: + - name: raw_api_events + description: Test source raw_api_events. + managed: true + write_strategy: table + columns: + - name: event_id + type: INTEGER + """ + ).strip() + + "\n", +} + +SECRET_FREE_SINK: dict[str, str] = { + "providers/event_log.py": dedent( + """ + from sqlbuild.providers import Provider + + + class EventLog(Provider): + '''Test provider EventLog.''' + region: str = "eu" + """ + ).strip() + + "\n", + "sinks/events.py": dedent( + """ + from providers.event_log import EventLog + from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink + + + @lifecycle_event_sink + def log_event(event: LifecycleEvent, event_log: EventLog) -> None: + del event, event_log + """ + ).strip() + + "\n", +} +API_SINK: dict[str, str] = { + "sinks/events.py": dedent( + """ + from providers.api import ApiProvider + from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink + + + @lifecycle_event_sink + def export_event(event: LifecycleEvent, api_provider: ApiProvider) -> None: + del event, api_provider + """ + ).strip() + + "\n", +} +POST_BUILD_CHECK: dict[str, str] = { + "python/checks/no_api_check.py": dedent( + """ + import os + from pathlib import Path + + from providers.api import ApiProvider + from python.tasks.no_api import no_api + from sqlbuild.checks import check + + + @check(depends_on=(no_api,)) + def no_api_check(ctx, api_provider: ApiProvider): + '''Test check no_api_check.''' + with Path(os.environ["MARKER_PATH"]).open("a", encoding="utf-8") as marker: + marker.write("no_api_check\\n") + return True + """ + ).strip() + + "\n", +} + +MISSING_ERROR: str = ( + f"provider 'api_provider' (providers/api.py) is missing secret {SECRET_VARIABLE}" +) +ALLOWED_NOTICE: str = "1 provider has missing secrets (allowed): api_provider" +SELECTION_ERROR: str = "selected nodes need secrets that are missing:" +UNBOUND_NOTICE: str = ( + f"sink 'export_event' is not bound: provider 'api_provider' is missing {SECRET_VARIABLE}" +) +BUILD_WITH_CHECKS: tuple[str, ...] = ("--no-color", "build", "--no-tests", "--select") +BUILD: tuple[str, ...] = ("--no-color", "build", "--no-tests", "--no-audits", "--select") + + +@pytest.mark.parametrize( + "test_case", + [ + ProviderSecretE2ETestCase( + description="compile fails and names the variable", + command=("--no-color", "compile"), + env={}, + extra_repo_files={}, + expected_exit_code=1, + expected_output_fragments=(MISSING_ERROR, "--allow-missing-secrets"), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="compile passes with the flag", + command=("--no-color", "compile", "--allow-missing-secrets"), + env={}, + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=(ALLOWED_NOTICE,), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="compile passes with the environment variable", + command=("--no-color", "compile"), + env={"SQLBUILD_ALLOW_MISSING_SECRETS": "true"}, + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=(ALLOWED_NOTICE,), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="compile passes with the local setting", + command=("--no-color", "compile"), + env={}, + extra_repo_files={"sqlbuild_local.toml": "[settings]\nallow_missing_secrets = true\n"}, + expected_exit_code=0, + expected_output_fragments=(ALLOWED_NOTICE,), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="build of a selection that does not use the provider runs", + command=(*BUILD, "no_api"), + env={}, + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=("Completed successfully",), + expected_marker_entries=("no_api",), + ), + ProviderSecretE2ETestCase( + description="build of a selection that uses the provider runs nothing", + command=(*BUILD, "no_api", "uses_api"), + env={}, + extra_repo_files={}, + expected_exit_code=1, + expected_output_fragments=(SELECTION_ERROR, SECRET_VARIABLE, "nothing was run"), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="build of a selection that uses the provider runs once the secret is set", + command=(*BUILD, "uses_api"), + env={SECRET_VARIABLE: "token"}, + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=("Completed successfully",), + expected_marker_entries=("uses_api",), + ), + ProviderSecretE2ETestCase( + description="check that uses the provider runs nothing", + command=("--no-color", "check", "--select", "api_check"), + env={}, + extra_repo_files={}, + expected_exit_code=1, + expected_output_fragments=(SELECTION_ERROR, SECRET_VARIABLE, "nothing was run"), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="load that uses the provider runs nothing", + command=("--no-color", "load", "--select", "raw_api_events"), + env={}, + extra_repo_files={}, + expected_exit_code=1, + expected_output_fragments=(SELECTION_ERROR, SECRET_VARIABLE, "nothing was run"), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="build whose post-build check uses the provider runs nothing", + command=(*BUILD_WITH_CHECKS, "no_api"), + env={}, + extra_repo_files=POST_BUILD_CHECK, + expected_exit_code=1, + expected_output_fragments=(SELECTION_ERROR, SECRET_VARIABLE, "nothing was run"), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="build whose post-build check uses the provider runs it once the secret is set", + command=(*BUILD_WITH_CHECKS, "no_api"), + env={SECRET_VARIABLE: "token"}, + extra_repo_files=POST_BUILD_CHECK, + expected_exit_code=0, + expected_output_fragments=("Completed successfully",), + expected_marker_entries=("no_api", "no_api_check"), + ), + ProviderSecretE2ETestCase( + description="compile with a sink still fails on another provider's missing secret", + command=("--no-color", "compile"), + env={}, + extra_repo_files=SECRET_FREE_SINK, + expected_exit_code=1, + expected_output_fragments=(MISSING_ERROR,), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="compile with a sink and the flag notes the allowed provider", + command=("--no-color", "compile", "--allow-missing-secrets"), + env={}, + extra_repo_files=SECRET_FREE_SINK, + expected_exit_code=0, + expected_output_fragments=(ALLOWED_NOTICE,), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="compile whose sink's provider is missing the secret fails with the coded error", + command=("--no-color", "compile"), + env={}, + extra_repo_files=API_SINK, + expected_exit_code=1, + expected_output_fragments=(MISSING_ERROR,), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="compile with the flag leaves the sink unbound", + command=("--no-color", "compile", "--allow-missing-secrets"), + env={}, + extra_repo_files=API_SINK, + expected_exit_code=0, + expected_output_fragments=(UNBOUND_NOTICE, ALLOWED_NOTICE), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="build whose sink's provider is missing the secret runs nothing", + command=(*BUILD, "no_api"), + env={}, + extra_repo_files=API_SINK, + expected_exit_code=1, + expected_output_fragments=(SELECTION_ERROR, "export_event", "nothing was run"), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="debug with a sink lists the missing secret", + command=("--no-color", "debug", "--no-connection"), + env={}, + extra_repo_files=API_SINK, + expected_exit_code=0, + expected_output_fragments=(f" api_provider: {SECRET_VARIABLE} [WARN missing]\n",), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="debug lists the missing secret", + command=("--no-color", "debug", "--no-connection"), + env={}, + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=( + " providers: 0 of 1 ready [WARN missing secrets]\n" + f" api_provider: {SECRET_VARIABLE} [WARN missing]\n", + ), + expected_marker_entries=(), + ), + ProviderSecretE2ETestCase( + description="verbose debug lists set secrets", + command=("--no-color", "debug", "--no-connection", "--verbose"), + env={SECRET_VARIABLE: "token"}, + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=( + " providers: 1 of 1 ready [OK secrets set]\n" + f" api_provider: {SECRET_VARIABLE} [OK set]\n", + ), + expected_marker_entries=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_provider_secret_unset_when_running_commands_then_only_users_of_it_fail( + test_case: ProviderSecretE2ETestCase, + tmp_path: Path, +) -> None: + marker_path: Path = tmp_path / "marker.log" + marker_path.write_text("", encoding="utf-8") + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="provider_secret_project", + repo_files={**PROJECT_FILES, **test_case.extra_repo_files}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, + project_dir=project_dir, + env={"MARKER_PATH": str(marker_path), **test_case.env}, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_output_fragments: + assert fragment in output, output + marker_entries: tuple[str, ...] = tuple(marker_path.read_text(encoding="utf-8").splitlines()) + assert marker_entries == test_case.expected_marker_entries + + +@pytest.mark.parametrize( + "test_case", + [ + ProviderCommandDiagnosticE2ETestCase( + description="the project file cannot allow missing secrets", + command=("--no-color", "compile"), + expected_error_fragment="cannot set [settings] allow_missing_secrets", + ) + ], + ids=lambda case: case.description, +) +def test_given_project_file_allows_missing_secrets_when_compiling_then_fails( + test_case: ProviderCommandDiagnosticE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="provider_secret_project", + repo_files={ + **PROJECT_FILES, + "sqlbuild_project.toml": PROJECT_TOML + "\n[settings]\nallow_missing_secrets = true\n", + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + assert result.returncode != 0 + assert test_case.expected_error_fragment in result.stdout + result.stderr diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index f04798925..9f8e2bd4a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -613,3 +613,28 @@ class PythonUdfConnectionsE2ETestCase: database: str concurrency: int expected_stdout_fragment: str + + +@dataclass(frozen=True) +class PythonTestsE2ETestCase: + """One `sqb test` run over a project with SQL tests and Python tests.""" + + description: str + command: tuple[str, ...] + extra_repo_files: Mapping[str, str] + expected_exit_code: int + expected_output_fragments: tuple[str, ...] + expected_absent_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PythonTestsJsonE2ETestCase: + """One `sqb test --json` run and the document's overall and Python test status.""" + + description: str + command: tuple[str, ...] + extra_repo_files: Mapping[str, str] + expected_exit_code: int + expected_status: str + expected_python_status: str + expected_python_files: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py new file mode 100644 index 000000000..52396e05f --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_tests.py @@ -0,0 +1,374 @@ +"""E2E tests for the project Python tests `sqb test` runs with pytest.""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + PythonTestsE2ETestCase, + PythonTestsJsonE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + run_sqb, +) + +PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "python_tests_project"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "python_tests_project.duckdb"\n' + ), + "models/orders.sql": ( + 'MODEL (description "Test model orders.");\n\nSELECT status FROM __source("raw_orders")\n' + ), + "sources/raw_orders.yml": ( + "sources:\n - name: raw_orders\n description: Test source raw_orders.\n" + " schema: main\n table: raw_orders\n" + ), + "tests/unit/test_orders.sql": ( + 'TEST (name "order_status");\n\n' + "WITH\n" + "__source__raw_orders AS (SELECT 'open' AS status),\n" + "__expected__orders AS (SELECT 'open' AS status)\n" + "SELECT 1\n" + ), + "python/orders/storefront_import/node.py": ( + "from sqlbuild.tasks import task\n\n\n" + "@task(description='Import storefront orders.', tags=['ingest'])\n" + "def storefront_import(ctx):\n return None\n" + ), + "python/orders/storefront_import/_helpers/row_mapping.py": ( + "def map_row(raw):\n return {'status': str(raw['state']).lower()}\n" + ), + "python/inventory/stock_sync/node.py": ( + "from sqlbuild.tasks import task\n\n\n" + "@task(description='Sync stock.')\n" + "def stock_sync(ctx):\n return None\n" + ), +} + +PASSING_PYTHON_TEST: dict[str, str] = { + "tests/python/orders/storefront_import/_helpers/test_row_mapping.py": ( + "from python.orders.storefront_import._helpers.row_mapping import map_row\n\n\n" + "def test_given_paid_order_when_mapping_then_status_is_paid():\n" + " assert map_row({'state': 'PAID'}) == {'status': 'paid'}\n" + ), +} + +FAILING_PYTHON_TEST: dict[str, str] = { + "tests/python/inventory/stock_sync/test_node.py": ( + "def test_given_stock_when_syncing_then_fails():\n assert False\n" + ), +} + +BROKEN_MODEL: dict[str, str] = { + "models/broken.sql": ( + 'MODEL (description "Test model broken.");\n\nSELECT missing_column FROM __ref("orders")\n' + ), +} + +PASSING_FILE: str = "tests/python/orders/storefront_import/_helpers/test_row_mapping.py" +FAILING_FILE: str = "tests/python/inventory/stock_sync/test_node.py" + +SQL_PASSED: str = "PASS=1 FAIL=0" +PYTHON_PASSED: str = "Python tests passed 1 file" +PYTHON_STARTED: str = "Running Python tests with pytest" +PYTEST_MISSING: str = "Python tests found but pytest isn't installed" + +HIDDEN_PYTEST_SITECUSTOMIZE: str = """ +import sys + + +class _HidePytest: + def __init__(self, finder): + self.finder = finder + + def find_spec(self, name, path=None, target=None): + return None if name.split(".")[0] == "pytest" else self.finder.find_spec(name, path, target) + + def __getattr__(self, name): + return getattr(self.finder, name) + + +sys.meta_path[:] = [_HidePytest(finder) for finder in sys.meta_path] +""" + + +@pytest.mark.parametrize( + "test_case", + [ + PythonTestsE2ETestCase( + description="without Python tests only SQL tests run", + command=("--no-color", "test"), + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=(PYTHON_STARTED,), + ), + PythonTestsE2ETestCase( + description="with Python tests both SQL and Python tests run", + command=("--no-color", "test"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED, PYTHON_PASSED, "1 passed"), + expected_absent_fragments=(), + ), + PythonTestsE2ETestCase( + description="a failing Python test fails the command after the SQL tests", + command=("--no-color", "test"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=1, + expected_output_fragments=( + SQL_PASSED, + "Python tests failed pytest exit code 1", + "1 failed, 1 passed", + ), + expected_absent_fragments=(), + ), + PythonTestsE2ETestCase( + description="--sql runs only SQL tests", + command=("--no-color", "test", "--sql"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=(PYTHON_STARTED,), + ), + PythonTestsE2ETestCase( + description="--python runs only Python tests", + command=("--no-color", "test", "--python"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=0, + expected_output_fragments=(PYTHON_PASSED,), + expected_absent_fragments=(SQL_PASSED,), + ), + PythonTestsE2ETestCase( + description="--python without Python tests is an error", + command=("--no-color", "test", "--python"), + extra_repo_files={}, + expected_exit_code=1, + expected_output_fragments=("no Python tests found",), + expected_absent_fragments=(SQL_PASSED,), + ), + PythonTestsE2ETestCase( + description="--select of a Python node runs only the tests in its mirrored folder", + command=("--no-color", "test", "--select", "storefront_import"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(PYTHON_PASSED, "1 passed"), + expected_absent_fragments=(SQL_PASSED, "failed"), + ), + PythonTestsE2ETestCase( + description="--select of a glob reaching a Python node runs its mirrored tests", + command=("--no-color", "test", "--select", "storefront_*"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(PYTHON_PASSED, "1 passed"), + expected_absent_fragments=("failed",), + ), + PythonTestsE2ETestCase( + description="--select of a tag on a Python node runs its mirrored tests", + command=("--no-color", "test", "--select", "tag:ingest"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(PYTHON_PASSED, "1 passed"), + expected_absent_fragments=("failed",), + ), + PythonTestsE2ETestCase( + description="--select of a Python node's path runs its mirrored tests", + command=("--no-color", "test", "--select", "python/orders/storefront_import"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(PYTHON_PASSED, "1 passed"), + expected_absent_fragments=("failed",), + ), + PythonTestsE2ETestCase( + description="--select of a glob reaching only models leaves an unrelated broken model unanalysed", + command=("--no-color", "test", "--select", "ord*"), + extra_repo_files={**PASSING_PYTHON_TEST, **BROKEN_MODEL}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=("B002", PYTHON_STARTED), + ), + PythonTestsE2ETestCase( + description="--select of a model runs its SQL tests and no Python tests", + command=("--no-color", "test", "--select", "orders"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=(PYTHON_STARTED,), + ), + PythonTestsE2ETestCase( + description="--select of a model leaves an unrelated broken model unanalysed", + command=("--no-color", "test", "--select", "orders"), + extra_repo_files={**PASSING_PYTHON_TEST, **BROKEN_MODEL}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=("B002", PYTHON_STARTED), + ), + PythonTestsE2ETestCase( + description="--select of a unit test runs it and no Python tests", + command=("--no-color", "test", "--select", "order_status"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=(PYTHON_STARTED,), + ), + PythonTestsE2ETestCase( + description="--python with --inspect is an error", + command=("--no-color", "test", "--python", "--inspect"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=1, + expected_output_fragments=("--python cannot be combined with --inspect or --since",), + expected_absent_fragments=(PYTHON_STARTED,), + ), + PythonTestsE2ETestCase( + description="build never runs Python tests", + command=("--no-color", "build", "--select", "storefront_import"), + extra_repo_files={**PASSING_PYTHON_TEST, **FAILING_PYTHON_TEST}, + expected_exit_code=0, + expected_output_fragments=("Completed successfully",), + expected_absent_fragments=(PYTHON_STARTED,), + ), + ], + ids=lambda case: case.description, +) +def test_given_project_python_tests_when_running_sqb_test_then_runs_the_requested_kinds( + test_case: PythonTestsE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_tests_project", + repo_files={**PROJECT_FILES, **test_case.extra_repo_files}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_output_fragments: + assert fragment in output, output + for fragment in test_case.expected_absent_fragments: + assert fragment not in output, output + + +@pytest.mark.parametrize( + "test_case", + [ + PythonTestsE2ETestCase( + description="without Python tests pytest is not needed", + command=("--no-color", "test"), + extra_repo_files={}, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=(PYTEST_MISSING,), + ), + PythonTestsE2ETestCase( + description="Python tests without pytest fail before any test runs", + command=("--no-color", "test"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=1, + expected_output_fragments=(PYTEST_MISSING, "sqb test --sql"), + expected_absent_fragments=(SQL_PASSED,), + ), + PythonTestsE2ETestCase( + description="--sql runs SQL tests without pytest", + command=("--no-color", "test", "--sql"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=0, + expected_output_fragments=(SQL_PASSED,), + expected_absent_fragments=(PYTEST_MISSING,), + ), + ], + ids=lambda case: case.description, +) +def test_given_pytest_missing_when_running_sqb_test_then_only_python_tests_need_it( + test_case: PythonTestsE2ETestCase, + tmp_path: Path, +) -> None: + hidden: Path = tmp_path / "hidden_pytest" + hidden.mkdir() + (hidden / "sitecustomize.py").write_text(HIDDEN_PYTEST_SITECUSTOMIZE, encoding="utf-8") + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_tests_project", + repo_files={**PROJECT_FILES, **test_case.extra_repo_files}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, + project_dir=project_dir, + env={"PYTHONPATH": os.pathsep.join((str(hidden), os.environ.get("PYTHONPATH", "")))}, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_output_fragments: + assert fragment in output, output + for fragment in test_case.expected_absent_fragments: + assert fragment not in output, output + + +@pytest.mark.parametrize( + "test_case", + [ + PythonTestsJsonE2ETestCase( + description="SQL and Python tests passing give one passing document", + command=("--no-color", "test", "--json"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=0, + expected_status="success", + expected_python_status="success", + expected_python_files=(PASSING_FILE,), + ), + PythonTestsJsonE2ETestCase( + description="a failing Python test fails the document whose SQL tests passed", + command=("--no-color", "test", "--json"), + extra_repo_files=FAILING_PYTHON_TEST, + expected_exit_code=1, + expected_status="failed", + expected_python_status="failed", + expected_python_files=(FAILING_FILE,), + ), + PythonTestsJsonE2ETestCase( + description="--python still writes the document with no SQL tests", + command=("--no-color", "test", "--json", "--python"), + extra_repo_files=PASSING_PYTHON_TEST, + expected_exit_code=0, + expected_status="success", + expected_python_status="success", + expected_python_files=(PASSING_FILE,), + ), + ], + ids=lambda case: case.description, +) +def test_given_json_output_when_running_python_tests_then_stdout_is_one_document_with_them( + test_case: PythonTestsJsonE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_tests_project", + repo_files={**PROJECT_FILES, **test_case.extra_repo_files}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + document: dict[str, object] = json.loads(result.stdout) + assert document["status"] == test_case.expected_status, document + assert document["python_tests"] == { + "status": test_case.expected_python_status, + "exit_code": test_case.expected_exit_code, + "files": list(test_case.expected_python_files), + }, document + assert "1 passed" in result.stderr or "1 failed" in result.stderr, result.stderr diff --git a/tests/goldens/compiler/failure/local-config-unknown-key.json b/tests/goldens/compiler/failure/local-config-unknown-key.json index 5b94d037c..60b80858d 100644 --- a/tests/goldens/compiler/failure/local-config-unknown-key.json +++ b/tests/goldens/compiler/failure/local-config-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_local.toml settings contains unknown key(s): sql_analysiss. Allowed keys: auto_load_sources, check_declared_columns, column_contract_mode, concurrency, default_audit_run_scope, default_audit_severity, delta_compile, max_concurrency, microbatch_concurrency, microbatch_unaccounted_partition_policy, query_change_tracking, require_sql_analysis, scenario_timeout, sql_analysis, sql_validation, table_promotion_mode, test_timeout, unit_test_max_real_models", + "message": "/sqlbuild_local.toml settings contains unknown key(s): sql_analysiss. Allowed keys: allow_missing_secrets, auto_load_sources, check_declared_columns, column_contract_mode, concurrency, default_audit_run_scope, default_audit_severity, delta_compile, max_concurrency, microbatch_concurrency, microbatch_unaccounted_partition_policy, query_change_tracking, require_sql_analysis, scenario_timeout, sql_analysis, sql_validation, table_promotion_mode, test_timeout, unit_test_max_real_models", "help": "did you mean 'sql_analysis'?" } ] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index d86e8149c..d243810b9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -3425,6 +3425,28 @@ def test_given_star_in_set_operation_branches_when_compiling_then_each_branch_is "stale rule exception suppresses no finding: SQBRPYTHON101 at python/report.py" ), ), + PythonRulesIntegrationTestCase( + description="an approved loader boundary stops compile-time purity at that function", + fixture="violations", + rules_toml=( + 'select = ["SQBRPYTHON022"]\n\n[rules.rule_options.SQBRPYTHON022]\n' + "approved_loader_boundaries = " + '["python.inventory.restock._helpers.thresholds.load_thresholds"]\n' + ), + expected_exit_code=0, + expected_findings=(), + ), + PythonRulesIntegrationTestCase( + description="a loader boundary glob fails configuration", + fixture="violations", + rules_toml=( + 'select = ["SQBRPYTHON022"]\n\n[rules.rule_options.SQBRPYTHON022]\n' + 'approved_loader_boundaries = ["python.inventory.*"]\n' + ), + expected_exit_code=1, + expected_findings=(("C000", None, None, None),), + expected_text="globs are not accepted", + ), PythonRulesIntegrationTestCase( description="a rule option tightens a Python threshold", fixture="violations", @@ -3520,7 +3542,7 @@ def test_given_project_python_when_running_python_rules_then_findings_match_comp _PYTHON_ANNOTATED_ALERT, ), ), - expected_hits_and_misses=((0, 1), (0, 14), (13, 1), (0, 14), (15, 0)), + expected_hits_and_misses=((0, 1), (0, 21), (20, 1), (0, 21), (22, 0)), expected_python_cache=(False, True, True, True, True), ) ], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 406627370..75ee9b840 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -323,6 +323,13 @@ class SnapshotDiscoveryTestCase: expected_trigger_declared_at: dict[str, dict[str, object] | None] = field(default_factory=dict) +@dataclass(frozen=True) +class SnapshotProcessReuseTestCase: + description: str + fixtures: tuple[str, ...] + expected_sensors: tuple[tuple[str, ...], ...] + + @dataclass(frozen=True) class SnapshotDiscoveryRejectionTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index aefebb42f..d21ca1661 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -36,6 +36,7 @@ SnapshotExpressionRangeTestCase, SnapshotIdentityTestCase, SnapshotLineageTestCase, + SnapshotProcessReuseTestCase, SnapshotProvidersTestCase, SnapshotRejectionTestCase, SnapshotScheduleTestCase, @@ -483,6 +484,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": True, "env": "FULFILLMENT_API_KEY", "description": None, + "missing": False, }, { "name": "backup_key", @@ -491,6 +493,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": True, "env": "FULFILLMENT_BACKUP_KEY", "description": None, + "missing": False, }, { "name": "regions", @@ -499,6 +502,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": False, "env": "FULFILLMENT_REGIONS", "description": None, + "missing": False, }, { "name": "compression", @@ -507,6 +511,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": False, "env": "FULFILLMENT_COMPRESSION", "description": None, + "missing": False, }, { "name": "endpoint", @@ -515,6 +520,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": False, "env": "FULFILLMENT_ENDPOINT", "description": "Base URL", + "missing": False, }, { "name": "timeout_seconds", @@ -523,6 +529,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": False, "env": "SHIP_TIMEOUT", "description": None, + "missing": False, }, ], "used_by": [ @@ -567,6 +574,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": False, "env": "REGION", "description": None, + "missing": False, }, { "name": "token", @@ -575,6 +583,7 @@ def test_given_a_compiled_project_when_publishing_then_its_column_lineage_is_a_s "secret": False, "env": "INVENTORY_TOKEN", "description": None, + "missing": False, }, ], "used_by": [], @@ -1301,6 +1310,28 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject } }, ), + SnapshotDiscoveryTestCase( + description="DuckShop sensors read support modules in a private folder of their domain", + files={**duckshop_orchestration("private_support_orders")}, + expected_jobs=("inventory_job",), + expected_sensors={ + "on_orders_sensor": { + "description": "Build inventory for every genuine orders rebuild.", + "job_key": "inventory_job", + "when": { + "kind": "node_event", + "selection": ["model:orders"], + "kinds": ["success"], + }, + "minimum_interval_seconds": 30, + "default_status": "running", + "declared_at": { + "path": "orchestration/sensors/inventory/on_orders_sensor.py", + "line": 7, + }, + } + }, + ), SnapshotDiscoveryTestCase( description="failure and recovery selectors preserve their individual transition kinds", files={ @@ -1597,6 +1628,15 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects "Failed to import orchestration file orchestration/sensors/broken_orders_sensor.py" ), ), + SnapshotDiscoveryRejectionTestCase( + description="a DuckShop sensor declared in a private folder fails instead of never running", + files=duckshop_orchestration("private_folder_sensor"), + expected_error=ValueError, + expected_message=( + "orchestration/sensors/inventory/_helpers/on_orders_sensor.py.*" + "declares a sensor inside the private folder _helpers/" + ), + ), SnapshotDiscoveryRejectionTestCase( description="DuckShop settings cannot replace the selection", files={**duckshop_orchestration("invalid_settings")}, @@ -1871,6 +1911,44 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho ) +@pytest.mark.parametrize( + "test_case", + [ + SnapshotProcessReuseTestCase( + description="a project whose sensor imports orchestration/ leaves the next project compilable", + fixtures=("private_support_orders", "node_orders", "private_support_orders"), + expected_sensors=(("on_orders_sensor",), ("on_orders_sensor",), ("on_orders_sensor",)), + ), + ], + ids=lambda case: case.description, +) +def test_given_projects_compiled_in_one_process_when_one_imports_orchestration_then_each_compiles( + tmp_path: Path, test_case: SnapshotProcessReuseTestCase +) -> None: + sensors: list[tuple[str, ...]] = [] + for index, fixture in enumerate(test_case.fixtures): + project_dir: Path = tmp_path / f"project_{index}" + write_files( + project_dir=project_dir, files=TYPE_PROOF_FILES | duckshop_orchestration(fixture) + ) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + publication: dict[str, Any] = json.loads( + compile_snapshot( + project=build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ), + metadata_json=json.dumps(METADATA), + project_dir=project_dir, + discovered_inputs=discovered, + ).to_json() + ) + sensors.append(tuple(publication["documents"].get("triggers", {}))) + + assert tuple(sensors) == test_case.expected_sensors + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 299b15387..1d2445a75 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -1289,9 +1289,10 @@ def run_debug( no_connection: bool, selected_target: str | None, json_output: bool, + verbose: bool, ) -> int: del selected_target - received_args.append((project_dir, no_color, no_connection, json_output)) + received_args.append((project_dir, no_color, no_connection, json_output, verbose)) return test_case.expected_exit_code exit_code: int = _main_with_dependencies( @@ -1300,7 +1301,7 @@ def run_debug( ) assert exit_code == test_case.expected_exit_code - assert received_args == [(None, True, True, True)] + assert received_args == [(None, True, True, True, False)] @pytest.mark.parametrize( diff --git a/ui/src/lib/api/http-locations.test.ts b/ui/src/lib/api/http-locations.test.ts index 0a4342f57..1b03b9d81 100644 --- a/ui/src/lib/api/http-locations.test.ts +++ b/ui/src/lib/api/http-locations.test.ts @@ -25,6 +25,13 @@ describe('location health', () => { expect(healthOf([seen(5, { kind: 'connection', generation: 4, healthy: true }), failed], now).health).toBe('healthy'); }); + it('reports missing provider secrets ahead of a failed connection until a newer report finds none', () => { + const missing = seen(10, { kind: 'provider_secrets', generation: 2, missing: 1 }); + const unhealthy = seen(10, { kind: 'connection', generation: 2, healthy: false }); + expect(healthOf([missing, unhealthy], now).health).toBe('missing_secrets'); + expect(healthOf([seen(5, { kind: 'provider_secrets', generation: 3, missing: 0 }), missing], now).health).toBe('healthy'); + }); + it('needs attention when the latest connection check failed', () => { const observations = [seen(5, { kind: 'connection', generation: 2, healthy: false }), seen(60, { kind: 'connection', generation: 1, healthy: true })]; expect(healthOf(observations, now).health).toBe('degraded'); diff --git a/ui/src/lib/api/http-locations.ts b/ui/src/lib/api/http-locations.ts index a276a4516..bce103588 100644 --- a/ui/src/lib/api/http-locations.ts +++ b/ui/src/lib/api/http-locations.ts @@ -11,19 +11,22 @@ function latestOf(observations: Wire[' return observations.find((o) => o.report.health.kind === kind) as (Wire['LocationHealthObservation'] & { report: { health: Extract } }) | undefined; } -/** Health from the newest observations: down without a recent check-in, then load failures, then connection failures. */ +/** Health from the newest observations: down without a recent check-in, then load failures, then missing provider secrets, then connection failures. */ export function healthOf(observations: Wire['LocationHealthObservation'][], now: number): { health: LocationHealth; liveInstances: number; lastSeenAt: string | null } { const live = new Set(observations.filter((o) => now - Date.parse(o.received_at) <= LIVE_WINDOW_MS).map((o) => o.report.session_id)); const lastSeenAt = observations.reduce((latest, o) => (!latest || o.received_at > latest ? o.received_at : latest), null); const failed = latestOf(observations, 'compile_failed'); const connection = latestOf(observations, 'connection'); + const secrets = latestOf(observations, 'provider_secrets'); const health: LocationHealth = !live.size ? 'broken' : failed && (!connection || failed.report.health.generation >= connection.report.health.generation) ? 'falling_back' - : connection && !connection.report.health.healthy - ? 'degraded' - : 'healthy'; + : secrets && secrets.report.health.missing > 0 + ? 'missing_secrets' + : connection && !connection.report.health.healthy + ? 'degraded' + : 'healthy'; return { health, liveInstances: live.size, lastSeenAt }; } diff --git a/ui/src/lib/api/http-providers.test.ts b/ui/src/lib/api/http-providers.test.ts index b2e319b58..4c40d63e7 100644 --- a/ui/src/lib/api/http-providers.test.ts +++ b/ui/src/lib/api/http-providers.test.ts @@ -14,7 +14,7 @@ describe('toProviders', () => { class: 'providers.fulfillment.FulfillmentApi', description: 'Client for the fulfillment service.', path: 'providers/fulfillment.py', - settings: [{ name: 'api_key', type: 'SecretStr', required: true, secret: true, env: 'FULFILLMENT_API_KEY', description: null }], + settings: [{ name: 'api_key', type: 'SecretStr', required: true, secret: true, env: 'FULFILLMENT_API_KEY', description: null, missing: true }], used_by: [ { kind: 'task', name: 'publish_orders' }, { kind: 'sink', name: 'order_events' }, @@ -26,7 +26,7 @@ describe('toProviders', () => { }; const [provider] = toProviders(wire); expect(provider.className).toBe('providers.fulfillment.FulfillmentApi'); - expect(provider.settings[0]).toEqual({ name: 'api_key', type: 'SecretStr', required: true, secret: true, env: 'FULFILLMENT_API_KEY', description: null }); + expect(provider.settings[0]).toEqual({ name: 'api_key', type: 'SecretStr', required: true, secret: true, env: 'FULFILLMENT_API_KEY', description: null, missing: true }); expect(provider.usedBy.map((c) => `${c.kind}:${c.name}`)).toEqual(['task:publish_orders', 'sink:order_events', 'event_exporter:run_events']); expect(provider.jobs).toEqual(['order_exports']); }); diff --git a/ui/src/lib/api/http-providers.ts b/ui/src/lib/api/http-providers.ts index 773d59cdc..1d343173b 100644 --- a/ui/src/lib/api/http-providers.ts +++ b/ui/src/lib/api/http-providers.ts @@ -9,7 +9,7 @@ export function toProviders(wire: Wire['ProvidersRead']): ProviderConnection[] { className: p.class, description: p.description, sourceFile: p.path, - settings: p.settings.map((s) => ({ name: s.name, type: s.type, required: s.required, secret: s.secret, env: s.env, description: s.description ?? null })), + settings: p.settings.map((s) => ({ name: s.name, type: s.type, required: s.required, secret: s.secret, env: s.env, description: s.description ?? null, missing: s.missing ?? false })), usedBy: p.used_by.map((c) => ({ kind: c.kind as ProviderConsumer['kind'], name: c.name })), jobs: p.jobs ?? [] })); diff --git a/ui/src/lib/api/mock/platform.ts b/ui/src/lib/api/mock/platform.ts index 6a18bedfa..a22aa2065 100644 --- a/ui/src/lib/api/mock/platform.ts +++ b/ui/src/lib/api/mock/platform.ts @@ -376,6 +376,7 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { secret: type.includes('Secret'), env, description, + missing: false, ...extra }); const providers: ProviderConnection[] = [ @@ -430,7 +431,7 @@ function createPlatformFixtures(now: number, role: BuiltInRole) { description: 'Incoming webhook of the team chat room.', sourceFile: 'providers/chat.py', settings: [ - setting('url', 'SecretStr', 'CHAT_WEBHOOK_URL', 'Webhook URL', { required: true }), + setting('url', 'SecretStr', 'CHAT_WEBHOOK_URL', 'Webhook URL', { required: true, missing: true }), setting('timeout_seconds', 'int', 'CHAT_WEBHOOK_TIMEOUT_SECONDS', 'Request timeout') ], usedBy: [], diff --git a/ui/src/lib/api/types.ts b/ui/src/lib/api/types.ts index d627752c5..03ceda96b 100644 --- a/ui/src/lib/api/types.ts +++ b/ui/src/lib/api/types.ts @@ -81,7 +81,7 @@ export type LocationTarget = { connectionSummary: string | null; }; -export type LocationHealth = 'healthy' | 'degraded' | 'falling_back' | 'broken'; +export type LocationHealth = 'healthy' | 'degraded' | 'missing_secrets' | 'falling_back' | 'broken'; /** A project deployed against exactly one target. Staging and dev are separate locations. */ export type Location = { /** Opaque server-generated UUID; the name is a mutable label. */ diff --git a/ui/src/lib/api/types/platform.ts b/ui/src/lib/api/types/platform.ts index 581f53722..52b68cdb7 100644 --- a/ui/src/lib/api/types/platform.ts +++ b/ui/src/lib/api/types/platform.ts @@ -251,6 +251,8 @@ export type ProviderSetting = { /** The environment variable the setting is read from. */ env: string; description: string | null; + /** Whether this required secret is unset in the location's environment. */ + missing: boolean; }; /** Python code that injects a provider. None of it is a catalog node, so it has no page to open. */ diff --git a/ui/src/lib/platform/labels.ts b/ui/src/lib/platform/labels.ts index f94438962..0283b3dbf 100644 --- a/ui/src/lib/platform/labels.ts +++ b/ui/src/lib/platform/labels.ts @@ -20,6 +20,7 @@ import type { export const healthTone: Record = { healthy: 'var(--sb-success)', degraded: 'var(--sb-warning)', + missing_secrets: 'var(--sb-error)', falling_back: 'var(--sb-warning)', broken: 'var(--sb-error)' }; @@ -27,6 +28,7 @@ export const healthTone: Record = { export const healthLabel: Record = { healthy: 'Healthy', degraded: 'Needs attention', + missing_secrets: 'Missing provider secrets', falling_back: 'Running previous version', broken: 'Not starting scheduled runs' }; diff --git a/ui/src/routes/providers/[name]/+page.svelte b/ui/src/routes/providers/[name]/+page.svelte index 9bc9cac8b..5f871cf64 100644 --- a/ui/src/routes/providers/[name]/+page.svelte +++ b/ui/src/routes/providers/[name]/+page.svelte @@ -63,7 +63,7 @@ {#if f.description}
{f.description}
{/if} {f.type} - {f.env} + {f.env}{#if f.missing}missing{/if} {f.required ? 'Required' : 'Optional'} {/each} @@ -128,4 +128,9 @@ color: var(--muted-foreground); font-size: 11px; } + .missing { + margin-left: 6px; + color: var(--sb-error); + font-size: 11px; + } diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index d323e30c6..d1a145086 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -35,7 +35,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | Flag | Description | |------|-------------| | `--target` | Build against a configured target instead of the active/default target | -| `--no-tests` | Skip SQL unit tests | +| `--no-tests` | Skip SQL unit tests; a build never runs [Python tests](/docs/cli/test#python-tests) | | `--no-audits` | Skip audits | | `--no-python` | Skip read-side Python tasks and assets (loader-side Python still runs for selected sources) | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index ed2f9227a..ee1d7602f 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -50,6 +50,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--exclude` | Remove resources from the selected scope | | `--no-sql-analysis` | Turn off SQL syntax, binding, type inference and semantic checks (`--no-sql-validation` is an alias) | | `--no-cache` | Compile in full, without reusing earlier compiles or the Rules cache; see [Forcing a fresh compile](#forcing-a-fresh-compile) | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](/docs/concepts/python-nodes/providers#secrets) | | `--defer-to` | Resolve unselected model references against another target | | `--json` | Print the full compile report as JSON | | `--manifest` | Also write `target/manifest.json` with project metadata | diff --git a/website/src/content/docs/docs/cli/dag.mdx b/website/src/content/docs/docs/cli/dag.mdx index aacd2cb35..dbefae2d5 100644 --- a/website/src/content/docs/docs/cli/dag.mdx +++ b/website/src/content/docs/docs/cli/dag.mdx @@ -21,6 +21,7 @@ sqb --project-dir dag [flags] |------|-------------| | `--json` | Print the full DAG artifact as JSON to stdout | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](/docs/concepts/python-nodes/providers#secrets) | | `--vars` | JSON object of project variable overrides | Without `--json`, the command prints a summary: diff --git a/website/src/content/docs/docs/cli/debug.mdx b/website/src/content/docs/docs/cli/debug.mdx index 488c1e6f4..2905c8980 100644 --- a/website/src/content/docs/docs/cli/debug.mdx +++ b/website/src/content/docs/docs/cli/debug.mdx @@ -23,7 +23,14 @@ The command reports four groups of checks: **Configuration** - Finds and validates `sqlbuild_project.toml`, loads `sqlbuild_local.toml` if present, resolves the adapter, active target, connection, database, and schema. For Snowflake it also shows the warehouse each [command group](/docs/concepts/adapters/snowflake#command-group-warehouses) connects with and where that value came from. -**Providers** - Lists the [providers](/docs/concepts/python-nodes/providers) the project defines. +**Providers** - Counts the [providers](/docs/concepts/python-nodes/providers) whose [secrets](/docs/concepts/python-nodes/providers#secrets) are all set, and lists each missing secret as `WARN`. When every secret is set it prints one line. `--verbose` lists every provider and secret. + +```text +Providers: + providers: 2 of 3 ready [WARN missing secrets] + slack_alerts: SLACK_WEBHOOK_URL [WARN missing] + more: sqb debug --verbose lists every provider [SKIP] +``` **Connection** - Displays connection settings as `sqb debug` itself uses them, with the `query` warehouse group applied (secrets are masked), attempts to connect to the warehouse, and runs `SELECT 1` to verify query execution. @@ -32,7 +39,8 @@ The command reports four groups of checks: | Flag | Description | |------|-------------| | `--no-connection` | Skip the connection and query tests. Useful for validating config without warehouse access. | -| `--json` | Output results as JSON instead of formatted text. | +| `--json` | Output results as JSON instead of formatted text, with `missing_secrets` counting the unset provider secrets. | +| `--verbose` | List every provider and secret setting, set or missing. | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/adapters/snowflake#command-group-warehouses) and the connection warehouse | ## Example @@ -80,7 +88,7 @@ Connection: ## Exit codes -Returns `0` when all checks pass, `1` when any check fails (e.g. connection refused, invalid config). +Returns `0` when all checks pass, `1` when any check fails (e.g. connection refused, invalid config). A missing provider secret is a warning and doesn't fail the command. A failed run is recorded in [Recall](/docs/concepts/observability/recall) with the first failed check as its cause: diff --git a/website/src/content/docs/docs/cli/format.mdx b/website/src/content/docs/docs/cli/format.mdx index e46f501a4..ea9b8a207 100644 --- a/website/src/content/docs/docs/cli/format.mdx +++ b/website/src/content/docs/docs/cli/format.mdx @@ -18,6 +18,7 @@ sqb format [flags] | `--fix` | Also apply compiler-verified, meaning-preserving Rule fixes before layout formatting | | `--json` | Print structured results | | `--no-cache` | Format every file instead of skipping files a previous run found clean | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](/docs/concepts/python-nodes/providers#secrets) | | `--select`, `-s` | Format selected models | | `--select-file` | Read selectors from a file | | `--exclude` | Exclude models from the selected scope | diff --git a/website/src/content/docs/docs/cli/lineage.mdx b/website/src/content/docs/docs/cli/lineage.mdx index e1fd442f0..664d900e9 100644 --- a/website/src/content/docs/docs/cli/lineage.mdx +++ b/website/src/content/docs/docs/cli/lineage.mdx @@ -34,6 +34,7 @@ Exactly one of a positional target or `--select` is required. | `--json` | Shorthand for `--format json`. | | `--mode` | Column lineage analysis mode: `rich` (default) or `fast`. | | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias). | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](/docs/concepts/python-nodes/providers#secrets) | | `--select`, `-s` | Select resources using the same selector syntax as `sqb compile`, `sqb plan` and `sqb build`, for example `path:models/marts`, `tag:finance` or `stg_*`. | | `--exclude` | Exclude resources from the selection. | diff --git a/website/src/content/docs/docs/cli/location.mdx b/website/src/content/docs/docs/cli/location.mdx index c40cac7fb..e1717bcf7 100644 --- a/website/src/content/docs/docs/cli/location.mdx +++ b/website/src/content/docs/docs/cli/location.mdx @@ -33,6 +33,7 @@ sqb location health --server https://control.example.com --location-id "$LOCATIO | --- | --- | --- | | `healthy` | An instance checked in within 90 seconds and its last load published | `0` | | `degraded` | The warehouse connection check after the last load failed | `0` | +| `missing_secrets` | A provider [secret](/docs/concepts/python-nodes/providers#secrets) is unset in the location's environment, so runs that use it fail | `1` | | `falling_back` | The last load failed, so runs use the previous snapshot | `1` | | `broken` | No instance has checked in for 90 seconds | `1` | @@ -50,3 +51,6 @@ healthcheck: A `falling_back` location still runs scheduled jobs from its last good snapshot. Restarting it does not fix the failed load; fix the project and deploy it again. + +A `missing_secrets` location publishes and runs jobs whose nodes have their secrets. Set the +missing variables in its environment and restart it; `sqb debug` in the location lists them. diff --git a/website/src/content/docs/docs/cli/rules.mdx b/website/src/content/docs/docs/cli/rules.mdx index 98088c8cc..f28a6b402 100644 --- a/website/src/content/docs/docs/cli/rules.mdx +++ b/website/src/content/docs/docs/cli/rules.mdx @@ -33,7 +33,7 @@ Lifecycle progress is written to stderr so stdout remains valid JSON. - `run`: compile and evaluate only one exact code or family prefix. - `skills`: install guidance derived from configured Rules; `--check` verifies freshness without writing. -`run` accepts normal SQLBuild model selectors. Native built-ins execute before custom Python Rules. +`run` accepts normal SQLBuild model selectors and `--allow-missing-secrets`, which runs without provider [secrets](/docs/concepts/python-nodes/providers#secrets). Native built-ins execute before custom Python Rules. ## Exit codes diff --git a/website/src/content/docs/docs/cli/scope.mdx b/website/src/content/docs/docs/cli/scope.mdx index ad9fee78f..39ae1ed19 100644 --- a/website/src/content/docs/docs/cli/scope.mdx +++ b/website/src/content/docs/docs/cli/scope.mdx @@ -68,6 +68,7 @@ These identities are for inspection. SQL authored inside the owning model contin | `--kind KIND` | Keep `macro`, `enum`, or `constant` declarations. Repeat to include multiple kinds. | | `--match GLOB` | Match declaration names or qualified identities with a deterministic, case-sensitive glob. | | `--used-only` | Keep declarations used by the target. | +| `--allow-missing-secrets` | Run without provider secrets; see [Secrets](/docs/concepts/python-nodes/providers#secrets) | | `--include-nearby` | Include bounded nearby declarations that are unavailable to the target. | | `--nearby-depth N` | Set filesystem proximity for nearby descendant and sibling discovery (default: `1`). | | `--dependency-depth N` | Expand dependencies from declarations in the used section by `N` edges (default: `0`). | diff --git a/website/src/content/docs/docs/cli/test.mdx b/website/src/content/docs/docs/cli/test.mdx index d810a8fec..67cfd4fd0 100644 --- a/website/src/content/docs/docs/cli/test.mdx +++ b/website/src/content/docs/docs/cli/test.mdx @@ -1,12 +1,13 @@ --- title: "sqb test" -description: "Run SQL unit tests and multi-model tests in isolation." +description: "Run SQL unit tests, multi-model tests and the project's Python tests in isolation." sidebar: label: "test" --- Runs SQL unit tests, independently reported parameterized cases, and multi-model tests without -building models. Useful for validating test logic independently. +building models, then the project's [Python tests](#python-tests) with pytest. Useful for +validating test logic independently. ## Usage @@ -28,6 +29,8 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--test-timeout ` | Time limit for each test in this run, such as `30s` or `2m`, or `disabled`; overrides `TEST(timeout ...)`, the target and `[settings] test_timeout` | | `--since ` | Run only the tests affected by changes since the merge base of `` and `HEAD` (see [Run only affected tests](#run-only-affected-tests)); `@base` reads `[git] base_ref` | | `--explain` | With `--since`, print why each selected test was selected | +| `--sql` | Run only SQL tests | +| `--python` | Run only [Python tests](#python-tests); cannot be combined with `--inspect` or `--since` | Test names are globally unique, so a bare test name selects exactly that test. `test:` is the explicit form and accepts the same glob patterns as names, such as `test:orders_*`. A bare pattern @@ -52,6 +55,40 @@ counted under `TIMEOUT=` and excluded from `FAIL=` in the summary line output, `summary.fail_count` still includes timed-out tests, and `summary.timeout_count` reports how many of them timed out. A timeout still fails the run: `sqb test` exits non-zero. +## Python tests + +Python tests are pytest modules named `test_*.py` under `tests/python/` and `tests/providers/`. Each +sits in the folder that mirrors the code it tests, so the tests of +`python/orders/storefront_import/node.py` live in `tests/python/orders/storefront_import/`. + +| Command | SQL unit tests | Python tests | +|---------|----------------|--------------| +| `sqb test` | Yes | Yes, when any exist; needs pytest installed | +| `sqb test --sql` | Yes | No | +| `sqb test --python` | No | Yes | +| `sqb test --since ` | The affected tests | No | +| `sqb build` | Yes | Never | + +pytest runs from the project directory with the same Python as SQLBuild and with +`--import-mode=importlib`, so tests import project code as `python.orders...` or `providers...`. +Its output appears after the SQL test summary, followed by `Python tests passed` or +`Python tests failed`. `sqb test` exits non-zero when either kind fails. + +With `--json`, pytest writes to stderr so stdout stays one JSON document. When Python tests ran, the +document has a `python_tests` object with `status` (`success` or `failed`), pytest's `exit_code` +and the test `files`, and the top-level `status` is `failed` when either kind failed. + +Selecting a Python node, such as `sqb test storefront_import`, runs the Python tests in that node's +mirrored folder. Selecting only models or unit tests runs no Python tests. When Python tests exist +but pytest is not installed, `sqb test` fails before running anything; install it, or run +`sqb test --sql`. + +[Ad-hoc runs](/docs/cli/server#run-ad-hoc-commands) on a server run SQL tests only, and reject +`--python`. + +[Python Rules](/docs/concepts/rules/python-rules#testing-python) check where Python tests sit and +how they are written. + ## Run only affected tests `--since ` runs only the tests a change can affect, which keeps CI on branch pushes from diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index 61c8651bd..30acbc814 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -30,8 +30,12 @@ sqb build nightly_orders_job # build what the job builds, locally - Subfolders are allowed, such as `orchestration/jobs/finance/revenue_job.toml`. - A name must end with its folder's suffix. Names are unique across the whole project, with models, sources and every other resource. -- A Python module whose file name starts with `_` is a private helper and declares nothing. +- A Python module whose file name starts with `_`, or that sits in a folder starting with `_` under + `orchestration/calendars/` or `orchestration/sensors/` (such as `sensors/inventory/_helpers/`), is + private support and declares nothing. Declaring a calendar or sensor there fails compilation. - Any other file under `orchestration/`, or a folder not in the table, is a `D017` error. +- The [Python Rules](/docs/concepts/rules/python-rules#laying-out-sensors-and-calendars) add a + domain-folder layout for sensor and calendar modules. ## Declare a job @@ -124,7 +128,7 @@ another system. The code location evaluates it when it publishes and again befor out. ```python -# orchestration/calendars/fulfillment_calendar.py +# orchestration/calendars/fulfillment/cutoffs.py from sqlbuild.orchestration import calendar @@ -146,7 +150,7 @@ returns timezone-aware datetimes inside it. Other parameters name A sensor starts a job or job group when something happens, named by `run`. ```python -# orchestration/sensors/on_new_orders_sensor.py +# orchestration/sensors/orders/new_orders.py from sqlbuild.orchestration import sensor, source_new_data on_new_orders = sensor( diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 5951af7c2..b43a5fbcc 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -491,6 +491,7 @@ delta_compile = true | `test_timeout` | `30s` | Time limit for each SQL unit test, or `"disabled"` | | `scenario_timeout` | none | Time limit for each scenario, or `"disabled"` | | `delta_compile` | `true` | After an edit to model SQL or SQL test files only, compile just the changed files against the previous compile. See [Compiling after an edit](/docs/cli/compile#compiling-after-an-edit). | +| `allow_missing_secrets` | `false` | Let `compile`, `rules run`, `format`, `dag`, `lineage` and `scope` run without provider secrets. `sqlbuild_local.toml` only. See [Secrets](/docs/concepts/python-nodes/providers#secrets). | ### Unit test and scenario limits @@ -744,10 +745,11 @@ Code that extends SQLBuild itself keeps its own root: Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create an unofficial project structure. -Keep repository pytest tests outside the SQLBuild project's `tests/` directory, which is reserved for -SQLBuild SQL tests and scenarios. Orchestrator code such as Dagster or Rivers definitions lives outside -the SQLBuild project directory and points at it; see [Dagster](/docs/integrations/dagster) and -[Rivers](/docs/integrations/rivers). +The project's `tests/` directory holds SQL tests and scenarios, and pytest tests of project Python +under `tests/python/` and `tests/providers/`, which `sqb test` runs; see +[Python tests](/docs/cli/test#python-tests). Orchestrator code such as Dagster or Rivers definitions +lives outside the SQLBuild project directory and points at it; see +[Dagster](/docs/integrations/dagster) and [Rivers](/docs/integrations/rivers). ## sqlbuild_local.toml diff --git a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx index d8f140c41..b9b6efd79 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx @@ -133,8 +133,8 @@ Both approaches (parameter injection and `ctx.providers`) are equivalent. Parame Providers follow a lazy setup, reverse-teardown lifecycle scoped to the command invocation: -1. **Discovery** - on compile, SQLBuild discovers all `Provider` subclasses under `providers/` and validates their settings (from environment variables or field defaults). -2. **Lazy setup** - `setup(ctx)` is called the first time a provider is accessed during a build, not at startup. Providers that are never used are never set up. +1. **Discovery** - on compile, SQLBuild discovers all `Provider` subclasses under `providers/` and validates their settings (from environment variables or field defaults). Missing [secrets](#secrets) follow the command's rules. +2. **Lazy setup** - the first time a provider is accessed during a build, SQLBuild reads its settings and calls `setup(ctx)`. Providers that are never used are never set up. 3. **Teardown** - after the command completes, `teardown()` is called on all providers that were set up, in reverse setup order. Teardown runs even if the build failed. ```python @@ -166,13 +166,62 @@ class SlackNotifier(Provider): See the [pydantic-settings documentation](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) for advanced configuration like custom env prefixes, `.env` file support, and nested settings. +## Secrets + +Declare a secret as a field typed `SecretStr`. Laptops and CI jobs can then compile without production secrets, and builds check only the secrets their selection needs. + +```python +from pydantic import SecretStr + + +class SlackAlerts(Provider): + """Posts alerts to Slack.""" + + channel: str = "#data-alerts" + slack_webhook_url: SecretStr # reads SLACK_WEBHOOK_URL +``` + +| Commands | When a required secret is unset | +| --- | --- | +| `compile`, `rules run`, `format`, `dag`, `lineage`, `scope` | Error `D012`, unless missing secrets are allowed | +| `build`, `load`, `check`, and runs on `sqb server` workers | Error before anything runs, only when a selected node or a sink uses the provider | +| `sqb debug` | Lists the missing secrets as warnings | +| `sqb location health` | Reports `missing_secrets` and exits `1` | + +```text +$ sqb compile +error[D012]: provider 'slack_alerts' (providers/slack_alerts/provider.py) is missing secret SLACK_WEBHOOK_URL + = help: set SLACK_WEBHOOK_URL, or pass --allow-missing-secrets where secrets aren't available +``` + +Allow missing secrets for the static commands in any of these ways: + +| Where | Setting | +| --- | --- | +| Command line | `--allow-missing-secrets` | +| Environment | `SQLBUILD_ALLOW_MISSING_SECRETS=1` | +| `sqlbuild_local.toml` | `allow_missing_secrets = true` under `[settings]` | + +The command then lists the providers it skipped on stderr, such as +`1 provider has missing secrets (allowed): slack_alerts`. Settings that aren't secrets are still +validated. Execution commands accept none of these. + +```text +$ sqb build --select python/orders +error[D012]: selected nodes need secrets that are missing: + slack_alerts SLACK_WEBHOOK_URL (partner_export_alert) +nothing was run +``` + +Run [`sqb debug`](/docs/cli/debug/) to see which secrets this machine has. + ## Discovery rules - Provider classes are discovered from `.py` files under `providers/` recursively - Files named `__init__.py` or starting with `_` are skipped - Each concrete (non-abstract) subclass of `Provider` is registered - Provider names must be globally unique across project resources -- Settings are validated at discovery time. Missing required fields (without environment variables set) raise a discovery error immediately, not at runtime +- Settings are validated at discovery time. A missing or invalid setting that isn't a secret is a `D012` error; missing secrets follow [Secrets](#secrets) ## Plan output @@ -193,7 +242,7 @@ provider with its class and docstring. A provider's page shows: | Tab | Shows | | --- | --- | -| Settings | Each setting's type, environment variable and whether it is required; secret settings are marked | +| Settings | Each setting's type, environment variable and whether it is required; secret settings are marked, and flagged **missing** when the location's environment doesn't set them | | Used by | The loaders, tasks, hooks and other code that take the provider, and the jobs that run that code | Setting values, defaults included, are never published. diff --git a/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx b/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx index 8567dc956..289190d85 100644 --- a/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx +++ b/website/src/content/docs/docs/concepts/rules/custom-rules/testing-and-determinism.mdx @@ -50,7 +50,9 @@ passes, the case fails with `custom rule is not deterministic`. The usual module-level state, such as a set of already-seen models, that carries over from one subject to the next. Compute findings from the subject and `ctx` alone. -Repository pytest files remain outside the SQLBuild project's SQL `tests/` directory. +Keep Rule tests under the project's `tests/` directory, outside `tests/python/` and +`tests/providers/`, such as in `tests/rules/`. `sqb test` runs only those two folders, so run Rule +tests with pytest. ## Parallel evaluation diff --git a/website/src/content/docs/docs/concepts/rules/python-rules.mdx b/website/src/content/docs/docs/concepts/rules/python-rules.mdx index 5e2c3263b..319fc4de9 100644 --- a/website/src/content/docs/docs/concepts/rules/python-rules.mdx +++ b/website/src/content/docs/docs/concepts/rules/python-rules.mdx @@ -46,10 +46,12 @@ sqb rules run SQBRPYTHON The Python Rules read every supported project Python file: the runtime package under `python/` and the folders that extend SQLBuild, such as `hooks/python/`, `functions/python/`, `providers/`, `rules/` and `macros/`. See [Project Python](/docs/concepts/project-configuration#project-python) -for the full list. Files under `tests/` are not checked. +for the full list. Under `tests/`, only the [Python tests](#testing-python) in `tests/python/` and +`tests/providers/` are checked, and only by the Rules marked `tests/`. Rules marked `python/` in the [reference](#rule-reference) check only the runtime package, because -they describe how its packages are organised. The others check every file. +they describe how its packages are organised. Rules marked `tests/` check only Python tests. The +others check every file except Python tests. ## Laying out `python/` @@ -174,6 +176,138 @@ A capability folder may also hold `types.py`, `constants.py`, `exceptions.py`, ` `Provider` through an alias or a base class in `_classes/`. A class whose base comes from another package is never reported as not being a provider. +## Laying out sensors and calendars + +`SQBRPYTHON016` to `SQBRPYTHON020` keep each sensor and calendar module in a domain folder, named +for what it checks, with its private support beside it: + +```text +orchestration/ + sensors/ + inventory/ + stock_feed.py # one sensor + _constants.py + _helpers/ + feed_age.py + calendars/ + finance/ + month_end.py # one calendar +``` + +A domain folder's private support is `_models.py`, `_types.py`, `_constants.py`, +`_exceptions.py` and `_helpers/`. These are reported: + +| Code | Finding | Fix | +|---|---|---| +| `SQBRPYTHON016` | A module directly in `sensors/` or `calendars/`, nested below its domain folder, or named `main.py`, `definitions.py`, `sensors.py`, `calendars.py` or `processing.py` | Move it to `//.py` | +| `SQBRPYTHON017` | Other private files or folders, private support outside a domain folder, or an import of another domain's private support | Use the support files or `_helpers/`; share code through `python/utils/` or a provider | +| `SQBRPYTHON018` | A sensor or calendar module holding a function, a class, a public constant or another statement besides its declaration | Move it to `_helpers/`, `_models.py` or `_constants.py` | +| `SQBRPYTHON019` | A sensor or calendar importing `python/` outside `utils/`, or code outside `orchestration/` importing it | Read node results through the sensor's context; share code through `python/utils/` or a provider | +| `SQBRPYTHON020` | A sensor or calendar function with a docstring | Keep the text in `description=` only | + +Providers importing `orchestration/` are reported by `SQBRPYTHON015` instead. + +## Testing Python + +`sqb test` runs the pytest modules under `tests/python/` and `tests/providers/`; see +[Python tests](/docs/cli/test#python-tests). `SQBRPYTHON021` keeps each test beside a mirror of the +code it tests, and `SQBRPYTHON701` to `SQBRPYTHON724` keep tests table-driven: + +```text +python/ + orders/ + storefront_import/ + node.py + _helpers/ + row_mapping.py +tests/ + python/ + orders/ + storefront_import/ + _helpers/ + _test_types.py # the test-case dataclasses + test_row_mapping.py # tests row_mapping.py +``` + +```python +# tests/python/orders/storefront_import/_helpers/_test_types.py +from dataclasses import dataclass + + +@dataclass(frozen=True) +class RowMappingTestCase: + description: str + status: str + expected_label: str +``` + +```python +# tests/python/orders/storefront_import/_helpers/test_row_mapping.py +import pytest + +from python.orders.storefront_import._helpers.row_mapping import status_label +from tests.python.orders.storefront_import._helpers._test_types import RowMappingTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + RowMappingTestCase(description="shipped orders", status="S", expected_label="shipped"), + RowMappingTestCase(description="open orders", status="O", expected_label="open"), + ], + ids=lambda case: case.description, +) +def test_given_a_status_when_mapping_then_returns_its_label(test_case: RowMappingTestCase) -> None: + assert status_label(test_case.status) == test_case.expected_label +``` + +| Code | Finding | Fix | +|---|---|---| +| `SQBRPYTHON021` | A test module whose folder does not mirror one under `python/` or `providers/`, or a `test_.py` with no `.py` or `/` package beside the code it tests | Move the test to the mirrored folder, or rename it for the module it tests | +| `SQBRPYTHON701` to `SQBRPYTHON711` | A test folder or module whose layout, imports, helpers, conditionals or test-case dataclasses break the pattern above | Follow the pattern above | +| `SQBRPYTHON712` to `SQBRPYTHON724` | A test function or `pytest.mark.parametrize` that is not named `test_given__when__then_`, or does not take dataclass cases through `test_case` with ids from each `description` | Follow the pattern above | + +## Keeping compile-time Python pure + +`sqb compile` imports project Python with no warehouse connection, so the code that runs then must +not reach outside the machine. `SQBRPYTHON022` checks every call made at module scope, in a class +body, or directly in a `@factory` body, in every file it reads, and follows the project functions +those calls reach. A call that reaches a subprocess, an HTTP request, a socket, a database or +warehouse driver, or a cloud or SSH client is reported at the call where the walk started: + +```python +# python/orders/storefront_import/constants.py +import requests + +STOREFRONT_REGIONS = requests.get("https://example.com/regions").json() # reported +``` + +```text +error[SQBRPYTHON022]: Python that runs while the project compiles must not reach a subprocess, the network, a database or a remote client: calls requests.get + --> python/orders/storefront_import/constants.py:3:22 +``` + +When the external call sits in a project function, the message names it and its line instead, such +as `reaches requests.get at python/orders/storefront_import/_helpers/regions.py:8`. + +Calls inside node bodies, `main()`, the nodes a factory generates, and provider methods run when +the node runs, so they are not checked. Only calls whose target is known from the imports are +followed, so a call through a variable or a method of an object is never reported. + +Keep the data a factory loops over in its unit's `constants.py` or `models.py`. When a factory must +discover what to generate, put that discovery behind one function and approve it. The walk does not +enter an approved function, and an approved factory's body is not checked: + +```toml +[rules.rule_options.SQBRPYTHON022] +approved_loader_boundaries = [ + "python.inventory.warehouse_snapshots._helpers.spec_loading.load_warehouse_specs", +] +``` + +Each entry is the fully qualified name of a project function. A glob fails configuration with +`C000`, and an entry that names a module fails the run. + ## Main modules and entry modules Some Rules apply only to a domain's main modules: files under a `main/` folder, and a unit's @@ -204,7 +338,7 @@ reason = "Carriers are looked up by name from the shipment's carrier code." ## Rule options -Configure the size limits by exact code: +Configure the size limits and approved boundaries by exact code: ```toml [rules.rule_options.SQBRPYTHON504] @@ -224,8 +358,10 @@ max_arguments = 6 | `SQBRPYTHON611` | `max_role_depth` | 1 | How many package levels may sit below a role package. | | `SQBRPYTHON615` | `min_shared_domain_prefix_packages` | 2 | Fewest sibling domains sharing a name prefix that are reported; 0 turns the rule off. | | `SQBRPYTHON627` | `max_file_lines` | 2000 | Most lines one Python file may hold. | +| `SQBRPYTHON022` | `approved_loader_boundaries` | none | Project functions the [compile-time purity](#keeping-compile-time-python-pure) walk does not enter. | -Each option takes a whole number of 0 or more. Unknown option names, text values and negative +`approved_loader_boundaries` takes a list of fully qualified function names; every other option +takes a whole number of 0 or more. Unknown option names, values of the wrong type and negative numbers fail configuration with `C000`. ## Rule reference @@ -263,6 +399,52 @@ remediation. | `SQBRPYTHON014` | provider-module-shape | All | yes | `provider.py` holds imports, private constants and one public `Provider` subclass | | `SQBRPYTHON015` | provider-import-boundaries | All | yes | A provider's `_helpers/` and `_classes/` are private to it; providers import only `python/utils/` from `python/` and nothing from `orchestration/` | +### Sensors and calendars + +| Code | Name | Files | Default | Checks | +|---|---|---|---|---| +| `SQBRPYTHON016` | orchestration-module-paths | All | yes | Sensor and calendar modules sit at `orchestration///.py` with a specific name | +| `SQBRPYTHON017` | orchestration-private-support | All | yes | A domain folder's private support is its support files and `_helpers/`, imported only from that folder | +| `SQBRPYTHON018` | orchestration-module-surface | All | yes | A sensor or calendar module holds imports, private constants and its one declaration | +| `SQBRPYTHON019` | orchestration-import-direction | All | yes | Sensors and calendars import only providers, `python/utils/` and their own support; nothing outside `orchestration/` imports them | +| `SQBRPYTHON020` | orchestration-no-docstring | All | yes | Sensor and calendar functions have no docstring | + +### Compile-time purity + +| Code | Name | Files | Default | Checks | +|---|---|---|---|---| +| `SQBRPYTHON022` | compile-time-purity | All | yes | Module scope, class bodies and `@factory` bodies reach no subprocess, network, database or remote client | + +### Python tests + +| Code | Name | Files | Default | Checks | +|---|---|---|---|---| +| `SQBRPYTHON021` | python-test-placement | `tests/` | yes | Python tests must mirror an existing module under `python/` or `providers/` | +| `SQBRPYTHON701` | test-init-module-empty | `tests/` | yes | Test package `__init__.py` files must be empty or docstring-only | +| `SQBRPYTHON702` | test-absolute-imports | `tests/` | yes | Tests must use absolute imports | +| `SQBRPYTHON703` | test-no-top-level-helpers | `tests/` | yes | Test modules may contain only tests, imports, and declarations | +| `SQBRPYTHON704` | test-no-conditionals | `tests/` | yes | Tests and local test helpers must not contain conditional control flow | +| `SQBRPYTHON705` | test-private-constant-order | `tests/` | yes | Private test constants must appear before test functions | +| `SQBRPYTHON706` | test-no-complex-comprehensions | `tests/` | yes | Nested or multi-generator comprehensions hide control flow and data shapes | +| `SQBRPYTHON707` | test-case-description | `tests/` | yes | Test-case dataclasses must define a `description` field | +| `SQBRPYTHON708` | test-case-expected-field | `tests/` | yes | Test-case dataclasses must define at least one `expected_` field | +| `SQBRPYTHON709` | test-local-types-import | `tests/` | yes | Tests must import test-case types from their local `_test_types.py` | +| `SQBRPYTHON710` | test-local-types-file | `tests/` | yes | Test directories must provide a local `_test_types.py` | +| `SQBRPYTHON711` | test-file-name | `tests/` | yes | Test modules must use a `test_` filename | +| `SQBRPYTHON712` | test-function-name | `tests/` | yes | Test functions must use `test_given__when__then_` | +| `SQBRPYTHON713` | test-dataclass-parametrize | `tests/` | yes | Tests must use dataclass-backed pytest parameterization | +| `SQBRPYTHON714` | test-accepts-test-case | `tests/` | yes | Parametrized tests must accept a `test_case` argument | +| `SQBRPYTHON715` | test-case-annotation | `tests/` | yes | `test_case` parameters must use a local test-case dataclass annotation | +| `SQBRPYTHON716` | test-expected-field-assertion | `tests/` | yes | Tests must assert against an `expected_` field from `test_case` | +| `SQBRPYTHON717` | test-parametrize-arguments | `tests/` | yes | Pytest parametrize decorators must define parameter names and values | +| `SQBRPYTHON718` | test-parametrize-test-case | `tests/` | yes | Pytest parametrize must expose cases through the `test_case` parameter | +| `SQBRPYTHON719` | test-parametrize-ids | `tests/` | yes | Pytest parametrize decorators must define readable case ids | +| `SQBRPYTHON720` | test-inline-parametrize-values | `tests/` | yes | Pytest parametrize values must be a visible list, tuple, or local comprehension | +| `SQBRPYTHON721` | test-nonempty-parametrize-values | `tests/` | yes | Pytest parametrize case sequences must not be empty | +| `SQBRPYTHON722` | test-no-dict-cases | `tests/` | yes | Pytest cases must use typed dataclasses instead of dictionaries | +| `SQBRPYTHON723` | test-local-case-constructors | `tests/` | yes | Pytest cases must construct dataclasses from the local `_test_types.py` | +| `SQBRPYTHON724` | test-description-ids | `tests/` | yes | Pytest case ids must come from each test case `description` | + ### Type annotations | Code | Name | Files | Default | Checks | diff --git a/website/src/content/docs/docs/concepts/testing.mdx b/website/src/content/docs/docs/concepts/testing.mdx index 221b09aa9..c6e72e476 100644 --- a/website/src/content/docs/docs/concepts/testing.mdx +++ b/website/src/content/docs/docs/concepts/testing.mdx @@ -517,6 +517,9 @@ sqb test sqb test --select stg_orders ``` +`sqb test` also runs the project's pytest modules under `tests/python/` and `tests/providers/`; +`sqb build` never runs them. See [Python tests](/docs/cli/test#python-tests). + ## More test features [Testing reference](/docs/concepts/testing-reference) covers macro, UDF and table function tests,