From aadbc98ff95b3e16417a407138cddd315b374cc3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:44:23 +0000 Subject: [PATCH 01/34] test: reproduce Python selectors in published job selections --- .../compiler/compiled_project/_test_types.py | 7 +++ .../compiler/compiled_project/helpers.py | 22 +++++++ .../test_orchestration_snapshot.py | 59 +++++++++++++++++++ 3 files changed, 88 insertions(+) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index f9fc57f34..048a0e910 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -262,6 +262,13 @@ class SnapshotProvidersTestCase: expected_providers: dict[str, object] | None +@dataclass(frozen=True) +class PythonJobSelectionTestCase: + description: str + select: str + expected_selection: tuple[str, ...] + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 22a57cf53..03050f151 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -66,6 +66,28 @@ "documents": {}, } +PYTHON_PIPELINE_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', tags [orders]);\n\nSELECT 1 AS order_id\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', tags [orders]);\n\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + "python/orders/partner_export/node.py": ( + "from sqlbuild.assets import asset\nfrom sqlbuild.refs import model\n\n\n" + "@asset(depends_on=model('orders'), tags=['orders'], description='Partner order file.')\n" + "def partner_orders_file(ctx):\n return None\n" + ), + "python/fulfillment/carrier_sync/node.py": ( + "from python.orders.partner_export.node import partner_orders_file\n" + "from sqlbuild.tasks import task\n\n\n" + "@task(depends_on=partner_orders_file, tags=['orders'], description='Book collection.')\n" + "def request_carrier_collection(ctx):\n return None\n" + ), +} + TYPE_PROOF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRCONTRACT105"]\n\n' diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 770f4cbd4..10556cef2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -44,12 +44,14 @@ SnapshotLineageTestCase, SnapshotProvidersTestCase, SnapshotRejectionTestCase, + PythonJobSelectionTestCase, SnapshotScheduleTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( FUNCTION_PROJECT_FILES, METADATA, PARITY_FILES, + PYTHON_PIPELINE_FILES, TYPE_PROOF_FILES, published_texts, ) @@ -1846,3 +1848,60 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho project_dir=tmp_path, discovered_inputs=discovered, ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonJobSelectionTestCase( + description="a model selector selects models", + select="+orders", + expected_selection=("model:orders", "model:stg_orders"), + ), + PythonJobSelectionTestCase( + description="a task selector selects the task", + select="task:request_carrier_collection", + expected_selection=("task:request_carrier_collection",), + ), + PythonJobSelectionTestCase( + description="a python folder selects every node under it", + select="python/orders", + expected_selection=("asset:partner_orders_file",), + ), + PythonJobSelectionTestCase( + description="a tag shared by models and Python nodes selects both", + select="tag:orders", + expected_selection=( + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_cli( + tmp_path: Path, test_case: PythonJobSelectionTestCase +) -> None: + write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + + snapshot: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=[job("orders_job", description="Orders.", select=test_case.select)], + discovered_inputs=discovered, + ).to_json() + ) + + assert snapshot["documents"]["jobs"]["orders_job"]["selection"] == list( + test_case.expected_selection + ) From 889f3d499fc74f6cc0acec466eaceeea8f64acb8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 02:11:12 +0000 Subject: [PATCH 02/34] wip: resolve published Python selectors with the CLI's unified resolver Work in progress, not ready for review. - Publish-time job, sensor-condition and role selections resolve with the same unified SQL/Python resolver `sqb build --select` uses. - Python snapshot nodes publish deps (Python and SQL) and tags; an intermediate loader publishes the source node it writes, and loader chains list it in `loads`. - Server-side ad-hoc selection rebuilds the Python selection graph from the snapshot and resolves with the same resolver. - The worker selects Python nodes by typed key. Validated so far: test_orchestration_snapshot.py passes (99 tests, including the Python selector reproduction). The Rust unit tests for the ad-hoc planner and planner scope have not finished yet; no review and no full gate. --- .../src/planner_scope/main/node_loads.rs | 3 +- .../src/planner_scope/tests/test_loaders.rs | 9 +- .../src/python_selection/_helpers/unified.rs | 32 ++++- .../src/python_selection/main/mod.rs | 2 + .../main/published_python_graph.rs | 52 ++++++++ .../main/resolve_python_sql_nodes.rs | 21 +++ .../src/python_selection/models.rs | 13 ++ .../src/adhoc/_helpers/planner.rs | 75 ++++++++++- .../sqlbuild-cli/src/adhoc/tests/helpers.rs | 1 + .../src/adhoc/tests/test_adhoc_planner.rs | 123 ++++++++++++++++++ .../src/runs/_helpers/requests/adhoc.rs | 21 ++- .../sqlbuild-orchestration/src/runs/models.rs | 3 +- .../compiled_project/snapshot_declarations.rs | 15 +-- .../compiled_project/snapshot_roles.rs | 23 +--- .../_helpers/graph/python_selection.rs | 89 +++++++++++++ .../_helpers/location_command/snapshot.rs | 88 ++++++++++++- .../_helpers/location_command/worker.rs | 10 +- .../_helpers/native_host/constants.rs | 2 - .../native_host/orchestration_triggers.rs | 47 ++----- .../test_orchestration_snapshot.py | 18 ++- 20 files changed, 555 insertions(+), 92 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs b/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs index 4d2f33a41..e80d101d5 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs @@ -5,7 +5,7 @@ use crate::planner_scope::constants::{MODEL, SOURCE}; use crate::planner_scope::models::LoaderGraph; use std::collections::{BTreeMap, BTreeSet}; -/// Managed sources each model's execution deps read and each managed source's declared loader chain, as `expand_loaders` keys them. +/// Managed sources each model's execution deps read and each managed source's loader chain, intermediate loaders included, as `expand_loaders` keys them. pub fn node_loads( graph: &LoaderGraph, upstream: &[(ObjectKey, Vec)], @@ -24,7 +24,6 @@ pub fn node_loads( graph .upstream(chain) .iter() - .filter(|name| !graph.intermediate(name)) .map(|name| graph.key(name)) .collect() } else { diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs index b90277e27..8fd13b676 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs @@ -147,8 +147,8 @@ fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_dec expected_loads: vec![("model:orders", vec!["source:raw_returns"])], }, NodeLoadsTestCase { - // stage_orders has no declaring source, so a run that loads it still fails in the worker. - description: "a chain keeps declared loaders behind an intermediate one and drops the intermediate", + // stage_orders has no declaring source; the snapshot publishes it as source:stage_orders. + description: "a chain keeps an intermediate loader and the declared loaders behind it", loaders: vec![ ("raw_orders", vec!["stage_orders"]), ("stage_orders", vec!["raw_customers"]), @@ -162,7 +162,10 @@ fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_dec ("source:raw_orders", vec![]), ("source:raw_customers", vec![]), ], - expected_loads: vec![("source:raw_orders", vec!["source:raw_customers"])], + expected_loads: vec![( + "source:raw_orders", + vec!["source:stage_orders", "source:raw_customers"], + )], }, ]; let key = |text: &str| { diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index b98e48dca..7bd1a6351 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -102,15 +102,20 @@ pub(crate) fn run_selection( .filter(|atom| !is_check(atom)) .collect(); validate_selected_dependencies(context, &runnable)?; + Ok(selection(context, sql_keys(&runnable), &runnable)) +} + +/// The SQL keys, with the source each selected intermediate loader writes, and the selected Python node names. +fn selection(context: &SelectionContext<'_>, sql: Vec, atoms: &Atoms) -> RunSelection { let terminal: HashSet<&str> = context .sql .sources .iter() .filter_map(|(_, loader)| loader.as_deref()) .collect(); - let mut keys: BTreeSet = sql_keys(&runnable).into_iter().collect(); + let mut keys: BTreeSet = sql.into_iter().collect(); let mut names: Vec = Vec::new(); - for atom in &runnable { + for atom in atoms { let Atom::Python(name) = atom else { continue; }; @@ -119,10 +124,29 @@ pub(crate) fn run_selection( } names.push(name.clone()); } - Ok(RunSelection { + RunSelection { sql_keys: keys.into_iter().collect(), python_node_names: names, - }) + } +} + +/// The SQL keys with their build functions and the Python node names, checks included, that selectors name; nothing is refused. +pub(crate) fn node_selection( + context: &SelectionContext<'_>, + select: &[String], + exclude: &[String], +) -> Result { + validate_boundaries(context)?; + let selected: Atoms = groups(context, select)? + .difference(&groups(context, exclude)?) + .cloned() + .collect(); + let sql: Vec = build_resources( + context.sql.graph, + &sql_keys(&selected), + BuildResources::SELECTION, + ); + Ok(selection(context, sql, &selected)) } fn groups(context: &SelectionContext<'_>, raw_groups: &[String]) -> Resolved { diff --git a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs index 884bfa0b8..3dace0806 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs @@ -1,4 +1,6 @@ +pub mod published_python_graph; pub mod python_node_graph_indexes; pub mod resolve_python_node_excludes; pub mod resolve_python_node_selectors; +pub mod resolve_python_sql_nodes; pub mod resolve_python_sql_run_selection; diff --git a/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs new file mode 100644 index 000000000..482d1689f --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs @@ -0,0 +1,52 @@ +use crate::definition_changes::constants::PYTHON_NODE_KINDS; +use crate::python_selection::_helpers::inventory::indexes; +use crate::python_selection::models::{ + InventoryIndexes, InventoryNode, PublishedPythonNode, PythonSelectionGraph, PythonSelectionNode, +}; + +/// The selection graph of Python nodes rebuilt from a published snapshot, so the server selects them as the CLI does. +pub fn published_python_graph(nodes: &[PublishedPythonNode]) -> PythonSelectionGraph { + let is_python = |kind: &str| PYTHON_NODE_KINDS.contains(&kind); + let inventory: Vec = nodes + .iter() + .map(|node| InventoryNode { + name: node.name.clone(), + tags: node.tags.clone(), + relative_path: String::new(), + upstream: node + .deps + .iter() + .filter(|(kind, _)| is_python(kind)) + .map(|(_, name)| name.clone()) + .collect(), + }) + .collect(); + let InventoryIndexes { + upstream, + downstream, + tags, + .. + } = indexes(&inventory); + PythonSelectionGraph { + nodes: nodes + .iter() + .map(|node| PythonSelectionNode { + name: node.name.clone(), + kind: node.kind.clone(), + sql_deps: node + .deps + .iter() + .filter(|(kind, _)| !is_python(kind)) + .cloned() + .collect(), + }) + .collect(), + upstream, + downstream, + tags, + paths: nodes + .iter() + .map(|node| (node.name.clone(), node.folder.clone())) + .collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs new file mode 100644 index 000000000..6a2d99a3c --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs @@ -0,0 +1,21 @@ +use crate::graph::errors::SelectorError; +use crate::python_selection::_helpers::nodes::sql_names; +use crate::python_selection::_helpers::unified::node_selection; +use crate::python_selection::models::{ + NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, +}; + +/// The SQL resources and Python nodes, checks included, that selectors name, as grants and sensor conditions read them. +pub fn resolve_python_sql_nodes( + sql: SqlSelectionGraph<'_>, + python: &PythonSelectionGraph, + select: &[String], + exclude: &[String], +) -> Result { + let context: SelectionContext<'_> = SelectionContext { + sql, + nodes: NodeIndex::new(python), + sql_names: sql_names(sql.graph), + }; + node_selection(&context, select, exclude) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/models.rs b/crates/sqlbuild-analysis/src/python_selection/models.rs index b0d615e1d..63b7aacf0 100644 --- a/crates/sqlbuild-analysis/src/python_selection/models.rs +++ b/crates/sqlbuild-analysis/src/python_selection/models.rs @@ -52,6 +52,19 @@ pub struct InventoryNode { pub upstream: Vec, } +/// One Python node as a published snapshot lists it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PublishedPythonNode { + pub name: String, + /// `task`, `asset`, `loader` or `check`. + pub kind: String, + pub tags: Vec, + /// The folder of the node's file relative to the project, in POSIX form. + pub folder: String, + /// The Python nodes and SQL resources it depends on, as published keys. + pub deps: Vec, +} + /// Python's `PythonNodeGraph` edges and indexes, each in Python's dict order. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct InventoryIndexes { diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index 93c34ee62..fd215a452 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -3,11 +3,14 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::{Value as Json, json}; +use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; use sqlbuild_analysis::graph::models::{GraphResource, JobSelection}; +use sqlbuild_analysis::python_selection::main::published_python_graph::published_python_graph; +use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; +use sqlbuild_analysis::python_selection::models::{PublishedPythonNode, SqlSelectionGraph}; use sqlbuild_core::text::main::python_str_repr::python_str_repr; use sqlbuild_orchestration::runs::models::{CommandPlan, PlannerResource}; use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs}; @@ -203,19 +206,81 @@ fn resolved( select: &[String], exclude: &[String], ) -> Result, String> { - let graph_resources: Vec = resources.iter().filter_map(graph_resource).collect(); + let (python, sql): (Vec<&PlannerResource>, Vec<&PlannerResource>) = resources + .iter() + .partition(|resource| python_kind(&resource.key).is_some()); + let graph_resources: Vec = sql + .iter() + .filter_map(|resource| graph_resource(resource)) + .collect(); let mut graph = build_project_graph(&graph_resources); let selectors: Vec = select.iter().chain(exclude).cloned().collect(); - if selectors_need_jobs(&graph, &selectors, &[]) { + let node_names: Vec<&str> = python + .iter() + .filter_map(|resource| object_key(&resource.key).map(|_| name_of(&resource.key))) + .collect(); + if selectors_need_jobs(&graph, &selectors, &node_names) { graph = attach_jobs(graph, published_jobs(jobs)); } - Ok(resolve_selectors(&graph, select, exclude) - .map_err(|error| format!("{}: {}", error.code, error.message))? + let sources: Vec<(String, Option)> = sql + .iter() + .filter(|resource| resource.key.starts_with("source:")) + .map(|resource| (name_of(&resource.key).to_owned(), resource.loader.clone())) + .collect(); + let nodes: Vec = python + .iter() + .filter_map(|resource| { + Some(PublishedPythonNode { + name: name_of(&resource.key).to_owned(), + kind: python_kind(&resource.key)?.to_owned(), + tags: resource.tags.clone(), + folder: resource.folder.clone().unwrap_or_else(|| ".".to_owned()), + deps: resource + .deps + .iter() + .filter_map(|dep| object_key(dep)) + .collect(), + }) + }) + .collect(); + let python_graph = published_python_graph(&nodes); + let selection = resolve_python_sql_run_selection( + SqlSelectionGraph { + graph: &graph, + sources: &sources, + }, + &python_graph, + select, + exclude, + ) + .map_err(|error| format!("{}: {}", error.code, error.message))?; + let kinds: BTreeMap<&str, &str> = nodes + .iter() + .map(|node| (node.name.as_str(), node.kind.as_str())) + .collect(); + Ok(selection + .sql_keys .into_iter() .map(|(kind, name)| format!("{kind}:{name}")) + .chain( + selection + .python_node_names + .iter() + .filter_map(|name| Some(format!("{}:{name}", kinds.get(name.as_str())?))), + ) .collect()) } +/// The Python node kind of a published key, such as `task` for `task:export_orders`. +fn python_kind(key: &str) -> Option<&'static str> { + let (kind, _) = key.split_once(':')?; + PYTHON_NODE_KINDS.into_iter().find(|python| *python == kind) +} + +fn name_of(key: &str) -> &str { + key.split_once(':').map_or(key, |(_, name)| name) +} + /// Published jobs as already-resolved graph selections. fn published_jobs(jobs: &PublishedJobs) -> BTreeMap { let mut selections: BTreeMap = BTreeMap::new(); diff --git a/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs b/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs index 0725ac938..3cd4087f5 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs @@ -13,6 +13,7 @@ pub(super) fn resource( deps: deps.iter().map(|dep| (*dep).to_owned()).collect(), tags: tags.iter().map(|tag| (*tag).to_owned()).collect(), folder: folder.map(str::to_owned), + loader: None, } } diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index 8631f1766..47d375311 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -289,3 +289,126 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select ); } } + +#[test] +fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_command_line() { + let resources = [ + resource("model:stg_orders", &[], &["orders"], Some("models/staging")), + resource( + "model:orders", + &["model:stg_orders"], + &["orders"], + Some("models/marts"), + ), + resource("model:customers", &[], &[], Some("models/marts")), + resource( + "asset:partner_orders_file", + &["model:orders"], + &["orders"], + Some("python/orders/partner_export"), + ), + resource( + "task:request_carrier_collection", + &["asset:partner_orders_file"], + &["orders"], + Some("python/fulfillment/carrier_sync"), + ), + ]; + let jobs: PublishedJobs = BTreeMap::new(); + let test_cases = [ + AdhocPlanTestCase { + description: "typed selectors select the Python nodes", + command: &[ + "build", + "--select", + "task:request_carrier_collection", + "asset:partner_orders_file", + ], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "a task without the Python node it depends on is refused, as on the command line", + command: &["build", "--select", "task:request_carrier_collection"], + expected_plan: Err("depends on unselected Python node 'partner_orders_file'"), + }, + AdhocPlanTestCase { + description: "a python folder selects the nodes under it", + command: &["build", "--select", "python/orders"], + expected_plan: Ok((&["asset:partner_orders_file"], json!({}))), + }, + AdhocPlanTestCase { + description: "a tag shared by models and Python nodes selects both", + command: &["build", "--select", "tag:orders"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "upstream of a task crosses its Python and SQL dependencies", + command: &["build", "--select", "+task:request_carrier_collection"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "downstream of a model reaches the Python nodes that read it", + command: &["build", "--select", "orders+", "--exclude", "stg_orders"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + ]; + let planner = adhoc_planner(); + for test_case in test_cases { + let command: Vec = test_case + .command + .iter() + .map(|token| (*token).to_owned()) + .collect(); + let actual = planner + .plan(&command, (&resources, &jobs)) + .map(|plan| { + ( + plan.selection.into_iter().collect::>(), + json!(plan.settings), + ) + }) + .map_err(|error| { + test_case + .expected_plan + .as_ref() + .err() + .filter(|fragment| error.contains(**fragment)) + .map_or(error.clone(), |fragment| (*fragment).to_owned()) + }); + assert_eq!( + actual, + owned_plan(&test_case.expected_plan), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index bcb1eddb6..6de44e69d 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -30,6 +30,10 @@ use crate::store::models::{ControlTransaction, Parameter}; /// Graph kinds in the order the CLI builds its project graph: models, sources, seeds, then functions. const GRAPH_KINDS: [&str; 5] = ["model", "source", "seed", "udf", "table_fn"]; +/// Python node kinds, in the order the CLI discovers them. +const PYTHON_NODE_KINDS: [&str; 4] = ["task", "asset", "loader", "check"]; +/// Marks the source node an intermediate loader writes, which the CLI's SQL graph does not list. +const INTERMEDIATE_LOADER_FIELD: &str = "intermediate_loader"; /// The frozen request and its verb, or why the command cannot run; `run_id` comes from the caller. pub(crate) async fn frozen_adhoc_request( @@ -141,23 +145,26 @@ pub(crate) async fn publication_plan( )) } -/// The selector graph rebuilt from published definitions, in the CLI's kind order. +/// The selector graph rebuilt from published definitions, in the CLI's kind order; sources intermediate loaders write are selected through their loader. fn planner_resources( nodes: &std::collections::BTreeMap, ) -> Vec { let mut resources: Vec = Vec::new(); - for kind in GRAPH_KINDS { + for kind in GRAPH_KINDS.iter().chain(&PYTHON_NODE_KINDS) { for (key, node) in nodes { - if node.kind != kind { + let definition = &node.definition; + if node.kind != *kind || definition.get(INTERMEDIATE_LOADER_FIELD).is_some() { continue; } - let definition = &node.definition; - let (tags, folder) = match kind { + let (tags, folder) = match *kind { "model" => ( definition.pointer("/config/tags"), Some(model_folder(definition)), ), "seed" => (definition.pointer("/schema/tags"), None), + _ if PYTHON_NODE_KINDS.contains(kind) => { + (definition.get("tags"), Some(model_folder(definition))) + } _ => (definition.get("tags"), None), }; resources.push(PlannerResource { @@ -165,6 +172,10 @@ fn planner_resources( deps: texts(definition.get("deps")), tags: texts(tags), folder, + loader: definition + .pointer("/entry/loader") + .and_then(Value::as_str) + .map(str::to_owned), }); } } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 6b9216161..48ac16d01 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -760,13 +760,14 @@ pub struct AdhocCheckRequest { pub priority: i64, } -/// One graph resource selectors resolve over: its `kind:name` key, upstream keys, tags and model folder. +/// One SQL resource or Python node selectors resolve over: its `kind:name` key, upstream keys, tags, file folder and a source's loader. #[derive(Clone, Debug, PartialEq, Eq)] pub struct PlannerResource { pub key: String, pub deps: Vec, pub tags: Vec, pub folder: Option, + pub loader: Option, } /// A parsed command: its verb, the exact node keys it selects and the flags it sets, by parser destination. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index a63c08cd1..ca9901107 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -6,7 +6,6 @@ use std::path::Path; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use serde_json::Value; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; @@ -20,6 +19,7 @@ use sqlbuild_orchestration::declarations::models::{ }; use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::orchestration_declarations::{ AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, declared_at, module_jobs, run_controls, @@ -60,7 +60,7 @@ pub(crate) fn discover_declarations<'py>( pub(crate) fn compile_declarations( py: Python<'_>, mut input: SnapshotInput, - (project_dir, graph): (&Path, &ProjectGraph), + (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), declarations: Vec, ) -> PyResult { if declarations.is_empty() { @@ -113,12 +113,9 @@ pub(crate) fn compile_declarations( job.name ))); } - let selection: BTreeSet = - resolve_selectors(graph, &tokens(&job.select), &tokens(&job.exclude)) - .map_err(|error| PyValueError::new_err(error.message))? - .into_iter() - .map(|(kind, name)| format!("{kind}:{name}")) - .collect(); + let selection: BTreeSet = selected + .run_keys(graph, &tokens(&job.select), &tokens(&job.exclude)) + .map_err(|error| PyValueError::new_err(format!("job {}: {error}", job.name)))?; let declared_at: Option = declared_at(project_dir, job.declared_at)?; let controls = run_controls( &job.name, @@ -229,7 +226,7 @@ pub(crate) fn compile_declarations( .. } = compile_sensors( py, - (project_dir, &input, graph), + (project_dir, &input, (graph, selected)), authored_sensors, compile_calendars( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs index 4867f861a..26e7bc126 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs @@ -1,35 +1,23 @@ //! Project roles from the reviewed `roles.toml` file, their selectors resolved against this snapshot. -use std::collections::BTreeSet; use std::path::Path; use pyo3::exceptions::PyValueError; use pyo3::prelude::{PyResult, Python}; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_orchestration::authorization::main::compile_roles::compile_roles; use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::host::write_console; /// The project-root file that defines project roles. pub(crate) const ROLES_FILE: &str = "roles.toml"; -/// Node keys of one grant selector, in the job selection grammar. -fn selector_nodes(graph: &ProjectGraph, selector: &str) -> Result, String> { - let tokens: Vec = selector.split_whitespace().map(str::to_owned).collect(); - let keys = resolve_selectors(graph, &tokens, &[]).map_err(|error| error.message)?; - let mut nodes: BTreeSet = BTreeSet::new(); - for (kind, name) in keys { - nodes.insert(format!("{kind}:{name}")); - } - Ok(nodes) -} - pub(crate) fn compile_project_roles( py: Python<'_>, mut input: SnapshotInput, - (project_dir, graph): (&Path, &ProjectGraph), + (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), ) -> PyResult { let path = project_dir.join(ROLES_FILE); if !path.is_file() { @@ -42,8 +30,11 @@ pub(crate) fn compile_project_roles( } let text = std::fs::read_to_string(&path) .map_err(|error| PyValueError::new_err(format!("roles.toml: {error}")))?; - let (document, warnings) = compile_roles(&text, |selector| selector_nodes(graph, selector)) - .map_err(PyValueError::new_err)?; + let (document, warnings) = compile_roles(&text, |selector| { + let tokens: Vec = selector.split_whitespace().map(str::to_owned).collect(); + selected.node_keys(graph, &tokens, &[]) + }) + .map_err(PyValueError::new_err)?; for warning in warnings { write_console(py, "stderr", &format!("Warning: {warning}\n"))?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 1dd9d85d8..385913077 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -1,5 +1,7 @@ //! Python node and unified SQL/Python selector resolution, and the Python node graph indexes. +use std::collections::BTreeSet; + use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -7,9 +9,11 @@ use pyo3::types::{PyDict, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; +use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection as run_selection; use sqlbuild_analysis::python_selection::models::{ InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, @@ -19,6 +23,7 @@ use sqlbuild_analysis::python_selection::models::{ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compiled_project::job_catalog::graph_with_jobs; use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_items; @@ -128,6 +133,90 @@ fn project_sources(project: &Bound<'_, PyAny>) -> PyResult)>, + pub(crate) nodes: PythonSelectionGraph, +} + +impl ProjectSelection { + pub(crate) fn new( + project: &Bound<'_, PyAny>, + discovered_inputs: &Bound<'_, PyAny>, + ) -> PyResult { + Ok(Self { + sources: project_sources(project)?, + nodes: selection_graph(&python_node_graph(discovered_inputs)?)?, + }) + } + + /// The `kind:name` keys `sqb build --select` runs for these selectors. + pub(crate) fn run_keys( + &self, + graph: &ProjectGraph, + select: &[String], + exclude: &[String], + ) -> Result, String> { + self.keys(run_selection(self.sql(graph), &self.nodes, select, exclude)) + } + + /// The `kind:name` keys these selectors name, Python checks included. + pub(crate) fn node_keys( + &self, + graph: &ProjectGraph, + select: &[String], + exclude: &[String], + ) -> Result, String> { + self.keys(resolve_python_sql_nodes( + self.sql(graph), + &self.nodes, + select, + exclude, + )) + } + + /// The published key of a Python node, such as `task:export_orders`. + pub(crate) fn node_key(&self, name: &str) -> Option { + self.nodes + .nodes + .iter() + .rev() + .find(|node| node.name == name) + .map(|node| format!("{}:{name}", node.kind)) + } + + /// Whether a loader writes no declared source and so runs only as part of another loader's chain. + pub(crate) fn intermediate_loader(&self, name: &str) -> bool { + !self + .sources + .iter() + .any(|(_, loader)| loader.as_deref() == Some(name)) + } + + fn sql<'graph>(&'graph self, graph: &'graph ProjectGraph) -> SqlSelectionGraph<'graph> { + SqlSelectionGraph { + graph, + sources: &self.sources, + } + } + + fn keys( + &self, + selection: Result, + ) -> Result, String> { + let selection: RunSelection = selection.map_err(|error| error.message)?; + let mut keys: BTreeSet = selection + .sql_keys + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + for name in &selection.python_node_names { + keys.extend(self.node_key(name)); + } + Ok(keys) + } +} + /// The Python node names `select` minus `exclude` resolve to; every node when unselected. pub(crate) fn python_node_names<'py>( graph: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 830e43a84..40a0682a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -1,6 +1,6 @@ //! Location publication assembles catalog data and planner-owned local identities. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; use pyo3::exceptions::PyValueError; @@ -24,6 +24,7 @@ use crate::bindings::_helpers::compiled_project::snapshot_roles::compile_project use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::discovery::project_facts::{named_node_rows, project_facts}; use crate::bindings::_helpers::discovery::project_validation::validation_error; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; use crate::bindings::_helpers::native_host::orchestration_declarations::{ @@ -36,6 +37,9 @@ use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; use crate::bindings::_helpers::planner_scope::values::project_graph; use crate::bindings::_helpers::records::fields::field; +/// The Python node kind whose functions load sources. +const LOADER_KIND: &str = "loader"; + /// Where declarations passed to the snapshot directly are reported as declared. const DECLARATIONS_ARGUMENT: &str = "declarations argument"; @@ -111,6 +115,9 @@ pub(crate) fn orchestration_snapshot<'py>( } input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; drop(owner); + let selected = + ProjectSelection::new((&project).into_pyobject(py)?.as_any(), &discovered_inputs)?; + input.nodes = python_nodes(&selected, input.nodes)?; if let Some(providers) = providers_document(&discovered_inputs, &input.nodes)? { input.documents.insert("providers".into(), providers); } @@ -137,11 +144,15 @@ pub(crate) fn orchestration_snapshot<'py>( input = compile_declarations( py, input, - (&project_dir, graph.bind(py).get().graph()), + (&project_dir, graph.bind(py).get().graph(), &selected), authored, )?; } - input = compile_project_roles(py, input, (&project_dir, graph.bind(py).get().graph()))?; + input = compile_project_roles( + py, + input, + (&project_dir, graph.bind(py).get().graph(), &selected), + )?; let project = project.into_pyobject(py)?; if let Some(lineage) = column_lineage_document(project.as_any())? { input.documents.insert("lineage".into(), lineage); @@ -151,6 +162,77 @@ pub(crate) fn orchestration_snapshot<'py>( Ok(NativeOrchestrationSnapshot { input }) } +/// Python nodes publish their `deps` and tags, and each intermediate loader publishes the source it writes, so the server orders, selects and grants them as the CLI does. +fn python_nodes( + selected: &ProjectSelection, + mut nodes: BTreeMap, +) -> PyResult> { + let graph = &selected.nodes; + let mut tags: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for (tag, names) in &graph.tags { + for name in names { + tags.entry(name.as_str()).or_default().push(tag.as_str()); + } + } + let upstream: BTreeMap<&str, &[String]> = graph + .upstream + .iter() + .map(|(name, names)| (name.as_str(), names.as_slice())) + .collect(); + let mut outputs: Vec<(String, SnapshotNode)> = Vec::new(); + for node in &graph.nodes { + let key = format!("{}:{}", node.kind, node.name); + let Some(published) = nodes.get_mut(&key) else { + return Err(PyValueError::new_err(format!( + "Python node {key} is missing from the snapshot" + ))); + }; + let deps: BTreeSet = upstream + .get(node.name.as_str()) + .into_iter() + .flat_map(|names| names.iter()) + .filter_map(|name| selected.node_key(name)) + .chain( + node.sql_deps + .iter() + .map(|(kind, name)| format!("{kind}:{name}")), + ) + .collect(); + published.definition["deps"] = json!(deps); + published.definition["tags"] = + json!(tags.get(node.name.as_str()).cloned().unwrap_or_default()); + if node.kind == LOADER_KIND && selected.intermediate_loader(&node.name) { + let mut definition = json!({ + "name": node.name, + "entry": {"loader": node.name}, + "intermediate_loader": true, + "deps": [key], + }); + if let Some(path) = published.definition.get("path") { + definition["path"] = path.clone(); + } + outputs.push(( + format!("source:{}", node.name), + SnapshotNode { + kind: "source".into(), + definition, + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + )); + } + } + for (key, node) in outputs { + if nodes.insert(key.clone(), node).is_some() { + return Err(PyValueError::new_err(format!( + "intermediate loader {key} collides with a published node" + ))); + } + } + Ok(nodes) +} + fn identity_nodes( project: &Bound<'_, PyAny>, mut nodes: BTreeMap, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 4d28d969a..ec62b33a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -8,6 +8,7 @@ use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; use serde_json::Value; +use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; use sqlbuild_cli::argv::main::command_settings_arguments::command_settings_arguments; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; @@ -148,10 +149,15 @@ fn command_request<'py>( if execution.skipped_nodes.contains_key(key) { continue; } - let (_, name) = key + let (kind, name) = key .split_once(':') .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; - argv.extend([PyString::new(py, "--select"), PyString::new(py, name)]); + let selector = if PYTHON_NODE_KINDS.contains(&kind) { + key.as_str() + } else { + name + }; + argv.extend([PyString::new(py, "--select"), PyString::new(py, selector)]); } let parsed = parse_cli_invocation(py, argv, false)?; let args = field(&parsed, "args")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index d19d66207..d0ef09dcb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -4,8 +4,6 @@ pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; -pub(crate) const NODE_EVENT_CONDITION: &str = "node_event"; -pub(crate) const CHECK_KIND: &str = "check"; pub(crate) const SOURCE_NEW_DATA_CONDITION: &str = "source_new_data"; pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 8779ef860..b8923950a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -1,6 +1,6 @@ //! Authored trigger controls and bounded calendars evaluated only in the location interpreter. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use std::path::Path; use chrono::{DateTime, TimeDelta, Timelike, Utc}; @@ -10,21 +10,20 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use serde_json::{Value, json}; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; use sqlbuild_orchestration::sensors::models::SensorDefinition; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::constants::{ - CALENDAR_WINDOW_PARAMETER, CHECK_KIND, NODE_EVENT_CONDITION, SOURCE_NEW_DATA_CONDITION, - SOURCE_STALE_CONDITION, + CALENDAR_WINDOW_PARAMETER, SOURCE_NEW_DATA_CONDITION, SOURCE_STALE_CONDITION, }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::{ @@ -253,7 +252,7 @@ pub(crate) fn compile_calendars( pub(crate) fn compile_sensors( py: Python<'_>, - (project_dir, input, graph): (&Path, &SnapshotInput, &ProjectGraph), + (project_dir, input, graph): (&Path, &SnapshotInput, (&ProjectGraph, &ProjectSelection)), authored: Vec, mut definitions: Definitions, ) -> PyResult { @@ -281,12 +280,11 @@ pub(crate) fn compile_sensors( if let Some(select) = when.select { let selectors: Vec = select.split_whitespace().map(str::to_owned).collect(); - condition["selection"] = - sensor_nodes(graph, &input.nodes, &selectors, &when.kind)?; + condition["selection"] = sensor_nodes(graph, &selectors, &when.kind)?; } if let Some(selectors) = when.executed_successfully { condition["executed_successfully"] = - sensor_nodes(graph, &input.nodes, &selectors, "executed_successfully")?; + sensor_nodes(graph, &selectors, "executed_successfully")?; } condition } @@ -314,8 +312,7 @@ pub(crate) fn compile_sensors( } fn sensor_nodes( - graph: &ProjectGraph, - available: &BTreeMap, + (graph, selected): (&ProjectGraph, &ProjectSelection), selectors: &[String], field: &str, ) -> PyResult { @@ -324,31 +321,9 @@ fn sensor_nodes( "{field} needs nonempty selectors" ))); } - let mut nodes: BTreeSet = BTreeSet::new(); - let mut sql_selectors: Vec = Vec::new(); - for selector in selectors { - if field == NODE_EVENT_CONDITION && selector.starts_with("check:") { - if !available - .get(selector) - .is_some_and(|node| node.kind == CHECK_KIND) - { - return Err(PyValueError::new_err(format!( - "unknown check node {selector}" - ))); - } - nodes.insert(selector.clone()); - } else { - sql_selectors.push(selector.clone()); - } - } - if !sql_selectors.is_empty() { - nodes.extend( - resolve_selectors(graph, &sql_selectors, &[]) - .map_err(|error| PyValueError::new_err(error.message))? - .into_iter() - .map(|(kind, name)| format!("{kind}:{name}")), - ); - } + let nodes: BTreeSet = selected + .node_keys(graph, selectors, &[]) + .map_err(|error| PyValueError::new_err(format!("{field}: {error}")))?; if nodes.is_empty() { return Err(PyValueError::new_err(format!( "{field} needs nonempty selectors" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 10556cef2..9cce168cc 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -1789,7 +1789,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects .replace('("orders",)', '("unknown_*",)') }, expected_error=ValueError, - expected_message="unknown selector pattern", + expected_message="executed_successfully: unknown selector name 'unknown_\\*'", ), SnapshotDiscoveryRejectionTestCase( description="invalid execution selector syntax fails closed", @@ -1859,9 +1859,19 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho expected_selection=("model:orders", "model:stg_orders"), ), PythonJobSelectionTestCase( - description="a task selector selects the task", - select="task:request_carrier_collection", - expected_selection=("task:request_carrier_collection",), + description="typed selectors select the task and the asset it depends on", + select="task:request_carrier_collection asset:partner_orders_file", + expected_selection=("asset:partner_orders_file", "task:request_carrier_collection"), + ), + PythonJobSelectionTestCase( + description="upstream of a task crosses its Python and SQL dependencies", + select="+task:request_carrier_collection", + expected_selection=( + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ), ), PythonJobSelectionTestCase( description="a python folder selects every node under it", From cc4a6c59b08c71ba2d38d6d1fa40dc159fd91018 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:22:56 +0000 Subject: [PATCH 03/34] feat: let a selected Python node read an unselected upstream's result A Python node can now be selected without its Python upstream in every path that uses the shared resolver: sqb build --select, published jobs, sensors, roles and ad-hoc runs. It reads the upstream's latest persisted result, and fails at run time with an error naming the upstream and the command to build it when the upstream has never succeeded. An intermediate loader without a write_strategy that returns rows now writes them to __loader__ as a table, as the loaders docs describe. A self-managed intermediate that returns nothing keeps its behaviour. The published snapshot rebuilds the Python selection graph from node deps, and job groups order jobs across Python and Python-to-SQL edges. Validated locally: python_selection and adhoc planner Rust tests, clippy on both crates, fensu check, the worker claim tests, orchestration snapshot, Python node selection E2E, load E2E (sqb load and sqb build on the docs example) and the python_nodes and executor/load unit tests. Not yet run: make check-ci, the full suite, review. --- .../loaders/intermediate_default_accounts.py | 25 ++++++ .../loaders/intermediate_default_accounts.yml | 5 ++ .../intermediate_loader_sources.py | 7 +- .../python_selection/_helpers/inventory.rs | 17 ++++ .../python_selection/_helpers/validation.rs | 15 +--- .../main/published_python_graph.rs | 42 ++++----- .../src/python_selection/tests/helpers.rs | 46 +++++++++- .../tests/test_python_selection.rs | 25 +++--- .../src/adhoc/_helpers/planner.rs | 30 ++++--- .../src/adhoc/tests/test_adhoc_planner.rs | 4 +- .../executor/load/_helpers/execution.py | 14 ++- .../load/_helpers/loader_invocation.py | 3 +- src/sqlbuild/executor/load/main/_execute.py | 3 +- .../python_nodes/_helpers/execution.py | 43 ++++++++++ src/sqlbuild/executor/python_nodes/models.py | 8 +- .../cli/commands/main/build/_test_types.py | 10 +++ .../cli/commands/main/build/helpers.py | 9 ++ .../main/build/test_python_node_selection.py | 49 +++++++++++ .../cli/commands/main/load/_test_types.py | 8 ++ .../cli/commands/main/load/test_load.py | 60 +++++++++++++ .../compiler/compiled_project/_test_types.py | 14 +++ .../compiler/compiled_project/helpers.py | 23 +++++ .../publication/_test_types.py | 6 ++ .../compiled_project/publication/helpers.py | 16 ++++ .../publication/test_worker.py | 47 +++++----- .../test_orchestration_snapshot.py | 86 ++++++++++++++++++- .../_helpers/test_run_selectors.py | 16 ++-- .../docs/concepts/python-nodes/overview.mdx | 14 ++- 28 files changed, 528 insertions(+), 117 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py create mode 100644 benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml diff --git a/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py new file mode 100644 index 000000000..6b01434f4 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py @@ -0,0 +1,25 @@ +"""The loader-dependencies example from the loaders docs, as written. + +raw_accounts is an intermediate loader without a write_strategy that returns rows, so it writes +them to __loader__raw_accounts as a table. +""" + +from sqlbuild.executor.load.models import LoaderContext +from sqlbuild.loaders import loader + + +@loader +def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: + """Load customer accounts.""" + return [ + {"account_id": 1, "account_name": "Pawnee Parks"}, + {"account_id": 2, "account_name": "Eagleton"}, + ] + + +@loader(depends_on=[raw_accounts]) +def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: + """Account metrics.""" + accounts = ctx.loader(raw_accounts) + rows = ctx.query(f"SELECT account_id FROM {accounts.destination}") + return [{"account_id": row[0], "metric": "active"} for row in rows.fetchall()] diff --git a/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml new file mode 100644 index 000000000..90e7a2605 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml @@ -0,0 +1,5 @@ +sources: + - name: raw_account_metrics + description: Account metrics read from the intermediate raw_accounts loader. + managed: true + write_strategy: table diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py index 9512d788c..70db157dd 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py @@ -1,9 +1,12 @@ -"""A source loader that depends on an intermediate loader no source declares.""" +"""A source loader that depends on an intermediate loader no source declares. + +A job that selects the model must claim the intermediate loader's relation as well as the source. +""" from sqlbuild.loaders import loader -@loader +@loader(write_strategy="table") def stage_shipments(ctx): """Stage synthetic shipments in the intermediate loader's own relation.""" return [{"order_id": 9}] diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs index 9a416f0c4..944eb5294 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs @@ -2,6 +2,9 @@ use std::collections::HashMap; +use crate::assembly::project::types::ObjectKey; +use crate::definition_changes::constants::PYTHON_NODE_KINDS; + use crate::python_selection::constants::{CURRENT_DIRECTORY, ROOT_DIRECTORY}; use crate::python_selection::models::{InventoryIndexes, InventoryNode}; @@ -83,3 +86,17 @@ pub(crate) fn indexes(nodes: &[InventoryNode]) -> InventoryIndexes { paths: paths.rows, } } + +/// A published node's dependency keys split into its Python upstream names and its SQL keys. +pub(crate) fn split_published_deps(deps: &[ObjectKey]) -> (Vec, Vec) { + let mut python: Vec = Vec::new(); + let mut sql: Vec = Vec::new(); + for (kind, name) in deps { + if PYTHON_NODE_KINDS.contains(&kind.as_str()) { + python.push(name.clone()); + } else { + sql.push((kind.clone(), name.clone())); + } + } + (python, sql) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs index ddacd2783..9bed1e962 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs @@ -157,7 +157,7 @@ fn validate_loader_upstreams(context: &SelectionContext<'_>) -> Result<(), Selec Ok(()) } -/// Python's `_validate_selected_dependencies`: selected sources and nodes have their loaders. +/// Selected managed sources have their loaders selected; a Python node may run without its Python upstream, whose latest persisted result it reads. pub(crate) fn validate_selected_dependencies( context: &SelectionContext<'_>, atoms: &BTreeSet, @@ -188,18 +188,5 @@ pub(crate) fn validate_selected_dependencies( select it directly or use upstream expansion" ))); } - for atom in atoms { - let Atom::Python(name) = atom else { - continue; - }; - for upstream in context.nodes.edges(name, EdgeDirection::Upstream) { - if !selected.contains(upstream.as_str()) { - return Err(failure(format!( - "Python node '{name}' depends on unselected Python node '{upstream}'; \ - select it directly or use upstream expansion" - ))); - } - } - } Ok(()) } diff --git a/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs index 482d1689f..695361c4e 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs @@ -1,26 +1,26 @@ -use crate::definition_changes::constants::PYTHON_NODE_KINDS; -use crate::python_selection::_helpers::inventory::indexes; +use crate::python_selection::_helpers::inventory::{indexes, split_published_deps}; use crate::python_selection::models::{ InventoryIndexes, InventoryNode, PublishedPythonNode, PythonSelectionGraph, PythonSelectionNode, }; /// The selection graph of Python nodes rebuilt from a published snapshot, so the server selects them as the CLI does. pub fn published_python_graph(nodes: &[PublishedPythonNode]) -> PythonSelectionGraph { - let is_python = |kind: &str| PYTHON_NODE_KINDS.contains(&kind); - let inventory: Vec = nodes - .iter() - .map(|node| InventoryNode { + let mut inventory: Vec = Vec::with_capacity(nodes.len()); + let mut selection_nodes: Vec = Vec::with_capacity(nodes.len()); + for node in nodes { + let (upstream, sql_deps) = split_published_deps(&node.deps); + inventory.push(InventoryNode { name: node.name.clone(), tags: node.tags.clone(), relative_path: String::new(), - upstream: node - .deps - .iter() - .filter(|(kind, _)| is_python(kind)) - .map(|(_, name)| name.clone()) - .collect(), - }) - .collect(); + upstream, + }); + selection_nodes.push(PythonSelectionNode { + name: node.name.clone(), + kind: node.kind.clone(), + sql_deps, + }); + } let InventoryIndexes { upstream, downstream, @@ -28,19 +28,7 @@ pub fn published_python_graph(nodes: &[PublishedPythonNode]) -> PythonSelectionG .. } = indexes(&inventory); PythonSelectionGraph { - nodes: nodes - .iter() - .map(|node| PythonSelectionNode { - name: node.name.clone(), - kind: node.kind.clone(), - sql_deps: node - .deps - .iter() - .filter(|(kind, _)| !is_python(kind)) - .cloned() - .collect(), - }) - .collect(), + nodes: selection_nodes, upstream, downstream, tags, diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs index 92b68329c..2ac7b7de0 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs @@ -2,13 +2,14 @@ use crate::assembly::project::types::ObjectKey; use crate::graph::errors::SelectorError; use crate::graph::main::build_project_graph::build_project_graph; use crate::graph::models::{GraphResource, ProjectGraph}; +use crate::python_selection::main::published_python_graph::published_python_graph; use crate::python_selection::main::python_node_graph_indexes::python_node_graph_indexes; use crate::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use crate::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; use crate::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; use crate::python_selection::models::{ - InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, - SqlSelectionGraph, + InventoryIndexes, InventoryNode, PublishedPythonNode, PythonSelectionGraph, + PythonSelectionNode, RunSelection, SqlSelectionGraph, }; use crate::python_selection::tests::test_types::{ExpectedRun, SelectorFailure}; @@ -130,6 +131,36 @@ fn orders_sql_graph() -> ProjectGraph { ]) } +/// The orders Python node graph as the server rebuilds it from published snapshot nodes. +pub(super) fn published_orders_graph() -> PythonSelectionGraph { + let nodes: Vec = ORDERS_NODES + .iter() + .map( + |(name, kind, path, tags, upstream, sql_deps)| PublishedPythonNode { + name: (*name).to_owned(), + kind: (*kind).to_owned(), + tags: owned(tags), + folder: path + .rsplit_once('/') + .map_or(".", |(folder, _)| folder) + .to_owned(), + deps: upstream + .iter() + .map(|upstream| { + let kind: &str = ORDERS_NODES + .iter() + .find(|node| node.0 == *upstream) + .map_or("task", |node| node.1); + key(&(kind, upstream)) + }) + .chain(sql_deps.iter().map(key)) + .collect(), + }, + ) + .collect(); + published_python_graph(&nodes) +} + /// The Python node names the selectors resolve to over the orders graph. pub(super) fn node_names(select: &[&str], exclude: &[&str]) -> Result, OwnedFailure> { resolve_python_node_selectors(&orders_python_graph(), &owned(select), &owned(exclude)) @@ -138,6 +169,15 @@ pub(super) fn node_names(select: &[&str], exclude: &[&str]) -> Result Result { + run_over(&orders_python_graph(), select, exclude) +} + +/// The run selection over the orders SQL graph and the given Python graph. +pub(super) fn run_over( + python: &PythonSelectionGraph, + select: &[&str], + exclude: &[&str], +) -> Result { let graph: ProjectGraph = orders_sql_graph(); let sources: Vec<(String, Option)> = vec![("raw_orders".to_owned(), Some("raw_orders".to_owned()))]; @@ -146,7 +186,7 @@ pub(super) fn run(select: &[&str], exclude: &[&str]) -> Result = select.iter().chain(exclude).cloned().collect(); let node_names: Vec<&str> = python .iter() - .filter_map(|resource| object_key(&resource.key).map(|_| name_of(&resource.key))) + .map(|resource| name_of(&resource.key)) .collect(); if selectors_need_jobs(&graph, &selectors, &node_names) { graph = attach_jobs(graph, published_jobs(jobs)); @@ -229,19 +229,7 @@ fn resolved( .collect(); let nodes: Vec = python .iter() - .filter_map(|resource| { - Some(PublishedPythonNode { - name: name_of(&resource.key).to_owned(), - kind: python_kind(&resource.key)?.to_owned(), - tags: resource.tags.clone(), - folder: resource.folder.clone().unwrap_or_else(|| ".".to_owned()), - deps: resource - .deps - .iter() - .filter_map(|dep| object_key(dep)) - .collect(), - }) - }) + .filter_map(|resource| published_python_node(resource)) .collect(); let python_graph = published_python_graph(&nodes); let selection = resolve_python_sql_run_selection( @@ -277,6 +265,20 @@ fn python_kind(key: &str) -> Option<&'static str> { PYTHON_NODE_KINDS.into_iter().find(|python| *python == kind) } +fn published_python_node(resource: &PlannerResource) -> Option { + Some(PublishedPythonNode { + name: name_of(&resource.key).to_owned(), + kind: python_kind(&resource.key)?.to_owned(), + tags: resource.tags.clone(), + folder: resource.folder.clone().unwrap_or_else(|| ".".to_owned()), + deps: resource + .deps + .iter() + .filter_map(|dep| object_key(dep)) + .collect(), + }) +} + fn name_of(key: &str) -> &str { key.split_once(':').map_or(key, |(_, name)| name) } diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index 47d375311..9855c4cbc 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -333,9 +333,9 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co )), }, AdhocPlanTestCase { - description: "a task without the Python node it depends on is refused, as on the command line", + description: "a task runs without the Python node it depends on, as on the command line", command: &["build", "--select", "task:request_carrier_collection"], - expected_plan: Err("depends on unselected Python node 'partner_orders_file'"), + expected_plan: Ok((&["task:request_carrier_collection"], json!({}))), }, AdhocPlanTestCase { description: "a python folder selects the nodes under it", diff --git a/src/sqlbuild/executor/load/_helpers/execution.py b/src/sqlbuild/executor/load/_helpers/execution.py index 955a267d8..673e81e4a 100644 --- a/src/sqlbuild/executor/load/_helpers/execution.py +++ b/src/sqlbuild/executor/load/_helpers/execution.py @@ -3,8 +3,9 @@ from __future__ import annotations from collections.abc import Callable +from dataclasses import replace -from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry, SourceWriteStrategy from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.executor.load.models import LoadExecutionIndexes, LoadExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -191,3 +192,14 @@ def is_untargeted_self_managed_intermediate( and source_entry.write_strategy is None and loader_function.destination is None ) + + +def row_write_entry(source_entry: SourceEntry) -> SourceEntry: + """Return the entry rows are written with: an intermediate loader defaults to `table`.""" + + if ( + source_entry.meta.get("sqlbuild_loader_node") is True + and source_entry.write_strategy is None + ): + return replace(source_entry, write_strategy=SourceWriteStrategy.TABLE) + return source_entry diff --git a/src/sqlbuild/executor/load/_helpers/loader_invocation.py b/src/sqlbuild/executor/load/_helpers/loader_invocation.py index c07f0fd41..d287c8681 100644 --- a/src/sqlbuild/executor/load/_helpers/loader_invocation.py +++ b/src/sqlbuild/executor/load/_helpers/loader_invocation.py @@ -18,6 +18,7 @@ from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.execution import ( is_untargeted_self_managed_intermediate, + row_write_entry, ) from sqlbuild.executor.load._helpers.relation_refs import ( build_loader_relation_refs, @@ -238,7 +239,7 @@ def interpret_loader_return( duration_ms=int((time.monotonic() - start) * 1000), lifecycle_events=statement_recorder.snapshot(), ) - if source_entry.write_strategy is None: + if row_write_entry(source_entry).write_strategy is None: raise ExecutorInputError( f"Source '{source_entry.name}' loader '{loader_function.name}' returned rows " "but source has no write_strategy" diff --git a/src/sqlbuild/executor/load/main/_execute.py b/src/sqlbuild/executor/load/main/_execute.py index f4ccaa036..eef2c5c56 100644 --- a/src/sqlbuild/executor/load/main/_execute.py +++ b/src/sqlbuild/executor/load/main/_execute.py @@ -26,7 +26,7 @@ infer_cursor_kind, load_staging_cursor_bounds, ) -from sqlbuild.executor.load._helpers.execution import load_resource_kind +from sqlbuild.executor.load._helpers.execution import load_resource_kind, row_write_entry from sqlbuild.executor.load._helpers.external import execute_external_source_load from sqlbuild.executor.load._helpers.loader_invocation import ( build_loader_context, @@ -176,6 +176,7 @@ def _execute_source_load( ) if early_result is not None: return early_result + source_entry = row_write_entry(source_entry) rows_loaded: int = write_loader_rows_to_staging( loader_return_value=raw_rows, source_entry=source_entry, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 43bd63753..dafc8e882 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -189,6 +189,18 @@ def _execute_ready_node_in_cost_scope( result_store=result_store, warnings=warnings, ) + missing: str | None = _missing_upstream_result( + node=node, node_kind=node_kind, context=context, upstream_results=upstream_results + ) + if missing is not None: + result = PythonNodeExecutionResult( + node_name=node.name, + kind=node_kind, + status=PythonNodeStatus.FAILED, + error_message=missing, + ) + _persist_python_node_result(result_store=result_store, result=result, run_id=run_id) + return result, None try: result: PythonNodeExecutionResult = _call_node_with_retry( node=node, @@ -344,6 +356,37 @@ def _call_node_with_retry( attempt += 1 +def _missing_upstream_result( + *, + node: ExecutablePythonNode, + node_kind: PythonNodeKind, + context: TaskContext | AssetContext, + upstream_results: tuple[PythonNodeExecutionResult, ...], +) -> str | None: + """Why the node cannot run: a Python upstream outside this run that has never succeeded.""" + + python_dependencies: tuple[Callable[..., object], ...] = tuple( + dependency for dependency in node.depends_on if not isinstance(dependency, SqlResourceRef) + ) + if len(upstream_results) == len(python_dependencies): + return None + in_run: frozenset[str] = frozenset(result.node_name for result in upstream_results) + for dependency in python_dependencies: + kind, name = context.result_dependency_identity(dependency) + if name in in_run or context.result_of(node_function=dependency, default=None) is not None: + continue + both: str = f"'sqb build --select +{node_kind.value}:{node.name}'" + if kind == PythonNodeKind.LOADER.value: + return ( + f"Upstream loader '{name}' has no successful result to read. Build it with {both}." + ) + return ( + f"Upstream {kind} '{name}' has no successful result to read. Build it first with " + f"'sqb build --select {kind}:{name}', or build both with {both}." + ) + return None + + def _build_context( *, node: ExecutablePythonNode, diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index e41a47874..245fab2a9 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -267,7 +267,7 @@ def result_of( if default is not MISSING_DEFAULT: return default raise ExecutorInputError("No Python node result store is available") - node_type, node_name = self._result_dependency_identity(node_function) + node_type, node_name = self.result_dependency_identity(node_function) return self.result_store.result_of( node_type=node_type, node_name=node_name, @@ -285,14 +285,16 @@ def results_of( if self.result_store is None: return () - node_type, node_name = self._result_dependency_identity(node_function) + node_type, node_name = self.result_dependency_identity(node_function) return self.result_store.results_of( node_type=node_type, node_name=node_name, limit=limit, ) - def _result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: + def result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: + """Return the `(kind, name)` a dependency's results are stored under.""" + task_definition: object = getattr(node_function, "__sqlbuild_task__", None) if task_definition is not None: return PythonNodeKind.TASK.value, self._definition_name(task_definition) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index ae15ae5ad..64837288d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1614,3 +1614,13 @@ class ChangedTwoUpE2ETestCase: warehouse: ChangedWarehouse expected_crm_run: frozenset[str] expected_downstream: frozenset[str] + + +@dataclass(frozen=True) +class PythonUpstreamResultE2ETestCase: + """A Python node selected without its Python upstream, before and after the upstream builds.""" + + description: str + select: str + upstream_select: str + expected_error: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index fc8da19a0..95f42565c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -895,6 +895,15 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: 'MODEL (description "Test model order_summary.", ' 'materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' ), + "python/tasks/reports.py": ( + "from python.tasks.counts import count_orders\nfrom sqlbuild.tasks import task\n\n\n" + "@task(depends_on=count_orders)\n" + "def report_order_count(ctx):\n" + " '''Test task report_order_count.'''\n" + " upstream = ctx.result_of(node_function=count_orders)\n" + ' ctx.execute_sql(f"CREATE OR REPLACE TABLE main.report_order_count_result AS "\n' + " f\"SELECT '{upstream.status}' AS status\")\n" + ), } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py index c792b221a..1ec298fe6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py @@ -10,6 +10,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( PythonNodeSelectionE2ETestCase, + PythonUpstreamResultE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( PYTHON_NODE_SELECTION_DATABASE, @@ -174,3 +175,51 @@ def test_given_python_node_selection_when_building_then_it_gates_only_on_schedul table_exists(db_path=database, table_name=table) for table in test_case.expected_missing_tables ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonUpstreamResultE2ETestCase( + description="a task reads its unselected upstream task's latest result", + select="task:report_order_count", + upstream_select="task:count_orders", + expected_error=( + "Upstream task 'count_orders' has no successful result to read. Build it first " + "with 'sqb build --select task:count_orders', or build both with " + "'sqb build --select +task:report_order_count'." + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_task_selected_without_its_python_upstream_when_building_then_it_reads_the_persisted_result( + test_case: PythonUpstreamResultE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="order_nodes", + repo_files=python_node_selection_project_files( + orders_sql=PythonNodeSelectionE2ETestCase.orders_sql + ), + ) + database: Path = project_dir / PYTHON_NODE_SELECTION_DATABASE + execute_duckdb(db_path=database, sql=_EXISTING_ROWS + _EXISTING_ORDERS) + report: tuple[str, ...] = ("--no-color", "build", "--select", test_case.select) + + never_built: subprocess.CompletedProcess[str] = run_sqb(command=report, project_dir=project_dir) + upstream: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", test_case.upstream_select), + project_dir=project_dir, + ) + rebuilt: subprocess.CompletedProcess[str] = run_sqb(command=report, project_dir=project_dir) + + output: str = never_built.stdout + never_built.stderr + assert never_built.returncode == 1, test_case.description + output + assert test_case.expected_error in output, test_case.description + assert upstream.returncode == 0, upstream.stdout + upstream.stderr + assert rebuilt.returncode == 0, rebuilt.stdout + rebuilt.stderr + node: str = test_case.select.split(":", 1)[1] + assert query_duckdb(db_path=database, sql=f"SELECT status FROM main.{node}_result") == [ + ("success",) + ] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py index 79bad7194..7a71fd98f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py @@ -127,3 +127,11 @@ class LoaderRelationGuardE2ETestCase: enforce_explicit: bool expected_warnings: tuple[str, ...] expected_final_line_prefix: str + + +@dataclass(frozen=True) +class IntermediateLoaderDefaultE2ETestCase: + description: str + command: tuple[str, ...] + expected_intermediate_rows: tuple[tuple[object, ...], ...] + expected_terminal_rows: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py index 40ffb4dc4..483e1c785 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py @@ -10,6 +10,7 @@ ChainedLoaderFailureE2ETestCase, ChainedLoaderPruningE2ETestCase, ChainedLoaderSelectionE2ETestCase, + IntermediateLoaderDefaultE2ETestCase, IntermediateLoaderStrategyE2ETestCase, LoaderWaffleShopE2ETestCase, SourceLoaderErrorE2ETestCase, @@ -1191,6 +1192,65 @@ def test_given_chained_loader_project_when_intermediate_uses_strategy_then_appli assert tuple(terminal_rows) == test_case.expected_terminal_rows +@pytest.mark.parametrize( + "test_case", + [ + IntermediateLoaderDefaultE2ETestCase( + description="sqb load writes the docs' intermediate loader as a table", + command=("--no-color", "load", "--select", "+raw_account_metrics"), + expected_intermediate_rows=((1, "Pawnee Parks"), (2, "Eagleton")), + expected_terminal_rows=((1, "active"), (2, "active")), + ), + IntermediateLoaderDefaultE2ETestCase( + description="sqb build writes the docs' intermediate loader as a table", + command=("--no-color", "build", "--select", "+source:raw_account_metrics"), + expected_intermediate_rows=((1, "Pawnee Parks"), (2, "Eagleton")), + expected_terminal_rows=((1, "active"), (2, "active")), + ), + ], + ids=lambda case: case.description, +) +def test_given_intermediate_loader_without_write_strategy_when_loading_twice_then_rows_replace_a_table( + tmp_path: Path, + test_case: IntermediateLoaderDefaultE2ETestCase, +) -> None: + edge_cases: Path = Path("benchmarks/duckshop/edge_cases/loaders") + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="source_loader_schema_behavior", + repo_files=build_schema_behavior_project_files( + source_yaml=(edge_cases / "intermediate_default_accounts.yml").read_text(), + loader_py=(edge_cases / "intermediate_default_accounts.py").read_text(), + ), + ) + db_path: Path = project_dir / "source_loader_schema_behavior.duckdb" + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=test_case.command, project_dir=project_dir) for _ in range(2) + ] + + for result in results: + assert result.returncode == 0, test_case.description + result.stdout + result.stderr + assert ( + tuple( + query_duckdb( + db_path=db_path, + sql="SELECT account_id, account_name FROM __loader__raw_accounts ORDER BY 1", + ) + ) + == test_case.expected_intermediate_rows + ) + assert ( + tuple( + query_duckdb( + db_path=db_path, + sql="SELECT account_id, metric FROM raw_account_metrics ORDER BY 1", + ) + ) + == test_case.expected_terminal_rows + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 048a0e910..01776fdad 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -269,6 +269,20 @@ class PythonJobSelectionTestCase: expected_selection: tuple[str, ...] +@dataclass(frozen=True) +class PythonJobGroupTestCase: + description: str + jobs: tuple[tuple[str, str], ...] + expected_groups: dict[str, dict[str, list[str]]] + + +@dataclass(frozen=True) +class PythonJobGroupCycleTestCase: + description: str + jobs: tuple[tuple[str, str], ...] + expected_message: str + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 03050f151..57ceccc80 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -26,6 +26,7 @@ LocalConfig, MeasurementThresholdBound, MeasurementThresholds, + NativeOrchestrationSnapshot, ProjectConfig, SourceColumnEntry, SqlLexicalSyntax, @@ -44,6 +45,8 @@ CompiledModelView, CompiledProjectView, ) +from sqlbuild.orchestration import compile_snapshot +from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files if TYPE_CHECKING: @@ -750,3 +753,23 @@ def published_texts(test_case: SnapshotExpressionRangeTestCase, tmp_path: Path) column: compiled[slice(*metadata["expression_range"])].decode() for column, metadata in columns.items() } + + +def python_pipeline_snapshot( + tmp_path: Path, declarations: list[Job | JobGroup] +) -> NativeOrchestrationSnapshot: + """Publish the Python pipeline project with the given job declarations.""" + write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + return compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=declarations, + discovered_inputs=discovered, + ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index b230cebb8..4f1c56819 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -125,6 +125,12 @@ class WorkerExecutionTestCase: warehouse_setup: Callable[[Path], None] = lambda _: None +@dataclass(frozen=True) +class ClaimedLoadTestCase(WorkerExecutionTestCase): + description: str + expected_write_set: frozenset[str] = frozenset() + + @dataclass(frozen=True) class UnclaimedLoadTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index abf17ff92..60fb29529 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1569,6 +1569,22 @@ def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: } +INTERMEDIATE_LOADER_FILES: dict[str, str] = { + "sources/shipments.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml" + ).read_text(), + "python/loaders/shipments.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py" + ).read_text(), + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql" + ).read_text(), + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" + ).read_text(), +} + + def without_loads(control: sqlite3.Connection, node_key: str) -> None: """Drop a published node's `loads` in place; its content address and the worker's check stay.""" for node_hash, body in control.execute( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index ff3cfb341..48db2d8bd 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -30,6 +30,7 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( CheckExecutionTestCase, + ClaimedLoadTestCase, ClaimedWorker, GroupExecutionTestCase, SnapshotWorkerTestCase, @@ -37,6 +38,7 @@ WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + INTERMEDIATE_LOADER_FILES, LOADED_ORDER_FILES, cancel_running_statement, claimed_worker, @@ -605,26 +607,38 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re @pytest.mark.parametrize( "test_case", [ - WorkerExecutionTestCase( + ClaimedLoadTestCase( "a tag-selected job claims its loader chain and hook-read source, not unread ones", "", "SELECT amount FROM orders", "success", [(7,)], files=LOADED_ORDER_FILES, - ) + expected_write_set=frozenset( + {"model:orders", "source:raw_orders", "source:raw_customers", "source:raw_returns"} + ), + ), + ClaimedLoadTestCase( + "a job claims an intermediate loader no source declares", + "", + "SELECT amount FROM orders", + "success", + [(9,)], + files=INTERMEDIATE_LOADER_FILES, + expected_write_set=frozenset( + {"model:orders", "source:raw_shipments", "source:stage_shipments"} + ), + ), ], ids=lambda case: case.description, ) def test_given_loads_when_a_worker_executes_then_the_run_claims_exactly_what_it_loads( - test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + test_case: ClaimedLoadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: run: dict[str, Any] = worker_result(worker, test_case) assert run["status"] == test_case.expected_status, worker.log.read_text() - assert run_write_set(worker.control_store, run["run_id"]) == frozenset( - {"model:orders", "source:raw_orders", "source:raw_customers", "source:raw_returns"} - ) + assert run_write_set(worker.control_store, run["run_id"]) == test_case.expected_write_set @pytest.mark.parametrize( @@ -639,27 +653,6 @@ def test_given_loads_when_a_worker_executes_then_the_run_claims_exactly_what_it_ {"model:orders", "source:raw_orders", "source:raw_customers"} ), ), - UnclaimedLoadTestCase( - # A known gap: no snapshot node exists for an intermediate loader, so `loads` leaves it out. - "an intermediate loader no source declares is never claimed, so the run still fails", - files={ - "sources/shipments.yml": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml" - ).read_text(), - "python/loaders/shipments.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py" - ).read_text(), - "models/orders.sql": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql" - ).read_text(), - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" - ).read_text(), - }, - store_edit=lambda _: None, - refused_writer="source:stage_shipments", - expected_write_set=frozenset({"model:orders", "source:raw_shipments"}), - ), ], ids=lambda case: case.description, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 9cce168cc..76d37d2d1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -32,8 +32,11 @@ schedule, sensor, ) -from sqlbuild.orchestration_declarations.models.authoring import Job +from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + PythonJobGroupCycleTestCase, + PythonJobGroupTestCase, + PythonJobSelectionTestCase, SnapshotCatalogTestCase, SnapshotDeclarationRejectionTestCase, SnapshotDiscoveryRejectionTestCase, @@ -44,7 +47,6 @@ SnapshotLineageTestCase, SnapshotProvidersTestCase, SnapshotRejectionTestCase, - PythonJobSelectionTestCase, SnapshotScheduleTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( @@ -54,6 +56,7 @@ PYTHON_PIPELINE_FILES, TYPE_PROOF_FILES, published_texts, + python_pipeline_snapshot, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -1858,6 +1861,11 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho select="+orders", expected_selection=("model:orders", "model:stg_orders"), ), + PythonJobSelectionTestCase( + description="a task selector selects only the task", + select="task:request_carrier_collection", + expected_selection=("task:request_carrier_collection",), + ), PythonJobSelectionTestCase( description="typed selectors select the task and the asset it depends on", select="task:request_carrier_collection asset:partner_orders_file", @@ -1915,3 +1923,77 @@ def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_c assert snapshot["documents"]["jobs"]["orders_job"]["selection"] == list( test_case.expected_selection ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonJobGroupTestCase( + description="a job waits for the job building the Python node its task depends on", + jobs=( + ("orders_export_job", "python/orders/partner_export"), + ("carrier_sync_job", "python/fulfillment/carrier_sync"), + ), + expected_groups={ + "orders_nightly": { + "carrier_sync_job": ["orders_export_job"], + "orders_export_job": [], + } + }, + ), + PythonJobGroupTestCase( + description="a Python node's SQL dependency orders it after the model's job", + jobs=(("orders_export_job", "asset:partner_orders_file"), ("models_job", "+orders")), + expected_groups={ + "orders_nightly": {"models_job": [], "orders_export_job": ["models_job"]} + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_dependencies_across_jobs_when_publishing_a_group_then_jobs_wait_for_them( + tmp_path: Path, test_case: PythonJobGroupTestCase +) -> None: + jobs: list[Job] = [ + job(name, description="Orders.", select=select) for name, select in test_case.jobs + ] + group: JobGroup = job_group("orders_nightly", description="Nightly orders.", jobs=jobs) + + snapshot: dict[str, Any] = json.loads( + python_pipeline_snapshot(tmp_path, [*jobs, group]).to_json() + ) + + assert { + name: definition["jobs"] for name, definition in snapshot["documents"]["groups"].items() + } == test_case.expected_groups + + +@pytest.mark.parametrize( + "test_case", + [ + PythonJobGroupCycleTestCase( + description="a Python edge that closes a cycle names both jobs and the edges", + jobs=( + ("staging_job", "stg_orders task:request_carrier_collection"), + ("export_job", "asset:partner_orders_file"), + ), + expected_message=( + "group dependencies contain a cycle in job group orders_nightly: " + "export_job waits for staging_job (model:orders reads model:stg_orders); " + "staging_job waits for export_job " + "(task:request_carrier_collection reads asset:partner_orders_file)" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_python_dependency_cycle_across_jobs_when_publishing_a_group_then_it_is_rejected( + tmp_path: Path, test_case: PythonJobGroupCycleTestCase +) -> None: + jobs: list[Job] = [ + job(name, description="Orders.", select=select) for name, select in test_case.jobs + ] + group: JobGroup = job_group("orders_nightly", description="Nightly orders.", jobs=jobs) + + with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): + python_pipeline_snapshot(tmp_path, [*jobs, group]) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py index c36f6956b..ae3937e7d 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py @@ -35,6 +35,13 @@ expected_sql_names=frozenset(), expected_python_node_names=frozenset({"prepare_orders", "export_orders"}), ), + PythonSqlSelectorTestCase( + description="selects a Python asset without its unselected Python dependency", + select=("path:python/assets",), + exclude=(), + expected_sql_names=frozenset(), + expected_python_node_names=frozenset({"export_orders"}), + ), PythonSqlSelectorTestCase( description="selects expanded source terminal loader requirement for run", select=("+source:raw_orders",), @@ -79,15 +86,6 @@ def test_given_run_selectors_when_resolving_then_excludes_python_checks( expected_error_type=ValueError, expected_error_fragment=("Python checks are not selectable here: check_orders_export"), ), - PythonSqlSelectorErrorTestCase( - description="rejects direct asset path with unselected Python dependency for run", - select=("path:python/assets",), - exclude=(), - expected_error_type=ValueError, - expected_error_fragment=( - "Python node 'export_orders' depends on unselected Python node 'prepare_orders'" - ), - ), ], ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 3f20e3ab2..ddc395623 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -127,7 +127,7 @@ def export_orders(ctx): - `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.tasks`/`sqlbuild.assets`. - Assets may pass `materialized=True`/`False` to record whether an artifact was produced. -Downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. +Within one run, downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. ## Result persistence @@ -161,6 +161,18 @@ dependency fails or is skipped. A dependency that is not part of the run (not se unmanaged source) is read as it currently exists, so `sqb build --select task:export_orders` runs the task against the relations already in the warehouse. +Python dependencies follow the same rule. A task or asset selected without its Python upstream +reads that upstream's latest successful result through `ctx.result_of`. If the upstream has never +succeeded for the target, the node fails with an error that names the upstream, for example: + +```text +Upstream task 'prepare_orders' has no successful result to read. Build it first with +'sqb build --select task:prepare_orders', or build both with 'sqb build --select +asset:export_orders'. +``` + +Jobs, sensors, roles and ad-hoc runs on the server resolve selectors the same way, so a job can +select one Python node and a job group orders it after the job that builds its upstream. + The commands differ in what they include by default: | Command | SQL | Loaders / tasks / assets | Checks | Audits | From 0e665a98b2eeb2b5025f3eaccd813594be32ff93 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:20:11 +0000 Subject: [PATCH 04/34] feat: select Python-only jobs and SQL tests across CLI and server runs - `job:` and bare job names resolve jobs through the Python-aware resolver publication uses, so a job that selects only Python nodes builds them. - Server ad-hoc runs accept `test:` selectors; a selected SQL test reports as a run node rather than as an attached check. - A provider function whose context is positional-only (`ctx, /`) receives it by position instead of failing at run time. - A role must grant the intermediate loader a job loads to run that job. --- .../intermediate_loader_roles.toml | 15 ++++ .../orchestration/python_node_jobs.py | 10 +++ .../python_pipeline_carrier_sync.py | 10 +++ .../python_pipeline_partner_export.py | 9 +++ .../src/graph/_helpers/jobs.rs | 14 +++- .../src/graph/main/attach_jobs_with.rs | 18 +++++ .../sqlbuild-analysis/src/graph/main/mod.rs | 1 + .../src/python_selection/_helpers/unified.rs | 48 +++++++++++- .../main/attach_python_jobs.rs | 40 ++++++++++ .../src/python_selection/main/mod.rs | 1 + .../tests/test_python_selection.rs | 2 +- .../src/adhoc/_helpers/planner.rs | 59 ++++++++++++++- crates/sqlbuild-cli/src/adhoc/constants.rs | 6 ++ .../src/adhoc/tests/test_adhoc_planner.rs | 58 ++++++++++++++- .../src/runs/_helpers/requests/adhoc.rs | 24 ++++++ .../_helpers/compiled_project/job_catalog.rs | 20 +++-- .../compiled_project/snapshot_execution.rs | 8 +- .../_helpers/graph/python_selection.rs | 17 ++++- .../_helpers/location_command/constants.rs | 3 + .../_helpers/location_command/worker.rs | 13 ++-- src/sqlbuild/provider/_helpers/injection.py | 7 +- .../cli/commands/main/build/helpers.py | 3 + .../main/build/test_python_node_selection.py | 24 ++++++ .../compiler/compiled_project/helpers.py | 17 ++--- .../publication/_test_types.py | 7 ++ .../publication/person/helpers.py | 16 ++++ .../publication/test_adhoc.py | 8 ++ .../publication/test_project_roles.py | 73 +++++++++++++++++++ .../src/sqlbuild/provider/_helpers/helpers.py | 8 ++ .../provider/_helpers/test_injection.py | 12 +++ .../content/docs/docs/concepts/selectors.mdx | 2 +- 31 files changed, 517 insertions(+), 36 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py create mode 100644 crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml new file mode 100644 index 000000000..78acf5802 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml @@ -0,0 +1,15 @@ +# Roles over a loader chain whose middle loader no source declares: running the job needs a grant +# on the intermediate loader's node as well as the model and the source it feeds. +[roles.shipments_runner] +description = "Run the shipments build, including its staging loader" + +[[roles.shipments_runner.grants]] +nodes = ["orders", "source:raw_shipments", "loader:stage_shipments"] +permissions = ["job.run"] + +[roles.orders_runner] +description = "Run the orders build without the staging loader" + +[[roles.orders_runner.grants]] +nodes = ["orders", "source:raw_shipments"] +permissions = ["job.run"] diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py new file mode 100644 index 000000000..d153e434f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py @@ -0,0 +1,10 @@ +"""Jobs whose selections are Python nodes, which `--select job:` must select as published.""" + +from sqlbuild.orchestration import job + +customer_counts = job( + "customer_counts", + description="Count customers.", + select="task:count_customers asset:customers_extract", +) +order_counts = job("order_counts", description="Count orders.", select="+task:count_orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py new file mode 100644 index 000000000..221447d1c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py @@ -0,0 +1,10 @@ +"""A task whose Python upstream another job builds, so a job group must order the two jobs.""" + +from python.orders.partner_export.node import partner_orders_file + +from sqlbuild.tasks import task + + +@task(depends_on=partner_orders_file, tags=["orders"], description="Book collection.") +def request_carrier_collection(ctx): + return None diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py new file mode 100644 index 000000000..dfe6194d2 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py @@ -0,0 +1,9 @@ +"""An asset one job builds that a task in another job of the same group depends on.""" + +from sqlbuild.assets import asset +from sqlbuild.refs import model + + +@asset(depends_on=model("orders"), tags=["orders"], description="Partner order file.") +def partner_orders_file(ctx): + return None diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs b/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs index 329bbbc64..f1c9480ac 100644 --- a/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs +++ b/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs @@ -8,17 +8,23 @@ use crate::graph::errors::SelectorError; use crate::graph::models::{JobSelection, ProjectGraph}; /// Resolve every job on the graph without jobs attached, as publication does, then attach them. -pub(crate) fn attach( +pub(crate) fn attach(graph: ProjectGraph, catalog: BTreeMap) -> ProjectGraph { + attach_with(graph, catalog, |graph, select, exclude| { + resolve(graph, select, exclude).map(|keys| keys.into_iter().collect()) + }) +} + +/// [`attach`] with each job's selectors resolved by `resolve_job` on the graph without jobs. +pub(crate) fn attach_with( mut graph: ProjectGraph, catalog: BTreeMap, + resolve_job: impl Fn(&ProjectGraph, &[String], &[String]) -> Result, ) -> ProjectGraph { graph.jobs = None; let mut resolved: BTreeMap> = BTreeMap::new(); for (name, selection) in &catalog { let keys: Result = match selection { - JobSelection::Job { select, exclude } => { - resolve(&graph, select, exclude).map(|keys| keys.into_iter().collect()) - } + JobSelection::Job { select, exclude } => resolve_job(&graph, select, exclude), JobSelection::Resolved { keys } => Ok(keys.iter().cloned().collect()), JobSelection::Group { .. } => continue, }; diff --git a/crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs b/crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs new file mode 100644 index 000000000..091081bc5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs @@ -0,0 +1,18 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::models::{JobSelection, ProjectGraph}; + +/// [`super::attach_jobs::attach_jobs`] with each job's selectors resolved by `resolve_job`. +pub fn attach_jobs_with( + graph: ProjectGraph, + catalog: BTreeMap, + resolve_job: impl Fn( + &ProjectGraph, + &[String], + &[String], + ) -> Result, SelectorError>, +) -> ProjectGraph { + crate::graph::_helpers::jobs::attach_with(graph, catalog, resolve_job) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/mod.rs b/crates/sqlbuild-analysis/src/graph/main/mod.rs index ccd21dc17..05d0272f4 100644 --- a/crates/sqlbuild-analysis/src/graph/main/mod.rs +++ b/crates/sqlbuild-analysis/src/graph/main/mod.rs @@ -1,4 +1,5 @@ pub mod attach_jobs; +pub mod attach_jobs_with; pub mod build_project_graph; pub mod build_resources; pub mod downstream_edges; diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index 7bd1a6351..4c94cd2f8 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -7,6 +7,7 @@ use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; use sqlbuild_core::text::main::glob_match::glob_match; use crate::assembly::project::types::ObjectKey; +use crate::definition_changes::constants::PYTHON_NODE_KINDS; use crate::graph::errors::SelectorError; use crate::graph::main::build_resources::build_resources; use crate::graph::main::edge_keys::edge_keys; @@ -282,7 +283,8 @@ fn single_side(context: &SelectionContext<'_>, raw: &str) -> Resolved { let atoms: Atoms = match kind.as_str() { TEST_KIND => return Err(unit_test_error(raw)), PATH_KIND => resolve_path(context, raw, &value)?, - SEED_KIND | SOURCE_KIND | JOB_KIND => resolve_sql(context, raw)?, + SEED_KIND | SOURCE_KIND => resolve_sql(context, raw)?, + JOB_KIND => resolve_job(context, raw)?, TASK_KIND | ASSET_KIND | LOADER_KIND | CHECK_KIND => { return resolve_typed(context, raw, (&kind, &value)); } @@ -314,6 +316,48 @@ fn resolve_sql(context: &SelectionContext<'_>, raw: &str) -> Resolved { )?)) } +/// A job's SQL keys and Python nodes, with `+` also expanding them over the Python node graph. +fn resolve_job(context: &SelectionContext<'_>, raw: &str) -> Resolved { + let ParsedSelector::Kind { + upstream, + downstream, + .. + } = parse_selector(raw)? + else { + return resolve_sql(context, raw); + }; + let mut atoms: Atoms = Atoms::new(); + let mut members: Vec = Vec::new(); + for atom in resolve_sql(context, raw)? { + match atom { + Atom::Sql((kind, name)) + if PYTHON_NODE_KINDS.contains(&kind.as_str()) + && context.nodes.node(&name).is_some() => + { + members.push(name); + } + atom => { + atoms.insert(atom); + } + } + } + let mut names: Names = members.iter().cloned().collect(); + for (expand, direction) in [ + (upstream, EdgeDirection::Upstream), + (downstream, EdgeDirection::Downstream), + ] { + if expand { + names.extend( + context + .nodes + .closure(members.iter().map(String::as_str), direction), + ); + } + } + atoms.extend(python_atoms(names)); + Ok(atoms) +} + fn resolve_python(context: &SelectionContext<'_>, raw: &str) -> Resolved { Ok(python_atoms(select_nodes( &context.nodes, @@ -407,7 +451,7 @@ fn resolve_name(context: &SelectionContext<'_>, raw: &str, value: &str) -> Resol (true, true) => Err(both_error(value)), (true, false) => resolve_sql(context, raw), (false, true) => resolve_python(context, raw), - (false, false) if is_job(context, value) => resolve_sql(context, raw), + (false, false) if is_job(context, value) => resolve_job(context, raw), (false, false) => Err(unknown_name(value)), }; } diff --git a/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs new file mode 100644 index 000000000..6e3095f6f --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs @@ -0,0 +1,40 @@ +use std::collections::BTreeMap; + +use crate::graph::main::attach_jobs_with::attach_jobs_with; +use crate::graph::models::{JobSelection, ProjectGraph}; +use crate::python_selection::_helpers::nodes::sql_names; +use crate::python_selection::_helpers::unified::run_selection; +use crate::python_selection::models::{ + NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, +}; + +/// Attach jobs as publication resolves them, keeping each job's Python nodes as `(kind, name)` keys. +pub fn attach_python_jobs( + sql: SqlSelectionGraph<'_>, + python: &PythonSelectionGraph, + catalog: BTreeMap, +) -> ProjectGraph { + let kinds: BTreeMap<&str, &str> = python + .nodes + .iter() + .map(|node| (node.name.as_str(), node.kind.as_str())) + .collect(); + attach_jobs_with(sql.graph.clone(), catalog, |graph, select, exclude| { + let context: SelectionContext<'_> = SelectionContext { + sql: SqlSelectionGraph { + graph, + sources: sql.sources, + }, + nodes: NodeIndex::new(python), + sql_names: sql_names(graph), + }; + let RunSelection { + sql_keys, + python_node_names, + } = run_selection(&context, select, exclude)?; + let python_keys = python_node_names + .into_iter() + .filter_map(|name| Some((kinds.get(name.as_str())?.to_string(), name))); + Ok(sql_keys.into_iter().chain(python_keys).collect()) + }) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs index 3dace0806..487d121f6 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs @@ -1,3 +1,4 @@ +pub mod attach_python_jobs; pub mod published_python_graph; pub mod python_node_graph_indexes; pub mod resolve_python_node_excludes; diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs index 27bc2914b..7a2c24b30 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs @@ -1,5 +1,5 @@ use crate::python_selection::tests::helpers::{ - excluded_names, expected_names, expected_run, indexed_folder, node_names, orders_python_graph, + excluded_names, expected_names, expected_run, indexed_folder, node_names, published_orders_graph, run, run_over, }; use crate::python_selection::tests::test_types::{ diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index 389c0fcd7..ddb519349 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -8,6 +8,8 @@ use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; use sqlbuild_analysis::graph::models::{GraphResource, JobSelection}; +use sqlbuild_analysis::planner_selection::main::split_unit_test_selectors::split_unit_test_selectors; +use sqlbuild_analysis::planner_selection::models::{SelectorGroups, UnitTestSelectorNames}; use sqlbuild_analysis::python_selection::main::published_python_graph::published_python_graph; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; use sqlbuild_analysis::python_selection::models::{PublishedPythonNode, SqlSelectionGraph}; @@ -18,7 +20,7 @@ use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs}; use crate::adhoc::constants::{ ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, CHANGED_COUNTS_DEST, CHANGED_DEST, COMMAND_DEST, FRESHNESS_VERB, INTEGER_CONVERTERS, LOAD_SOURCES_DEST, POSITIVE_INTEGER_CONVERTER, - REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, + REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND, UNIT_TEST_VERBS, }; use crate::argv::main::command_settings_arguments::command_settings_arguments; use crate::argv::main::job_settings_arguments::job_settings_arguments; @@ -84,7 +86,7 @@ impl CommandPlanner for CliPlanner { seeds } FRESHNESS_VERB => freshness_sources(publication, &select, &exclude)?, - _ => resolve(publication, &select, &exclude)?, + _ => resolve_with_tests(verb, publication, &select, &exclude)?, }; Ok(CommandPlan { verb: verb.clone(), @@ -201,6 +203,58 @@ fn resolve( Ok(selected) } +/// [`resolve`] with `test:` and unit-test names selecting `sql_test:` keys, as `sqb test` does. +fn resolve_with_tests( + verb: &str, + (resources, jobs): (&[PlannerResource], &PublishedJobs), + select: &[String], + exclude: &[String], +) -> Result, String> { + let names = UnitTestSelectorNames { + tests: resources + .iter() + .filter(|resource| is_sql_test(resource)) + .map(|resource| name_of(&resource.key).to_owned()) + .collect(), + resources: resources + .iter() + .filter(|resource| !is_sql_test(resource)) + .map(|resource| name_of(&resource.key).to_owned()) + .chain(jobs.keys().cloned()) + .collect(), + }; + let split = split_unit_test_selectors( + SelectorGroups { select, exclude }, + &names, + UNIT_TEST_VERBS.contains(&verb), + ) + .map_err(|error| format!("{}: {}", error.code, error.message))?; + if !split.excluded_test_names.is_empty() { + return Err( + "an ad-hoc run cannot exclude unit tests; select the unit tests to run instead".into(), + ); + } + let mut keys: BTreeSet = if split.models_selected { + resolved((resources, jobs), &split.select, &split.exclude)? + } else { + BTreeSet::new() + }; + keys.extend( + split + .test_names + .iter() + .map(|name| format!("{SQL_TEST_KIND}:{name}")), + ); + if keys.is_empty() { + return Err("the selectors match no nodes in the current publication".into()); + } + Ok(keys) +} + +fn is_sql_test(resource: &PlannerResource) -> bool { + resource.key.split_once(':').map(|(kind, _)| kind) == Some(SQL_TEST_KIND) +} + fn resolved( (resources, jobs): (&[PlannerResource], &PublishedJobs), select: &[String], @@ -208,6 +262,7 @@ fn resolved( ) -> Result, String> { let (python, sql): (Vec<&PlannerResource>, Vec<&PlannerResource>) = resources .iter() + .filter(|resource| !is_sql_test(resource)) .partition(|resource| python_kind(&resource.key).is_some()); let graph_resources: Vec = sql .iter() diff --git a/crates/sqlbuild-cli/src/adhoc/constants.rs b/crates/sqlbuild-cli/src/adhoc/constants.rs index c4fc1ca67..0de828474 100644 --- a/crates/sqlbuild-cli/src/adhoc/constants.rs +++ b/crates/sqlbuild-cli/src/adhoc/constants.rs @@ -6,6 +6,12 @@ pub(crate) const ADHOC_VERBS: [&str; 5] = ["build", "test", "seed", "audit", "fr /// The verb whose settings must also form valid job settings. pub(crate) const BUILD_VERB: &str = "build"; +/// Verbs that select SQL unit tests by `test:` or by name, as on the command line. +pub(crate) const UNIT_TEST_VERBS: [&str; 2] = ["build", "test"]; + +/// The published kind of a SQL unit test. +pub(crate) const SQL_TEST_KIND: &str = "sql_test"; + /// The verb that runs only the seeds of its selection. pub(crate) const SEED_VERB: &str = "seed"; diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index 9855c4cbc..fe0b38f40 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -28,6 +28,7 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select resource("model:customers", &[], &[], Some("models/staging")), resource("source:shop.raw_orders", &[], &[], None), resource("seed:currencies", &[], &[], None), + resource("sql_test:orders_keep_amounts", &["model:orders"], &[], None), ]; let jobs: PublishedJobs = BTreeMap::from([ ( @@ -229,6 +230,40 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select command: &["test", "-s", "orders", "--case", "happy_path"], expected_plan: Ok((&["model:orders"], json!({"case": "happy_path"}))), }, + AdhocPlanTestCase { + description: "test: selects one unit test", + command: &["test", "--select", "test:orders_keep_amounts"], + expected_plan: Ok((&["sql_test:orders_keep_amounts"], json!({}))), + }, + AdhocPlanTestCase { + description: "a unit test's bare name selects it, as on the command line", + command: &["test", "--select", "orders_keep_amounts"], + expected_plan: Ok((&["sql_test:orders_keep_amounts"], json!({}))), + }, + AdhocPlanTestCase { + description: "build selects a model and a unit test together", + command: &["build", "--select", "customers", "test:orders_keep_amounts"], + expected_plan: Ok(( + &["model:customers", "sql_test:orders_keep_amounts"], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "excluding a unit test is refused, since a run freezes only what it selects", + command: &[ + "test", + "--select", + "orders", + "--exclude", + "test:orders_keep_amounts", + ], + expected_plan: Err("an ad-hoc run cannot exclude unit tests"), + }, + AdhocPlanTestCase { + description: "audit refuses unit-test selectors as the command line does", + command: &["audit", "--select", "test:orders_keep_amounts"], + expected_plan: Err("selects a unit test"), + }, AdhocPlanTestCase { description: "audit settings come from the audit parser", command: &["audit", "-s", "orders", "--concurrency", "2"], @@ -314,8 +349,29 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co Some("python/fulfillment/carrier_sync"), ), ]; - let jobs: PublishedJobs = BTreeMap::new(); + let jobs: PublishedJobs = BTreeMap::from([( + "carrier_sync".to_owned(), + BTreeSet::from(["task:request_carrier_collection".to_owned()]), + )]); let test_cases = [ + AdhocPlanTestCase { + description: "a published job selects its Python nodes", + command: &["build", "--select", "job:carrier_sync"], + expected_plan: Ok((&["task:request_carrier_collection"], json!({}))), + }, + AdhocPlanTestCase { + description: "upstream of a published job crosses its Python node's dependencies", + command: &["build", "--select", "+job:carrier_sync"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, AdhocPlanTestCase { description: "typed selectors select the Python nodes", command: &[ diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index 6de44e69d..45ca1649d 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -32,6 +32,10 @@ use crate::store::models::{ControlTransaction, Parameter}; const GRAPH_KINDS: [&str; 5] = ["model", "source", "seed", "udf", "table_fn"]; /// Python node kinds, in the order the CLI discovers them. const PYTHON_NODE_KINDS: [&str; 4] = ["task", "asset", "loader", "check"]; +/// SQL unit tests, which `test:` selects; each lists the models it tests. +const SQL_TEST_KIND: &str = "sql_test"; +/// Where a SQL unit test's definition lists the models it tests. +const SQL_TEST_MODELS_POINTER: &str = "/test/payload/expected_model_names"; /// Marks the source node an intermediate loader writes, which the CLI's SQL graph does not list. const INTERMEDIATE_LOADER_FIELD: &str = "intermediate_loader"; @@ -179,6 +183,18 @@ fn planner_resources( }); } } + resources.extend( + nodes + .iter() + .filter(|(_, node)| node.kind == SQL_TEST_KIND) + .map(|(key, node)| PlannerResource { + key: key.clone(), + deps: tested_models(&node.definition), + tags: Vec::new(), + folder: None, + loader: None, + }), + ); resources } @@ -193,6 +209,14 @@ fn model_folder(definition: &Value) -> String { .map_or_else(|| ".".to_owned(), |(folder, _)| folder.to_owned()) } +/// The model keys a SQL unit test tests. +fn tested_models(definition: &Value) -> Vec { + texts(definition.pointer(SQL_TEST_MODELS_POINTER)) + .into_iter() + .map(|model| format!("model:{model}")) + .collect() +} + fn texts(value: Option<&Value>) -> Vec { value .and_then(Value::as_array) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs index f5122b631..79b2b2e30 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs @@ -61,15 +61,25 @@ pub(crate) fn graph_with_jobs<'g>( selectors: &[String], also_known: &[&str], ) -> PyResult> { - if graph.jobs.is_some() || !selectors_need_jobs(graph, selectors, also_known) { - return Ok(Cow::Borrowed(graph)); - } - Ok(match job_catalog(py)? { - Some((catalog, _)) => Cow::Owned(attach_jobs(graph.clone(), catalog)), + Ok(match needed_jobs(py, graph, selectors, also_known)? { + Some(catalog) => Cow::Owned(attach_jobs(graph.clone(), catalog)), None => Cow::Borrowed(graph), }) } +/// The command's jobs when `selectors` name one and `graph` has none attached yet. +pub(crate) fn needed_jobs( + py: Python<'_>, + graph: &ProjectGraph, + selectors: &[String], + also_known: &[&str], +) -> PyResult>> { + if graph.jobs.is_some() || !selectors_need_jobs(graph, selectors, also_known) { + return Ok(None); + } + Ok(job_catalog(py)?.map(|(catalog, _)| catalog)) +} + /// The command's job and job group names, loaded on first use; empty outside a command. pub(crate) fn job_names(py: Python<'_>) -> PyResult> { Ok(job_catalog(py)? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index 509d5b753..a9392f573 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -260,12 +260,15 @@ impl WorkerBuild { self.complete(py, key, (outcome, None, (None, None))) } - /// A test or audit the snapshot records has started; the node it attaches to starts with it. + /// A test or audit the snapshot records has started; a selected SQL test reports as a node. pub(crate) fn start_attached(&self, py: Python<'_>, key: &str) -> PyResult<()> { self.check_cancelled(py)?; let Some(nodes) = self.attached.get(key) else { return Ok(()); }; + if self.execution.claim.request.selection.contains(key) { + return self.start_key(py, key); + } if self .checks .lock() @@ -313,6 +316,9 @@ impl WorkerBuild { } else { outcome }; + if self.execution.claim.request.selection.contains(key) { + return self.complete(py, key, (outcome, None, (None, None))); + } self.finish_check(py, key, outcome)?; if outcome == NodeOutcome::Skipped { return Ok(()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 385913077..3a5d3b1e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -1,5 +1,6 @@ //! Python node and unified SQL/Python selector resolution, and the Python node graph indexes. +use std::borrow::Cow; use std::collections::BTreeSet; use pyo3::prelude::{ @@ -10,6 +11,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_analysis::python_selection::main::attach_python_jobs::attach_python_jobs; use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; @@ -21,7 +23,7 @@ use sqlbuild_analysis::python_selection::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_guard; -use crate::bindings::_helpers::compiled_project::job_catalog::graph_with_jobs; +use crate::bindings::_helpers::compiled_project::job_catalog::needed_jobs; use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::records::construct::error_class; @@ -272,7 +274,18 @@ pub(crate) fn resolve_python_sql_run_selection( let nodes: PythonSelectionGraph = selection_graph(python_graph)?; let node_names: Vec<&str> = nodes.nodes.iter().map(|node| node.name.as_str()).collect(); let selectors: Vec = select.iter().chain(&exclude).cloned().collect(); - let graph = graph_with_jobs(py, native.get().graph(), &selectors, &node_names)?; + let graph: Cow<'_, ProjectGraph> = + match needed_jobs(py, native.get().graph(), &selectors, &node_names)? { + Some(catalog) => Cow::Owned(attach_python_jobs( + SqlSelectionGraph { + graph: native.get().graph(), + sources: &sources, + }, + &nodes, + catalog, + )), + None => Cow::Borrowed(native.get().graph()), + }; let selection: RunSelection = compiler_guard(|| { Ok(run_selection( SqlSelectionGraph { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index b9e42eaa9..fb1c70fdf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -33,3 +33,6 @@ pub(crate) const REACH_SETTING: &str = "reach"; pub(crate) const JOB_RUN_REACH: &str = "all"; /// What a job run with `changed: True` counts when `[changes] counts` is not set: every change. pub(crate) const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; + +/// A SQL unit test's published kind; a frozen run selects it back with `test:`. +pub(crate) const SQL_TEST_NODE_KIND: &str = "sql_test"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index ec62b33a1..889158c12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -34,6 +34,7 @@ use crate::bindings::_helpers::discovery::project_config::configured_changes; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::constants::{ CHANGED_SETTING, COMMAND_IMPLEMENTATIONS, JOB_RUN_COUNTS, JOB_RUN_REACH, REACH_SETTING, + SQL_TEST_NODE_KIND, }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::recall::console::write_stderr; @@ -59,7 +60,7 @@ fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap definition + SQL_TEST_NODE_KIND => definition .pointer("/test/payload/expected_model_names") .and_then(Value::as_array) .into_iter() @@ -152,12 +153,14 @@ fn command_request<'py>( let (kind, name) = key .split_once(':') .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; - let selector = if PYTHON_NODE_KINDS.contains(&kind) { - key.as_str() + let selector: String = if PYTHON_NODE_KINDS.contains(&kind) { + key.clone() + } else if kind == SQL_TEST_NODE_KIND { + format!("test:{name}") } else { - name + name.to_owned() }; - argv.extend([PyString::new(py, "--select"), PyString::new(py, selector)]); + argv.extend([PyString::new(py, "--select"), PyString::new(py, &selector)]); } let parsed = parse_cli_invocation(py, argv, false)?; let args = field(&parsed, "args")?; diff --git a/src/sqlbuild/provider/_helpers/injection.py b/src/sqlbuild/provider/_helpers/injection.py index e7680a16e..471870041 100644 --- a/src/sqlbuild/provider/_helpers/injection.py +++ b/src/sqlbuild/provider/_helpers/injection.py @@ -73,7 +73,12 @@ def call_with_provider_injection( if not context_bound and parameter.default is inspect.Parameter.empty: kwargs[parameter.name] = context context_bound = True - return function(**kwargs) + positional: list[object] = [] + for parameter in signature.parameters.values(): + if parameter.kind != inspect.Parameter.POSITIONAL_ONLY or parameter.name not in kwargs: + break + positional.append(kwargs.pop(parameter.name)) + return function(*positional, **kwargs) def _provider_for_parameter( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 95f42565c..c1e0f9211 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -895,6 +895,9 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: 'MODEL (description "Test model order_summary.", ' 'materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' ), + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py" + ).read_text(), "python/tasks/reports.py": ( "from python.tasks.counts import count_orders\nfrom sqlbuild.tasks import task\n\n\n" "@task(depends_on=count_orders)\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py index 1ec298fe6..80803f98f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py @@ -104,6 +104,30 @@ expected_row_counts={"orders": 2}, expected_node_counts={"orders_extract": 2}, ), + PythonNodeSelectionE2ETestCase( + description="a job selecting only Python nodes builds them", + select=("job:customer_counts",), + expected_exit_code=0, + expected_output_patterns=(r"count_customers\s+OK", r"customers_extract\s+OK"), + expected_row_counts={"raw_customers": 3}, + expected_node_counts={"count_customers": 3, "customers_extract": 3}, + ), + PythonNodeSelectionE2ETestCase( + description="a job named without job: builds its Python nodes", + select=("customer_counts",), + expected_exit_code=0, + expected_output_patterns=(r"count_customers\s+OK", r"customers_extract\s+OK"), + expected_row_counts={"raw_customers": 3}, + expected_node_counts={"count_customers": 3, "customers_extract": 3}, + ), + PythonNodeSelectionE2ETestCase( + description="a job selecting a task with its upstream model builds both", + select=("job:order_counts",), + expected_exit_code=0, + expected_output_patterns=(r"count_orders\s+OK",), + expected_row_counts={"orders": 2}, + expected_node_counts={"count_orders": 2}, + ), PythonNodeSelectionE2ETestCase( description="task is skipped when its selected upstream model fails", select=("+task:count_orders",), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 57ceccc80..bbaca4925 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -78,17 +78,12 @@ "MODEL (description 'Orders.', tags [orders]);\n\n" 'SELECT order_id FROM __ref("stg_orders")\n' ), - "python/orders/partner_export/node.py": ( - "from sqlbuild.assets import asset\nfrom sqlbuild.refs import model\n\n\n" - "@asset(depends_on=model('orders'), tags=['orders'], description='Partner order file.')\n" - "def partner_orders_file(ctx):\n return None\n" - ), - "python/fulfillment/carrier_sync/node.py": ( - "from python.orders.partner_export.node import partner_orders_file\n" - "from sqlbuild.tasks import task\n\n\n" - "@task(depends_on=partner_orders_file, tags=['orders'], description='Book collection.')\n" - "def request_carrier_collection(ctx):\n return None\n" - ), + "python/orders/partner_export/node.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py" + ).read_text(), + "python/fulfillment/carrier_sync/node.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py" + ).read_text(), } TYPE_PROOF_FILES: dict[str, str] = { diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 4f1c56819..ddeb7ebde 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -371,6 +371,13 @@ class ProjectRolesTestCase: expected_denial: str +@dataclass(frozen=True) +class ProjectRoleRunTestCase: + description: str + role: str + expected_response: tuple[int, Any] + + @dataclass(frozen=True, kw_only=True) class ChangedJobTestCase(WorkerExecutionTestCase): description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index 512b7a78d..8357d5d10 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -265,6 +265,22 @@ def person_request( return json.load(response) +def request_person_run( + api: SnapshotApi, session: tuple[str, str], body: dict[str, Any] +) -> tuple[int, Any]: + """A person's run request: the response on success, else the denial without its decision.""" + request: Request = Request( + api.url + "/api/control/runs", + data=json.dumps(body).encode(), + headers=person_headers(session), + ) + try: + with urlopen(request, timeout=5) as response: + return 200, json.load(response) + except HTTPError as denied: + return denied.code, json.load(denied)["error"].split("; decision ")[0] + + def person_login(api: SnapshotApi, name: str) -> tuple[str, str]: request: Request = Request( api.url + "/api/auth/password/login", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index cd3ce745b..703856d11 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -260,6 +260,14 @@ def test_given_an_ad_hoc_command_when_checked_and_launched_then_the_worker_runs_ ], rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, ), + AdhocVerbTestCase( + description="test: runs only the selected verified SQL test", + prerequisites=[], + command=["test", "--select", "test:customers_have_ids"], + expected_selection=["sql_test:customers_have_ids"], + expected_run_nodes=[("sql_test:customers_have_ids", "node", "success")], + rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, + ), AdhocVerbTestCase( description="freshness observes the verified sources upstream of the selection", prerequisites=[], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py index fc6221eb2..0215796c8 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py @@ -3,13 +3,16 @@ import subprocess from pathlib import Path from typing import Any +from uuid import uuid4 import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ProjectRoleRunTestCase, ProjectRolesTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + INTERMEDIATE_LOADER_FILES, publish_generation, read_person_sql, snapshot_api, @@ -19,6 +22,7 @@ person_login, person_request, provision_person, + request_person_run, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -100,3 +104,72 @@ def test_given_project_roles_when_published_then_person_sql_follows_selector_cov assert rewritten.returncode == 0, rewritten.stderr assert read_person_sql(api, session, "model:customers") == (200, "model:customers") assert read_person_sql(api, session, "model:orders")[0] == 403 + + +@pytest.mark.parametrize( + "test_case", + [ + ProjectRoleRunTestCase( + description="a role granting the loader chain runs the job", + role="shipments_runner", + expected_response=(200, {"applied": True}), + ), + ProjectRoleRunTestCase( + description="a role missing the intermediate loader is refused", + role="orders_runner", + expected_response=(403, "missing job.run permission for this target"), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_loading_an_intermediate_loader_when_a_role_runs_it_then_the_grant_must_cover_it( + test_case: ProjectRoleRunTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "roles.toml": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml" + ).read_text(), + } + | INTERMEDIATE_LOADER_FILES, + ) + with snapshot_api( + tmp_path / "control", + server_arguments=("--password-origin", "https://control.example.com"), + ) as api: + published: subprocess.CompletedProcess[str] = publish_generation(project, api, 1) + assert published.returncode == 0, published.stderr + admin: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-runner" + ) + assert person_request( + api, + "/api/admin/roles", + body={ + "principal_id": person["principal_id"], + "role": test_case.role, + "location_id": api.location_id, + "target": "development", + }, + session=person_login(api, admin["name"]), + ) == {"applied": True} + request: dict[str, Any] = { + "run_id": str(uuid4()), + "location_id": api.location_id, + "target": "development", + "job_key": "annual_orders", + "reason": "Refresh shipments", + "priority": 0, + } + session: tuple[str, str] = person_login(api, person["name"]) + assert request_person_run(api, session, request) == test_case.expected_response diff --git a/tests/unit/src/sqlbuild/provider/_helpers/helpers.py b/tests/unit/src/sqlbuild/provider/_helpers/helpers.py index 26a1bbdb8..23b22ab3c 100644 --- a/tests/unit/src/sqlbuild/provider/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/provider/_helpers/helpers.py @@ -30,6 +30,14 @@ def context_and_provider(ctx: object, slack_provider: SlackProvider) -> tuple[ob return ctx, slack_provider.label +def positional_only_context(ctx: object, /, *, slack_provider: SlackProvider) -> tuple[object, str]: + return ctx, slack_provider.label + + +def keyword_only_provider(ctx: object, *, slack_provider: SlackProvider) -> tuple[object, str]: + return ctx, slack_provider.label + + def provider_only(slack_provider: SlackProvider) -> str: return slack_provider.label diff --git a/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py b/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py index a930e5384..0ba9419ac 100644 --- a/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py +++ b/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py @@ -20,9 +20,11 @@ SlackProvider, context_and_provider, context_only, + keyword_only_provider, load_module, mismatched_provider, missing_provider, + positional_only_context, provider_only, reserved_context_conflict, unannotated_provider, @@ -65,6 +67,16 @@ def test_given_context_only_function_when_calling_with_provider_injection_then_c function=provider_only, expected_result="slack", ), + ProviderInjectionTestCase( + description="keyword-only provider is injected by name", + function=keyword_only_provider, + expected_result=("runtime-context", "slack"), + ), + ProviderInjectionTestCase( + description="positional-only context is passed by position", + function=positional_only_context, + expected_result=("runtime-context", "slack"), + ), ), ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/concepts/selectors.mdx b/website/src/content/docs/docs/concepts/selectors.mdx index a10fac80a..7f598d7a3 100644 --- a/website/src/content/docs/docs/concepts/selectors.mdx +++ b/website/src/content/docs/docs/concepts/selectors.mdx @@ -69,7 +69,7 @@ sqb build --select source:raw__orders ### Job -Select the nodes a job or job group declared under `orchestration/` builds. Job names are globally +Select the SQL and Python nodes a job or job group declared under `orchestration/` builds. Job names are globally unique, so a bare name works; `job:` is the explicit form and accepts glob patterns: ```bash From a96db64de6743be9b1ba8155a6bf93b8896eb9a1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:10:19 +0000 Subject: [PATCH 05/34] fix: keep Python selections runnable on workers and require checked nodes Selections now publish a managed source instead of its loader, ad-hoc test, audit, seed and freshness runs resolve SQL only, a selected SQL test reports its own outcome, and a selected model folds in its failing test. A selector that matches a Python check must also select the nodes the check depends on, in jobs, roles and the CLI alike. --- .../python_pipeline_orders_check.py | 16 +++++ .../orchestration/upstream_loaded_orders.py | 15 +++++ .../src/python_selection/_helpers/unified.rs | 44 +++++++++++--- .../src/python_selection/main/mod.rs | 1 + .../python_selection/main/python_run_key.rs | 13 +++++ .../tests/test_python_selection.rs | 13 ++++- .../src/adhoc/_helpers/planner.rs | 46 ++++++++++----- crates/sqlbuild-cli/src/adhoc/constants.rs | 3 + .../src/adhoc/tests/test_adhoc_planner.rs | 58 +++++++++++++++++++ .../compiled_project/snapshot_execution.rs | 23 +++++--- .../_helpers/graph/python_selection.rs | 14 ++++- .../build/no_tests_no_audits/_test_types.py | 1 + .../test_build_no_tests_no_audits.py | 30 +++++++--- .../compiler/compiled_project/_test_types.py | 14 +++++ .../compiler/compiled_project/helpers.py | 3 + .../publication/_test_types.py | 2 + .../publication/test_adhoc.py | 23 +++++++- .../publication/test_worker.py | 17 ++++++ .../test_orchestration_snapshot.py | 49 ++++++++++++++++ .../_helpers/test_run_selectors.py | 21 ++++--- .../docs/concepts/python-nodes/checks.mdx | 6 +- .../docs/concepts/python-nodes/overview.mdx | 2 +- 22 files changed, 357 insertions(+), 57 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py new file mode 100644 index 000000000..2f8b9b832 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py @@ -0,0 +1,16 @@ +"""A Python check on the orders model that a job's tag selector matches. + +A job matching the check and the model it checks publishes and runs the check with the model. A job +matching the check alone fails publication, naming the check and the model it leaves out. +""" + +from sqlbuild.checks import check +from sqlbuild.refs import model + + +@check(depends_on=model("orders"), tags=["order_checks"]) +def orders_present(ctx): + """Orders are present.""" + orders = ctx.relation(model("orders")) + count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0] + return ctx.pass_() if count else ctx.fail(message="no orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py new file mode 100644 index 000000000..0723fa403 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py @@ -0,0 +1,15 @@ +"""A job that reaches a managed source through `+`, so its run selects the source's loader chain. + +`+orders` selects the model, its managed source raw_shipments, the raw_shipments loader and the +intermediate stage_shipments loader it depends on. The run records each loader as the source it +writes, which is the node its load reports under. +""" + +from sqlbuild.orchestration import job, schedule + +annual_orders = job( + "annual_orders", + description="Build orders and everything upstream of it, loaders included.", + select="+orders", + schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), +) diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index 4c94cd2f8..06eaa1048 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -89,14 +89,10 @@ pub(crate) fn run_selection( Atom::Sql(_) => None, }) .collect(); - if !select.is_empty() && !checks.is_empty() { - return Err(error( - PLANNER_DEFAULT_CODE, - format!( - "Python checks are not selectable here: {}. Use sqb check instead.", - checks.join(", ") - ), - )); + if !select.is_empty() + && let Some(message) = unselected_checked_nodes(context, &selected, &checks) + { + return Err(error(PLANNER_DEFAULT_CODE, message)); } let runnable: Atoms = selected .into_iter() @@ -106,6 +102,38 @@ pub(crate) fn run_selection( Ok(selection(context, sql_keys(&runnable), &runnable)) } +/// A matched check runs with the nodes it checks, so each of them must be selected too. +fn unselected_checked_nodes( + context: &SelectionContext<'_>, + selected: &Atoms, + checks: &[&str], +) -> Option { + checks.iter().find_map(|check| { + let python = context + .nodes + .edges(check, EdgeDirection::Upstream) + .iter() + .filter(|name| !selected.contains(&Atom::Python((*name).clone()))) + .filter_map(|name| Some(format!("{}:{name}", context.nodes.node(name)?.kind))); + let sql = context + .nodes + .node(check) + .map_or(&[][..], |node| &node.sql_deps); + let sql = sql + .iter() + .filter(|key| !selected.contains(&Atom::Sql((*key).clone()))) + .map(|(kind, name)| format!("{kind}:{name}")); + let missing: Vec = python.chain(sql).collect(); + (!missing.is_empty()).then(|| { + format!( + "Python check '{check}' checks {}, which the selection does not include; \ + select it too, or narrow the selector so it no longer matches the check.", + missing.join(", ") + ) + }) + }) +} + /// The SQL keys, with the source each selected intermediate loader writes, and the selected Python node names. fn selection(context: &SelectionContext<'_>, sql: Vec, atoms: &Atoms) -> RunSelection { let terminal: HashSet<&str> = context diff --git a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs index 487d121f6..4630bc0f8 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs @@ -1,6 +1,7 @@ pub mod attach_python_jobs; pub mod published_python_graph; pub mod python_node_graph_indexes; +pub mod python_run_key; pub mod resolve_python_node_excludes; pub mod resolve_python_node_selectors; pub mod resolve_python_sql_nodes; diff --git a/crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs b/crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs new file mode 100644 index 000000000..a23f1075f --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs @@ -0,0 +1,13 @@ +use crate::python_selection::constants::LOADER_KIND; + +/// The key a selected Python node's run reports under; a loader reports as the source it writes. +pub fn python_run_key(sources: &[(String, Option)], (kind, name): (&str, &str)) -> String { + if kind != LOADER_KIND { + return format!("{kind}:{name}"); + } + let source = sources + .iter() + .find(|(_, loader)| loader.as_deref() == Some(name)) + .map_or(name, |(source, _)| source.as_str()); + format!("source:{source}") +} diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs index 7a2c24b30..6e89230f2 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs @@ -139,13 +139,20 @@ fn given_run_selectors_when_resolving_then_selection_matches_python() { expected_selection: Ok((&[("model", "orders")], &["export_orders", "prepare_orders"])), }, RunSelectionTestCase { - description: "checks are rejected", + description: "a check whose checked node is selected runs with it", + select: &["tag:exports"], + exclude: &[], + expected_selection: Ok((&[], &["export_orders"])), + }, + RunSelectionTestCase { + description: "a check whose checked node is unselected names both", select: &["check:check_orders_export"], exclude: &[], expected_selection: Err(( "S000", - "Python checks are not selectable here: check_orders_export. \ - Use sqb check instead.", + "Python check 'check_orders_export' checks asset:export_orders, which the \ + selection does not include; select it too, or narrow the selector so it no \ + longer matches the check.", )), }, RunSelectionTestCase { diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index ddb519349..241a2eaf8 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -11,6 +11,7 @@ use sqlbuild_analysis::graph::models::{GraphResource, JobSelection}; use sqlbuild_analysis::planner_selection::main::split_unit_test_selectors::split_unit_test_selectors; use sqlbuild_analysis::planner_selection::models::{SelectorGroups, UnitTestSelectorNames}; use sqlbuild_analysis::python_selection::main::published_python_graph::published_python_graph; +use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; use sqlbuild_analysis::python_selection::models::{PublishedPythonNode, SqlSelectionGraph}; use sqlbuild_core::text::main::python_str_repr::python_str_repr; @@ -19,8 +20,9 @@ use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs}; use crate::adhoc::constants::{ ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, CHANGED_COUNTS_DEST, CHANGED_DEST, COMMAND_DEST, - FRESHNESS_VERB, INTEGER_CONVERTERS, LOAD_SOURCES_DEST, POSITIVE_INTEGER_CONVERTER, - REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND, UNIT_TEST_VERBS, + FRESHNESS_VERB, INCLUDE_PYTHON_DEST, INTEGER_CONVERTERS, LOAD_SOURCES_DEST, + POSITIVE_INTEGER_CONVERTER, REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND, + UNIT_TEST_VERBS, }; use crate::argv::main::command_settings_arguments::command_settings_arguments; use crate::argv::main::job_settings_arguments::job_settings_arguments; @@ -86,7 +88,11 @@ impl CommandPlanner for CliPlanner { seeds } FRESHNESS_VERB => freshness_sources(publication, &select, &exclude)?, - _ => resolve_with_tests(verb, publication, &select, &exclude)?, + _ => { + let python = verb == BUILD_VERB + && settings.get(INCLUDE_PYTHON_DEST) != Some(&Json::Bool(false)); + resolve_with_tests((verb, python), publication, &select, &exclude)? + } }; Ok(CommandPlan { verb: verb.clone(), @@ -196,7 +202,7 @@ fn resolve( select: &[String], exclude: &[String], ) -> Result, String> { - let selected: BTreeSet = resolved(publication, select, exclude)?; + let selected: BTreeSet = resolved(publication, select, exclude, false)?; if selected.is_empty() { return Err("the selectors match no nodes in the current publication".into()); } @@ -205,7 +211,7 @@ fn resolve( /// [`resolve`] with `test:` and unit-test names selecting `sql_test:` keys, as `sqb test` does. fn resolve_with_tests( - verb: &str, + (verb, python): (&str, bool), (resources, jobs): (&[PlannerResource], &PublishedJobs), select: &[String], exclude: &[String], @@ -235,7 +241,7 @@ fn resolve_with_tests( ); } let mut keys: BTreeSet = if split.models_selected { - resolved((resources, jobs), &split.select, &split.exclude)? + resolved((resources, jobs), &split.select, &split.exclude, python)? } else { BTreeSet::new() }; @@ -255,15 +261,20 @@ fn is_sql_test(resource: &PlannerResource) -> bool { resource.key.split_once(':').map(|(kind, _)| kind) == Some(SQL_TEST_KIND) } +/// The published keys the selectors match; Python nodes only when the command runs them. fn resolved( (resources, jobs): (&[PlannerResource], &PublishedJobs), select: &[String], exclude: &[String], + include_python: bool, ) -> Result, String> { - let (python, sql): (Vec<&PlannerResource>, Vec<&PlannerResource>) = resources + let (mut python, sql): (Vec<&PlannerResource>, Vec<&PlannerResource>) = resources .iter() .filter(|resource| !is_sql_test(resource)) .partition(|resource| python_kind(&resource.key).is_some()); + if !include_python { + python.clear(); + } let graph_resources: Vec = sql .iter() .filter_map(|resource| graph_resource(resource)) @@ -275,7 +286,7 @@ fn resolved( .map(|resource| name_of(&resource.key)) .collect(); if selectors_need_jobs(&graph, &selectors, &node_names) { - graph = attach_jobs(graph, published_jobs(jobs)); + graph = attach_jobs(graph, published_jobs(jobs, include_python)); } let sources: Vec<(String, Option)> = sql .iter() @@ -306,10 +317,9 @@ fn resolved( .into_iter() .map(|(kind, name)| format!("{kind}:{name}")) .chain( - selection - .python_node_names - .iter() - .filter_map(|name| Some(format!("{}:{name}", kinds.get(name.as_str())?))), + selection.python_node_names.iter().filter_map(|name| { + Some(python_run_key(&sources, (kinds.get(name.as_str())?, name))) + }), ) .collect()) } @@ -339,10 +349,14 @@ fn name_of(key: &str) -> &str { } /// Published jobs as already-resolved graph selections. -fn published_jobs(jobs: &PublishedJobs) -> BTreeMap { +fn published_jobs(jobs: &PublishedJobs, include_python: bool) -> BTreeMap { let mut selections: BTreeMap = BTreeMap::new(); for (name, keys) in jobs { - let keys: Vec<(String, String)> = keys.iter().filter_map(|key| object_key(key)).collect(); + let keys: Vec<(String, String)> = keys + .iter() + .filter(|key| include_python || python_kind(key).is_none()) + .filter_map(|key| object_key(key)) + .collect(); selections.insert(name.clone(), JobSelection::Resolved { keys }); } selections @@ -355,11 +369,11 @@ fn freshness_sources( exclude: &[String], ) -> Result, String> { let selected: BTreeSet = - upstream_sources(publication.0, &resolved(publication, select, &[])?); + upstream_sources(publication.0, &resolved(publication, select, &[], false)?); let excluded: BTreeSet = if exclude.is_empty() { BTreeSet::new() } else { - upstream_sources(publication.0, &resolved(publication, exclude, &[])?) + upstream_sources(publication.0, &resolved(publication, exclude, &[], false)?) }; let sources: BTreeSet = selected.difference(&excluded).cloned().collect(); if sources.is_empty() { diff --git a/crates/sqlbuild-cli/src/adhoc/constants.rs b/crates/sqlbuild-cli/src/adhoc/constants.rs index 0de828474..dc93f3bcb 100644 --- a/crates/sqlbuild-cli/src/adhoc/constants.rs +++ b/crates/sqlbuild-cli/src/adhoc/constants.rs @@ -27,6 +27,9 @@ pub(crate) const CHANGED_COUNTS_DEST: &str = "changed_counts"; /// The job setting `--changed` becomes. pub(crate) const CHANGED_DEST: &str = "changed"; +/// The destination `--no-python` clears; only a build with Python nodes resolves them. +pub(crate) const INCLUDE_PYTHON_DEST: &str = "include_python"; + /// The destination `--load` and `--no-load` share; only `--no-load` is accepted. pub(crate) const LOAD_SOURCES_DEST: &str = "load_sources"; diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index fe0b38f40..df7501c9c 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::json; +use sqlbuild_orchestration::runs::models::PlannerResource; use sqlbuild_orchestration::runs::types::PublishedJobs; use crate::adhoc::main::adhoc_planner::adhoc_planner; @@ -348,6 +349,23 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co &["orders"], Some("python/fulfillment/carrier_sync"), ), + PlannerResource { + loader: Some("raw_shipments".to_owned()), + ..resource("source:raw_shipments", &[], &[], None) + }, + resource( + "model:shipments", + &["source:raw_shipments"], + &[], + Some("models/marts"), + ), + resource( + "loader:raw_shipments", + &["loader:stage_shipments"], + &[], + Some("python/loaders"), + ), + resource("loader:stage_shipments", &[], &[], Some("python/loaders")), ]; let jobs: PublishedJobs = BTreeMap::from([( "carrier_sync".to_owned(), @@ -436,6 +454,46 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co json!({}), )), }, + AdhocPlanTestCase { + description: "upstream of a model runs its source's loader chain as the sources they write", + command: &["build", "--select", "+shipments"], + expected_plan: Ok(( + &[ + "model:shipments", + "source:raw_shipments", + "source:stage_shipments", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "an intermediate loader runs as the source it writes", + command: &["build", "--select", "loader:stage_shipments"], + expected_plan: Ok((&["source:stage_shipments"], json!({}))), + }, + AdhocPlanTestCase { + description: "test selects only the SQL nodes a tag shared with Python nodes matches", + command: &["test", "--select", "tag:orders"], + expected_plan: Ok((&["model:orders", "model:stg_orders"], json!({}))), + }, + AdhocPlanTestCase { + description: "test downstream of a model stops at the Python nodes that read it", + command: &["test", "--select", "orders+", "--exclude", "stg_orders"], + expected_plan: Ok((&["model:orders"], json!({}))), + }, + AdhocPlanTestCase { + description: "a build without Python nodes selects only the SQL nodes a shared tag matches", + command: &["build", "--no-python", "--select", "tag:orders"], + expected_plan: Ok(( + &["model:orders", "model:stg_orders"], + json!({"include_python": false}), + )), + }, + AdhocPlanTestCase { + description: "test of a job with only Python nodes has nothing to test", + command: &["test", "--select", "job:carrier_sync"], + expected_plan: Err("the selectors match no nodes in the current publication"), + }, ]; let planner = adhoc_planner(); for test_case in test_cases { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index b27d9b4b2..033b08c4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -357,9 +357,10 @@ impl WorkerBuild { outcome }; if self.execution.claim.request.selection.contains(key) { - return self.complete(py, key, (outcome, None, (None, None))); + self.complete(py, key, (outcome, None, (None, None)))?; + } else { + self.finish_check(py, key, outcome)?; } - self.finish_check(py, key, outcome)?; if outcome == NodeOutcome::Skipped { return Ok(()); } @@ -395,15 +396,12 @@ impl WorkerBuild { Ok(None) } - /// A failing SQL test, found by its compiled name such as `parent [case]`, fails the models it blocks with its error. + /// A finished SQL test completes itself when selected; a failing one fails the models it blocks. fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { let py = result.py(); let outcome: String = field(result, "outcome")?.str()?.extract()?; - if SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass) - || self.worker.bind(py).get().refresh_cancellation(py)? - { - return Ok(()); - } + let passed = SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass); + let cancelled = !passed && self.worker.bind(py).get().refresh_cancellation(py)?; let name: String = field(result, "test_name")?.extract()?; let key = format!("{SQL_TEST_KEY_PREFIX}{name}"); let error: Option = match result.getattr_opt("error_code")? { @@ -414,6 +412,15 @@ impl WorkerBuild { Some(value) => value.extract()?, None => None, }; + let node = match (passed, cancelled) { + (true, _) => NodeOutcome::Success, + (false, true) => NodeOutcome::Cancelled, + (false, false) => NodeOutcome::Failed, + }; + self.complete(py, &key, (node, None, (error.clone(), message.clone())))?; + if passed || cancelled { + return Ok(()); + } for model in self.blocked.get(&key).into_iter().flatten() { self.complete( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 3a5d3b1e6..f9d1b4e03 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -13,6 +13,7 @@ use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_analysis::python_selection::main::attach_python_jobs::attach_python_jobs; use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; +use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; @@ -187,6 +188,17 @@ impl ProjectSelection { .map(|node| format!("{}:{name}", node.kind)) } + /// The key a selected Python node's run reports under; a loader reports as the source it writes. + fn run_key(&self, name: &str) -> Option { + let node = self + .nodes + .nodes + .iter() + .rev() + .find(|node| node.name == name)?; + Some(python_run_key(&self.sources, (&node.kind, name))) + } + /// Whether a loader writes no declared source and so runs only as part of another loader's chain. pub(crate) fn intermediate_loader(&self, name: &str) -> bool { !self @@ -213,7 +225,7 @@ impl ProjectSelection { .map(|(kind, name)| format!("{kind}:{name}")) .collect(); for name in &selection.python_node_names { - keys.extend(self.node_key(name)); + keys.extend(self.run_key(name)); } Ok(keys) } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py index bfc5361cf..5e38ffcc8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py @@ -18,3 +18,4 @@ class RunE2ETestCase: expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] = field( default_factory=tuple ) + select: tuple[str, ...] = field(default_factory=tuple) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index 3100cd5be..b3114af93 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -2185,15 +2185,29 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou "test_case", [ RunE2ETestCase( - description="run rejects selected check node", + description="build rejects a selected check whose task is not selected", expected_exit_code=1, expected_table_names=(), expected_view_names=(), - ) + expected_output_fragments=( + "Python check 'check_orders_export' checks task:prepare_orders, which the " + "selection does not include; select it too, or narrow the selector so it no " + "longer matches the check.", + ), + select=("check:check_orders_export",), + ), + RunE2ETestCase( + description="build runs a selected check with its selected task", + expected_exit_code=0, + expected_table_names=(), + expected_view_names=(), + expected_output_fragments=("check check_orders_export START", "PASS=2"), + select=("check:check_orders_export", "task:prepare_orders"), + ), ], ids=lambda case: case.description, ) -def test_given_check_selector_when_running_run_then_command_rejects_check( +def test_given_check_selector_when_running_build_then_check_needs_its_checked_node( test_case: RunE2ETestCase, tmp_path: Path, ) -> None: @@ -2230,14 +2244,12 @@ def test_given_check_selector_when_running_run_then_command_rejects_check( "--no-tests", "--no-audits", "--select", - "check:check_orders_export", + *test_case.select, ), project_dir=project_dir, ) - assert result.returncode == test_case.expected_exit_code combined_output: str = result.stdout + result.stderr - assert ( - "Python checks are not selectable here: " - "check_orders_export. Use sqb check instead." in combined_output - ) + assert result.returncode == test_case.expected_exit_code, combined_output + for fragment in test_case.expected_output_fragments: + assert fragment in combined_output diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 01776fdad..fa3f650f9 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -283,6 +283,20 @@ class PythonJobGroupCycleTestCase: expected_message: str +@dataclass(frozen=True) +class PythonCheckJobTestCase: + description: str + select: str + expected_selection: list[str] + + +@dataclass(frozen=True) +class PythonCheckJobRejectionTestCase: + description: str + select: str + expected_message: str + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index bbaca4925..d114213a6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -84,6 +84,9 @@ "python/fulfillment/carrier_sync/node.py": Path( "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py" ).read_text(), + "python/checks/orders_present.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py" + ).read_text(), } TYPE_PROOF_FILES: dict[str, str] = { diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 1da98aee9..be2c12860 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -406,6 +406,8 @@ class AdhocVerbTestCase: # Files the worker's checkout gains after it verified the final run, which must not run. rewrites: dict[str, str] = field(default_factory=dict) expected_status: str = "success" + # Files that replace the published project's own before publication. + files: dict[str, str] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index 703856d11..b7247fd0a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -268,6 +268,26 @@ def test_given_an_ad_hoc_command_when_checked_and_launched_then_the_worker_runs_ expected_run_nodes=[("sql_test:customers_have_ids", "node", "success")], rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, ), + AdhocVerbTestCase( + description="build test: runs and reports only the selected verified SQL test", + prerequisites=[], + command=["build", "--select", "test:customers_have_ids"], + expected_selection=["sql_test:customers_have_ids"], + expected_run_nodes=[("sql_test:customers_have_ids", "node", "success")], + rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, + ), + AdhocVerbTestCase( + description="test of a model and its failing SQL test fails both", + prerequisites=[], + command=["test", "--select", "customers", "test:customers_have_ids"], + expected_selection=["model:customers", "sql_test:customers_have_ids"], + expected_run_nodes=[ + ("model:customers", "node", "failed"), + ("sql_test:customers_have_ids", "node", "failed"), + ], + files={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, + expected_status="failed", + ), AdhocVerbTestCase( description="freshness observes the verified sources upstream of the selection", prerequisites=[], @@ -303,7 +323,8 @@ def test_given_an_ad_hoc_verb_when_the_checkout_changes_after_verification_then_ "seeds/seeds.yml": SEEDS_YML.format(region_type="VARCHAR"), "sources/raw.yml": SOURCES_YML.format(table="raw_orders"), "python/rewrite_checkout.py": REWRITE_CHECKOUT, - }, + } + | test_case.files, ) with scheduled_worker( tmp_path, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 0169dd24b..4231bf9a7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -851,6 +851,23 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re {"model:orders", "source:raw_shipments", "source:stage_shipments"} ), ), + ClaimedLoadTestCase( + "a job reaching a managed source through + runs the source's loader", + "", + "SELECT amount FROM orders", + "success", + [(9,)], + files=INTERMEDIATE_LOADER_FILES + | { + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py" + ).read_text() + }, + expected_node_count=3, + expected_write_set=frozenset( + {"model:orders", "source:raw_shipments", "source:stage_shipments"} + ), + ), ], ids=lambda case: case.description, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 76d37d2d1..179999d18 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -34,6 +34,8 @@ ) from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + PythonCheckJobRejectionTestCase, + PythonCheckJobTestCase, PythonJobGroupCycleTestCase, PythonJobGroupTestCase, PythonJobSelectionTestCase, @@ -1997,3 +1999,50 @@ def test_given_a_python_dependency_cycle_across_jobs_when_publishing_a_group_the with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): python_pipeline_snapshot(tmp_path, [*jobs, group]) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonCheckJobTestCase( + description="a job matching a check and the model it checks publishes the model", + select="tag:order_checks orders", + expected_selection=["model:orders"], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_matching_a_check_and_its_node_when_publishing_then_the_check_runs_with_it( + tmp_path: Path, test_case: PythonCheckJobTestCase +) -> None: + nightly: Job = job("nightly_orders", description="Orders.", select=test_case.select) + + snapshot: dict[str, Any] = json.loads(python_pipeline_snapshot(tmp_path, [nightly]).to_json()) + + assert snapshot["documents"]["jobs"]["nightly_orders"]["selection"] == ( + test_case.expected_selection + ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonCheckJobRejectionTestCase( + description="a job matching a check without the model it checks names both", + select="tag:order_checks", + expected_message=( + "job nightly_orders: Python check 'orders_present' checks model:orders, which " + "the selection does not include; select it too, or narrow the selector so it no " + "longer matches the check." + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_matching_a_check_without_its_node_when_publishing_then_it_is_rejected( + tmp_path: Path, test_case: PythonCheckJobRejectionTestCase +) -> None: + nightly: Job = job("nightly_orders", description="Orders.", select=test_case.select) + + with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): + python_pipeline_snapshot(tmp_path, [nightly]) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py index ae3937e7d..f42a9bed9 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py @@ -42,6 +42,13 @@ expected_sql_names=frozenset(), expected_python_node_names=frozenset({"export_orders"}), ), + PythonSqlSelectorTestCase( + description="accepts a check whose checked node the selection also matches", + select=("tag:exports",), + exclude=(), + expected_sql_names=frozenset(), + expected_python_node_names=frozenset({"export_orders"}), + ), PythonSqlSelectorTestCase( description="selects expanded source terminal loader requirement for run", select=("+source:raw_orders",), @@ -73,18 +80,14 @@ def test_given_run_selectors_when_resolving_then_excludes_python_checks( "test_case", [ PythonSqlSelectorErrorTestCase( - description="rejects explicit check selector for run", + description="rejects a check whose checked node the selection leaves out", select=("check:check_orders_export",), exclude=(), expected_error_type=ValueError, - expected_error_fragment=("Python checks are not selectable here: check_orders_export"), - ), - PythonSqlSelectorErrorTestCase( - description="rejects tag selector that includes a check for run", - select=("tag:exports",), - exclude=(), - expected_error_type=ValueError, - expected_error_fragment=("Python checks are not selectable here: check_orders_export"), + expected_error_fragment=( + "Python check 'check_orders_export' checks asset:export_orders, which the " + "selection does not include" + ), ), ], ids=lambda case: case.description, diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx index dd93a67b3..c1acdf2f8 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx @@ -109,7 +109,11 @@ Checks do not support `columns`, `column_lineage`, or `retry`. ## Running checks -Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](/docs/cli/check): +Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. + +A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. Jobs, roles and server runs resolve selectors the same way. + +To run checks on their own, use [`sqb check`](/docs/cli/check): ```bash # Run all checks diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index ddc395623..9d215b702 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -141,7 +141,7 @@ Python nodes are selected like SQL resources, by bare name or typed selector: sqb build --select export_orders # bare name sqb build --select task:export_orders # typed sqb build --select asset:orders_export -sqb build --select check:check_orders +sqb build --select export_orders check:check_orders_exported # a check with the node it checks sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` From 879f9316b6be7d9a9aae48d9e0ca2d9504fed148 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:31:15 +0000 Subject: [PATCH 06/34] fix: refuse selecting a Python check on a source no loader writes A build never loads such a source, so a matched check on it would be accepted and then never run. The resolver now names the check and the source and points to sqb check. --- .../src/python_selection/_helpers/unified.rs | 79 +++++++++++++------ .../src/python_selection/tests/helpers.rs | 14 ++++ .../tests/test_python_selection.rs | 12 +++ .../test_build_no_tests_no_audits.py | 22 ++++++ .../docs/concepts/python-nodes/checks.mdx | 2 +- 5 files changed, 104 insertions(+), 25 deletions(-) diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index 06eaa1048..56bf52afc 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -102,35 +102,66 @@ pub(crate) fn run_selection( Ok(selection(context, sql_keys(&runnable), &runnable)) } -/// A matched check runs with the nodes it checks, so each of them must be selected too. +/// The first matched check a build cannot run after the nodes it checks, as an error message. fn unselected_checked_nodes( context: &SelectionContext<'_>, selected: &Atoms, checks: &[&str], ) -> Option { - checks.iter().find_map(|check| { - let python = context - .nodes - .edges(check, EdgeDirection::Upstream) - .iter() - .filter(|name| !selected.contains(&Atom::Python((*name).clone()))) - .filter_map(|name| Some(format!("{}:{name}", context.nodes.node(name)?.kind))); - let sql = context - .nodes - .node(check) - .map_or(&[][..], |node| &node.sql_deps); - let sql = sql - .iter() - .filter(|key| !selected.contains(&Atom::Sql((*key).clone()))) - .map(|(kind, name)| format!("{kind}:{name}")); - let missing: Vec = python.chain(sql).collect(); - (!missing.is_empty()).then(|| { - format!( - "Python check '{check}' checks {}, which the selection does not include; \ - select it too, or narrow the selector so it no longer matches the check.", - missing.join(", ") - ) - }) + let loaded: HashSet<&str> = source_loaders(context.sql) + .into_iter() + .map(|(source, _)| source) + .collect(); + for check in checks { + if let Some(message) = unrunnable_check(context, selected, &loaded, check) { + return Some(message); + } + } + None +} + +/// Why a build cannot run `check`: it checks a source no loader writes, or an unselected node. +fn unrunnable_check( + context: &SelectionContext<'_>, + selected: &Atoms, + loaded: &HashSet<&str>, + check: &str, +) -> Option { + let sql: &[ObjectKey] = context + .nodes + .node(check) + .map_or(&[][..], |node| &node.sql_deps); + let unloaded: Vec = sql + .iter() + .filter(|(kind, name)| kind == SOURCE_RESOURCE && !loaded.contains(name.as_str())) + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + if !unloaded.is_empty() { + return Some(format!( + "Python check '{check}' checks {}, which a build does not load, so the check \ + would not run; run it with sqb check, or narrow the selector so it no longer \ + matches the check.", + unloaded.join(", ") + )); + } + let missing: Vec = context + .nodes + .edges(check, EdgeDirection::Upstream) + .iter() + .filter(|name| !selected.contains(&Atom::Python((*name).clone()))) + .filter_map(|name| Some(format!("{}:{name}", context.nodes.node(name)?.kind))) + .chain( + sql.iter() + .filter(|key| !selected.contains(&Atom::Sql((*key).clone()))) + .map(|(kind, name)| format!("{kind}:{name}")), + ) + .collect(); + (!missing.is_empty()).then(|| { + format!( + "Python check '{check}' checks {}, which the selection does not include; \ + select it too, or narrow the selector so it no longer matches the check.", + missing.join(", ") + ) }) } diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs index 2ac7b7de0..2e8c3134f 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs @@ -67,6 +67,14 @@ const ORDERS_NODES: &[NodeSpec] = &[ &[], &[], ), + ( + "check_raw_customers", + "check", + "python/checks/customers.py", + &["customer_checks"], + &[], + &[("source", "raw_customers")], + ), ]; fn owned(values: &[&str]) -> Vec { @@ -128,6 +136,12 @@ fn orders_sql_graph() -> ProjectGraph { tags: Vec::new(), folder: None, }, + GraphResource { + key: key(&("source", "raw_customers")), + deps: Vec::new(), + tags: Vec::new(), + folder: None, + }, ]) } diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs index 6e89230f2..fad17b2ca 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs @@ -113,6 +113,7 @@ fn given_run_selectors_when_resolving_then_selection_matches_python() { &[ ("model", "orders"), ("source", "load_events"), + ("source", "raw_customers"), ("source", "raw_orders"), ], &[ @@ -144,6 +145,17 @@ fn given_run_selectors_when_resolving_then_selection_matches_python() { exclude: &[], expected_selection: Ok((&[], &["export_orders"])), }, + RunSelectionTestCase { + description: "a check on a source a build does not load is refused even when selected", + select: &["tag:customer_checks", "source:raw_customers"], + exclude: &[], + expected_selection: Err(( + "S000", + "Python check 'check_raw_customers' checks source:raw_customers, which a build \ + does not load, so the check would not run; run it with sqb check, or narrow the \ + selector so it no longer matches the check.", + )), + }, RunSelectionTestCase { description: "a check whose checked node is unselected names both", select: &["check:check_orders_export"], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index b3114af93..3af739ac2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -2204,6 +2204,18 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou expected_output_fragments=("check check_orders_export START", "PASS=2"), select=("check:check_orders_export", "task:prepare_orders"), ), + RunE2ETestCase( + description="build rejects a selected check on a source no loader writes", + expected_exit_code=1, + expected_table_names=(), + expected_view_names=(), + expected_output_fragments=( + "Python check 'check_raw_events' checks source:raw_events, which a build does " + "not load, so the check would not run; run it with sqb check, or narrow the " + "selector so it no longer matches the check.", + ), + select=("check:check_raw_events", "source:raw_events"), + ), ], ids=lambda case: case.description, ) @@ -2234,6 +2246,16 @@ def test_given_check_selector_when_running_build_then_check_needs_its_checked_no "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n return ctx.result()\n" ), + "python/checks/events.py": ( + "from sqlbuild.checks import check\n" + "from sqlbuild.refs import source\n\n" + "@check(depends_on=source('raw_events'))\n" + "def check_raw_events(ctx):\n" + " '''Test check check_raw_events.'''\n return ctx.pass_()\n" + ), + "sources/raw.yml": ( + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + ), }, ) diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx index c1acdf2f8..17a34a194 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx @@ -111,7 +111,7 @@ Checks do not support `columns`, `column_lineage`, or `retry`. Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. -A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. Jobs, roles and server runs resolve selectors the same way. +A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. A check on a source that no loader writes never runs in a build, so a selector that matches it fails too; run that check with `sqb check`. Jobs and server runs resolve selectors the same way. To run checks on their own, use [`sqb check`](/docs/cli/check): From 059f8cdb7380c82e8d9e3a4449b1dfa517f1c724 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:56:00 +0000 Subject: [PATCH 07/34] refactor: share upstream result reads between loader and node contexts Loader and Python node contexts read upstream results through one mixin, and the selection entry points build their context in one place. Regenerates the packaged docs skill for the selector and check pages. --- .../src/python_selection/_helpers/nodes.rs | 17 ++++- .../main/attach_python_jobs.rs | 12 ++-- .../main/resolve_python_sql_nodes.rs | 10 +-- .../main/resolve_python_sql_run_selection.rs | 10 +-- .../docs/concepts/python-nodes/checks.md | 6 +- .../docs/concepts/python-nodes/overview.md | 16 ++++- .../references/docs/concepts/selectors.md | 2 +- src/sqlbuild/executor/load/models.py | 63 +------------------ .../classes/dependency_result_reader.py | 61 ++++++++++++++++++ .../executor/node_results/constants.py | 7 +++ src/sqlbuild/executor/python_nodes/models.py | 63 +------------------ 11 files changed, 118 insertions(+), 149 deletions(-) create mode 100644 src/sqlbuild/executor/node_results/classes/dependency_result_reader.py diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs index e2390a571..7fc0e9cd4 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs @@ -17,7 +17,9 @@ use crate::python_selection::constants::{ UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, }; -use crate::python_selection::models::{NodeIndex, PythonSelectionGraph, PythonSelectionNode}; +use crate::python_selection::models::{ + NodeIndex, PythonSelectionGraph, PythonSelectionNode, SelectionContext, SqlSelectionGraph, +}; use sqlbuild_core::constants::PYTHON_NODE_ROOT as PYTHON_NODE_PATH_ROOT; pub(crate) type Names = BTreeSet; @@ -59,6 +61,19 @@ fn keyed(rows: &[(String, Vec)]) -> HashMap<&str, &[String]> { .collect() } +impl<'graph> SelectionContext<'graph> { + pub(crate) fn new( + sql: SqlSelectionGraph<'graph>, + python: &'graph PythonSelectionGraph, + ) -> Self { + Self { + sql, + nodes: NodeIndex::new(python), + sql_names: sql_names(sql.graph), + } + } +} + impl<'graph> NodeIndex<'graph> { pub(crate) fn new(graph: &'graph PythonSelectionGraph) -> Self { Self { diff --git a/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs index 6e3095f6f..21edb1f7e 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs @@ -2,10 +2,9 @@ use std::collections::BTreeMap; use crate::graph::main::attach_jobs_with::attach_jobs_with; use crate::graph::models::{JobSelection, ProjectGraph}; -use crate::python_selection::_helpers::nodes::sql_names; use crate::python_selection::_helpers::unified::run_selection; use crate::python_selection::models::{ - NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, + PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, }; /// Attach jobs as publication resolves them, keeping each job's Python nodes as `(kind, name)` keys. @@ -20,14 +19,13 @@ pub fn attach_python_jobs( .map(|node| (node.name.as_str(), node.kind.as_str())) .collect(); attach_jobs_with(sql.graph.clone(), catalog, |graph, select, exclude| { - let context: SelectionContext<'_> = SelectionContext { - sql: SqlSelectionGraph { + let context: SelectionContext<'_> = SelectionContext::new( + SqlSelectionGraph { graph, sources: sql.sources, }, - nodes: NodeIndex::new(python), - sql_names: sql_names(graph), - }; + python, + ); let RunSelection { sql_keys, python_node_names, diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs index 6a2d99a3c..b482df85e 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs @@ -1,8 +1,7 @@ use crate::graph::errors::SelectorError; -use crate::python_selection::_helpers::nodes::sql_names; use crate::python_selection::_helpers::unified::node_selection; use crate::python_selection::models::{ - NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, + PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, }; /// The SQL resources and Python nodes, checks included, that selectors name, as grants and sensor conditions read them. @@ -12,10 +11,5 @@ pub fn resolve_python_sql_nodes( select: &[String], exclude: &[String], ) -> Result { - let context: SelectionContext<'_> = SelectionContext { - sql, - nodes: NodeIndex::new(python), - sql_names: sql_names(sql.graph), - }; - node_selection(&context, select, exclude) + node_selection(&SelectionContext::new(sql, python), select, exclude) } diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs index a820fd41d..fef9983c7 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs @@ -1,8 +1,7 @@ use crate::graph::errors::SelectorError; -use crate::python_selection::_helpers::nodes::sql_names; use crate::python_selection::_helpers::unified::run_selection; use crate::python_selection::models::{ - NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, + PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, }; /// Python's `resolve_python_sql_run_selectors`: run selectors over SQL resources and Python nodes. @@ -12,10 +11,5 @@ pub fn resolve_python_sql_run_selection( select: &[String], exclude: &[String], ) -> Result { - let context: SelectionContext<'_> = SelectionContext { - sql, - nodes: NodeIndex::new(python), - sql_names: sql_names(sql.graph), - }; - run_selection(&context, select, exclude) + run_selection(&SelectionContext::new(sql, python), select, exclude) } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md index 4dc3e6997..fe54d4880 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md @@ -120,7 +120,11 @@ Checks do not support `columns`, `column_lineage`, or `retry`. ## Running checks -Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](../../cli/check.md): +Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. + +A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. A check on a source that no loader writes never runs in a build, so a selector that matches it fails too; run that check with `sqb check`. Jobs and server runs resolve selectors the same way. + +To run checks on their own, use [`sqb check`](../../cli/check.md): ```bash # Run all checks diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index 17cdf8438..c4c2db12e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -143,7 +143,7 @@ def export_orders(ctx): - `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.tasks`/`sqlbuild.assets`. - Assets may pass `materialized=True`/`False` to record whether an artifact was produced. -Downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. +Within one run, downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. ## Result persistence @@ -157,7 +157,7 @@ Python nodes are selected like SQL resources, by bare name or typed selector: sqb build --select export_orders # bare name sqb build --select task:export_orders # typed sqb build --select asset:orders_export -sqb build --select check:check_orders +sqb build --select export_orders check:check_orders_exported # a check with the node it checks sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` @@ -177,6 +177,18 @@ dependency fails or is skipped. A dependency that is not part of the run (not se unmanaged source) is read as it currently exists, so `sqb build --select task:export_orders` runs the task against the relations already in the warehouse. +Python dependencies follow the same rule. A task or asset selected without its Python upstream +reads that upstream's latest successful result through `ctx.result_of`. If the upstream has never +succeeded for the target, the node fails with an error that names the upstream, for example: + +```text +Upstream task 'prepare_orders' has no successful result to read. Build it first with +'sqb build --select task:prepare_orders', or build both with 'sqb build --select +asset:export_orders'. +``` + +Jobs, sensors, roles and ad-hoc runs on the server resolve selectors the same way, so a job can +select one Python node and a job group orders it after the job that builds its upstream. + The commands differ in what they include by default: | Command | SQL | Loaders / tasks / assets | Checks | Audits | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index cf563c84a..09e66562e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -82,7 +82,7 @@ sqb build --select source:raw__orders ### Job -Select the nodes a job or job group declared under `orchestration/` builds. Job names are globally +Select the SQL and Python nodes a job or job group declared under `orchestration/` builds. Job names are globally unique, so a bare name works; `job:` is the explicit form and accepts glob patterns: ```bash diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 39097300b..f64d2156b 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -17,13 +17,12 @@ from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind, SkipMode +from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.constants import LOADER_RELATION_QUALIFIER_SEPARATOR from sqlbuild.executor.load.types import LoadProgressCallback -from sqlbuild.executor.node_results.models import NodeResultEnvelope +from sqlbuild.executor.node_results.classes.dependency_result_reader import DependencyResultReader from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard -from sqlbuild.executor.python_nodes.constants import MISSING_DEFAULT from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef @@ -95,7 +94,7 @@ class LoaderResult: @dataclass(frozen=True) -class LoaderContext: +class LoaderContext(DependencyResultReader): """Runtime context passed to a source loader function.""" adapter: BaseAdapter @@ -164,44 +163,6 @@ def result( materialized=materialized, ) - def result_of( - self, - *, - node_function: Callable[..., object], - run_id: str | None = None, - default: object = MISSING_DEFAULT, - ) -> NodeResultEnvelope | object: - """Return the latest persisted upstream result by Python node function reference.""" - - if self.result_store is None: - if default is not MISSING_DEFAULT: - return default - raise ExecutorInputError("No Python node result store is available") - node_type, node_name = self._result_dependency_identity(node_function) - return self.result_store.result_of( - node_type=node_type, - node_name=node_name, - run_id=run_id, - default=default, - ) - - def results_of( - self, - *, - node_function: Callable[..., object], - limit: int, - ) -> tuple[NodeResultEnvelope, ...]: - """Return persisted successful upstream result history, newest first.""" - - if self.result_store is None: - return () - node_type, node_name = self._result_dependency_identity(node_function) - return self.result_store.results_of( - node_type=node_type, - node_name=node_name, - limit=limit, - ) - def qualify_name( self, *, @@ -251,24 +212,6 @@ def _record_resolved(self, relation_ref: LoaderRelationRef) -> None: SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=relation_ref.name) ) - def _result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: - task_definition: object = getattr(node_function, "__sqlbuild_task__", None) - if task_definition is not None: - return PythonNodeKind.TASK.value, self._definition_name(task_definition) - asset_definition: object = getattr(node_function, "__sqlbuild_asset__", None) - if asset_definition is not None: - return PythonNodeKind.ASSET.value, self._definition_name(asset_definition) - loader_definition: object = getattr(node_function, "__sqlbuild_loader__", None) - if loader_definition is not None: - return PythonNodeKind.LOADER.value, self._definition_name(loader_definition) - raise ExecutorInputError("Python node result dependency must be a task, asset, or loader") - - def _definition_name(self, definition: object) -> str: - name: object = getattr(definition, "name", None) - if isinstance(name, str): - return name - raise ExecutorInputError("Python node result dependency has no resolved name") - @dataclass(frozen=True) class LoadExecutionResult: diff --git a/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py new file mode 100644 index 000000000..7e5a36393 --- /dev/null +++ b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py @@ -0,0 +1,61 @@ +"""Reads of a Python node's upstream results, looked up by the upstream's decorated function.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.node_results.constants import RESULT_DEPENDENCY_ATTRIBUTES +from sqlbuild.executor.node_results.models import NodeResultEnvelope +from sqlbuild.executor.python_nodes.constants import MISSING_DEFAULT + + +class DependencyResultReader: + """Mixin for node contexts that read their upstream results from a `result_store`.""" + + result_store: Any | None + + def result_of( + self, + *, + node_function: Callable[..., object], + run_id: str | None = None, + default: object = MISSING_DEFAULT, + ) -> NodeResultEnvelope | object: + """Return the latest persisted upstream result by Python node function reference.""" + + if self.result_store is None: + if default is not MISSING_DEFAULT: + return default + raise ExecutorInputError("No Python node result store is available") + node_type, node_name = self.result_dependency_identity(node_function) + return self.result_store.result_of( + node_type=node_type, node_name=node_name, run_id=run_id, default=default + ) + + def results_of( + self, + *, + node_function: Callable[..., object], + limit: int, + ) -> tuple[NodeResultEnvelope, ...]: + """Return persisted successful upstream result history, newest first.""" + + if self.result_store is None: + return () + node_type, node_name = self.result_dependency_identity(node_function) + return self.result_store.results_of(node_type=node_type, node_name=node_name, limit=limit) + + def result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: + """Return the `(kind, name)` a dependency's results are stored under.""" + + for attribute, kind in RESULT_DEPENDENCY_ATTRIBUTES: + definition: object = getattr(node_function, attribute, None) + if definition is None: + continue + name: object = getattr(definition, "name", None) + if not isinstance(name, str): + raise ExecutorInputError("Python node result dependency has no resolved name") + return kind, name + raise ExecutorInputError("Python node result dependency must be a task, asset, or loader") diff --git a/src/sqlbuild/executor/node_results/constants.py b/src/sqlbuild/executor/node_results/constants.py index b2e744499..3b2d79202 100644 --- a/src/sqlbuild/executor/node_results/constants.py +++ b/src/sqlbuild/executor/node_results/constants.py @@ -1,6 +1,7 @@ """Runtime node result storage constants.""" from sqlbuild._native import StateSqlValueType +from sqlbuild.compiler.python_nodes.types import PythonNodeKind NODE_RESULT_MATERIALIZED_TRUE_VALUE: str = "true" @@ -36,3 +37,9 @@ **{column: StateSqlValueType.STRING for column in NODE_RESULT_COLUMNS}, COLUMN_TIMESTAMP: StateSqlValueType.TEXT_TIMESTAMP, } + +RESULT_DEPENDENCY_ATTRIBUTES: tuple[tuple[str, str], ...] = ( + ("__sqlbuild_task__", PythonNodeKind.TASK.value), + ("__sqlbuild_asset__", PythonNodeKind.ASSET.value), + ("__sqlbuild_loader__", PythonNodeKind.LOADER.value), +) diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 245fab2a9..4d7ffae69 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -21,10 +21,9 @@ ) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.node_results.models import NodeResultEnvelope +from sqlbuild.executor.node_results.classes.dependency_result_reader import DependencyResultReader from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard from sqlbuild.executor.python_nodes.constants import ( - MISSING_DEFAULT, PYTHON_NODE_RELATION_QUALIFIER_SEPARATOR, ) from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder @@ -165,7 +164,7 @@ class IngressCallbacks: @dataclass(frozen=True, kw_only=True) -class BasePythonNodeContext: +class BasePythonNodeContext(DependencyResultReader): """Shared runtime helpers for framework-owned Python nodes.""" adapter: BaseAdapter @@ -254,64 +253,6 @@ def skip( metadata={} if metadata is None else metadata, ) - def result_of( - self, - *, - node_function: Callable[..., object], - run_id: str | None = None, - default: object = MISSING_DEFAULT, - ) -> NodeResultEnvelope | object: - """Return the latest persisted upstream result by Python node function reference.""" - - if self.result_store is None: - if default is not MISSING_DEFAULT: - return default - raise ExecutorInputError("No Python node result store is available") - node_type, node_name = self.result_dependency_identity(node_function) - return self.result_store.result_of( - node_type=node_type, - node_name=node_name, - run_id=run_id, - default=default, - ) - - def results_of( - self, - *, - node_function: Callable[..., object], - limit: int, - ) -> tuple[NodeResultEnvelope, ...]: - """Return persisted successful upstream result history, newest first.""" - - if self.result_store is None: - return () - node_type, node_name = self.result_dependency_identity(node_function) - return self.result_store.results_of( - node_type=node_type, - node_name=node_name, - limit=limit, - ) - - def result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: - """Return the `(kind, name)` a dependency's results are stored under.""" - - task_definition: object = getattr(node_function, "__sqlbuild_task__", None) - if task_definition is not None: - return PythonNodeKind.TASK.value, self._definition_name(task_definition) - asset_definition: object = getattr(node_function, "__sqlbuild_asset__", None) - if asset_definition is not None: - return PythonNodeKind.ASSET.value, self._definition_name(asset_definition) - loader_definition: object = getattr(node_function, "__sqlbuild_loader__", None) - if loader_definition is not None: - return PythonNodeKind.LOADER.value, self._definition_name(loader_definition) - raise ExecutorInputError("Python node result dependency must be a task, asset, or loader") - - def _definition_name(self, definition: object) -> str: - name: object = getattr(definition, "name", None) - if isinstance(name, str): - return name - raise ExecutorInputError("Python node result dependency must have a string name") - @dataclass(frozen=True, kw_only=True) class TaskContext(BasePythonNodeContext): From e0c01ba2ca20b28aa0e9c78247ed97b9f7f04853 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:15:28 +0000 Subject: [PATCH 08/34] feat!: order job groups by lineage only and skip checks under --no-audits JobGroup no longer takes after: a group's order comes only from the lineage between the nodes its jobs build, and declaring after fails with an error that names the group. build --no-audits now skips Python checks alongside audits, as documented. --- .../src/declarations/_helpers/group_waits.rs | 2 +- .../declarations/main/group_dependencies.rs | 26 +++++------------ .../declarations/main/snapshot_definitions.rs | 3 +- .../src/declarations/models.rs | 4 +-- .../_helpers/build_command/execution.rs | 5 +++- .../compiled_project/snapshot_declarations.rs | 12 +------- .../native_host/orchestration_declarations.rs | 15 ---------- .../models/authoring.py | 14 +++++++--- .../cli/commands/main/build/helpers.py | 12 ++++++++ .../build/test_no_tests_no_audits_flags.py | 6 ++-- .../test_orchestration_snapshot.py | 28 +++++-------------- ui/src/lib/api/mock/operate-store.ts | 5 +--- ui/src/lib/api/mock/operate.ts | 15 +--------- ui/src/lib/api/types/operate.ts | 4 +-- ui/src/lib/lineage/job-graph.test.ts | 12 ++++---- ui/src/lib/lineage/job-graph.ts | 15 +++++----- 16 files changed, 66 insertions(+), 112 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs index 76a2b097a..fcc0b6ece 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; -/// Keep the first reason found for a wait: an explicit `after`, or the first lineage edge. +/// Keep the first reason found for a wait: the first lineage edge. pub(crate) fn record_wait( mut reasons: BTreeMap<(String, String), String>, (job, dependency): (&str, &str), diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs index d2103f243..6e78d95cf 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs @@ -1,4 +1,4 @@ -//! Freeze job boundaries from snapshot lineage plus explicit waits, rejecting cycles. +//! Freeze job boundaries from snapshot lineage, rejecting cycles. use std::collections::{BTreeMap, BTreeSet}; @@ -10,19 +10,12 @@ use crate::store::errors::ControlError; pub fn group_dependencies( (group, input): (&str, &SnapshotInput), jobs: &BTreeMap, - (members, after): (&BTreeSet, &BTreeMap>), + members: &BTreeSet, ) -> Result>, ControlError> { - if members.is_empty() - || members.iter().any(|name| !jobs.contains_key(name)) - || after - .iter() - .any(|(name, dependencies)| !members.contains(name) || !dependencies.is_subset(members)) - { - return Err(ControlError::Invalid( - "group needs known jobs and dependencies within its membership".into(), - )); + if members.is_empty() || members.iter().any(|name| !jobs.contains_key(name)) { + return Err(ControlError::Invalid("group needs known jobs".into())); } - let waits = lineage_dependencies(input, jobs, (members, after)); + let waits = lineage_dependencies(input, jobs, members); ensure_acyclic(group, &waits.edges, &waits.reasons)?; Ok(waits.edges) } @@ -31,7 +24,7 @@ pub fn group_dependencies( fn lineage_dependencies( input: &SnapshotInput, jobs: &BTreeMap, - (members, after): (&BTreeSet, &BTreeMap>), + members: &BTreeSet, ) -> GroupWaits { let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); for name in members { @@ -42,12 +35,7 @@ fn lineage_dependencies( let mut edges: BTreeMap> = BTreeMap::new(); let mut reasons: BTreeMap<(String, String), String> = BTreeMap::new(); for name in members { - let mut dependencies = after.get(name).cloned().unwrap_or_default(); - for dependency in &dependencies { - reasons = record_wait(reasons, (name, dependency), || { - "declared in after".to_owned() - }); - } + let mut dependencies: BTreeSet = BTreeSet::new(); let mut seen: BTreeSet<&str> = BTreeSet::new(); let mut pending: Vec<&str> = jobs[name].selection.iter().map(String::as_str).collect(); while let Some(key) = pending.pop() { diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs index a415f62d1..4d42f2baa 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -30,8 +30,7 @@ pub fn snapshot_definitions(input: &SnapshotInput) -> Result = group.jobs.keys().cloned().collect(); - let dependencies = - group_dependencies((name, input), &definitions.jobs, (&members, &group.jobs))?; + let dependencies = group_dependencies((name, input), &definitions.jobs, &members)?; let selection: BTreeSet = group .jobs .keys() diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index b0a475a46..91ab53873 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -201,7 +201,7 @@ pub struct JobDetailRecord { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct JobMember { pub job_key: String, - /// Members this one waits for, from lineage and from the group's `after`. + /// Members this one waits for, from the lineage between their nodes. pub waits_for: BTreeSet, } @@ -288,7 +288,7 @@ pub(crate) struct PreviousTriggerState { pub(crate) status_override: Option, } -/// Each member job's waits, and why it waits: an explicit `after`, or the first lineage edge found. +/// Each member job's waits, and why it waits: the first lineage edge found. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct GroupWaits { pub(crate) edges: BTreeMap>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 3e5a9123b..66f70a70d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -668,7 +668,10 @@ pub(crate) fn run_post_build_python_checks<'py>( ) -> PyResult> { let py: Python<'py> = context.py(); let result = &outcome.result; - if result.status != BuildStatus::Success { + // `--no-audits` skips Python checks alongside audits. + if result.status != BuildStatus::Success + || !native_field(&context.request, "run_audits")?.extract::()? + { return tuple(py, &[]); } let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index ca9901107..ee439627f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -162,17 +162,7 @@ pub(crate) fn compile_declarations( if members.len() != group.jobs.len() { return Err(PyValueError::new_err("group members must be distinct")); } - let mut after: BTreeMap> = BTreeMap::new(); - for (job, dependencies) in group.after { - after.insert( - job.name, - dependencies - .into_iter() - .map(|dependency| dependency.name) - .collect(), - ); - } - let dependencies = group_dependencies((&group.name, &input), &jobs, (&members, &after)) + let dependencies = group_dependencies((&group.name, &input), &jobs, &members) .map_err(|error| PyValueError::new_err(error.to_string()))?; let selection: BTreeSet = members .iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 89f8f159b..82a61b348 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -49,7 +49,6 @@ pub(crate) struct AuthoredGroup { pub(crate) name: String, pub(crate) description: String, pub(crate) jobs: Vec, - pub(crate) after: BTreeMap>, pub(crate) concurrency_groups: Vec, pub(crate) schedule: Option>, pub(crate) targets: Vec, @@ -328,20 +327,6 @@ pub(crate) fn collect_declarations<'py>( } pending.push(member); } - for (member, dependencies) in declaration.getattr("after")?.cast::()?.iter() { - if !member.is_instance(&job)? { - return Err(PyTypeError::new_err( - "group dependencies must be Job objects", - )); - } - for dependency in dependencies.try_iter()? { - if !dependency?.is_instance(&job)? { - return Err(PyTypeError::new_err( - "group dependencies must be Job objects", - )); - } - } - } AuthoredDeclaration::Group(declaration.extract()?) } else if !declaration.is_instance(&job)? { return Err(PyTypeError::new_err( diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index dbc3f4e8c..3300af9aa 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Callable, Iterable, Sequence -from dataclasses import dataclass, field, replace +from dataclasses import InitVar, dataclass, field, replace from datetime import datetime from typing import Literal @@ -53,12 +53,12 @@ def __post_init__(self) -> None: @dataclass(frozen=True, eq=False) class JobGroup: - """Jobs ordered by compiled lineage and optional explicit dependencies.""" + """Jobs ordered by the lineage between the nodes they build.""" name: str description: str = field(kw_only=True) jobs: Sequence[Job] = field(kw_only=True) - after: dict[Job, Sequence[Job]] = field(default_factory=dict, kw_only=True) + after: InitVar[object] = field(default=None, kw_only=True) concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) @@ -67,7 +67,13 @@ class JobGroup: on_failure: OnFailure | None = field(default=None, kw_only=True) _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) - def __post_init__(self) -> None: + def __post_init__(self, after: object) -> None: + if after is not None: + raise ValueError( + f"job group '{self.name}' declares after, which is no longer supported: groups " + "order jobs only by lineage. Declare the dependency on the nodes instead, for " + "example depends_on on the task that must wait." + ) object.__setattr__(self, "_declared_at", declaration_site()) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 712fe48b8..8140e645d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -318,6 +318,18 @@ def prepare_build_test_audit_flag_project(*, tmp_path: Path, project_name: str) """ ).strip() + "\n", + "python/checks/orders.py": dedent( + """ + from sqlbuild.checks import check + from sqlbuild.refs import model + + + @check(depends_on=model("orders")) + def check_orders_exist(ctx): + '''Test check check_orders_exist.''' + return ctx.pass_() + """ + ).lstrip(), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py index 467368d17..ee86817ce 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py @@ -21,15 +21,15 @@ description="no tests skips SQL tests but runs audits", project_name="build_no_tests_project", command=("--no-color", "build", "--no-tests"), - expected_stdout_fragments=("audit not_null",), + expected_stdout_fragments=("audit not_null", "check check_orders_exist"), unexpected_stdout_fragments=("test test_orders",), ), BuildNoTestsNoAuditsFlagE2ETestCase( - description="no audits skips audits but runs SQL tests", + description="no audits skips audits and Python checks but runs SQL tests", project_name="build_no_audits_project", command=("--no-color", "build", "--no-audits"), expected_stdout_fragments=("test test_orders",), - unexpected_stdout_fragments=("audit not_null",), + unexpected_stdout_fragments=("audit not_null", "check_orders_exist"), ), ), ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 179999d18..70b2bbf39 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -1102,7 +1102,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject expected_jobs=("daily", "staging"), ), SnapshotDiscoveryTestCase( - description="group-only references retain lineage and explicit waits", + description="group-only references retain lineage waits", files={ "orchestration/jobs.py": ( "from sqlbuild.orchestration import job, job_group\n" @@ -1111,14 +1111,14 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject " marts = job('marts', description='Build orders.', select='orders')\n" " inventory = job('inventory', description='Repeat staging.', select='stg_orders')\n" " return job_group('daily', description='Ordered jobs.', " - "jobs=[marts, inventory, staging], after={inventory: [staging]})\n" + "jobs=[marts, inventory, staging])\n" "daily = factory()\n" ), }, expected_jobs=("inventory", "marts", "staging"), expected_groups={ "daily": { - "inventory": ["staging"], + "inventory": [], "marts": ["inventory", "staging"], "staging": [], } @@ -1505,36 +1505,22 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_message="callbacks with when are not supported yet", ), SnapshotDiscoveryRejectionTestCase( - description="explicit group cycles cannot publish", + description="a group declaring after is refused and names the group", files={ "orchestration/jobs.py": ( "from sqlbuild.orchestration import job, job_group\n" "staging = job('staging', description='Stage orders.', select='stg_orders')\n" "marts = job('marts', description='Build orders.', select='orders')\n" - "daily = job_group('daily', description='Cyclic jobs.', " + "daily = job_group('daily', description='Ordered jobs.', " "jobs=[marts, staging], after={staging: [marts]})\n" ), }, expected_error=ValueError, expected_message=re.escape( - "group dependencies contain a cycle in job group daily: marts waits for staging " - "(model:orders reads model:stg_orders); staging waits for marts (declared in after)" + "job group 'daily' declares after, which is no longer supported: groups order " + "jobs only by lineage. Declare the dependency on the nodes instead" ), ), - SnapshotDiscoveryRejectionTestCase( - description="group waits cannot refer to nonmember jobs", - files={ - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job, job_group\n" - "staging = job('staging', description='Stage orders.', select='stg_orders')\n" - "marts = job('marts', description='Build orders.', select='orders')\n" - "daily = job_group('daily', description='Unknown upstream.', " - "jobs=[marts], after={marts: [staging]})\n" - ), - }, - expected_error=ValueError, - expected_message="group needs known jobs and dependencies within its membership", - ), SnapshotDiscoveryRejectionTestCase( description="distinct objects cannot share a job name", files={ diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index ab0a7b05a..aacde0be9 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -1155,7 +1155,6 @@ export class MockOperate { if (a === b) continue; if ([...nodesOf.get(a)!].some((n) => needs.has(n))) ups.push({ jobKey: a, source: 'lineage' }); } - for (const a of job.after[b] ?? []) if (!ups.some((u) => u.jobKey === a)) ups.push({ jobKey: a, source: 'after' }); upstream.set(b, ups); } const stage = new Map(); @@ -1176,10 +1175,8 @@ export class MockOperate { const view = latest ? this.#view(latest, scope) : null; const holders = this.#records.filter((r) => HOLDING.has(r.run.status)); const resolved = job.kind === 'group' ? jobNodes(key) : resolveJobSelection(job).nodes; - const { after, ...rest } = job; - void after; return structuredClone({ - ...rest, + ...job, nodeCount: resolved.filter((n) => snapshotNode(n)?.kind !== 'source').length, resolvedNodes: resolved, lastRun: view ? { id: view.id, status: view.status, startedAt: view.startedAt, endedAt: view.endedAt } : null, diff --git a/ui/src/lib/api/mock/operate.ts b/ui/src/lib/api/mock/operate.ts index 28c0da7c9..168130a8f 100644 --- a/ui/src/lib/api/mock/operate.ts +++ b/ui/src/lib/api/mock/operate.ts @@ -37,9 +37,7 @@ type Known = Omit & { [P in K]-?: NonNullable type JobDef = Known< Omit, 'members' | 'overlap' | 'priority' | 'sourceFile' | 'declaration' | 'settings' -> & { - after: Record; -}; +>; export const CONCURRENCY_LIMITS: Record = { warehouse_heavy: 1, 'target:prod': 4 }; @@ -62,7 +60,6 @@ export const JOB_DEFS: JobDef[] = [ targets: null, owner: 'ingest', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/jobs.py', 14, @@ -91,7 +88,6 @@ export const JOB_DEFS: JobDef[] = [ targets: null, owner: 'data-platform', criticalNodes: ['fact_orders'], - after: {}, declaration: decl( 'orchestration/jobs.py', 24, @@ -124,7 +120,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod', 'staging'], owner: 'analytics', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/jobs.py', 37, @@ -153,7 +148,6 @@ export const JOB_DEFS: JobDef[] = [ targets: null, owner: 'analytics', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/activity.py', 8, @@ -184,7 +178,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod'], owner: 'finance-data', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/finance.py', 11, @@ -215,7 +208,6 @@ export const JOB_DEFS: JobDef[] = [ targets: null, owner: 'analytics', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/finance.py', 23, @@ -243,7 +235,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod'], owner: 'analytics', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/reporting.py', 6, @@ -273,7 +264,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod'], owner: 'data-platform', criticalNodes: ['fact_orders'], - after: {}, declaration: decl( 'orchestration/groups.py', 5, @@ -303,7 +293,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod'], owner: 'finance-data', criticalNodes: ['daily_revenue'], - after: { customer_segments: ['revenue_refresh'] }, declaration: decl( 'orchestration/groups.py', 16, @@ -311,8 +300,6 @@ export const JOB_DEFS: JobDef[] = [ "close_of_business", description="Revenue first, then activity and customer segments", jobs=[revenue_refresh, activity_microbatch, customer_segments], - # Segments read closed revenue through an external lookup lineage can't see. - after={customer_segments: [revenue_refresh]}, on_failure=on_failure(retries=1, retry_delay_seconds=900), critical=["daily_revenue"], concurrency_groups=[warehouse_heavy], diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index c8f98f9ec..0dc7087af 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -688,8 +688,8 @@ export type ConcurrencyGroup = { export type JobGroupMember = { jobKey: string; - /** Jobs this member waits for: lineage-derived plus declared `after`; null where the cause isn't reported. */ - upstream: { jobKey: string; source: 'lineage' | 'after' | null }[]; + /** Jobs this member waits for, from lineage; null where the cause isn't reported. */ + upstream: { jobKey: string; source: 'lineage' | null }[]; /** Position in the derived order; members in the same stage run side by side. */ stage: number; }; diff --git a/ui/src/lib/lineage/job-graph.test.ts b/ui/src/lib/lineage/job-graph.test.ts index eac406715..00f4749e9 100644 --- a/ui/src/lib/lineage/job-graph.test.ts +++ b/ui/src/lib/lineage/job-graph.test.ts @@ -6,7 +6,7 @@ import { boxId, jobGraph, layoutJobGraph } from './job-graph'; const node = (key: string, upstream: string[] = []) => ({ key, kind: 'table', upstream, tags: [], stale: null, checks: null }) as unknown as CatalogNode; const measured = (nodes: Node[]) => nodes.map((n) => (n.type === 'job' ? n : { ...n, measured: { width: 236, height: 61 } })); -// staging builds raw → stg; orders reads stg; reports waits for orders only through `after`. +// staging builds raw → stg; orders reads stg; reports waits for orders through lineage outside the graph. const catalog = [node('model:raw'), node('model:stg', ['model:raw']), node('model:orders', ['model:stg']), node('model:report')]; const all = catalog.map((n) => n.key); const boxes = [ @@ -17,11 +17,11 @@ const boxes = [ const members = [ { jobKey: 'staging', upstream: [], stage: 0 }, { jobKey: 'orders', upstream: [{ jobKey: 'staging', source: 'lineage' as const }], stage: 1 }, - { jobKey: 'reports', upstream: [{ jobKey: 'orders', source: 'after' as const }, { jobKey: 'staging', source: 'lineage' as const }], stage: 2 } + { jobKey: 'reports', upstream: [{ jobKey: 'orders', source: null }, { jobKey: 'staging', source: 'lineage' as const }], stage: 2 } ]; describe('job graph', () => { - it('puts each node in the first box that builds it and draws `after` only where no lineage links two boxes', () => { + it('puts each node in the first box that builds it and draws a wait edge only where no drawn lineage links two boxes', () => { const { nodes, edges } = jobGraph(all, catalog, boxes, members); const parent = Object.fromEntries(nodes.filter((n) => n.type === 'asset').map((n) => [n.id, n.parentId])); expect(parent).toEqual({ 'model:raw': boxId('staging'), 'model:stg': boxId('staging'), 'model:orders': boxId('orders'), 'model:report': boxId('reports') }); @@ -64,7 +64,7 @@ describe('job graph', () => { expect(nodes.filter((n) => n.type === 'asset').map((n) => n.id).sort()).toEqual(['model:orders', 'model:report']); expect(nodes.find((n) => n.id === boxId('staging'))!.data.collapsed).toBe(true); // raw → stg sits inside the box and disappears; stg → orders now leaves the box. Staging and - // reports still have no lineage between them, so their `after` edge stays. + // reports still have no lineage between them, so their wait edge stays. expect(edges.map((e) => `${e.source}>${e.target}`).sort()).toEqual( [`${boxId('staging')}>model:orders`, `${boxId('orders')}>${boxId('reports')}`, `${boxId('staging')}>${boxId('reports')}`].sort() ); @@ -106,8 +106,8 @@ describe('job graph', () => { { jobKey: 'b', nodes: wide.slice(30).map((n) => n.key) }, { jobKey: 'empty', nodes: [] } ]; - const after = (jobKey: string) => ({ jobKey, upstream: [{ jobKey: 'a', source: 'after' as const }], stage: 1 }); - const { nodes, edges } = jobGraph(wide.map((n) => n.key), wide, groupBoxes, [{ jobKey: 'a', upstream: [], stage: 0 }, after('b'), after('empty')]); + const waiting = (jobKey: string) => ({ jobKey, upstream: [{ jobKey: 'a', source: null }], stage: 1 }); + const { nodes, edges } = jobGraph(wide.map((n) => n.key), wide, groupBoxes, [{ jobKey: 'a', upstream: [], stage: 0 }, waiting('b'), waiting('empty')]); const laid = layoutJobGraph(measured(nodes), edges); const box = (k: string) => laid.find((n) => n.id === boxId(k))!; expect(box('a').height).toBeLessThan(15 * (61 + 24) + 60); diff --git a/ui/src/lib/lineage/job-graph.ts b/ui/src/lib/lineage/job-graph.ts index 4917d59c7..b4d9ae3d8 100644 --- a/ui/src/lib/lineage/job-graph.ts +++ b/ui/src/lib/lineage/job-graph.ts @@ -26,7 +26,7 @@ const MARGIN = 24; export type JobBox = { jobKey: string; nodes: string[]; lastRun?: RunRef | null }; const edgeStyle = 'stroke: var(--sb-edge); stroke-width: 1.5;'; -const afterStyle = 'stroke: var(--sb-edge); stroke-width: 1.5; stroke-dasharray: 5 4;'; +const waitStyle = 'stroke: var(--sb-edge); stroke-width: 1.5; stroke-dasharray: 5 4;'; export function boxId(jobKey: string): string { return `job:${jobKey}`; @@ -35,8 +35,9 @@ export function boxId(jobKey: string): string { /** * Cards for `nodes` and the lineage between them. With `boxes`, a node built by several members sits * in the most specific box, the one building the fewest nodes (the earliest on a tie), so a small - * job inside a broad one keeps its nodes; a declared `after` between two members with no lineage - * between them is a dashed edge between their boxes. A box in `collapsed` shows only its header, + * job inside a broad one keeps its nodes; a member that waits for another with no drawn lineage + * between them, because the lineage runs through nodes outside the graph, gets a dashed edge between + * their boxes. A box in `collapsed` shows only its header, * and lineage to its cards meets the box instead. */ export function jobGraph( @@ -99,13 +100,13 @@ export function jobGraph( ); const jobOf = (id: string) => owner.get(id) ?? id.slice('job:'.length); const linked = new Set(lineage.map((e) => `${jobOf(e.source)}>${jobOf(e.target)}`)); - const after: Edge[] = members.flatMap((m) => + const waits: Edge[] = members.flatMap((m) => m.upstream .filter((u) => !linked.has(`${u.jobKey}>${m.jobKey}`) && boxes.some((b) => b.jobKey === u.jobKey)) - .map((u) => ({ id: `after:${u.jobKey}-${m.jobKey}`, source: boxId(u.jobKey), target: boxId(m.jobKey), type: 'bezier', style: afterStyle })) + .map((u) => ({ id: `wait:${u.jobKey}-${m.jobKey}`, source: boxId(u.jobKey), target: boxId(m.jobKey), type: 'bezier', style: waitStyle })) ); // Svelte Flow needs a parent before its children. - return { nodes: [...boxNodes, ...cards], edges: [...lineage, ...after] }; + return { nodes: [...boxNodes, ...cards], edges: [...lineage, ...waits] }; } /** The most cards a column holds before it wraps, growing with the graph so its shape stays near the canvas's. */ @@ -200,7 +201,7 @@ function layoutLayer(items: Item[], pairs: ReadonlyArray Date: Sat, 14 Nov 2026 06:18:48 +0000 Subject: [PATCH 09/34] docs: list orchestration/ in the Project Python table --- .../sqlbuild/references/docs/concepts/project-configuration.md | 1 + website/src/content/docs/docs/concepts/project-configuration.mdx | 1 + 2 files changed, 2 insertions(+) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 1d1b1add4..1bfea05be 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -738,6 +738,7 @@ Code that extends SQLBuild itself keeps its own root: | `hooks/python/` | Python model lifecycle hooks | | `functions/python/` | Python UDFs | | `macros/python/`, `enums/`, `constants/` and scoped `_macros/python/`, `_enums/`, `_constants/` | Python macros and declarations | +| `orchestration/` | Jobs, job groups, sensors and calendar schedules. SQLBuild imports these modules only when a selector names a job or an unknown name, and when a location publishes. See [Job selectors](selectors.md#job) | Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create an unofficial project structure. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 33a7d51f3..c7ec219c1 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -713,6 +713,7 @@ Code that extends SQLBuild itself keeps its own root: | `hooks/python/` | Python model lifecycle hooks | | `functions/python/` | Python UDFs | | `macros/python/`, `enums/`, `constants/` and scoped `_macros/python/`, `_enums/`, `_constants/` | Python macros and declarations | +| `orchestration/` | Jobs, job groups, sensors and calendar schedules. SQLBuild imports these modules only when a selector names a job or an unknown name, and when a location publishes. See [Job selectors](/docs/concepts/selectors#job) | Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create an unofficial project structure. From c6d766132f5292209ce6713335d547d2de6dd401 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:34:44 +0000 Subject: [PATCH 10/34] feat: apply max_description_lines to Python and orchestration descriptions A Python task, asset, loader, check, hook, provider or UDF description that wraps past [format] max_description_lines is a P027 compile error at its declaration, and a job, job group, sensor or calendar schedule with one cannot publish. --- .../compiled_project/attachment_inputs.rs | 2 + .../compiled_project/snapshot_declarations.rs | 44 +++++++++- .../_helpers/input_loops/attachment.rs | 16 ++++ .../input_loops/resource_diagnostics.rs | 37 +++++++- .../main/description_line_count.rs | 10 +++ .../src/dsl_headers/main/mod.rs | 1 + .../sqlbuild/references/docs/cli/format.md | 8 +- .../docs/concepts/models/configuration.md | 5 ++ .../compile/test_required_descriptions.py | 88 +++++++++++++++++++ .../test_orchestration_snapshot.py | 13 +++ website/src/content/docs/docs/cli/format.mdx | 8 +- .../docs/concepts/models/configuration.mdx | 5 ++ 12 files changed, 232 insertions(+), 5 deletions(-) create mode 100644 crates/sqlbuild-rules/src/dsl_headers/main/description_line_count.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs index 8f6eef321..4339117ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs @@ -226,6 +226,8 @@ pub(crate) struct DiagnosticSettings { pub(crate) loaders: HashMap, /// Python-defined resources: kind, name, description, location and help, read by Python. pub(crate) python_resources: Vec, + /// The project's `max_description_lines` and `line_width`, which Python descriptions keep to. + pub(crate) description_limit: (usize, i64), } /// The orphaned audit factory warnings, then the missing descriptions, then the cursor errors. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index ee439627f..a95fae69c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; @@ -27,6 +27,9 @@ use crate::bindings::_helpers::native_host::orchestration_declarations::{ use crate::bindings::_helpers::native_host::orchestration_triggers::{ compile_calendars, compile_cron, compile_sensors, }; +use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; +use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; +use sqlbuild_rules::lint_text::models::FormatSettings; /// Dedicated declaration modules, imported by publication and by selectors that name a job. pub(crate) fn discover_declarations<'py>( @@ -87,6 +90,45 @@ pub(crate) fn compile_declarations( AuthoredDeclaration::Sensor(sensor) => authored_sensors.push(sensor), } } + let format: FormatSettings = + resolve_format_settings(project_dir).map_err(PyValueError::new_err)?; + let mut described: Vec<(&str, String, String)> = Vec::new(); + described.extend( + authored_jobs + .iter() + .map(|job| ("job", job.name.clone(), job.description.clone())), + ); + described.extend( + authored_groups + .iter() + .map(|group| ("job group", group.name.clone(), group.description.clone())), + ); + described.extend( + authored_sensors + .iter() + .map(|sensor| ("sensor", sensor.name.clone(), sensor.description.clone())), + ); + for calendar in &authored_calendars { + let calendar: &Bound<'_, PyAny> = calendar.bind(py); + described.push(( + "calendar schedule", + calendar + .getattr("function")? + .getattr("__name__")? + .extract()?, + calendar.getattr("description")?.extract()?, + )); + } + for (kind, name, description) in described { + let limit: i64 = format.max_description_lines; + let lines: usize = description_line_count(&description, format.line_width); + if i64::try_from(lines).unwrap_or(i64::MAX) > limit { + return Err(PyValueError::new_err(format!( + "{kind} '{name}' description exceeds {limit} lines; shorten it to {limit} lines \ + and move detail into documentation" + ))); + } + } let mut triggers: BTreeMap = BTreeMap::new(); for job in authored_jobs { if !job.targets.is_empty() && !job.targets.contains(&input.target) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs index d34298388..bef8722e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs @@ -1,13 +1,17 @@ //! Seeds, functions, sources, SQL tests, scenarios and audits attached from discovered inputs. use std::collections::HashMap; +use std::path::PathBuf; +use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_attachments::audits::constants::BUILTIN_AUDITS; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost, UserCodeHost}; +use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; +use sqlbuild_rules::lint_text::models::FormatSettings; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, imported}; use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditBlockRow; @@ -564,6 +568,7 @@ fn diagnostic_settings<'py>( .collect::>()?; settings.set_item("sql_hooks", hooks)?; settings.set_item("loaders", loader_rows(discovered)?)?; + settings.set_item("description_limit", description_limit(discovered)?)?; settings.set_item( "python_resources", call_with( @@ -577,6 +582,17 @@ fn diagnostic_settings<'py>( Ok(settings) } +/// The project's `max_description_lines` and `line_width`, as `sqb format` resolves them. +fn description_limit(discovered: &Bound<'_, PyAny>) -> PyResult<(usize, i64)> { + let directory: Option = native_field(discovered, "project_dir")?.extract()?; + let settings: FormatSettings = + resolve_format_settings(&directory.unwrap_or_default()).map_err(PyValueError::new_err)?; + Ok(( + usize::try_from(settings.max_description_lines).unwrap_or(0), + settings.line_width, + )) +} + /// Attach every SQL test case, scenario and audit, then report every resource diagnostic. #[pyfunction] #[pyo3(signature = (**options))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index cae1f4f92..4866d5c37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -19,6 +19,7 @@ use sqlbuild_analysis::resource_diagnostics::main::yaml_help::yaml_help; use sqlbuild_analysis::resource_diagnostics::models::{CursorModel, HeaderHelp}; use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; use sqlbuild_pysource::definition_locations::main::decorator_location::decorator_location; +use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; use sqlbuild_sqltext::compiler::main::model_header_entry_span::model_header_entry_span; use crate::bindings::_helpers::compile_context::delta::{ @@ -35,6 +36,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use sqlbuild_project::models::ReferenceRow; const MISSING_DESCRIPTION_CODE: &str = "P010"; +const DESCRIPTION_LENGTH_CODE: &str = "P027"; const CURSOR_WITHOUT_INPUTS_CODE: &str = "P011"; const ORPHAN_FACTORY_CODE: &str = "C216"; const AUDIT_FACTORIES_KEY: &str = "audit_factories"; @@ -153,6 +155,37 @@ impl<'py> Diagnostics<'_, 'py> { location()?, )?)) } + + /// P010 for a Python `kind` `name` without a description, or P027 when it wraps past the limit. + fn described_python( + &self, + resource: (&str, &str, Option<&str>), + description: Option<&str>, + location: impl FnOnce() -> PyResult>, + help: &str, + ) -> PyResult>> { + let (kind, name, resource_type) = resource; + let (limit, line_width) = self.settings.description_limit; + let Some(text) = + description.filter(|_| missing_description(kind, name, description).is_none()) + else { + return self.described(resource, description, location, help); + }; + if description_line_count(text, line_width) <= limit { + return Ok(None); + } + Ok(Some(self.diagnostic( + (DESCRIPTION_LENGTH_CODE, "error"), + ( + &format!("{kind} '{name}' description exceeds {limit} lines"), + Some(&format!( + "Shorten the description to {limit} lines; move detail into documentation." + )), + ), + (resource_type, name), + location()?, + )?)) + } } /// Orphaned factory warnings, missing descriptions and cursor errors of the owner's resources. @@ -367,7 +400,7 @@ fn missing_descriptions<'py>( let text = description .as_ref() .map(|description| docstring_text(description.bind(py))); - found.push(diagnostics.described( + found.push(diagnostics.described_python( (kind, name, None), text.as_deref(), || location.bind(py).call0(), @@ -456,7 +489,7 @@ fn function_description<'py>( ), &[format!("@udf(description=\"What {name} returns\", ...)")], ); - return diagnostics.described( + return diagnostics.described_python( ("function", name, Some(resource_type)), attached.description.as_deref(), || { diff --git a/crates/sqlbuild-rules/src/dsl_headers/main/description_line_count.rs b/crates/sqlbuild-rules/src/dsl_headers/main/description_line_count.rs new file mode 100644 index 000000000..fb3ca72df --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/main/description_line_count.rs @@ -0,0 +1,10 @@ +//! `description_line_count`: the lines a description takes once wrapped at the line width. + +use crate::dsl_headers::_helpers::text::line_count; +use crate::dsl_headers::_helpers::wrap::wrap_description; + +/// The lines `description` takes wrapped at `line_width`, as the description-length rule counts them. +pub fn description_line_count(description: &str, line_width: i64) -> usize { + let characters: Vec = description.chars().collect(); + line_count(&wrap_description(&characters, line_width, line_width)) +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs b/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs index 862b8dc0f..19633251d 100644 --- a/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs +++ b/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs @@ -1,4 +1,5 @@ pub mod declares_table_returns; +pub mod description_line_count; pub mod fix_escaped_quotes; pub mod format_headers; pub mod inner_header_text; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md index 01cf27cd3..1c09acc5b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -53,13 +53,19 @@ formatted. `--fix` never skips files. Pass ## Line width and descriptions -Configure the maximum physical line width in `sqlbuild_project.toml` (the default is `100`): +Configure the maximum physical line width and the most lines a description may take in +`sqlbuild_project.toml` (the defaults are `100` and `10`): ```toml [format] line_width = 100 +max_description_lines = 10 ``` +A header description longer than `max_description_lines`, once wrapped at the line width, fails +`sqb format --check` with `description-length`. Python descriptions keep to the same limit at +compile; see [Descriptions](../concepts/models/configuration.md#descriptions). + `MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers need a non-empty `description`. When one is missing, `sqb format --check` fails with `description-present`, so pre-commit catches it before compile does. A model described by its `model_schema` or its `[path_defaults]` entry passes, as it diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index f1eaafc8d..0155b1385 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -87,6 +87,11 @@ error[P010]: model 'orders_summary' has no description | Loader | `@loader(description="...")` or the function docstring; a loader that feeds a declared source may instead rely on the source's YAML description | | Provider | The provider class docstring | +A description may take at most `[format] max_description_lines` lines (default `10`) once wrapped +at the line width, the same limit `sqb format --check` applies to headers. A longer Python +description, from a decorator or a docstring, is a `P027` compile error at its declaration. A job, +job group, sensor or calendar schedule with a longer description cannot publish. + SQL tests, audits, macros, enums, constants, materializations and event or output sinks do not need a description. Descriptions do not change fingerprints, so adding or editing one does not rebuild a model or redeploy a function (a Python UDF docstring is part of the deployed body, so editing it diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py index eab3e5a83..28bd54632 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py @@ -506,6 +506,94 @@ ) +def _paragraphs(count: int) -> str: + """A docstring of `count` one-line paragraphs, which wrap to 2 * count - 1 lines.""" + + body: str = "\n\n ".join(f"Orders paragraph {index}." for index in range(count)) + return f' """{body}"""\n' + + +# The default max_description_lines is 10: five paragraphs fit in 9 lines, six need 11. +_LENGTH_CASES: tuple[RequiredDescriptionCase, ...] = ( + RequiredDescriptionCase( + description="task docstring", + missing_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source( + module="tasks", + decorator="task", + name="refresh_exports", + docstring_line=_paragraphs(6), + ), + ), + ), + described_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source( + module="tasks", + decorator="task", + name="refresh_exports", + docstring_line=_paragraphs(5), + ), + ), + ), + expected_diagnostic=("P027", "python/tasks/refresh_exports.py", 4), + expected_message="task 'refresh_exports' description exceeds 10 lines", + expected_help_fragments=("Shorten the description to 10 lines",), + ), + RequiredDescriptionCase( + description="asset description argument", + missing_files=( + ( + "python/assets/order_report.py", + python_node_source( + module="assets", + decorator="asset", + name="order_report", + decorator_arguments=f"(description={'Order report. ' * 90!r})", + ), + ), + ), + described_files=( + ( + "python/assets/order_report.py", + python_node_source( + module="assets", + decorator="asset", + name="order_report", + decorator_arguments=f"(description={'Order report. ' * 60!r})", + ), + ), + ), + expected_diagnostic=("P027", "python/assets/order_report.py", 4), + expected_message="asset 'order_report' description exceeds 10 lines", + expected_help_fragments=("Shorten the description to 10 lines",), + ), +) + + +@pytest.mark.parametrize("test_case", _LENGTH_CASES, ids=lambda case: case.description) +def test_given_python_description_past_the_line_limit_when_compiling_then_reports_p027( + test_case: RequiredDescriptionCase, tmp_path: Path +) -> None: + too_long: subprocess.CompletedProcess[str] = compile_inline_files( + tmp_path=tmp_path / "too_long", files=test_case.missing_files + ) + at_limit: subprocess.CompletedProcess[str] = compile_inline_files( + tmp_path=tmp_path / "at_limit", files=test_case.described_files + ) + + diagnostics, text = require_sql_analysis_output(too_long) + assert too_long.returncode == 1, too_long.stdout + too_long.stderr + assert diagnostics == (test_case.expected_diagnostic,) + assert json.loads(too_long.stdout)["diagnostics"][0]["message"] == test_case.expected_message + assert all(fragment in text for fragment in test_case.expected_help_fragments), text + assert at_limit.returncode == 0, at_limit.stdout + at_limit.stderr + assert require_sql_analysis_output(at_limit)[0] == () + + @pytest.mark.parametrize( "test_case", [RequiredDescriptionCase(**asdict(case)) for case in _CASES], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 70b2bbf39..5d7b7b4ab 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -1504,6 +1504,19 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=ValueError, expected_message="callbacks with when are not supported yet", ), + SnapshotDiscoveryRejectionTestCase( + description="a job description past max_description_lines cannot publish", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "daily = job('daily', description='Build orders. ' * 100, select='orders')\n" + ), + }, + expected_error=ValueError, + expected_message=re.escape( + "job 'daily' description exceeds 10 lines; shorten it to 10 lines" + ), + ), SnapshotDiscoveryRejectionTestCase( description="a group declaring after is refused and names the group", files={ diff --git a/website/src/content/docs/docs/cli/format.mdx b/website/src/content/docs/docs/cli/format.mdx index 145dbab35..e46f501a4 100644 --- a/website/src/content/docs/docs/cli/format.mdx +++ b/website/src/content/docs/docs/cli/format.mdx @@ -44,13 +44,19 @@ formatted. `--fix` never skips files. Pass ## Line width and descriptions -Configure the maximum physical line width in `sqlbuild_project.toml` (the default is `100`): +Configure the maximum physical line width and the most lines a description may take in +`sqlbuild_project.toml` (the defaults are `100` and `10`): ```toml [format] line_width = 100 +max_description_lines = 10 ``` +A header description longer than `max_description_lines`, once wrapped at the line width, fails +`sqb format --check` with `description-length`. Python descriptions keep to the same limit at +compile; see [Descriptions](/docs/concepts/models/configuration#descriptions). + `MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers need a non-empty `description`. When one is missing, `sqb format --check` fails with `description-present`, so pre-commit catches it before compile does. A model described by its `model_schema` or its `[path_defaults]` entry passes, as it diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index e4ca878c5..88f265321 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -71,6 +71,11 @@ error[P010]: model 'orders_summary' has no description | Loader | `@loader(description="...")` or the function docstring; a loader that feeds a declared source may instead rely on the source's YAML description | | Provider | The provider class docstring | +A description may take at most `[format] max_description_lines` lines (default `10`) once wrapped +at the line width, the same limit `sqb format --check` applies to headers. A longer Python +description, from a decorator or a docstring, is a `P027` compile error at its declaration. A job, +job group, sensor or calendar schedule with a longer description cannot publish. + SQL tests, audits, macros, enums, constants, materializations and event or output sinks do not need a description. Descriptions do not change fingerprints, so adding or editing one does not rebuild a model or redeploy a function (a Python UDF docstring is part of the deployed body, so editing it From 1e3e9e8089a9841c85a1e24f5c72eac2470b8c6b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:15:52 +0000 Subject: [PATCH 11/34] fix: keep orchestration fixtures and description limits consistent with lineage-only groups Rust orchestration fixtures declare the node lineage their group waits come from. A Python description limit falls back to the defaults without a project directory, and a calendar schedule that never decorated a function no longer breaks publication for other targets; DuckShop's calendar edge case now declares one. --- .../edge_cases/orchestration/calendar_orders.py | 10 ++++++++++ .../tests/runs/test_group_admission.rs | 6 ++++++ .../tests/runs/test_groups.rs | 5 +++++ .../tests/runs/test_job_detail.rs | 3 +-- .../tests/runs/test_job_subsets.rs | 6 ++++++ .../tests/runs/test_reruns.rs | 13 ++++++++++++- .../compiled_project/snapshot_declarations.rs | 10 ++++++---- .../src/bindings/_helpers/input_loops/attachment.rs | 10 +++++++++- crates/sqlbuild-rules/src/lint_text/constants.rs | 4 ++-- 9 files changed, 57 insertions(+), 10 deletions(-) diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py index 80d336fbb..ce5a9b672 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py +++ b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py @@ -13,3 +13,13 @@ ) def fulfillment_calendar(window, fulfillment): return fulfillment.upcoming(window) + + +# Declared for another target without decorating a function: development publication skips it. +production_calendar = calendar_schedule( + job=orders, + description="Production fulfillment times, not yet attached to a provider.", + horizon="8s", + timezone="UTC", + targets=("production",), +) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs index 6f90bc757..ebe07fb98 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -491,12 +491,18 @@ fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_th for (index, parent) in test_case.parents.iter().enumerate() { let mut members = serde_json::Map::new(); let mut selection = BTreeSet::new(); + let mut previous: Option<&str> = None; for (stage, needs) in parent.children.iter().enumerate() { let name = format!("orders_job_{index}_{stage}"); let node = ["orders", "customers", "inventory", "products"][child_count % 4]; child_count += 1; selection.insert(node); + // Each stage reads the previous stage's node, so lineage orders the stages. + if let Some(upstream) = previous.replace(node) { + input.nodes.get_mut(node).expect("stage node").definition["deps"] = + json!([upstream]); + } jobs.insert(name.clone(), json!({"description": "Build a synthetic stage", "selection": [node], "concurrency_groups": needs})); let after: Vec = (0..stage) .map(|earlier| format!("orders_job_{index}_{earlier}")) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs index cef10c013..82d54108a 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -152,6 +152,11 @@ fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_a input.generation = 2; input.documents.get_mut("jobs").expect("jobs")["marts"]["selection"] = json!(["inventory"]); + input + .nodes + .get_mut("inventory") + .expect("inventory") + .definition["deps"] = json!(["customers"]); input.documents.get_mut("groups").expect("groups")["daily_orders"]["selection"] = json!(["customers", "inventory"]); publish_snapshot(store, &input) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs index e8586b285..f0cbec653 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs @@ -87,8 +87,7 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r .nodes .get_mut("customers") .expect("customers node") - .definition = - json!({"materialization": "table", "config": {"cursor_type": "timestamp"}}); + .definition = json!({"materialization": "table", "deps": ["orders"], "config": {"cursor_type": "timestamp"}}); publish_snapshot(store, &input).await.expect("publish jobs"); let administrator = (&roles.context, credentials(&roles.administrator)); let location = roles.location_id; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs index 34c162e5b..26cae64df 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs @@ -102,6 +102,12 @@ fn given_a_published_job_when_a_run_narrows_it_then_only_the_jobs_own_selected_n ip_address: None, }; let mut input = snapshot(location_id); + // Customers read orders, so customer_sync waits for daily_orders. + input + .nodes + .get_mut("customers") + .expect("customers model") + .definition["deps"] = json!(["orders"]); input.documents.insert("jobs".into(), json!({ "daily_orders": {"description": "Orders and stock", "selection": ["customers", "inventory", "orders"], "concurrency_groups": ["orders-load"]}, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index 7fff91790..f3c5e1af1 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -68,6 +68,17 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .get_mut("orders") .expect("orders model") .definition = json!({"materialization": "table", "deps": ["source:raw_orders"]}); + // Lineage orders the group: products reads orders, and inventory reads products. + input + .nodes + .get_mut("products") + .expect("products model") + .definition["deps"] = json!(["orders"]); + input + .nodes + .get_mut("inventory") + .expect("inventory model") + .definition["deps"] = json!(["products"]); input.documents.insert("jobs".into(), json!({ "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, @@ -75,7 +86,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ })); input.documents.insert("groups".into(), json!({"daily_orders": { "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products"], - "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed"], "completed": ["staging"]} + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed", "staging"], "completed": ["staging"]} }})); let published = publish_snapshot(store, &input).await.expect("publish"); let source_id = Uuid::now_v7(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 97466cf8a..fddcbbf85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -110,12 +110,14 @@ pub(crate) fn compile_declarations( ); for calendar in &authored_calendars { let calendar: &Bound<'_, PyAny> = calendar.bind(py); + // A calendar that never decorated a function has no name; `compile_calendars` handles it per target. + let function: Bound<'_, PyAny> = calendar.getattr("function")?; + if function.is_none() { + continue; + } described.push(( "calendar schedule", - calendar - .getattr("function")? - .getattr("__name__")? - .extract()?, + function.getattr("__name__")?.extract()?, calendar.getattr("description")?.extract()?, )); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs index bef8722e8..c6e02c9a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs @@ -10,6 +10,7 @@ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_attachments::audits::constants::BUILTIN_AUDITS; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost, UserCodeHost}; +use sqlbuild_rules::lint_text::constants::{DEFAULT_LINE_WIDTH, DEFAULT_MAX_DESCRIPTION_LINES}; use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; use sqlbuild_rules::lint_text::models::FormatSettings; @@ -585,8 +586,15 @@ fn diagnostic_settings<'py>( /// The project's `max_description_lines` and `line_width`, as `sqb format` resolves them. fn description_limit(discovered: &Bound<'_, PyAny>) -> PyResult<(usize, i64)> { let directory: Option = native_field(discovered, "project_dir")?.extract()?; + // Without a project directory there is no configuration to read, not the current directory's. + let Some(directory) = directory else { + return Ok(( + usize::try_from(DEFAULT_MAX_DESCRIPTION_LINES).unwrap_or(0), + DEFAULT_LINE_WIDTH, + )); + }; let settings: FormatSettings = - resolve_format_settings(&directory.unwrap_or_default()).map_err(PyValueError::new_err)?; + resolve_format_settings(&directory).map_err(PyValueError::new_err)?; Ok(( usize::try_from(settings.max_description_lines).unwrap_or(0), settings.line_width, diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index 3ead10485..a629cf9be 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -169,8 +169,8 @@ pub(crate) const NON_CONTIGUOUS_FIX: &str = pub(crate) const GENERATED_FIX: &str = "fix overlaps generated SQL"; pub(crate) const GENERATED_SQL_SUFFIX: &str = "(in generated SQL)"; pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; -pub(crate) const DEFAULT_LINE_WIDTH: i64 = 100; -pub(crate) const DEFAULT_MAX_DESCRIPTION_LINES: i64 = 10; +pub const DEFAULT_LINE_WIDTH: i64 = 100; +pub const DEFAULT_MAX_DESCRIPTION_LINES: i64 = 10; pub(crate) const DEFAULT_MAX_LITERAL_LENGTH: i64 = 100; pub(crate) const DEFAULT_MAX_RANKING_ORDER_BY: i64 = 5; pub(crate) const DEFAULT_DIALECT: &str = "generic"; From 66f2d7d9338f01a1c21c8d7d0eda3bffccf4419a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:26:03 +0000 Subject: [PATCH 12/34] test: keep the rerun wait-boundary case transitive under lineage-only groups --- .../tests/runs/test_reruns.rs | 30 +++++++++++-------- .../compiled_project/snapshot_declarations.rs | 3 +- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index f3c5e1af1..339285d19 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -68,12 +68,11 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .get_mut("orders") .expect("orders model") .definition = json!({"materialization": "table", "deps": ["source:raw_orders"]}); - // Lineage orders the group: products reads orders, and inventory reads products. - input - .nodes - .get_mut("products") - .expect("products model") - .definition["deps"] = json!(["orders"]); + // Lineage orders the group: shipments reads orders and inventory reads products, so + // marts waits for staging only through completed, never directly. + let mut shipments = input.nodes["products"].clone(); + shipments.definition["deps"] = json!(["orders"]); + input.nodes.insert("shipments".into(), shipments); input .nodes .get_mut("inventory") @@ -82,11 +81,11 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ input.documents.insert("jobs".into(), json!({ "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, - "completed": {"description": "Build products", "selection": ["products"]} + "completed": {"description": "Build products and shipments", "selection": ["products", "shipments"]} })); input.documents.insert("groups".into(), json!({"daily_orders": { - "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products"], - "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed", "staging"], "completed": ["staging"]} + "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products", "shipments"], + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed"], "completed": ["staging"]} }})); let published = publish_snapshot(store, &input).await.expect("publish"); let source_id = Uuid::now_v7(); @@ -142,6 +141,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .expect("admit source"); let node_outcomes = BTreeMap::from([ ("products", NodeOutcome::Success), + ("shipments", NodeOutcome::Success), ("orders", NodeOutcome::Failed), ("customers", NodeOutcome::Cancelled), ("inventory", NodeOutcome::Skipped), @@ -276,9 +276,15 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ..retry.clone() }, expected_applied: true, - expected_selection: ["orders", "inventory", "customers", "products"] - .map(str::to_owned) - .into(), + expected_selection: [ + "orders", + "inventory", + "customers", + "products", + "shipments", + ] + .map(str::to_owned) + .into(), expected_jobs: &["staging", "completed", "marts"], }, ] { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index fddcbbf85..93c4d93c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -110,7 +110,8 @@ pub(crate) fn compile_declarations( ); for calendar in &authored_calendars { let calendar: &Bound<'_, PyAny> = calendar.bind(py); - // A calendar that never decorated a function has no name; `compile_calendars` handles it per target. + // A calendar that never decorated a function has no name to report; on its own target, + // `compile_calendars` still fails publication for it. let function: Bound<'_, PyAny> = calendar.getattr("function")?; if function.is_none() { continue; From c949c54d95c3c47ea187a363ad46e9df0637ff92 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:39:03 +0000 Subject: [PATCH 13/34] refactor: report P010 and P027 from one description check --- .../input_loops/resource_diagnostics.rs | 78 ++++++++----------- 1 file changed, 31 insertions(+), 47 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index 4866d5c37..245343b85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -136,52 +136,36 @@ impl<'py> Diagnostics<'_, 'py> { .call((), Some(&fields)) } - /// P010 for `kind` `name` when its description is missing or blank. + /// P010 for `kind` `name` when its description is missing or blank; for a Python resource, + /// P027 when it wraps past the project's `max_description_lines`. fn described( &self, resource: (&str, &str, Option<&str>), - description: Option<&str>, + (description, python): (Option<&str>, bool), location: impl FnOnce() -> PyResult>, help: &str, ) -> PyResult>> { let (kind, name, resource_type) = resource; - let Some(message) = missing_description(kind, name, description) else { - return Ok(None); - }; - Ok(Some(self.diagnostic( - (MISSING_DESCRIPTION_CODE, "error"), - (&message, Some(help)), - (resource_type, name), - location()?, - )?)) - } - - /// P010 for a Python `kind` `name` without a description, or P027 when it wraps past the limit. - fn described_python( - &self, - resource: (&str, &str, Option<&str>), - description: Option<&str>, - location: impl FnOnce() -> PyResult>, - help: &str, - ) -> PyResult>> { - let (kind, name, resource_type) = resource; - let (limit, line_width) = self.settings.description_limit; - let Some(text) = - description.filter(|_| missing_description(kind, name, description).is_none()) - else { - return self.described(resource, description, location, help); + let (code, message, help): (&str, String, String) = if let Some(message) = + missing_description(kind, name, description) + { + (MISSING_DESCRIPTION_CODE, message, help.to_owned()) + } else { + let (limit, line_width) = self.settings.description_limit; + match description { + Some(text) if python && description_line_count(text, line_width) > limit => ( + DESCRIPTION_LENGTH_CODE, + format!("{kind} '{name}' description exceeds {limit} lines"), + format!( + "Shorten the description to {limit} lines; move detail into documentation." + ), + ), + _ => return Ok(None), + } }; - if description_line_count(text, line_width) <= limit { - return Ok(None); - } Ok(Some(self.diagnostic( - (DESCRIPTION_LENGTH_CODE, "error"), - ( - &format!("{kind} '{name}' description exceeds {limit} lines"), - Some(&format!( - "Shorten the description to {limit} lines; move detail into documentation." - )), - ), + (code, "error"), + (&message, Some(&help)), (resource_type, name), location()?, )?)) @@ -304,7 +288,7 @@ fn missing_descriptions<'py>( .and_then(|schema| schema.description.as_deref()); found.push(diagnostics.described( ("model", &name, Some("model")), - description, + (description, false), || diagnostics.header_location(&path, contents), &help, )?); @@ -333,7 +317,7 @@ fn missing_descriptions<'py>( }); found.push(diagnostics.described( ("scenario", name, Some("sql_scenario")), - description.as_deref(), + (description.as_deref(), false), || diagnostics.header_location(path, contents), &help, )?); @@ -355,7 +339,7 @@ fn missing_descriptions<'py>( let contents = contents_value.downcast::()?.to_str()?; found.push(diagnostics.described( ("seed", &name, Some("seed")), - description.as_deref(), + (description.as_deref(), false), || diagnostics.yaml_location(&path, contents, &name), &yaml_help("seed", &name, &posix(&path)), )?); @@ -384,7 +368,7 @@ fn missing_descriptions<'py>( }); found.push(diagnostics.described( ("hook", name, None), - description.as_deref(), + (description.as_deref(), false), || diagnostics.header_location(path, contents), &help, )?); @@ -400,9 +384,9 @@ fn missing_descriptions<'py>( let text = description .as_ref() .map(|description| docstring_text(description.bind(py))); - found.push(diagnostics.described_python( + found.push(diagnostics.described( (kind, name, None), - text.as_deref(), + (text.as_deref(), true), || location.bind(py).call0(), help, )?); @@ -456,7 +440,7 @@ fn source_description<'py>( } diagnostics.described( ("source", name, Some("source")), - description.as_deref(), + (description.as_deref(), false), || diagnostics.yaml_location(&path, contents, name), &help, ) @@ -489,9 +473,9 @@ fn function_description<'py>( ), &[format!("@udf(description=\"What {name} returns\", ...)")], ); - return diagnostics.described_python( + return diagnostics.described( ("function", name, Some(resource_type)), - attached.description.as_deref(), + (attached.description.as_deref(), true), || { let version = py.version_info(); let position: (usize, usize) = decorator_location( @@ -515,7 +499,7 @@ fn function_description<'py>( }); diagnostics.described( ("function", name, Some(resource_type)), - attached.description.as_deref(), + (attached.description.as_deref(), false), || diagnostics.header_location(&path, &contents), &help, ) From cc210da27fc84b1513fa3a659abbd2af79cc0761 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:44:56 +0000 Subject: [PATCH 14/34] chore: regenerate the control client for lineage-only group waits --- clients/typescript/src/schema.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 31884a29d..fd2e5a038 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1854,7 +1854,7 @@ export interface components { }; JobMember: { job_key: string; - /** @description Members this one waits for, from lineage and from the group's `after`. */ + /** @description Members this one waits for, from the lineage between their nodes. */ waits_for: string[]; }; /** @description A job or group of the current published snapshot. Settings are left out of person reads. */ From eed4c5528b7b62645b50b20c4abad79c062f7ce9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:56:19 +0000 Subject: [PATCH 15/34] refactor: route calendar descriptions through the native host and drop JobGroup after JobGroup no longer has an after parameter at all, so passing it fails at the call site. Calendar names and descriptions are read through the native host edge, the admission fixture builds stage lineage without branching, and the description-limit test keeps its cases beside the test. --- .../tests/runs/test_group_admission.rs | 11 +- .../_helpers/build_command/execution.rs | 1 - .../compiled_project/snapshot_declarations.rs | 21 +--- .../_helpers/input_loops/attachment.rs | 3 +- .../input_loops/resource_diagnostics.rs | 3 +- .../native_host/orchestration_triggers.rs | 19 +++ .../models/authoring.py | 11 +- .../cli/commands/main/compile/helpers.py | 7 ++ .../compile/test_required_descriptions.py | 109 +++++++++--------- 9 files changed, 93 insertions(+), 92 deletions(-) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs index ebe07fb98..643b151e5 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -491,18 +491,17 @@ fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_th for (index, parent) in test_case.parents.iter().enumerate() { let mut members = serde_json::Map::new(); let mut selection = BTreeSet::new(); - let mut previous: Option<&str> = None; + let mut earlier_nodes: Vec<&str> = Vec::new(); for (stage, needs) in parent.children.iter().enumerate() { let name = format!("orders_job_{index}_{stage}"); let node = ["orders", "customers", "inventory", "products"][child_count % 4]; child_count += 1; selection.insert(node); - // Each stage reads the previous stage's node, so lineage orders the stages. - if let Some(upstream) = previous.replace(node) { - input.nodes.get_mut(node).expect("stage node").definition["deps"] = - json!([upstream]); - } + // Each stage reads every earlier stage's node, so lineage orders the stages. + input.nodes.get_mut(node).expect("stage node").definition["deps"] = + json!(earlier_nodes); + earlier_nodes.push(node); jobs.insert(name.clone(), json!({"description": "Build a synthetic stage", "selection": [node], "concurrency_groups": needs})); let after: Vec = (0..stage) .map(|earlier| format!("orders_job_{index}_{earlier}")) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 66f70a70d..34e0873df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -668,7 +668,6 @@ pub(crate) fn run_post_build_python_checks<'py>( ) -> PyResult> { let py: Python<'py> = context.py(); let result = &outcome.result; - // `--no-audits` skips Python checks alongside audits. if result.status != BuildStatus::Success || !native_field(&context.request, "run_audits")?.extract::()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 93c4d93c4..c9482f9b3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::native_host::orchestration_declarations::{ module_jobs, run_controls, }; use crate::bindings::_helpers::native_host::orchestration_triggers::{ - compile_calendars, compile_cron, compile_sensors, + calendar_descriptions, compile_calendars, compile_cron, compile_sensors, }; use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; @@ -108,20 +108,11 @@ pub(crate) fn compile_declarations( .iter() .map(|sensor| ("sensor", sensor.name.clone(), sensor.description.clone())), ); - for calendar in &authored_calendars { - let calendar: &Bound<'_, PyAny> = calendar.bind(py); - // A calendar that never decorated a function has no name to report; on its own target, - // `compile_calendars` still fails publication for it. - let function: Bound<'_, PyAny> = calendar.getattr("function")?; - if function.is_none() { - continue; - } - described.push(( - "calendar schedule", - function.getattr("__name__")?.extract()?, - calendar.getattr("description")?.extract()?, - )); - } + described.extend( + calendar_descriptions(py, &authored_calendars)? + .into_iter() + .map(|(name, description)| ("calendar schedule", name, description)), + ); for (kind, name, description) in described { let limit: i64 = format.max_description_lines; let lines: usize = description_line_count(&description, format.line_width); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs index c6e02c9a8..bf3340298 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs @@ -583,10 +583,9 @@ fn diagnostic_settings<'py>( Ok(settings) } -/// The project's `max_description_lines` and `line_width`, as `sqb format` resolves them. +/// The project's `max_description_lines` and `line_width` as `sqb format` resolves them, else the defaults. fn description_limit(discovered: &Bound<'_, PyAny>) -> PyResult<(usize, i64)> { let directory: Option = native_field(discovered, "project_dir")?.extract()?; - // Without a project directory there is no configuration to read, not the current directory's. let Some(directory) = directory else { return Ok(( usize::try_from(DEFAULT_MAX_DESCRIPTION_LINES).unwrap_or(0), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index 245343b85..577882795 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -136,8 +136,7 @@ impl<'py> Diagnostics<'_, 'py> { .call((), Some(&fields)) } - /// P010 for `kind` `name` when its description is missing or blank; for a Python resource, - /// P027 when it wraps past the project's `max_description_lines`. + /// P010 when `kind` `name` has no description; for Python, P027 when it wraps past the limit. fn described( &self, resource: (&str, &str, Option<&str>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index b8923950a..9faf31f8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -184,6 +184,25 @@ fn calendar_times( }) } +/// The name and description of each calendar that decorates a function. +pub(crate) fn calendar_descriptions( + py: Python<'_>, + calendars: &[Py], +) -> PyResult> { + let mut described: Vec<(String, String)> = Vec::new(); + for calendar in calendars { + let calendar: &Bound<'_, PyAny> = calendar.bind(py); + let function: Bound<'_, PyAny> = calendar.getattr("function")?; + if !function.is_none() { + described.push(( + function.getattr("__name__")?.extract()?, + calendar.getattr("description")?.extract()?, + )); + } + } + Ok(described) +} + pub(crate) fn compile_calendars( py: Python<'_>, (project_dir, target): (&Path, &str), diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index 6d934a749..0bf9122d9 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Callable, Iterable, Sequence -from dataclasses import InitVar, dataclass, field, replace +from dataclasses import dataclass, field, replace from datetime import datetime from typing import Literal @@ -68,7 +68,6 @@ class JobGroup: name: str description: str = field(kw_only=True) jobs: Sequence[Job] = field(kw_only=True) - after: InitVar[object] = field(default=None, kw_only=True) concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) @@ -77,13 +76,7 @@ class JobGroup: on_failure: OnFailure | None = field(default=None, kw_only=True) _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) - def __post_init__(self, after: object) -> None: - if after is not None: - raise ValueError( - f"job group '{self.name}' declares after, which is no longer supported: groups " - "order jobs only by lineage. Declare the dependency on the nodes instead, for " - "example depends_on on the task that must wait." - ) + def __post_init__(self) -> None: object.__setattr__(self, "_declared_at", declaration_site()) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index e7e6b9703..d3cfbc3b5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -3265,6 +3265,13 @@ def hooked_model_file(*, model_name: str, hook: str) -> tuple[str, str]: ) +def description_paragraphs(count: int) -> str: + """A docstring of `count` one-line paragraphs, which wrap to 2 * count - 1 lines.""" + + body: str = "\n\n ".join(f"Orders paragraph {index}." for index in range(count)) + return f' """{body}"""\n' + + def python_node_source( *, module: str, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py index 28bd54632..fedac2da3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py @@ -22,6 +22,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( REQUIRED_DESCRIPTIONS_PROJECT, compile_inline_files, + description_paragraphs, hooked_model_file, python_node_source, require_sql_analysis_output, @@ -506,75 +507,69 @@ ) -def _paragraphs(count: int) -> str: - """A docstring of `count` one-line paragraphs, which wrap to 2 * count - 1 lines.""" - - body: str = "\n\n ".join(f"Orders paragraph {index}." for index in range(count)) - return f' """{body}"""\n' - - # The default max_description_lines is 10: five paragraphs fit in 9 lines, six need 11. -_LENGTH_CASES: tuple[RequiredDescriptionCase, ...] = ( - RequiredDescriptionCase( - description="task docstring", - missing_files=( - ( - "python/tasks/refresh_exports.py", - python_node_source( - module="tasks", - decorator="task", - name="refresh_exports", - docstring_line=_paragraphs(6), +@pytest.mark.parametrize( + "test_case", + [ + RequiredDescriptionCase( + description="task docstring", + missing_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source( + module="tasks", + decorator="task", + name="refresh_exports", + docstring_line=description_paragraphs(6), + ), ), ), - ), - described_files=( - ( - "python/tasks/refresh_exports.py", - python_node_source( - module="tasks", - decorator="task", - name="refresh_exports", - docstring_line=_paragraphs(5), + described_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source( + module="tasks", + decorator="task", + name="refresh_exports", + docstring_line=description_paragraphs(5), + ), ), ), + expected_diagnostic=("P027", "python/tasks/refresh_exports.py", 4), + expected_message="task 'refresh_exports' description exceeds 10 lines", + expected_help_fragments=("Shorten the description to 10 lines",), ), - expected_diagnostic=("P027", "python/tasks/refresh_exports.py", 4), - expected_message="task 'refresh_exports' description exceeds 10 lines", - expected_help_fragments=("Shorten the description to 10 lines",), - ), - RequiredDescriptionCase( - description="asset description argument", - missing_files=( - ( - "python/assets/order_report.py", - python_node_source( - module="assets", - decorator="asset", - name="order_report", - decorator_arguments=f"(description={'Order report. ' * 90!r})", + RequiredDescriptionCase( + description="asset description argument", + missing_files=( + ( + "python/assets/order_report.py", + python_node_source( + module="assets", + decorator="asset", + name="order_report", + decorator_arguments=f"(description={'Order report. ' * 90!r})", + ), ), ), - ), - described_files=( - ( - "python/assets/order_report.py", - python_node_source( - module="assets", - decorator="asset", - name="order_report", - decorator_arguments=f"(description={'Order report. ' * 60!r})", + described_files=( + ( + "python/assets/order_report.py", + python_node_source( + module="assets", + decorator="asset", + name="order_report", + decorator_arguments=f"(description={'Order report. ' * 60!r})", + ), ), ), + expected_diagnostic=("P027", "python/assets/order_report.py", 4), + expected_message="asset 'order_report' description exceeds 10 lines", + expected_help_fragments=("Shorten the description to 10 lines",), ), - expected_diagnostic=("P027", "python/assets/order_report.py", 4), - expected_message="asset 'order_report' description exceeds 10 lines", - expected_help_fragments=("Shorten the description to 10 lines",), - ), + ], + ids=lambda case: case.description, ) - - -@pytest.mark.parametrize("test_case", _LENGTH_CASES, ids=lambda case: case.description) def test_given_python_description_past_the_line_limit_when_compiling_then_reports_p027( test_case: RequiredDescriptionCase, tmp_path: Path ) -> None: From 3b40dcc9e44d43ca4534ed3f538d90a888fd4301 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:04:11 +0000 Subject: [PATCH 16/34] test: expect the unknown-argument error for JobGroup after --- .../_helpers/compiled_project/snapshot_declarations.rs | 2 +- .../compiled_project/test_orchestration_snapshot.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index c9482f9b3..47304d5c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 5d7b7b4ab..06e61dc87 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -1518,7 +1518,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects ), ), SnapshotDiscoveryRejectionTestCase( - description="a group declaring after is refused and names the group", + description="a group declaring after is refused at its module", files={ "orchestration/jobs.py": ( "from sqlbuild.orchestration import job, job_group\n" @@ -1530,8 +1530,8 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects }, expected_error=ValueError, expected_message=re.escape( - "job group 'daily' declares after, which is no longer supported: groups order " - "jobs only by lineage. Declare the dependency on the nodes instead" + "Failed to import orchestration file orchestration/jobs.py: JobGroup.__init__() " + "got an unexpected keyword argument 'after'" ), ), SnapshotDiscoveryRejectionTestCase( From 797bee75a4c4b1fe2ede9d4e752ed77951aa3c07 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:06:20 +0000 Subject: [PATCH 17/34] feat: read TOML jobs and job groups and check orchestration layout without importing Python Discovery reads orchestration/jobs/ and orchestration/job_groups/ TOML with source positions, scans calendar and sensor modules statically for their literal names and job references, and reports layout, suffix and reference errors as D017. Not wired into compilation yet. --- Cargo.lock | 4 + crates/sqlbuild-discovery/Cargo.toml | 4 + crates/sqlbuild-discovery/src/lib.rs | 1 + crates/sqlbuild-discovery/src/models.rs | 3 + .../src/orchestration_files/_helpers/mod.rs | 4 + .../_helpers/python_modules.rs | 150 +++++++++++++ .../src/orchestration_files/_helpers/raw.rs | 56 +++++ .../orchestration_files/_helpers/reading.rs | 194 +++++++++++++++++ .../_helpers/references.rs | 153 +++++++++++++ .../src/orchestration_files/constants.rs | 57 +++++ .../src/orchestration_files/main/mod.rs | 3 + .../main/orchestration_inventory.rs | 165 ++++++++++++++ .../main/read_group_file.rs | 41 ++++ .../orchestration_files/main/read_job_file.rs | 40 ++++ .../src/orchestration_files/mod.rs | 9 + .../src/orchestration_files/models.rs | 187 ++++++++++++++++ .../src/orchestration_files/tests.rs | 6 + .../tests/test_orchestration_inventory.rs | 201 ++++++++++++++++++ .../tests/test_read_files.rs | 149 +++++++++++++ .../orchestration_files/tests/test_types.rs | 6 + crates/sqlbuild-pysource/src/lib.rs | 1 + .../orchestration_modules/_helpers/calls.rs | 76 +++++++ .../src/orchestration_modules/_helpers/mod.rs | 1 + .../src/orchestration_modules/constants.rs | 25 +++ .../src/orchestration_modules/main/mod.rs | 1 + .../main/orchestration_module_declarations.rs | 95 +++++++++ .../src/orchestration_modules/mod.rs | 7 + .../src/orchestration_modules/models.rs | 41 ++++ .../src/orchestration_modules/tests.rs | 2 + .../test_orchestration_module_declarations.rs | 97 +++++++++ 30 files changed, 1779 insertions(+) create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/_helpers/mod.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/constants.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/main/read_group_file.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/main/read_job_file.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/mod.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/models.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/tests.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs create mode 100644 crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs create mode 100644 crates/sqlbuild-pysource/src/orchestration_modules/_helpers/mod.rs create mode 100644 crates/sqlbuild-pysource/src/orchestration_modules/constants.rs create mode 100644 crates/sqlbuild-pysource/src/orchestration_modules/main/mod.rs create mode 100644 crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs create mode 100644 crates/sqlbuild-pysource/src/orchestration_modules/mod.rs create mode 100644 crates/sqlbuild-pysource/src/orchestration_modules/models.rs create mode 100644 crates/sqlbuild-pysource/src/orchestration_modules/tests.rs create mode 100644 crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs diff --git a/Cargo.lock b/Cargo.lock index ff6d847d8..acde81235 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4288,12 +4288,16 @@ version = "0.0.0" dependencies = [ "junction", "rayon", + "serde", + "serde_json", "sqlbuild-cache", "sqlbuild-config", "sqlbuild-core", "sqlbuild-model-config", + "sqlbuild-pysource", "sqlbuild-sqltext", "tempfile", + "toml", ] [[package]] diff --git a/crates/sqlbuild-discovery/Cargo.toml b/crates/sqlbuild-discovery/Cargo.toml index 0cd082a7f..5ca9425e6 100644 --- a/crates/sqlbuild-discovery/Cargo.toml +++ b/crates/sqlbuild-discovery/Cargo.toml @@ -9,11 +9,15 @@ rust-version.workspace = true [dependencies] rayon.workspace = true +serde.workspace = true +serde_json.workspace = true sqlbuild-cache.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-pysource.workspace = true sqlbuild-sqltext.workspace = true +toml.workspace = true [target.'cfg(windows)'.dependencies] junction.workspace = true diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index 0f9a9b835..8bb9b900a 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -10,6 +10,7 @@ pub mod header_changes; pub mod key_checks; pub mod model_files; pub mod models; +pub mod orchestration_files; pub mod project_validation; pub mod python_paths; pub mod sql_tests; diff --git a/crates/sqlbuild-discovery/src/models.rs b/crates/sqlbuild-discovery/src/models.rs index f00fcbd4d..44627f8c2 100644 --- a/crates/sqlbuild-discovery/src/models.rs +++ b/crates/sqlbuild-discovery/src/models.rs @@ -30,6 +30,8 @@ pub enum FailureKind { Conflict, /// `SeedDiscoveryError` (D008). Seed, + /// `OrchestrationDeclarationError` (D017). + Orchestration, } impl FailureKind { @@ -48,6 +50,7 @@ impl FailureKind { Self::ResourceIdentity => "resource_identity", Self::Conflict => "conflict", Self::Seed => "seed", + Self::Orchestration => "orchestration", } } } diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/mod.rs new file mode 100644 index 000000000..68584816f --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod python_modules; +pub(crate) mod raw; +pub(crate) mod reading; +pub(crate) mod references; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs new file mode 100644 index 000000000..52741a0b2 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs @@ -0,0 +1,150 @@ +//! A calendar or sensor module checked against its folder, without running it. + +use sqlbuild_pysource::orchestration_modules::main::orchestration_module_declarations::orchestration_module_declarations; +use sqlbuild_pysource::orchestration_modules::models::{ + LiteralArgument, ModuleDeclarations, PythonDeclaration, PythonDeclarationKind, SourcePosition, +}; + +use crate::orchestration_files::constants::{ + CALENDAR_KIND, MIGRATE_HELP, PRIVATE_PREFIX, SENSOR_KIND, +}; +use crate::orchestration_files::models::{FileError, FilePosition, NamedModule, Positioned}; + +fn file_position(position: SourcePosition) -> FilePosition { + FilePosition { + line: position.line, + column: position.column, + } +} + +fn error(position: SourcePosition, message: String, help: Option) -> FileError { + FileError { + message, + position: file_position(position), + help, + } +} + +fn label(kind: PythonDeclarationKind) -> (&'static str, &'static str) { + match kind { + PythonDeclarationKind::Calendar => CALENDAR_KIND, + PythonDeclarationKind::Sensor => SENSOR_KIND, + } +} + +/// A literal string argument, or an error naming `what` and how to fix it. +fn literal(argument: LiteralArgument, what: &str) -> Result, FileError> { + match argument.value { + Some(value) if !value.trim().is_empty() => Ok(Positioned { + value, + position: file_position(argument.position), + }), + Some(_) => Err(error( + argument.position, + format!("{what} must not be blank"), + None, + )), + None => Err(error( + argument.position, + format!("{what} must be a string literal"), + Some(format!( + "write the {what} as a quoted string, such as \"orders_job\", so the compiler can check it without running the module" + )), + )), + } +} + +/// The one declaration a public module in a `kind` folder makes; `None` for a private support module. +pub(crate) fn module_declaration( + text: &str, + stem: &str, + kind: PythonDeclarationKind, + python_version: (u8, u8), +) -> Result, FileError> { + let start = SourcePosition { line: 1, column: 1 }; + let Some(ModuleDeclarations { + declarations, + removed_imports, + }) = orchestration_module_declarations(text, python_version) + else { + return Err(error( + start, + format!( + "is not valid Python {}.{}", + python_version.0, python_version.1 + ), + None, + )); + }; + if let Some((name, position)) = removed_imports.into_iter().next() { + return Err(error( + position, + format!( + "`{name}` was removed from sqlbuild.orchestration; jobs, job groups and schedules are declared in TOML" + ), + Some(MIGRATE_HELP.to_owned()), + )); + } + let (noun, suffix) = label(kind); + if let Some(other) = declarations + .iter() + .find(|declaration| declaration.kind != kind) + { + let (other_noun, _) = label(other.kind); + return Err(error( + other.position, + format!("declares a {other_noun}, which belongs in orchestration/{other_noun}s/"), + None, + )); + } + let private: bool = stem.starts_with(PRIVATE_PREFIX); + let mut declarations = declarations.into_iter(); + let first: Option = declarations.next(); + if let Some(second) = declarations.next() { + return Err(error( + second.position, + format!("declares a second {noun}; each module declares exactly one"), + None, + )); + } + let Some(declaration) = first else { + if private { + return Ok(None); + } + return Err(error( + start, + format!("declares no {noun}"), + Some(format!( + "declare one {noun}, or start the file name with `_` to make it a private support module" + )), + )); + }; + if private { + return Err(error( + declaration.position, + format!("private support module declares a {noun}"), + Some("remove the leading `_` from the file name".to_owned()), + )); + } + let name: Positioned = match declaration.name { + Some(name) => literal(name, &format!("{noun} name"))?, + None => { + return Err(error( + declaration.position, + format!("{noun} needs a name"), + Some(format!("pass name=\"{suffix}\"")), + )); + } + }; + let job_references = declaration + .job_references + .into_iter() + .map(|(argument, value)| literal(value, &format!("sensor {argument} job"))) + .collect::, _>>()?; + Ok(Some(NamedModule { + name: name.value, + path: String::new(), + position: name.position, + job_references, + })) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs new file mode 100644 index 000000000..d4473e2f7 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs @@ -0,0 +1,56 @@ +//! The file shapes serde reads before their values are checked. + +use serde::Deserialize; +use toml::Spanned; + +use crate::orchestration_files::models::{InitialStatus, Overlap}; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawSchedule { + pub(crate) cron: Option, + pub(crate) calendar: Option>, + pub(crate) timezone: String, + pub(crate) starts_at: Option, + #[serde(default)] + pub(crate) default_status: InitialStatus, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawOnFailure { + pub(crate) retries: i64, + #[serde(default)] + pub(crate) retry_delay_seconds: i64, +} + +#[derive(Deserialize)] +pub(crate) struct RawRun { + pub(crate) name: Spanned, + pub(crate) description: Spanned, + #[serde(default)] + pub(crate) targets: Vec, + pub(crate) owner: Option>, + #[serde(default)] + pub(crate) overlap: Overlap, + #[serde(default)] + pub(crate) concurrency_groups: Vec, + pub(crate) schedule: Option>, + pub(crate) on_failure: Option>, +} + +/// The job-only fields; the shared ones are read as `RawRun` from the same text. +#[derive(Deserialize)] +pub(crate) struct RawJob { + pub(crate) select: Spanned, + pub(crate) exclude: Option>, + pub(crate) resources: Option>, + #[serde(default)] + pub(crate) settings: toml::Table, +} + +/// The group-only fields; the shared ones are read as `RawRun` from the same text. +#[derive(Deserialize)] +pub(crate) struct RawGroup { + pub(crate) jobs: Vec>, +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs new file mode 100644 index 000000000..6e366565f --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs @@ -0,0 +1,194 @@ +//! Positions, unknown keys and value checks shared by job and group files. + +use std::collections::BTreeSet; + +use sqlbuild_core::text::main::close_matches::close_matches; +use toml::Spanned; +use toml::de::DeTable; + +use crate::orchestration_files::_helpers::raw::{RawOnFailure, RawRun, RawSchedule}; +use crate::orchestration_files::constants::{ + CONCURRENCY_KEY, RETRIES_RANGE, RETRY_DELAY_RANGE, SUGGESTION_CUTOFF, +}; +use crate::orchestration_files::models::{ + FileError, FilePosition, OnFailure, Positioned, RunFile, ScheduleFile, ScheduleSource, +}; + +/// The 1-based line and character column of byte `offset` in `text`. +pub(crate) fn position(text: &str, offset: usize) -> FilePosition { + let before: &str = &text[..offset.min(text.len())]; + let line_start: usize = before.rfind('\n').map_or(0, |index| index + 1); + FilePosition { + line: before.matches('\n').count() + 1, + column: before[line_start..].chars().count() + 1, + } +} + +/// An error at byte `offset`. +pub(crate) fn error_at(text: &str, offset: usize, message: String) -> FileError { + FileError { + message, + position: position(text, offset), + help: None, + } +} + +/// A TOML syntax or type error, at the span the parser reports. +pub(crate) fn toml_error(text: &str, error: &toml::de::Error) -> FileError { + error_at( + text, + error.span().map_or(0, |span| span.start), + error.message().trim_end().to_owned(), + ) +} + +/// The value with the position its span starts at. +pub(crate) fn positioned(text: &str, spanned: Spanned) -> Positioned { + let start: usize = spanned.span().start; + Positioned { + value: spanned.into_inner(), + position: position(text, start), + } +} + +/// Refuse a top-level key outside `allowed`, at the key, with the nearest known key as help. +pub(crate) fn check_keys(text: &str, allowed: &[&str], kind: &str) -> Result<(), FileError> { + let table = DeTable::parse(text).map_err(|error| toml_error(text, &error))?; + for key in table.get_ref().keys() { + let name: &str = key.get_ref(); + if allowed.contains(&name) { + continue; + } + let help: String = if name == CONCURRENCY_KEY { + format!("move it under [settings] as `{CONCURRENCY_KEY} = ...`") + } else if let Some(nearest) = close_matches(name, allowed, 1, SUGGESTION_CUTOFF).first() { + format!("did you mean '{nearest}'?") + } else { + let mut sorted: Vec<&str> = allowed.to_vec(); + sorted.sort_unstable(); + format!("supported keys: {}", sorted.join(", ")) + }; + return Err(FileError { + help: Some(help), + ..error_at( + text, + key.span().start, + format!("unknown {kind} key '{name}'"), + ) + }); + } + Ok(()) +} + +/// A string that must hold more than whitespace. +pub(crate) fn nonblank( + text: &str, + value: Spanned, + what: &str, +) -> Result, FileError> { + if value.get_ref().trim().is_empty() { + return Err(error_at( + text, + value.span().start, + format!("{what} must not be blank"), + )); + } + Ok(positioned(text, value)) +} + +fn schedule(text: &str, raw: Spanned) -> Result { + let start: usize = raw.span().start; + let raw: RawSchedule = raw.into_inner(); + let source: ScheduleSource = match (raw.cron, raw.calendar) { + (Some(cron), None) => ScheduleSource::Cron(cron), + (None, Some(calendar)) => { + ScheduleSource::Calendar(nonblank(text, calendar, "schedule calendar")?) + } + _ => { + return Err(error_at( + text, + start, + "schedule needs exactly one of `cron` or `calendar`".to_owned(), + )); + } + }; + Ok(ScheduleFile { + source, + timezone: raw.timezone, + starts_at: raw.starts_at, + default_status: raw.default_status, + }) +} + +fn on_failure(text: &str, raw: Spanned) -> Result { + let start: usize = raw.span().start; + let raw: RawOnFailure = raw.into_inner(); + let invalid = |field: &str, (low, high): (i64, i64)| { + error_at( + text, + start, + format!("on_failure {field} must be a whole number from {low} to {high}"), + ) + }; + let retries: u8 = u8::try_from(raw.retries) + .ok() + .filter(|value| (RETRIES_RANGE.0..=RETRIES_RANGE.1).contains(value)) + .ok_or_else(|| { + invalid( + "retries", + (i64::from(RETRIES_RANGE.0), i64::from(RETRIES_RANGE.1)), + ) + })?; + let retry_delay_seconds: u32 = u32::try_from(raw.retry_delay_seconds) + .ok() + .filter(|value| (RETRY_DELAY_RANGE.0..=RETRY_DELAY_RANGE.1).contains(value)) + .ok_or_else(|| { + invalid( + "retry_delay_seconds", + ( + i64::from(RETRY_DELAY_RANGE.0), + i64::from(RETRY_DELAY_RANGE.1), + ), + ) + })?; + Ok(OnFailure { + retries, + retry_delay_seconds, + }) +} + +/// The fields jobs and groups share, checked. +pub(crate) fn run_file(text: &str, raw: RawRun) -> Result { + let owner: Option = raw + .owner + .map(|owner| nonblank(text, owner, "owner").map(|owner| owner.value)) + .transpose()?; + let mut groups: BTreeSet<&str> = BTreeSet::new(); + if let Some(repeated) = raw + .concurrency_groups + .iter() + .find(|group| !groups.insert(group.as_str())) + { + return Err(error_at( + text, + 0, + format!("concurrency group '{repeated}' is listed twice"), + )); + } + Ok(RunFile { + name: nonblank(text, raw.name, "name")?, + description: nonblank(text, raw.description, "description")?.value, + targets: raw.targets, + owner, + overlap: raw.overlap, + concurrency_groups: raw.concurrency_groups, + schedule: raw + .schedule + .map(|value| schedule(text, value)) + .transpose()?, + on_failure: raw + .on_failure + .map(|value| on_failure(text, value)) + .transpose()?, + }) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs new file mode 100644 index 000000000..5e86cfd18 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs @@ -0,0 +1,153 @@ +//! Name suffixes and the names groups, schedules and sensors refer to. + +use std::collections::BTreeSet; + +use crate::orchestration_files::constants::{ + CALENDAR_KIND, CALENDARS_DIRECTORY, GROUP_KIND, JOB_KIND, JOBS_DIRECTORY, SENSOR_KIND, +}; +use crate::orchestration_files::models::{ + FileError, FilePosition, OrchestrationInventory, Positioned, RunFile, ScheduleSource, +}; + +/// The first problem with a name or reference, and the file it is in. +pub(crate) type ReferenceError = (String, FileError); + +fn at(position: FilePosition, message: String, help: Option) -> FileError { + FileError { + message, + position, + help, + } +} + +fn suffixed( + path: &str, + (noun, suffix): (&str, &str), + name: &str, + position: FilePosition, +) -> Result<(), ReferenceError> { + if name.ends_with(suffix) && name.len() > suffix.len() { + return Ok(()); + } + Err(( + path.to_owned(), + at( + position, + format!("{noun} '{name}' must be named with the suffix '{suffix}'"), + Some(format!("rename it to '{name}{suffix}'")), + ), + )) +} + +fn calendar_exists( + path: &str, + run: &RunFile, + calendars: &BTreeSet<&str>, +) -> Result<(), ReferenceError> { + let Some(ScheduleSource::Calendar(calendar)) = run.schedule.as_ref().map(|s| &s.source) else { + return Ok(()); + }; + if calendars.contains(calendar.value.as_str()) { + return Ok(()); + } + Err(( + path.to_owned(), + at( + calendar.position, + format!( + "schedule names '{}', which is not a calendar in orchestration/{CALENDARS_DIRECTORY}/", + calendar.value + ), + None, + ), + )) +} + +fn member( + path: &str, + reference: &Positioned, + known: &BTreeSet<&str>, + (owner, wanted): (&str, &str), +) -> Result<(), ReferenceError> { + if known.contains(reference.value.as_str()) { + return Ok(()); + } + Err(( + path.to_owned(), + at( + reference.position, + format!("{owner} names '{}', which is not {wanted}", reference.value), + None, + ), + )) +} + +/// Refuse a name without its kind's suffix, then a group member, calendar or sensor job that is not declared. +pub(crate) fn check_references(inventory: &OrchestrationInventory) -> Result<(), ReferenceError> { + for (path, job) in &inventory.jobs { + suffixed(path, JOB_KIND, &job.run.name.value, job.run.name.position)?; + } + for (path, group) in &inventory.groups { + suffixed( + path, + GROUP_KIND, + &group.run.name.value, + group.run.name.position, + )?; + } + for module in &inventory.calendars { + suffixed(&module.path, CALENDAR_KIND, &module.name, module.position)?; + } + for module in &inventory.sensors { + suffixed(&module.path, SENSOR_KIND, &module.name, module.position)?; + } + let jobs: BTreeSet<&str> = inventory + .jobs + .iter() + .map(|(_, job)| job.run.name.value.as_str()) + .collect(); + let runnable: BTreeSet<&str> = jobs + .iter() + .copied() + .chain( + inventory + .groups + .iter() + .map(|(_, group)| group.run.name.value.as_str()), + ) + .collect(); + let calendars: BTreeSet<&str> = inventory + .calendars + .iter() + .map(|module| module.name.as_str()) + .collect(); + for (path, job) in &inventory.jobs { + calendar_exists(path, &job.run, &calendars)?; + } + for (path, group) in &inventory.groups { + calendar_exists(path, &group.run, &calendars)?; + let wanted = format!("a job in orchestration/{JOBS_DIRECTORY}/"); + for reference in &group.jobs { + member( + path, + reference, + &jobs, + (&format!("job group '{}'", group.run.name.value), &wanted), + )?; + } + } + for module in &inventory.sensors { + for reference in &module.job_references { + member( + &module.path, + reference, + &runnable, + ( + &format!("sensor '{}'", module.name), + "a declared job or job group", + ), + )?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs new file mode 100644 index 000000000..2d256b1e7 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs @@ -0,0 +1,57 @@ +//! The keys job and job group files accept, and the bounds on their run controls. + +/// Top-level keys of a job file. +pub const JOB_KEYS: [&str; 12] = [ + "name", + "description", + "select", + "exclude", + "targets", + "owner", + "overlap", + "concurrency_groups", + "schedule", + "on_failure", + "resources", + "settings", +]; +/// Top-level keys of a job group file. +pub const GROUP_KEYS: [&str; 9] = [ + "name", + "description", + "jobs", + "targets", + "owner", + "overlap", + "concurrency_groups", + "schedule", + "on_failure", +]; +/// The fewest and most reruns `on_failure.retries` allows. +pub const RETRIES_RANGE: (u8, u8) = (1, 10); +/// The shortest and longest `on_failure.retry_delay_seconds`, a day. +pub const RETRY_DELAY_RANGE: (u32, u32) = (0, 86_400); +/// The job setting that moved under `[settings]`. +pub(crate) const CONCURRENCY_KEY: &str = "concurrency"; +/// How close an unknown key must be to a known one to be suggested. +pub(crate) const SUGGESTION_CUTOFF: f64 = 0.6; +/// The folders below `orchestration/`, one per declaration kind. +pub const JOBS_DIRECTORY: &str = "jobs"; +pub const GROUPS_DIRECTORY: &str = "job_groups"; +pub const CALENDARS_DIRECTORY: &str = "calendars"; +pub const SENSORS_DIRECTORY: &str = "sensors"; +/// The file suffix of job and job group declarations. +pub const TOML_FILE_SUFFIX: &str = ".toml"; +/// Each kind as name conflicts word it, and the suffix its names end with. +pub const JOB_KIND: (&str, &str) = ("job", "_job"); +pub const GROUP_KIND: (&str, &str) = ("job group", "_job_group"); +pub const CALENDAR_KIND: (&str, &str) = ("calendar", "_calendar"); +pub const SENSOR_KIND: (&str, &str) = ("sensor", "_sensor"); +/// What to run when a project still declares orchestration the old way. +pub const MIGRATE_HELP: &str = + "run `sqb migrate orchestration` to move Python jobs, job groups and schedules into TOML"; +/// The folder Python writes bytecode to, which declarations never live in. +pub(crate) const BYTECODE_DIRECTORY: &str = "__pycache__"; +/// The prefix of hidden files and folders, and of private support modules. +pub(crate) const HIDDEN_PREFIX: char = '.'; +pub(crate) const PRIVATE_PREFIX: char = '_'; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs new file mode 100644 index 000000000..619ee67d4 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs @@ -0,0 +1,3 @@ +pub mod orchestration_inventory; +pub mod read_group_file; +pub mod read_job_file; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs new file mode 100644 index 000000000..2d2936444 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs @@ -0,0 +1,165 @@ +//! Discover, read and check every orchestration declaration without importing project Python. + +use sqlbuild_pysource::orchestration_modules::models::PythonDeclarationKind; + +use crate::_helpers::reading::read_authored_text; +use crate::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::orchestration_files::_helpers::python_modules::module_declaration; +use crate::orchestration_files::_helpers::references::check_references; +use crate::orchestration_files::constants::{ + BYTECODE_DIRECTORY, CALENDARS_DIRECTORY, GROUPS_DIRECTORY, HIDDEN_PREFIX, JOBS_DIRECTORY, + SENSORS_DIRECTORY, TOML_FILE_SUFFIX, +}; +use crate::orchestration_files::main::read_group_file::read_group_file; +use crate::orchestration_files::main::read_job_file::read_job_file; +use crate::orchestration_files::models::{ + FileError, FilePosition, OrchestrationFailure, OrchestrationInventory, +}; +use crate::tree::main::children::children; +use crate::tree::main::rglob::rglob; +use crate::tree::models::{ProjectTree, TreeEntry}; + +const ROLE_DIRECTORIES: [&str; 4] = [ + JOBS_DIRECTORY, + GROUPS_DIRECTORY, + CALENDARS_DIRECTORY, + SENSORS_DIRECTORY, +]; + +/// `D017` at a file and position, with optional help. +fn failure(path: &str, error: FileError) -> OrchestrationFailure { + let FilePosition { line, column } = error.position; + OrchestrationFailure::Declaration(DiscoveryFailure { + kind: FailureKind::Orchestration, + message: format!("{path}:{line}:{column}: {}", error.message), + help: error.help, + }) +} + +fn layout(path: &str, message: String, help: Option) -> OrchestrationFailure { + failure( + path, + FileError { + message, + position: FilePosition { line: 1, column: 1 }, + help, + }, + ) +} + +fn ignored(entry: &TreeEntry) -> bool { + entry.name.starts_with(HIDDEN_PREFIX) || entry.name == BYTECODE_DIRECTORY +} + +fn read(tree: &ProjectTree, path: &str) -> Result { + if tree.is_undecodable(path) { + return Err(layout( + path, + "path is not valid UTF-8; rename it so SQLBuild can read it".to_owned(), + None, + )); + } + read_authored_text(&tree.absolute(path)).map_err(|failure| OrchestrationFailure::Unreadable { + relative_path: path.to_owned(), + failure, + }) +} + +/// The files below one role folder, refusing any without `suffix`. +fn role_files( + tree: &ProjectTree, + role: &str, + suffix: &str, +) -> Result, OrchestrationFailure> { + let root: String = format!("{ORCHESTRATION_DIRECTORY}/{role}"); + let files: Vec = rglob(tree, &root, |entry| !entry.is_dir)?; + let mut kept: Vec = Vec::new(); + for path in files { + let hidden: bool = path + .split('/') + .any(|part| part.starts_with(HIDDEN_PREFIX) || part == BYTECODE_DIRECTORY); + if hidden { + continue; + } + if !path.ends_with(suffix) { + return Err(layout( + &path, + format!("orchestration/{role}/ holds only {suffix} files"), + Some(match role { + JOBS_DIRECTORY | GROUPS_DIRECTORY => { + "jobs and job groups are TOML; calendars and sensors are Python in orchestration/calendars/ and orchestration/sensors/".to_owned() + } + _ => "jobs and job groups are TOML in orchestration/jobs/ and orchestration/job_groups/".to_owned(), + }), + )); + } + kept.push(path); + } + Ok(kept) +} + +/// Every job, group, calendar and sensor below `orchestration/`, or the first problem with them. +pub fn orchestration_inventory( + tree: &ProjectTree, + python_version: (u8, u8), +) -> Result { + let mut inventory = OrchestrationInventory::default(); + if !tree.is_dir(ORCHESTRATION_DIRECTORY) { + return Ok(inventory); + } + for (path, entry) in children(tree, ORCHESTRATION_DIRECTORY)? { + if ignored(&entry) { + continue; + } + if !entry.is_dir { + return Err(layout( + &path, + "files cannot sit directly in orchestration/".to_owned(), + Some(format!( + "move it into one of orchestration/{}/", + ROLE_DIRECTORIES.join("/, orchestration/") + )), + )); + } + if !ROLE_DIRECTORIES.contains(&entry.name.as_str()) { + return Err(layout( + &path, + format!("unknown orchestration folder '{}'", entry.name), + Some(format!("use one of: {}", ROLE_DIRECTORIES.join(", "))), + )); + } + } + for path in role_files(tree, JOBS_DIRECTORY, TOML_FILE_SUFFIX)? { + let job = read_job_file(&read(tree, &path)?).map_err(|error| failure(&path, error))?; + inventory.jobs.push((path, job)); + } + for path in role_files(tree, GROUPS_DIRECTORY, TOML_FILE_SUFFIX)? { + let group = read_group_file(&read(tree, &path)?).map_err(|error| failure(&path, error))?; + inventory.groups.push((path, group)); + } + for (role, kind) in [ + (CALENDARS_DIRECTORY, PythonDeclarationKind::Calendar), + (SENSORS_DIRECTORY, PythonDeclarationKind::Sensor), + ] { + for path in role_files(tree, role, PYTHON_FILE_SUFFIX)? { + let stem: &str = path + .rsplit('/') + .next() + .and_then(|name| name.strip_suffix(PYTHON_FILE_SUFFIX)) + .unwrap_or_default(); + let declared = module_declaration(&read(tree, &path)?, stem, kind, python_version) + .map_err(|error| failure(&path, error))?; + if let Some(mut module) = declared { + module.path = path.clone(); + match kind { + PythonDeclarationKind::Calendar => inventory.calendars.push(module), + PythonDeclarationKind::Sensor => inventory.sensors.push(module), + } + } + inventory.modules.push(path); + } + } + check_references(&inventory).map_err(|(path, error)| failure(&path, error))?; + Ok(inventory) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/read_group_file.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/read_group_file.rs new file mode 100644 index 000000000..503b8f2e0 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/read_group_file.rs @@ -0,0 +1,41 @@ +//! Read one job group file. + +use std::collections::BTreeSet; + +use crate::orchestration_files::_helpers::raw::{RawGroup, RawRun}; +use crate::orchestration_files::_helpers::reading::{ + check_keys, error_at, nonblank, run_file, toml_error, +}; +use crate::orchestration_files::constants::GROUP_KEYS; +use crate::orchestration_files::models::{FileError, GroupFile, Positioned}; + +/// The job group `text` declares, or the first problem with it and where it is. +pub fn read_group_file(text: &str) -> Result { + check_keys(text, &GROUP_KEYS, "job group")?; + let run: RawRun = toml::from_str(text).map_err(|error| toml_error(text, &error))?; + let raw: RawGroup = toml::from_str(text).map_err(|error| toml_error(text, &error))?; + if raw.jobs.is_empty() { + return Err(error_at( + text, + run.name.span().start, + "a job group needs at least one job in `jobs`".to_owned(), + )); + } + let mut seen: BTreeSet = BTreeSet::new(); + let mut jobs: Vec> = Vec::new(); + for job in raw.jobs { + let job: Positioned = nonblank(text, job, "job name")?; + if !seen.insert(job.value.clone()) { + return Err(FileError { + message: format!("job '{}' is listed twice", job.value), + position: job.position, + help: None, + }); + } + jobs.push(job); + } + Ok(GroupFile { + run: run_file(text, run)?, + jobs, + }) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/read_job_file.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/read_job_file.rs new file mode 100644 index 000000000..84d991a37 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/read_job_file.rs @@ -0,0 +1,40 @@ +//! Read one job file. + +use crate::orchestration_files::_helpers::raw::{RawJob, RawRun}; +use crate::orchestration_files::_helpers::reading::{ + check_keys, nonblank, position, positioned, run_file, toml_error, +}; +use crate::orchestration_files::constants::JOB_KEYS; +use crate::orchestration_files::models::{FileError, JobFile, Positioned}; + +/// The job `text` declares, or the first problem with it and where it is. +pub fn read_job_file(text: &str) -> Result { + check_keys(text, &JOB_KEYS, "job")?; + let run: RawRun = toml::from_str(text).map_err(|error| toml_error(text, &error))?; + let raw: RawJob = toml::from_str(text).map_err(|error| toml_error(text, &error))?; + let resources = raw.resources.map(|resources| { + let start: usize = resources.span().start; + let table = serde_json::to_value(resources.into_inner()) + .ok() + .and_then(|value| value.as_object().cloned()) + .unwrap_or_default(); + Positioned { + value: table, + position: position(text, start), + } + }); + let settings = serde_json::to_value(&raw.settings) + .ok() + .and_then(|value| serde_json::from_value(value).ok()) + .unwrap_or_default(); + Ok(JobFile { + run: run_file(text, run)?, + select: nonblank(text, raw.select, "select")?, + exclude: raw + .exclude + .filter(|exclude| !exclude.get_ref().trim().is_empty()) + .map(|exclude| positioned(text, exclude)), + resources, + settings, + }) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/mod.rs b/crates/sqlbuild-discovery/src/orchestration_files/mod.rs new file mode 100644 index 000000000..9ed28e529 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/mod.rs @@ -0,0 +1,9 @@ +//! Orchestration declaration files: job and job group TOML, read with source positions. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/models.rs b/crates/sqlbuild-discovery/src/orchestration_files/models.rs new file mode 100644 index 000000000..c54beba16 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/models.rs @@ -0,0 +1,187 @@ +//! Typed job and job group files. + +use std::collections::BTreeMap; + +use serde::Deserialize; + +use crate::models::{DiscoveryFailure, ReadFailure, StageFailure}; +use crate::project_validation::models::OrchestrationNameFact; + +/// Whether a run that starts while one is active waits (`queue`) or is dropped (`skip`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Overlap { + #[default] + Queue, + Skip, +} + +/// Whether a schedule starts running or paused when first published. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum InitialStatus { + #[default] + Running, + Paused, +} + +/// Reruns from failure, `retries` times, `retry_delay_seconds` apart. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct OnFailure { + pub retries: u8, + pub retry_delay_seconds: u32, +} + +/// A 1-based line and column in a declaration file. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +pub struct FilePosition { + pub line: usize, + pub column: usize, +} + +/// A value and where it is written. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Positioned { + pub value: T, + pub position: FilePosition, +} + +/// Why a declaration file cannot be read, at the position it points to. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FileError { + pub message: String, + pub position: FilePosition, + pub help: Option, +} + +/// What starts a job or group on its own: a cron expression or a named calendar. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ScheduleSource { + Cron(String), + Calendar(Positioned), +} + +/// A job's or group's `schedule` table. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScheduleFile { + pub source: ScheduleSource, + pub timezone: String, + /// RFC 3339 as written; publication parses it. + pub starts_at: Option, + pub default_status: InitialStatus, +} + +/// The fields jobs and groups share. +#[derive(Clone, Debug, PartialEq)] +pub struct RunFile { + pub name: Positioned, + pub description: String, + pub targets: Vec, + pub owner: Option, + pub overlap: Overlap, + pub concurrency_groups: Vec, + pub schedule: Option, + pub on_failure: Option, +} + +/// One `orchestration/jobs/**.toml` file. +#[derive(Clone, Debug, PartialEq)] +pub struct JobFile { + pub run: RunFile, + pub select: Positioned, + pub exclude: Option>, + /// Worker container limits as written; publication checks their values. + pub resources: Option>>, + /// `sqb build` flags by name, as JSON values. + pub settings: BTreeMap, +} + +/// One `orchestration/job_groups/**.toml` file. +#[derive(Clone, Debug, PartialEq)] +pub struct GroupFile { + pub run: RunFile, + pub jobs: Vec>, +} + +/// A calendar or sensor module and the declaration it makes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NamedModule { + pub name: String, + pub path: String, + pub position: FilePosition, + /// The jobs and groups a sensor starts or watches; empty for a calendar. + pub job_references: Vec>, +} + +/// Every orchestration declaration in a project, checked for layout, names and references. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct OrchestrationInventory { + pub jobs: Vec<(String, JobFile)>, + pub groups: Vec<(String, GroupFile)>, + pub calendars: Vec, + pub sensors: Vec, + /// Every Python module under `calendars/` and `sensors/`, which publication imports. + pub modules: Vec, +} + +impl OrchestrationInventory { + /// Each declared name with its kind and file, for the project-wide name check. + #[must_use] + pub fn names(&self) -> Vec { + let fact = |name: &str, kind: (&'static str, &str), path: &str| OrchestrationNameFact { + name: name.to_owned(), + kind: kind.0, + path: path.to_owned(), + }; + use crate::orchestration_files::constants::{ + CALENDAR_KIND, GROUP_KIND, JOB_KIND, SENSOR_KIND, + }; + let mut names: Vec = Vec::new(); + names.extend( + self.jobs + .iter() + .map(|(path, job)| fact(&job.run.name.value, JOB_KIND, path)), + ); + names.extend( + self.groups + .iter() + .map(|(path, group)| fact(&group.run.name.value, GROUP_KIND, path)), + ); + names.extend( + self.calendars + .iter() + .map(|module| fact(&module.name, CALENDAR_KIND, &module.path)), + ); + names.extend( + self.sensors + .iter() + .map(|module| fact(&module.name, SENSOR_KIND, &module.path)), + ); + names + } + + /// Whether the project declares anything to orchestrate. + #[must_use] + pub fn is_empty(&self) -> bool { + self.jobs.is_empty() && self.groups.is_empty() && self.sensors.is_empty() + } +} + +/// Why the orchestration declarations cannot be read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum OrchestrationFailure { + /// A layout, file, name or reference error, as `OrchestrationDeclarationError`. + Declaration(DiscoveryFailure), + /// A file could not be read the way Python's `read_text` reads it. + Unreadable { + relative_path: String, + failure: ReadFailure, + }, + Stage(StageFailure), +} + +impl From for OrchestrationFailure { + fn from(failure: StageFailure) -> Self { + Self::Stage(failure) + } +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests.rs new file mode 100644 index 000000000..5fc6ae351 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_orchestration_inventory.rs"] +mod orchestration_inventory; +#[path = "tests/test_read_files.rs"] +mod read_files; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs new file mode 100644 index 000000000..1df3aa263 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs @@ -0,0 +1,201 @@ +use std::path::Path; + +use crate::orchestration_files::main::orchestration_inventory::orchestration_inventory; +use crate::orchestration_files::models::{OrchestrationFailure, OrchestrationInventory}; +use crate::orchestration_files::tests::test_types::InventoryErrorCase; +use crate::tree::models::ProjectTree; + +const EXPORT_JOB: &str = "name = \"orders_export_job\"\ndescription = \"d\"\nselect = \"orders\"\nschedule = { calendar = \"trading_days_calendar\", timezone = \"UTC\" }\n"; +const NIGHTLY_GROUP: &str = + "name = \"orders_nightly_job_group\"\ndescription = \"d\"\njobs = [\"orders_export_job\"]\n"; +const CALENDAR: &str = "from sqlbuild.orchestration import calendar\n\n@calendar(name=\"trading_days_calendar\", description=\"d\", horizon=\"7d\")\ndef trading_days(window):\n return []\n"; +const SENSOR: &str = "from sqlbuild.orchestration import run_event, sensor\n\nstale = sensor(name=\"stock_stale_sensor\", description=\"d\", when=run_event(job=\"orders_export_job\", kinds=[]), run=\"orders_nightly_job_group\")\n"; + +fn project(files: &[(&str, &str)]) -> tempfile::TempDir { + let directory = tempfile::tempdir().expect("temporary project"); + for (path, text) in files { + let path = directory.path().join(path); + std::fs::create_dir_all(path.parent().expect("parent")).expect("folders"); + std::fs::write(path, text).expect("file"); + } + directory +} + +fn inventory(root: &Path) -> Result { + orchestration_inventory(&ProjectTree::new(root), (3, 12)) +} + +fn message(files: &[(&str, &str)]) -> (String, Option) { + let directory = project(files); + match inventory(directory.path()).expect_err("invalid project") { + OrchestrationFailure::Declaration(failure) => (failure.message, failure.help), + other => panic!("unexpected failure {other:?}"), + } +} + +fn valid() -> Vec<(&'static str, &'static str)> { + vec![ + ("orchestration/jobs/orders/export.toml", EXPORT_JOB), + ( + "orchestration/job_groups/orders/nightly.toml", + NIGHTLY_GROUP, + ), + ("orchestration/calendars/trading_days.py", CALENDAR), + ("orchestration/calendars/_holidays.py", "HOLIDAYS = []\n"), + ("orchestration/sensors/inventory/stock.py", SENSOR), + ( + "orchestration/sensors/__pycache__/stock.cpython-312.pyc", + "x", + ), + ] +} + +#[test] +fn given_valid_layout_when_inventorying_then_reads_each_kind_and_names_them() { + let directory = project(&valid()); + let found = inventory(directory.path()).expect("valid project"); + let names: Vec<(String, &str, String)> = found + .names() + .into_iter() + .map(|fact| (fact.name, fact.kind, fact.path)) + .collect(); + assert_eq!( + names, + [ + ( + "orders_export_job".into(), + "job", + "orchestration/jobs/orders/export.toml".into() + ), + ( + "orders_nightly_job_group".into(), + "job group", + "orchestration/job_groups/orders/nightly.toml".into() + ), + ( + "trading_days_calendar".into(), + "calendar", + "orchestration/calendars/trading_days.py".into() + ), + ( + "stock_stale_sensor".into(), + "sensor", + "orchestration/sensors/inventory/stock.py".into() + ), + ] + ); + assert_eq!( + found.modules, + [ + "orchestration/calendars/_holidays.py", + "orchestration/calendars/trading_days.py", + "orchestration/sensors/inventory/stock.py", + ] + ); +} + +#[test] +fn given_no_orchestration_folder_when_inventorying_then_is_empty() { + let directory = project(&[("models/orders.sql", "select 1")]); + assert!( + inventory(directory.path()) + .expect("no declarations") + .is_empty() + ); +} + +#[test] +fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_fix() { + let mut cases: Vec = Vec::new(); + let with = |path: &'static str, text: &'static str| { + let mut files = valid(); + files.retain(|(existing, _)| *existing != path); + files.push((path, text)); + files + }; + cases.push(InventoryErrorCase { + description: "loose file", + files: with("orchestration/jobs.py", "x = 1\n"), + expected_message: "orchestration/jobs.py:1:1: files cannot sit directly in orchestration/", + expected_help: Some("move it into one of orchestration/jobs/, orchestration/job_groups/, orchestration/calendars/, orchestration/sensors/"), + }); + cases.push(InventoryErrorCase { + description: "unknown folder", + files: with("orchestration/schedules/nightly.toml", EXPORT_JOB), + expected_message: "orchestration/schedules:1:1: unknown orchestration folder 'schedules'", + expected_help: Some("use one of: jobs, job_groups, calendars, sensors"), + }); + cases.push(InventoryErrorCase { + description: "python job", + files: with("orchestration/jobs/orders/legacy.py", "x = 1\n"), + expected_message: "orchestration/jobs/orders/legacy.py:1:1: orchestration/jobs/ holds only .toml files", + expected_help: Some("jobs and job groups are TOML; calendars and sensors are Python in orchestration/calendars/ and orchestration/sensors/"), + }); + cases.push(InventoryErrorCase { + description: "missing suffix", + files: with( + "orchestration/jobs/orders/export.toml", + "name = \"orders_export\"\ndescription = \"d\"\nselect = \"orders\"\n", + ), + expected_message: "orchestration/jobs/orders/export.toml:1:8: job 'orders_export' must be named with the suffix '_job'", + expected_help: Some("rename it to 'orders_export_job'"), + }); + cases.push(InventoryErrorCase { + description: "unknown group member", + files: with( + "orchestration/job_groups/orders/nightly.toml", + "name = \"orders_nightly_job_group\"\ndescription = \"d\"\njobs = [\"orders_export_job\", \"carrier_sync_job\"]\n", + ), + expected_message: "orchestration/job_groups/orders/nightly.toml:3:30: job group 'orders_nightly_job_group' names 'carrier_sync_job', which is not a job in orchestration/jobs/", + expected_help: None, + }); + cases.push(InventoryErrorCase { + description: "unknown calendar", + files: with( + "orchestration/calendars/trading_days.py", + "from sqlbuild.orchestration import calendar\n\n@calendar(name=\"working_days_calendar\", description=\"d\", horizon=\"7d\")\ndef f(window):\n return []\n", + ), + expected_message: "orchestration/jobs/orders/export.toml:4:25: schedule names 'trading_days_calendar', which is not a calendar in orchestration/calendars/", + expected_help: None, + }); + cases.push(InventoryErrorCase { + description: "non-literal sensor job", + files: with( + "orchestration/sensors/inventory/stock.py", + "from sqlbuild.orchestration import sensor\nJOB = \"orders_export_job\"\nstale = sensor(name=\"stock_stale_sensor\", description=\"d\", run=JOB)\n", + ), + expected_message: "orchestration/sensors/inventory/stock.py:3:64: sensor run job must be a string literal", + expected_help: Some("write the sensor run job as a quoted string, such as \"orders_job\", so the compiler can check it without running the module"), + }); + cases.push(InventoryErrorCase { + description: "sensor in calendars", + files: with("orchestration/calendars/stock.py", SENSOR), + expected_message: "orchestration/calendars/stock.py:3:9: declares a sensor, which belongs in orchestration/sensors/", + expected_help: None, + }); + cases.push(InventoryErrorCase { + description: "public module with nothing", + files: with("orchestration/sensors/helpers.py", "def helper():\n return 1\n"), + expected_message: "orchestration/sensors/helpers.py:1:1: declares no sensor", + expected_help: Some("declare one sensor, or start the file name with `_` to make it a private support module"), + }); + cases.push(InventoryErrorCase { + description: "removed Python job API", + files: with( + "orchestration/sensors/inventory/stock.py", + "from sqlbuild.orchestration import job, sensor\nstale = sensor(name=\"stock_stale_sensor\", description=\"d\", run=\"orders_export_job\")\n", + ), + expected_message: "orchestration/sensors/inventory/stock.py:1:36: `job` was removed from sqlbuild.orchestration; jobs, job groups and schedules are declared in TOML", + expected_help: Some("run `sqb migrate orchestration` to move Python jobs, job groups and schedules into TOML"), + }); + for case in cases { + let (actual, actual_help) = message(&case.files); + assert_eq!(actual, case.expected_message, "{}", case.description); + assert_eq!( + actual_help.as_deref(), + case.expected_help, + "{}", + case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs new file mode 100644 index 000000000..12e5bf25c --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs @@ -0,0 +1,149 @@ +use crate::orchestration_files::main::read_group_file::read_group_file; +use crate::orchestration_files::main::read_job_file::read_job_file; +use crate::orchestration_files::models::{ + FilePosition, InitialStatus, OnFailure, Overlap, ScheduleSource, +}; + +fn at(line: usize, column: usize) -> FilePosition { + FilePosition { line, column } +} + +const FULL_JOB: &str = r#"name = "orders_export_job" +description = "Write the daily partner file." +select = "python/orders/partner_export" +exclude = "tag:slow" +targets = ["production"] +owner = "Orders team" +overlap = "skip" +concurrency_groups = ["warehouse"] +schedule = { calendar = "trading_days_calendar", timezone = "Europe/London", default_status = "paused" } + +[on_failure] +retries = 2 + +[resources] +memory = "2g" +cpus = 1.5 + +[settings] +vars = { batch = 23 } +concurrency = 4 +"#; + +#[test] +fn given_full_job_when_reading_then_types_every_field_and_keeps_positions() { + let job = read_job_file(FULL_JOB).expect("valid job"); + assert_eq!(job.run.name.value, "orders_export_job"); + assert_eq!(job.run.name.position, at(1, 8)); + assert_eq!(job.select.position, at(3, 10)); + assert_eq!( + job.exclude.map(|exclude| exclude.value).as_deref(), + Some("tag:slow") + ); + assert_eq!(job.run.overlap, Overlap::Skip); + assert_eq!( + job.run.on_failure, + Some(OnFailure { + retries: 2, + retry_delay_seconds: 0 + }) + ); + let schedule = job.run.schedule.expect("schedule"); + assert_eq!(schedule.default_status, InitialStatus::Paused); + let ScheduleSource::Calendar(calendar) = schedule.source else { + panic!("calendar schedule expected"); + }; + assert_eq!(calendar.value, "trading_days_calendar"); + assert_eq!(calendar.position, at(9, 25)); + assert_eq!( + job.resources + .map(|resources| resources.value["cpus"].clone()), + Some(serde_json::json!(1.5)) + ); + assert_eq!(job.settings["concurrency"], serde_json::json!(4)); + assert_eq!(job.settings["vars"], serde_json::json!({"batch": 23})); +} + +#[test] +fn given_invalid_job_files_when_reading_then_reports_message_position_and_help() { + let cases: [(&str, &str, &str, FilePosition, Option<&str>); 6] = [ + ( + "misspelt key", + "name = \"a_job\"\ndescription = \"d\"\nselct = \"orders\"\n", + "unknown job key 'selct'", + at(3, 1), + Some("did you mean 'select'?"), + ), + ( + "top-level concurrency", + "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nconcurrency = 2\n", + "unknown job key 'concurrency'", + at(4, 1), + Some("move it under [settings] as `concurrency = ...`"), + ), + ( + "blank select", + "name = \"a_job\"\ndescription = \"d\"\nselect = \" \"\n", + "select must not be blank", + at(3, 10), + None, + ), + ( + "both cron and calendar", + "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nschedule = { cron = \"0 2 * * *\", calendar = \"c_calendar\", timezone = \"UTC\" }\n", + "schedule needs exactly one of `cron` or `calendar`", + at(4, 12), + None, + ), + ( + "retries out of range", + "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\non_failure = { retries = 0 }\n", + "on_failure retries must be a whole number from 1 to 10", + at(4, 14), + None, + ), + ( + "missing description", + "name = \"a_job\"\nselect = \"orders\"\n", + "missing field `description`", + at(1, 1), + None, + ), + ]; + for (description, text, message, position, help) in cases { + let error = read_job_file(text).expect_err(description); + assert_eq!(error.message, message, "{description}"); + assert_eq!(error.position, position, "{description}"); + assert_eq!(error.help.as_deref(), help, "{description}"); + } +} + +#[test] +fn given_group_when_reading_then_keeps_member_positions_and_refuses_repeats() { + let group = read_group_file( + "name = \"nightly_job_group\"\ndescription = \"d\"\njobs = [\"export_job\", \"carrier_job\"]\n", + ) + .expect("valid group"); + let members: Vec<(&str, FilePosition)> = group + .jobs + .iter() + .map(|job| (job.value.as_str(), job.position)) + .collect(); + assert_eq!( + members, + [("export_job", at(3, 9)), ("carrier_job", at(3, 23))] + ); + + let repeated = read_group_file( + "name = \"nightly_job_group\"\ndescription = \"d\"\njobs = [\"export_job\", \"export_job\"]\n", + ) + .expect_err("repeated member"); + assert_eq!(repeated.message, "job 'export_job' is listed twice"); + assert_eq!(repeated.position, at(3, 23)); + + let selecting = read_group_file( + "name = \"nightly_job_group\"\ndescription = \"d\"\njobs = [\"export_job\"]\nselect = \"orders\"\n", + ) + .expect_err("groups do not select"); + assert_eq!(selecting.message, "unknown job group key 'select'"); +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs new file mode 100644 index 000000000..338e12426 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs @@ -0,0 +1,6 @@ +pub(super) struct InventoryErrorCase { + pub(super) description: &'static str, + pub(super) files: Vec<(&'static str, &'static str)>, + pub(super) expected_message: &'static str, + pub(super) expected_help: Option<&'static str>, +} diff --git a/crates/sqlbuild-pysource/src/lib.rs b/crates/sqlbuild-pysource/src/lib.rs index 56220f830..0d5cbe413 100644 --- a/crates/sqlbuild-pysource/src/lib.rs +++ b/crates/sqlbuild-pysource/src/lib.rs @@ -8,6 +8,7 @@ pub mod import_rewrites; pub mod literal_sql; pub mod macro_modules; pub mod module_parse; +pub mod orchestration_modules; pub mod rule_evidence; pub mod string_literals; pub mod udf_functions; diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs new file mode 100644 index 000000000..dc5ef58b8 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs @@ -0,0 +1,76 @@ +//! Orchestration authoring calls and their literal arguments, read from the ruff tree. + +use std::collections::BTreeMap; + +use ruff_python_ast::{Expr, ExprCall, Stmt}; +use ruff_text_size::{Ranged, TextSize}; + +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::models::SourceLines; +use crate::orchestration_modules::constants::{ + CALENDAR_CALL, ORCHESTRATION_MODULE, RUN_EVENT_CALL, SENSOR_CALL, +}; +use crate::orchestration_modules::models::{LiteralArgument, SourcePosition}; + +/// The calls a module's `calendar`, `sensor` and `run_event` names can make. +const AUTHORING_CALLS: [&str; 3] = [CALENDAR_CALL, SENSOR_CALL, RUN_EVENT_CALL]; + +/// The 1-based line and column of `offset`. +pub(crate) fn position(lines: &SourceLines, offset: TextSize) -> SourcePosition { + let (line, column) = source_position(lines, offset); + SourcePosition { + line, + column: column + 1, + } +} + +/// Each local name a top-level `from sqlbuild.orchestration import ...` binds, with the name it imports and where. +pub(crate) fn imported_names(body: &[Stmt]) -> Vec<(String, String, TextSize)> { + let mut imported: Vec<(String, String, TextSize)> = Vec::new(); + for statement in body { + let Stmt::ImportFrom(import) = statement else { + continue; + }; + if import.level != 0 + || import + .module + .as_ref() + .is_none_or(|module| module.as_str() != ORCHESTRATION_MODULE) + { + continue; + } + for alias in &import.names { + let local = alias.asname.as_ref().unwrap_or(&alias.name); + imported.push(( + local.as_str().to_owned(), + alias.name.as_str().to_owned(), + alias.range().start(), + )); + } + } + imported +} + +/// Which authoring call `call` is: an imported `calendar`, `sensor` or `run_event`, or one reached as an attribute. +pub(crate) fn authoring_call( + call: &ExprCall, + imported: &BTreeMap<&str, &str>, +) -> Option<&'static str> { + let name: &str = match call.func.as_ref() { + Expr::Name(name) => imported.get(name.id.as_str())?, + Expr::Attribute(attribute) => attribute.attr.as_str(), + _ => return None, + }; + AUTHORING_CALLS.into_iter().find(|call| *call == name) +} + +/// `expression` as a string literal argument, keeping where it starts whatever it is. +pub(crate) fn literal(expression: &Expr, lines: &SourceLines) -> LiteralArgument { + LiteralArgument { + value: match expression { + Expr::StringLiteral(literal) => Some(literal.value.to_str().to_owned()), + _ => None, + }, + position: position(lines, expression.range().start()), + } +} diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/mod.rs b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/mod.rs new file mode 100644 index 000000000..aedb5c489 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod calls; diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/constants.rs b/crates/sqlbuild-pysource/src/orchestration_modules/constants.rs new file mode 100644 index 000000000..b7a876fe6 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/constants.rs @@ -0,0 +1,25 @@ +//! The orchestration authoring calls a module is read for. + +/// The decorator or call that declares a calendar. +pub(crate) const CALENDAR_CALL: &str = "calendar"; +/// The decorator or call that declares a sensor. +pub(crate) const SENSOR_CALL: &str = "sensor"; +/// The sensor condition that names the job whose runs it watches. +pub(crate) const RUN_EVENT_CALL: &str = "run_event"; +/// The keyword naming a declaration. +pub(crate) const NAME_KEYWORD: &str = "name"; +/// The sensor keyword naming the job it starts. +pub(crate) const RUN_KEYWORD: &str = "run"; +/// The `run_event` keyword naming the job it watches. +pub(crate) const JOB_KEYWORD: &str = "job"; +/// The public module the authoring API is imported from. +pub(crate) const ORCHESTRATION_MODULE: &str = "sqlbuild.orchestration"; +/// Authoring names that TOML declarations replaced. +pub(crate) const REMOVED_NAMES: [&str; 6] = [ + "job", + "job_group", + "schedule", + "calendar_schedule", + "on_failure", + "resources", +]; diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/main/mod.rs b/crates/sqlbuild-pysource/src/orchestration_modules/main/mod.rs new file mode 100644 index 000000000..8630a3052 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/main/mod.rs @@ -0,0 +1 @@ +pub mod orchestration_module_declarations; diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs b/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs new file mode 100644 index 000000000..6b1c55d6f --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs @@ -0,0 +1,95 @@ +//! The calendars and sensors a module declares, and the job names they use, without importing it. + +use std::collections::BTreeMap; + +use ruff_python_ast::Expr; +use ruff_text_size::{Ranged, TextSize}; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::orchestration_modules::_helpers::calls::{ + authoring_call, imported_names, literal, position, +}; +use crate::orchestration_modules::constants::{ + CALENDAR_CALL, JOB_KEYWORD, NAME_KEYWORD, REMOVED_NAMES, RUN_EVENT_CALL, RUN_KEYWORD, + SENSOR_CALL, +}; +use crate::orchestration_modules::models::{ + LiteralArgument, ModuleDeclarations, PythonDeclaration, PythonDeclarationKind, +}; + +/// Every `calendar(...)` and `sensor(...)` call with its name and job references, in source order; `None` when unparsable. +#[must_use] +pub fn orchestration_module_declarations( + contents: &str, + python_version: (u8, u8), +) -> Option { + let parsed: ParsedSource = parse_cpython_module(contents, python_version)?; + let imports: Vec<(String, String, TextSize)> = imported_names(&parsed.module.body); + let imported: BTreeMap<&str, &str> = imports + .iter() + .map(|(local, name, _)| (local.as_str(), name.as_str())) + .collect(); + let removed_imports = imports + .iter() + .filter(|(_, name, _)| REMOVED_NAMES.contains(&name.as_str())) + .map(|(_, name, start)| (name.clone(), position(&parsed.lines, *start))) + .collect(); + let mut declarations: Vec = Vec::new(); + for node in walk_breadth_first(parsed.module.body.iter().map(CpythonNode::Stmt).collect()) { + let CpythonNode::Expr(Expr::Call(call)) = node else { + continue; + }; + let kind = match authoring_call(call, &imported) { + Some(CALENDAR_CALL) => PythonDeclarationKind::Calendar, + Some(SENSOR_CALL) => PythonDeclarationKind::Sensor, + _ => continue, + }; + let name: Option = call + .arguments + .find_argument_value(NAME_KEYWORD, 0) + .map(|value| literal(value, &parsed.lines)); + let mut job_references: Vec<(&'static str, LiteralArgument)> = Vec::new(); + if kind == PythonDeclarationKind::Sensor { + if let Some(run) = call.arguments.find_keyword(RUN_KEYWORD) { + job_references.push((RUN_KEYWORD, literal(&run.value, &parsed.lines))); + } + let conditions: Vec> = call + .arguments + .keywords + .iter() + .filter(|keyword| { + keyword + .arg + .as_ref() + .is_some_and(|arg| arg.as_str() != RUN_KEYWORD) + }) + .map(|keyword| CpythonNode::Expr(&keyword.value)) + .collect(); + for inner in walk_breadth_first(conditions) { + let CpythonNode::Expr(Expr::Call(inner)) = inner else { + continue; + }; + if authoring_call(inner, &imported) != Some(RUN_EVENT_CALL) { + continue; + } + if let Some(job) = inner.arguments.find_keyword(JOB_KEYWORD) { + job_references.push((RUN_EVENT_CALL, literal(&job.value, &parsed.lines))); + } + } + } + declarations.push(PythonDeclaration { + kind, + name, + position: position(&parsed.lines, call.range().start()), + job_references, + }); + } + declarations + .sort_by_key(|declaration| (declaration.position.line, declaration.position.column)); + Some(ModuleDeclarations { + declarations, + removed_imports, + }) +} diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/mod.rs b/crates/sqlbuild-pysource/src/orchestration_modules/mod.rs new file mode 100644 index 000000000..1e64f3d11 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/models.rs b/crates/sqlbuild-pysource/src/orchestration_modules/models.rs new file mode 100644 index 000000000..05303ebf9 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/models.rs @@ -0,0 +1,41 @@ +//! The orchestration declarations a Python module makes, read without running it. + +/// What a module-level `calendar(...)` or `sensor(...)` call declares. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PythonDeclarationKind { + Calendar, + Sensor, +} + +/// A 1-based line and column in the module source. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SourcePosition { + pub line: usize, + pub column: usize, +} + +/// A string argument of a declaration; `None` when it is not a string literal. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LiteralArgument { + pub value: Option, + pub position: SourcePosition, +} + +/// One calendar or sensor and the job names it refers to. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PythonDeclaration { + pub kind: PythonDeclarationKind, + /// The `name` argument, or `None` when the call passes no name. + pub name: Option, + pub position: SourcePosition, + /// `run=` of a sensor, then each `run_event(job=...)`, labelled by the argument. + pub job_references: Vec<(&'static str, LiteralArgument)>, +} + +/// The orchestration facts of one module. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModuleDeclarations { + pub declarations: Vec, + /// Each removed authoring name the module imports from `sqlbuild.orchestration`. + pub removed_imports: Vec<(String, SourcePosition)>, +} diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/tests.rs b/crates/sqlbuild-pysource/src/orchestration_modules/tests.rs new file mode 100644 index 000000000..959e94a42 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/tests.rs @@ -0,0 +1,2 @@ +#[path = "tests/test_orchestration_module_declarations.rs"] +mod orchestration_module_declarations; diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs new file mode 100644 index 000000000..75681c782 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs @@ -0,0 +1,97 @@ +use crate::orchestration_modules::main::orchestration_module_declarations::orchestration_module_declarations; +use crate::orchestration_modules::models::{ + LiteralArgument, ModuleDeclarations, PythonDeclarationKind, SourcePosition, +}; + +fn at(line: usize, column: usize) -> SourcePosition { + SourcePosition { line, column } +} + +fn text(value: &str, line: usize, column: usize) -> LiteralArgument { + LiteralArgument { + value: Some(value.to_owned()), + position: at(line, column), + } +} + +fn scan(contents: &str) -> ModuleDeclarations { + orchestration_module_declarations(contents, (3, 12)).expect("module parses") +} + +#[test] +fn given_decorated_calendar_when_scanning_then_reads_its_literal_name() { + let module = scan( + "from sqlbuild.orchestration import calendar\n\n@calendar(name=\"trading_days_calendar\", description=\"d\", horizon=\"7d\")\ndef trading_days(window):\n return []\n", + ); + assert_eq!(module.declarations.len(), 1); + let calendar = &module.declarations[0]; + assert_eq!(calendar.kind, PythonDeclarationKind::Calendar); + assert_eq!(calendar.name, Some(text("trading_days_calendar", 3, 16))); + assert_eq!(calendar.position, at(3, 2)); + assert!(calendar.job_references.is_empty()); +} + +#[test] +fn given_sensor_when_scanning_then_reads_run_and_watched_job_in_order() { + let module = scan( + "from sqlbuild.orchestration import RunEventKind, run_event as watched, sensor\n\nstock = sensor(\n \"stock_sensor\",\n description=\"d\",\n when=watched(job=\"orders_job\", kinds=[RunEventKind.SUCCESS]),\n run=\"inventory_job\",\n)\n", + ); + let sensor = &module.declarations[0]; + assert_eq!(sensor.kind, PythonDeclarationKind::Sensor); + assert_eq!(sensor.name, Some(text("stock_sensor", 4, 5))); + assert_eq!( + sensor.job_references, + vec![ + ("run", text("inventory_job", 7, 9)), + ("run_event", text("orders_job", 6, 22)), + ] + ); +} + +#[test] +fn given_non_literal_references_when_scanning_then_keeps_their_positions_without_values() { + let module = scan( + "import sqlbuild.orchestration as orchestration\n\nNAME = \"x_sensor\"\nJOB = \"x_job\"\nx = orchestration.sensor(name=NAME, description=\"d\", run=JOB)\n", + ); + let sensor = &module.declarations[0]; + assert_eq!( + sensor.name, + Some(LiteralArgument { + value: None, + position: at(5, 31), + }) + ); + assert_eq!(sensor.job_references[0].1.value, None); + assert_eq!(sensor.job_references[0].1.position, at(5, 58)); +} + +#[test] +fn given_unrelated_calls_when_scanning_then_ignores_them() { + let module = scan( + "import calendar\nfrom other import sensor\n\ndays = calendar.monthrange(2026, 2)\nsensor(\"not_ours\")\n", + ); + assert!(module.declarations.is_empty()); + assert!(module.removed_imports.is_empty()); +} + +#[test] +fn given_removed_authoring_imports_when_scanning_then_reports_each() { + let module = scan( + "from sqlbuild.orchestration import (\n job,\n job_group as group,\n sensor,\n)\n", + ); + assert_eq!( + module.removed_imports, + vec![ + ("job".to_owned(), at(2, 5)), + ("job_group".to_owned(), at(3, 5)) + ] + ); +} + +#[test] +fn given_syntax_error_when_scanning_then_returns_none() { + assert_eq!( + orchestration_module_declarations("def f(:\n", (3, 12)), + None + ); +} From 4532fe16991a85c1a79337139b4809553529d355 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:12:20 +0000 Subject: [PATCH 18/34] feat: check orchestration declarations at compile as D017 and share their names with the project namespace --- .../_helpers/resource_names.rs | 2 +- .../src/project_validation/constants.rs | 4 +- .../main/validate_orchestration_names.rs | 2 +- .../src/project_validation/models.rs | 4 +- .../src/project_validation/tests/helpers.rs | 2 +- .../test_validate_orchestration_names.rs | 12 ++--- .../_helpers/compiler_errors/constants.rs | 9 ++++ .../src/bindings/_helpers/discovery/mod.rs | 1 + .../_helpers/discovery/orchestration_files.rs | 46 +++++++++++++++++++ .../_helpers/discovery/project_validation.rs | 7 ++- .../src/bindings/_helpers/discovery/raised.rs | 3 +- .../src/bindings/_helpers/records/ids.rs | 1 + src/sqlbuild/_native.pyi | 3 ++ 13 files changed, 81 insertions(+), 15 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs index 2d10e91f6..6191263dd 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs @@ -142,7 +142,7 @@ fn name_conflict(first: &DeclaredName<'_>, second: &DeclaredName<'_>) -> Validat }; conflict(format!( "Project resource name '{}' is declared as both {} in {} and {} in {}; model, source, seed, function, loader, task, asset, check, provider, \ - hook, unit test, scenario, job, job group, sensor, and calendar schedule names must be \ + hook, unit test, scenario, job, job group, sensor, and calendar names must be \ globally unique; rename one of them{advice}", first.name, first.kind, first.path, second.kind, second.path )) diff --git a/crates/sqlbuild-discovery/src/project_validation/constants.rs b/crates/sqlbuild-discovery/src/project_validation/constants.rs index e2dcb6a95..ff5222415 100644 --- a/crates/sqlbuild-discovery/src/project_validation/constants.rs +++ b/crates/sqlbuild-discovery/src/project_validation/constants.rs @@ -14,5 +14,5 @@ pub(crate) const INITIAL_SEED_READ_BYTES: u64 = 64 * 1024; pub(crate) const RECORDS_FOR_MANY_ROWS: usize = 3; /// How a colliding orchestration declaration is renamed, added to its name conflict. -pub(crate) const ORCHESTRATION_RENAME_ADVICE: &str = " (a job, job group or sensor takes its name \ - from its first argument, and a calendar schedule from the function it decorates)"; +pub(crate) const ORCHESTRATION_RENAME_ADVICE: &str = " (a job or job group takes its name from \ + `name` in its TOML file, and a calendar or sensor from its `name` argument)"; diff --git a/crates/sqlbuild-discovery/src/project_validation/main/validate_orchestration_names.rs b/crates/sqlbuild-discovery/src/project_validation/main/validate_orchestration_names.rs index ac3a340b3..edbb1d5cb 100644 --- a/crates/sqlbuild-discovery/src/project_validation/main/validate_orchestration_names.rs +++ b/crates/sqlbuild-discovery/src/project_validation/main/validate_orchestration_names.rs @@ -3,7 +3,7 @@ use crate::project_validation::_helpers::resource_names::validate_unique_project_resource_names; use crate::project_validation::models::{ProjectFacts, ValidationFailure}; -/// Reject the first name a job, group, sensor or calendar schedule shares with anything else. +/// Reject the first name a job, group, calendar or sensor shares with anything else. pub fn validate_orchestration_names(facts: &ProjectFacts) -> Result<(), ValidationFailure> { validate_unique_project_resource_names(facts) } diff --git a/crates/sqlbuild-discovery/src/project_validation/models.rs b/crates/sqlbuild-discovery/src/project_validation/models.rs index 2b185d25a..813c60d70 100644 --- a/crates/sqlbuild-discovery/src/project_validation/models.rs +++ b/crates/sqlbuild-discovery/src/project_validation/models.rs @@ -178,7 +178,7 @@ pub struct ProjectFacts { pub project_model_paths: Option>, /// Python's `csv.field_size_limit()`, or `None` for its default. pub csv_field_limit: Option, - /// Jobs, job groups, sensors and calendar schedules, known only when a location publishes. + /// Jobs, job groups, calendars and sensors, read from `orchestration/` without importing it. pub orchestration: Vec, } @@ -188,7 +188,7 @@ pub(crate) struct DeclaredName<'a> { pub(crate) name: &'a str, pub(crate) kind: &'a str, pub(crate) path: &'a str, - /// A job, job group, sensor or calendar schedule, whose conflicts say how it is named. + /// A job, job group, calendar or sensor, whose conflicts say how it is named. pub(crate) orchestration: bool, } diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs b/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs index 64c9bc747..d7b42c670 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs @@ -106,7 +106,7 @@ pub(super) fn collision(message: &str, advice: &str) -> Result<(), String> { FailureKind::Conflict, &format!( "{message}; model, source, seed, function, loader, task, asset, check, provider, \ - hook, unit test, scenario, job, job group, sensor, and calendar schedule names must \ + hook, unit test, scenario, job, job group, sensor, and calendar names must \ be globally unique; rename one of them{advice}" ), ) diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs index eb5505f44..f58a33ad7 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs @@ -5,8 +5,8 @@ use crate::project_validation::models::{ use crate::project_validation::tests::helpers::{collision, declared, model, named}; use crate::project_validation::tests::test_types::ValidationTestCase; -const RENAME_DECLARATION: &str = " (a job, job group or sensor takes its name from its first \ - argument, and a calendar schedule from the function it decorates)"; +const RENAME_DECLARATION: &str = " (a job or job group takes its name from `name` in its TOML \ + file, and a calendar or sensor from its `name` argument)"; #[test] fn given_orchestration_names_when_validating_then_they_share_the_project_namespace() { @@ -19,7 +19,7 @@ fn given_orchestration_names_when_validating_then_they_share_the_project_namespa declared("refresh_orders", "job"), declared("nightly", "job group"), declared("orders_arrived", "sensor"), - declared("fulfillment_calendar", "calendar schedule"), + declared("fulfillment_calendar", "calendar"), ], ..ProjectFacts::default() }, @@ -55,15 +55,15 @@ fn given_orchestration_names_when_validating_then_they_share_the_project_namespa ), }, ValidationTestCase { - description: "a calendar schedule named like a provider", + description: "a calendar named like a provider", facts: ProjectFacts { providers: vec![named("fulfillment", "providers/fulfillment.py")], - orchestration: vec![declared("fulfillment", "calendar schedule")], + orchestration: vec![declared("fulfillment", "calendar")], ..ProjectFacts::default() }, expected_failure: collision( "Project resource name 'fulfillment' is declared as both provider in \ - providers/fulfillment.py and calendar schedule in orchestration/daily.py", + providers/fulfillment.py and calendar in orchestration/daily.py", RENAME_DECLARATION, ), }, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs index b69fc3bbd..ba5fb5914 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs @@ -140,6 +140,15 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ optional: &[], fixed: &[], }, + ErrorSpec { + name: "OrchestrationDeclarationError", + parent: "DiscoveryError", + doc: "Raised when an orchestration job, job group, calendar or sensor declaration is invalid.", + class_attributes: &[("code", ClassValue::Text("D017"))], + required: &[], + optional: &[], + fixed: &[], + }, ErrorSpec { name: "ProjectPythonPathError", parent: "DiscoveryError", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 77a316516..60b884fc5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -17,6 +17,7 @@ pub(crate) mod key_checks; pub(crate) mod locations; pub(crate) mod model_file_record; pub(crate) mod model_records; +pub(crate) mod orchestration_files; pub(crate) mod project_config; pub(crate) mod project_facts; pub(crate) mod project_files; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs new file mode 100644 index 000000000..d4bd4cf51 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs @@ -0,0 +1,46 @@ +//! Orchestration declarations read natively, raised as the Python exceptions discovery raises. + +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use sqlbuild_discovery::orchestration_files::main::orchestration_inventory::orchestration_inventory; +use sqlbuild_discovery::orchestration_files::models::{ + OrchestrationFailure, OrchestrationInventory, +}; +use sqlbuild_discovery::tree::models::ProjectTree; + +use crate::bindings::_helpers::discovery::project_files::path_object; +use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; + +/// Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). +pub(crate) fn orchestration_declarations( + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = project_dir.py(); + let root: PathBuf = project_dir.extract()?; + let tree = ProjectTree::new(&root); + let version = py.version_info(); + let inventory = py.detach(|| orchestration_inventory(&tree, (version.major, version.minor))); + inventory.map_err(|failure| orchestration_error(&tree, project_dir, failure)) +} + +fn orchestration_error( + tree: &ProjectTree, + project_dir: &Bound<'_, PyAny>, + failure: OrchestrationFailure, +) -> PyErr { + let py: Python<'_> = project_dir.py(); + match failure { + OrchestrationFailure::Declaration(failure) => failure_error(py, failure), + OrchestrationFailure::Unreadable { + relative_path, + failure, + } => match path_object(py, tree, relative_path) + .and_then(|relative| project_dir.div(relative.bind(py))) + { + Ok(file_path) => read_error(failure, &file_path), + Err(error) => error, + }, + OrchestrationFailure::Stage(failure) => stage_error(tree, project_dir, failure), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs index b3b1a096e..cccf046b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs @@ -8,6 +8,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::project_validation::main::validate_project_inputs::validate_project_inputs; use sqlbuild_discovery::project_validation::models::{ProjectFacts, ValidationFailure}; +use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; use crate::bindings::_helpers::discovery::project_facts::{NodeRow, project_facts}; use crate::bindings::_helpers::discovery::raised::{failure_error, read_error}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -32,7 +33,11 @@ fn validate_discovered_project_inputs( inputs: Bound<'_, PyAny>, nodes: Vec, ) -> Result<(), PyErr> { - let facts: ProjectFacts = project_facts(py, &inputs, nodes)?; + let mut facts: ProjectFacts = project_facts(py, &inputs, nodes)?; + let project_dir: Bound<'_, PyAny> = native_field(&inputs, "project_dir")?; + if !project_dir.is_none() { + facts.orchestration = orchestration_declarations(&project_dir)?.names(); + } validate_project_inputs(&facts).map_err(|failure| validation_error(py, failure)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs index aca358755..2920bebf3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs @@ -19,7 +19,7 @@ const OS_MODULE: &str = "os"; const DECODE_ENCODING: &str = "utf-8"; /// The exception class of each native failure kind. -const FAILURE_CLASSES: [(&str, ErrorId); 13] = [ +const FAILURE_CLASSES: [(&str, ErrorId); 14] = [ ("model_sql", ErrorId::ModelSqlParse), ("declaration", ErrorId::DeclarationParse), ("sql_test", ErrorId::SqlTestParse), @@ -33,6 +33,7 @@ const FAILURE_CLASSES: [(&str, ErrorId); 13] = [ ("conflict", ErrorId::DiscoveryConflict), ("seed", ErrorId::SeedDiscovery), ("python_path", ErrorId::ProjectPythonPath), + ("orchestration", ErrorId::OrchestrationDeclaration), ]; fn raised(result: PyResult>) -> PyErr { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 7c48897c6..acf3e15c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -618,6 +618,7 @@ native_ids! { NonHermeticRule => "NonHermeticRuleError", OpaqueModuleState => "OpaqueModuleStateError", PlannerInput => "PlannerInputError", + OrchestrationDeclaration => "OrchestrationDeclarationError", ProjectCompile => "ProjectCompileError", ProjectConfig => "ProjectConfigError", ProjectPath => "ProjectPathError", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 53f0f7373..c72819394 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -8447,6 +8447,9 @@ class SqlHookParseError(DiscoveryError): class EventExporterDiscoveryError(DiscoveryError): """Raised when project lifecycle event exporters are invalid.""" +class OrchestrationDeclarationError(DiscoveryError): + """Raised when an orchestration job, job group, calendar or sensor declaration is invalid.""" + class ProjectPythonPathError(DiscoveryError): """Raised when project-owned Python lives outside a supported extension root.""" From e85c29ac40e97800280131f06dbfc62cb645f9a4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:59:40 +0000 Subject: [PATCH 19/34] feat!: publish TOML jobs and groups, and declare calendars and sensors by name Jobs and job groups publish from their TOML files. Selectors that name a job read the TOML without importing project Python. Publication imports only calendar and sensor modules. A calendar is @calendar(name, description, horizon), and a job's schedule names it and sets the timezone. Sensor run= and run_event(job=) take names. The removed records stay private for the migration command. --- .../_helpers/python_modules.rs | 5 +- .../_helpers/references.rs | 2 +- .../src/orchestration_files/models.rs | 6 +- .../orchestration_modules/_helpers/calls.rs | 16 + .../src/orchestration_modules/constants.rs | 2 + .../main/orchestration_module_declarations.rs | 9 +- .../src/orchestration_modules/models.rs | 2 + .../_helpers/compiled_project/constants.rs | 3 + .../_helpers/compiled_project/job_catalog.rs | 71 ++- .../compiled_project/snapshot_declarations.rs | 312 +++++++------ .../_helpers/location_command/command.rs | 5 +- .../_helpers/location_command/snapshot.rs | 50 +- .../_helpers/native_host/constants.rs | 13 +- .../native_host/orchestration_declarations.rs | 434 ++---------------- .../native_host/orchestration_sensors.rs | 13 +- .../native_host/orchestration_triggers.rs | 152 ++---- src/sqlbuild/_native.pyi | 1 - src/sqlbuild/orchestration.py | 22 +- .../main/compile_snapshot.py | 13 +- .../models/authoring.py | 93 +--- .../models/legacy_authoring.py | 132 ++++++ 21 files changed, 537 insertions(+), 819 deletions(-) create mode 100644 src/sqlbuild/orchestration_declarations/models/legacy_authoring.py diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs index 52741a0b2..c650e520f 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs @@ -139,12 +139,15 @@ pub(crate) fn module_declaration( let job_references = declaration .job_references .into_iter() - .map(|(argument, value)| literal(value, &format!("sensor {argument} job"))) + .map(|(argument, value)| { + Ok((argument, literal(value, &format!("sensor {argument} job"))?)) + }) .collect::, _>>()?; Ok(Some(NamedModule { name: name.value, path: String::new(), position: name.position, job_references, + targets: declaration.targets, })) } diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs index 5e86cfd18..acd076023 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs @@ -137,7 +137,7 @@ pub(crate) fn check_references(inventory: &OrchestrationInventory) -> Result<(), } } for module in &inventory.sensors { - for reference in &module.job_references { + for (_, reference) in &module.job_references { member( &module.path, reference, diff --git a/crates/sqlbuild-discovery/src/orchestration_files/models.rs b/crates/sqlbuild-discovery/src/orchestration_files/models.rs index c54beba16..95f4b3990 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/models.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/models.rs @@ -109,8 +109,10 @@ pub struct NamedModule { pub name: String, pub path: String, pub position: FilePosition, - /// The jobs and groups a sensor starts or watches; empty for a calendar. - pub job_references: Vec>, + /// The jobs and groups a sensor starts (`run`) or watches (`run_event`); empty for a calendar. + pub job_references: Vec<(&'static str, Positioned)>, + /// A sensor's targets: empty for every target, `None` when they are not written as literals. + pub targets: Option>, } /// Every orchestration declaration in a project, checked for layout, names and references. diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs index dc5ef58b8..4c1ec660d 100644 --- a/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs +++ b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs @@ -74,3 +74,19 @@ pub(crate) fn literal(expression: &Expr, lines: &SourceLines) -> LiteralArgument position: position(lines, expression.range().start()), } } + +/// `expression` as a literal tuple or list of strings; `None` for anything else. +pub(crate) fn literal_strings(expression: &Expr) -> Option> { + let elements = match expression { + Expr::Tuple(tuple) => &tuple.elts, + Expr::List(list) => &list.elts, + _ => return None, + }; + elements + .iter() + .map(|element| match element { + Expr::StringLiteral(literal) => Some(literal.value.to_str().to_owned()), + _ => None, + }) + .collect() +} diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/constants.rs b/crates/sqlbuild-pysource/src/orchestration_modules/constants.rs index b7a876fe6..bf2be39eb 100644 --- a/crates/sqlbuild-pysource/src/orchestration_modules/constants.rs +++ b/crates/sqlbuild-pysource/src/orchestration_modules/constants.rs @@ -12,6 +12,8 @@ pub(crate) const NAME_KEYWORD: &str = "name"; pub(crate) const RUN_KEYWORD: &str = "run"; /// The `run_event` keyword naming the job it watches. pub(crate) const JOB_KEYWORD: &str = "job"; +/// The sensor keyword naming the targets it publishes for. +pub(crate) const TARGETS_KEYWORD: &str = "targets"; /// The public module the authoring API is imported from. pub(crate) const ORCHESTRATION_MODULE: &str = "sqlbuild.orchestration"; /// Authoring names that TOML declarations replaced. diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs b/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs index 6b1c55d6f..8e0f6298f 100644 --- a/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs +++ b/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs @@ -9,11 +9,11 @@ use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; use crate::cpython_tree::models::{CpythonNode, ParsedSource}; use crate::orchestration_modules::_helpers::calls::{ - authoring_call, imported_names, literal, position, + authoring_call, imported_names, literal, literal_strings, position, }; use crate::orchestration_modules::constants::{ CALENDAR_CALL, JOB_KEYWORD, NAME_KEYWORD, REMOVED_NAMES, RUN_EVENT_CALL, RUN_KEYWORD, - SENSOR_CALL, + SENSOR_CALL, TARGETS_KEYWORD, }; use crate::orchestration_modules::models::{ LiteralArgument, ModuleDeclarations, PythonDeclaration, PythonDeclarationKind, @@ -51,6 +51,10 @@ pub fn orchestration_module_declarations( .find_argument_value(NAME_KEYWORD, 0) .map(|value| literal(value, &parsed.lines)); let mut job_references: Vec<(&'static str, LiteralArgument)> = Vec::new(); + let targets: Option> = match call.arguments.find_keyword(TARGETS_KEYWORD) { + Some(targets) => literal_strings(&targets.value), + None => Some(Vec::new()), + }; if kind == PythonDeclarationKind::Sensor { if let Some(run) = call.arguments.find_keyword(RUN_KEYWORD) { job_references.push((RUN_KEYWORD, literal(&run.value, &parsed.lines))); @@ -84,6 +88,7 @@ pub fn orchestration_module_declarations( name, position: position(&parsed.lines, call.range().start()), job_references, + targets, }); } declarations diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/models.rs b/crates/sqlbuild-pysource/src/orchestration_modules/models.rs index 05303ebf9..108760147 100644 --- a/crates/sqlbuild-pysource/src/orchestration_modules/models.rs +++ b/crates/sqlbuild-pysource/src/orchestration_modules/models.rs @@ -30,6 +30,8 @@ pub struct PythonDeclaration { pub position: SourcePosition, /// `run=` of a sensor, then each `run_event(job=...)`, labelled by the argument. pub job_references: Vec<(&'static str, LiteralArgument)>, + /// A sensor's `targets`: empty when omitted, `None` when not a literal tuple or list of strings. + pub targets: Option>, } /// The orchestration facts of one module. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs index 910140339..a265293a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs @@ -11,3 +11,6 @@ pub(crate) const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"]; pub(crate) const SELECTOR_INTERSECTION: char = ','; /// The key prefix of a SQL unit test's result, as the snapshot names it. pub(crate) const SQL_TEST_KEY_PREFIX: &str = "sql_test:"; + +/// The sensor argument naming the job it starts, as the orchestration inventory labels it. +pub(crate) const SENSOR_RUN_ARGUMENT: &str = "run"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs index 79b2b2e30..c6ef9d997 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs @@ -5,22 +5,21 @@ use std::cell::RefCell; use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; +use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::main::parse_selector::parse_selector; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; use sqlbuild_analysis::graph::models::{JobSelection, ParsedSelector, ProjectGraph}; use sqlbuild_cli::native_commands::main::effective_target::effective_target; +use sqlbuild_discovery::orchestration_files::models::{OrchestrationInventory, RunFile}; use crate::bindings::_helpers::compiled_project::constants::{ - JOB_SELECTOR_KINDS, SELECTOR_INTERSECTION, -}; -use crate::bindings::_helpers::compiled_project::snapshot_declarations::discover_declarations; -use crate::bindings::_helpers::native_host::orchestration_declarations::{ - job_selections, triggered_job_names, + JOB_SELECTOR_KINDS, SELECTOR_INTERSECTION, SENSOR_RUN_ARGUMENT, }; +use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; /// Each job and group's selection, and the names of those a schedule or sensor starts. type Catalog = (BTreeMap, BTreeSet); @@ -154,11 +153,9 @@ fn job_catalog(py: Python<'_>) -> PyResult> { }; let target: Option = effective_target(&project_dir, selected_target).map_err(PyValueError::new_err)?; - let roots: Vec<(Bound<'_, PyAny>, String)> = discover_declarations(py, &project_dir)?; - let catalog: Catalog = ( - job_selections(py, &roots, target.as_deref())?, - triggered_job_names(py, &roots, target.as_deref())?, - ); + let inventory: OrchestrationInventory = + orchestration_declarations(project_dir.clone().into_pyobject(py)?.as_any())?; + let catalog: Catalog = inventory_catalog(&inventory, target.as_deref()); JOB_SCOPE.with(|scope| { if let Some(scope) = scope.borrow_mut().as_mut() { scope.catalog = Some(catalog.clone()); @@ -166,3 +163,55 @@ fn job_catalog(py: Python<'_>) -> PyResult> { }); Ok(Some(catalog)) } + +/// Each job's selector tokens and each group's members declared for `target`, and the names a schedule or sensor starts there. +fn inventory_catalog(inventory: &OrchestrationInventory, target: Option<&str>) -> Catalog { + let declared_for = |targets: &[String]| { + targets.is_empty() || target.is_some_and(|target| targets.iter().any(|name| name == target)) + }; + let tokens = |value: &str| value.split_whitespace().map(str::to_owned).collect(); + let mut catalog: BTreeMap = BTreeMap::new(); + let mut triggered: BTreeSet = BTreeSet::new(); + let mut add = |run: &RunFile, selection: JobSelection| { + if declared_for(&run.targets) { + if run.schedule.is_some() { + triggered.insert(run.name.value.clone()); + } + catalog.insert(run.name.value.clone(), selection); + } + }; + for (_, job) in &inventory.jobs { + add( + &job.run, + JobSelection::Job { + select: tokens(&job.select.value), + exclude: job + .exclude + .as_ref() + .map(|exclude| tokens(&exclude.value)) + .unwrap_or_default(), + }, + ); + } + for (_, group) in &inventory.groups { + add( + &group.run, + JobSelection::Group { + members: group.jobs.iter().map(|job| job.value.clone()).collect(), + }, + ); + } + for sensor in &inventory.sensors { + // A sensor whose targets are not literals counts for every target. + if sensor.targets.as_deref().is_none_or(declared_for) { + triggered.extend( + sensor + .job_references + .iter() + .filter(|(argument, _)| *argument == SENSOR_RUN_ARGUMENT) + .map(|(_, job)| job.value.clone()), + ); + } + } + (catalog, triggered) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 47304d5c3..c031cf77f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -1,72 +1,129 @@ -//! Compile authoring records against retained graph data before publication. +//! Compile TOML jobs and groups and Python calendars and sensors against retained graph data before publication. use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; -use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; -use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; -use sqlbuild_discovery::tree::main::rglob::rglob; +use sqlbuild_discovery::orchestration_files::models::{ + FilePosition, InitialStatus, OrchestrationInventory, RunFile, ScheduleSource, +}; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - DeclaredAt, Definitions, GroupDefinition, JobDefinition, TriggerDeclaration, TriggerDefinition, + DeclaredAt, Definitions, GroupDefinition, JobDefinition, Schedule, TriggerDeclaration, + TriggerDefinition, TriggerStatus, }; use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; +use crate::bindings::_helpers::native_host::constants::{AUTHORING_MODULE, SENSOR_CLASS}; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, declared_at, job_resources, - module_jobs, run_controls, + AuthoredSensor, module_declarations, run_controls, run_resources, }; use crate::bindings::_helpers::native_host::orchestration_triggers::{ - calendar_descriptions, compile_calendars, compile_cron, compile_sensors, + CalendarRequest, compile_calendars, compile_sensors, }; use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; use sqlbuild_rules::lint_text::models::FormatSettings; -/// Dedicated declaration modules, imported by publication and by selectors that name a job. -pub(crate) fn discover_declarations<'py>( +/// The calendars and sensors of the inventory's modules, imported by publication and by sensor evaluation. +pub(crate) fn python_declarations<'py>( py: Python<'py>, project_dir: &Path, -) -> PyResult, String)>> { + inventory: &OrchestrationInventory, +) -> PyResult>> { let tree = ProjectTree::new(project_dir); - let files = py - .detach(|| { - rglob(&tree, ORCHESTRATION_DIRECTORY, |entry| { - !entry.is_dir - && entry.name.ends_with(PYTHON_FILE_SUFFIX) - && entry.name.strip_suffix(PYTHON_FILE_SUFFIX) != Some(PYTHON_INIT_MODULE_STEM) - }) - }) - .map_err(|error| { - PyValueError::new_err(format!("orchestration discovery failed: {error:?}")) - })?; - let mut declarations: Vec<(Bound<'py, PyAny>, String)> = Vec::new(); + let mut declarations: Vec> = Vec::new(); let mut seen: BTreeSet = BTreeSet::new(); - for file in files { - for job in module_jobs(py, &tree.absolute(&file), project_dir)? { - if seen.insert(job.as_ptr() as usize) { - declarations.push((job, file.clone())); + for module in inventory.calendars.iter().chain(&inventory.sensors) { + for declaration in module_declarations(py, &tree.absolute(&module.path), project_dir)? { + if seen.insert(declaration.as_ptr() as usize) { + declarations.push(declaration); } } } Ok(declarations) } +/// Where a TOML job or group is declared: its file and the line of its `name`. +fn file_site(path: &str, position: FilePosition) -> Option { + Some(DeclaredAt { + path: path.to_owned(), + line: u32::try_from(position.line).unwrap_or(u32::MAX), + }) +} + +/// Whether a declaration with `targets` applies to `target`. +fn declared_for(targets: &[String], target: &str) -> bool { + targets.is_empty() || targets.iter().any(|name| name == target) +} + +/// A run's cron trigger, or the calendar it asks for, keyed by the run's name. +fn run_schedule( + (path, run): (&str, &RunFile), + triggers: &mut BTreeMap, + calendars: &mut Vec, +) -> PyResult<()> { + let Some(schedule) = &run.schedule else { + return Ok(()); + }; + let name: &str = &run.name.value; + let starts_at = schedule + .starts_at + .as_ref() + .map(|starts_at| serde_json::from_value(Value::String(starts_at.clone()))) + .transpose() + .map_err(|error| PyValueError::new_err(format!("{path}: schedule starts_at: {error}")))?; + let trigger = TriggerDefinition { + description: run.description.clone(), + job_key: name.to_owned(), + schedule: Schedule::Cron { + expression: String::new(), + timezone: schedule.timezone.clone(), + }, + starts_at, + default_status: match schedule.default_status { + InitialStatus::Running => TriggerStatus::Running, + InitialStatus::Paused => TriggerStatus::Paused, + }, + targets: run.targets.iter().cloned().collect(), + declared_at: file_site(path, run.name.position), + }; + match &schedule.source { + ScheduleSource::Cron(expression) => { + triggers.insert( + name.to_owned(), + TriggerDefinition { + schedule: Schedule::Cron { + expression: expression.clone(), + timezone: schedule.timezone.clone(), + }, + ..trigger + }, + ); + } + ScheduleSource::Calendar(calendar) => calendars.push(CalendarRequest { + calendar: calendar.value.clone(), + timezone: schedule.timezone.clone(), + trigger, + }), + } + Ok(()) +} + pub(crate) fn compile_declarations( py: Python<'_>, mut input: SnapshotInput, (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), - declarations: Vec, + (inventory, declarations): (&OrchestrationInventory, Vec>), ) -> PyResult { - if declarations.is_empty() { + if inventory.is_empty() && declarations.is_empty() { return Ok(input); } if ["jobs", "groups", "triggers"] @@ -77,42 +134,40 @@ pub(crate) fn compile_declarations( "authored declarations cannot overwrite snapshot job or trigger documents", )); } - let mut jobs: BTreeMap = BTreeMap::new(); - let mut authored_groups: Vec = Vec::new(); - let mut authored_jobs: Vec = Vec::new(); - let mut authored_calendars: Vec> = Vec::new(); + let sensor_class = py.import(AUTHORING_MODULE)?.getattr(SENSOR_CLASS)?; + let mut authored_calendars: BTreeMap> = BTreeMap::new(); let mut authored_sensors: Vec = Vec::new(); for declaration in declarations { - match declaration { - AuthoredDeclaration::Job(job) => authored_jobs.push(job), - AuthoredDeclaration::Group(group) => authored_groups.push(group), - AuthoredDeclaration::Calendar(calendar) => authored_calendars.push(calendar), - AuthoredDeclaration::Sensor(sensor) => authored_sensors.push(sensor), + if declaration.is_instance(&sensor_class)? { + authored_sensors.push(declaration.extract()?); + } else { + let name: String = declaration.getattr("name")?.extract()?; + if authored_calendars.insert(name.clone(), declaration).is_some() { + return Err(PyValueError::new_err(format!( + "calendar '{name}' is declared twice" + ))); + } } } let format: FormatSettings = resolve_format_settings(project_dir).map_err(PyValueError::new_err)?; let mut described: Vec<(&str, String, String)> = Vec::new(); - described.extend( - authored_jobs - .iter() - .map(|job| ("job", job.name.clone(), job.description.clone())), - ); - described.extend( - authored_groups - .iter() - .map(|group| ("job group", group.name.clone(), group.description.clone())), - ); + described.extend(inventory.jobs.iter().map(|(_, job)| { + let run = &job.run; + ("job", run.name.value.clone(), run.description.clone()) + })); + described.extend(inventory.groups.iter().map(|(_, group)| { + let run = &group.run; + ("job group", run.name.value.clone(), run.description.clone()) + })); described.extend( authored_sensors .iter() .map(|sensor| ("sensor", sensor.name.clone(), sensor.description.clone())), ); - described.extend( - calendar_descriptions(py, &authored_calendars)? - .into_iter() - .map(|(name, description)| ("calendar schedule", name, description)), - ); + for (name, calendar) in &authored_calendars { + described.push(("calendar", name.clone(), calendar.getattr("description")?.extract()?)); + } for (kind, name, description) in described { let limit: i64 = format.max_description_lines; let lines: usize = description_line_count(&description, format.line_width); @@ -123,70 +178,53 @@ pub(crate) fn compile_declarations( ))); } } + let tokens = |value: &str| { + value + .split_whitespace() + .map(str::to_owned) + .collect::>() + }; + let mut jobs: BTreeMap = BTreeMap::new(); let mut triggers: BTreeMap = BTreeMap::new(); - for job in authored_jobs { - if !job.targets.is_empty() && !job.targets.contains(&input.target) { + let mut calendar_requests: Vec = Vec::new(); + for (path, job) in &inventory.jobs { + let run = &job.run; + if !declared_for(&run.targets, &input.target) { continue; } - let mut settings: BTreeMap = serde_json::from_value(job.settings) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - if !job.concurrency.is_null() { - if settings.contains_key("concurrency") { - return Err(PyValueError::new_err("job concurrency is declared twice")); - } - settings.insert("concurrency".into(), job.concurrency); - } - job_settings_arguments(&settings).map_err(PyValueError::new_err)?; - let tokens = |value: &str| { - value - .split_whitespace() - .map(str::to_owned) - .collect::>() - }; - if job.select.trim().is_empty() { - return Err(PyValueError::new_err(format!( - "job {} needs a selection", - job.name - ))); - } + let name: &str = &run.name.value; + job_settings_arguments(&job.settings) + .map_err(|error| PyValueError::new_err(format!("job {name} settings: {error}")))?; + let exclude: Option = job + .exclude + .as_ref() + .map(|exclude| exclude.value.clone()) + .filter(|exclude| !exclude.trim().is_empty()); let selection: BTreeSet = selected - .run_keys(graph, &tokens(&job.select), &tokens(&job.exclude)) - .map_err(|error| PyValueError::new_err(format!("job {}: {error}", job.name)))?; - let declared_at: Option = declared_at(project_dir, job.declared_at)?; - let controls = run_controls( - &job.name, - ( - job.owner, - &job.overlap, - job.on_failure.as_ref().map(|policy| policy.bind(py)), - ), - )?; - let resources = job_resources( - &job.name, - job.resources.as_ref().map(|resources| resources.bind(py)), - )?; - let targets: BTreeSet = job.targets.into_iter().collect(); - if let Some(schedule) = job.schedule { - triggers.insert( - job.name.clone(), - TriggerDefinition { - targets: targets.clone(), - declared_at: declared_at.clone(), - ..compile_cron(schedule.bind(py), &job.name, &job.description)? - }, - ); - } + .run_keys( + graph, + &tokens(&job.select.value), + &tokens(exclude.as_deref().unwrap_or_default()), + ) + .map_err(|error| PyValueError::new_err(format!("job {name}: {error}")))?; + let resources = job + .resources + .as_ref() + .map(|resources| run_resources(name, &resources.value)) + .transpose()?; + run_schedule((path, run), &mut triggers, &mut calendar_requests)?; + let controls = run_controls(run); jobs.insert( - job.name, + name.to_owned(), JobDefinition { - description: job.description, + description: run.description.clone(), selection, - settings, - concurrency_groups: job.concurrency_groups.into_iter().collect(), - select: Some(job.select), - exclude: Some(job.exclude).filter(|exclude| !exclude.trim().is_empty()), - targets, - declared_at, + settings: job.settings.clone(), + concurrency_groups: run.concurrency_groups.iter().cloned().collect(), + select: Some(job.select.value.clone()), + exclude, + targets: run.targets.iter().cloned().collect(), + declared_at: file_site(path, run.name.position), owner: controls.owner, overlap: controls.overlap, on_failure: controls.on_failure, @@ -195,52 +233,38 @@ pub(crate) fn compile_declarations( ); } let mut groups: BTreeMap = BTreeMap::new(); - for group in authored_groups { - if !group.targets.is_empty() && !group.targets.contains(&input.target) { + for (path, group) in &inventory.groups { + let run = &group.run; + if !declared_for(&run.targets, &input.target) { continue; } - let members: BTreeSet = group.jobs.iter().map(|job| job.name.clone()).collect(); + let name: &str = &run.name.value; + let members: BTreeSet = group.jobs.iter().map(|job| job.value.clone()).collect(); if members.len() != group.jobs.len() { - return Err(PyValueError::new_err("group members must be distinct")); + return Err(PyValueError::new_err(format!( + "job group {name} lists a job twice" + ))); } - let dependencies = group_dependencies((&group.name, &input), &jobs, &members) + let dependencies = group_dependencies((name, &input), &jobs, &members) .map_err(|error| PyValueError::new_err(error.to_string()))?; let selection: BTreeSet = members .iter() - .flat_map(|name| jobs[name].selection.iter().cloned()) + .flat_map(|member| jobs[member].selection.iter().cloned()) .collect(); - let declared_at: Option = declared_at(project_dir, group.declared_at)?; - let controls = run_controls( - &group.name, - ( - group.owner, - &group.overlap, - group.on_failure.as_ref().map(|policy| policy.bind(py)), - ), - )?; - let targets: BTreeSet = group.targets.into_iter().collect(); - if let Some(schedule) = group.schedule { - triggers.insert( - group.name.clone(), - TriggerDefinition { - targets: targets.clone(), - declared_at: declared_at.clone(), - ..compile_cron(schedule.bind(py), &group.name, &group.description)? - }, - ); - } + run_schedule((path, run), &mut triggers, &mut calendar_requests)?; + let controls = run_controls(run); groups.insert( - group.name, + name.to_owned(), GroupDefinition { job: JobDefinition { - description: group.description, + description: run.description.clone(), selection, settings: BTreeMap::new(), - concurrency_groups: group.concurrency_groups.into_iter().collect(), + concurrency_groups: run.concurrency_groups.iter().cloned().collect(), select: None, exclude: None, - targets, - declared_at, + targets: run.targets.iter().cloned().collect(), + declared_at: file_site(path, run.name.position), owner: controls.owner, overlap: controls.overlap, on_failure: controls.on_failure, @@ -262,8 +286,8 @@ pub(crate) fn compile_declarations( authored_sensors, compile_calendars( py, - (project_dir, &input.target), - authored_calendars, + project_dir, + (&authored_calendars, calendar_requests), Definitions { jobs, groups, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 1d6c1c9be..218cb98b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -145,11 +145,10 @@ fn compile_location<'py>( project_dir.extract()?, ( discovered.clone(), - arguments + !arguments .value("action")? .extract::>()? - .is_some_and(|action| action == EXECUTE_ACTION) - .then(Vec::new), + .is_some_and(|action| action == EXECUTE_ACTION), ), )?; status(py, "Location snapshot compiled.")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 40a0682a2..99b9680f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -10,13 +10,14 @@ use pyo3::{IntoPyObject, pyfunction}; use serde_json::{Value, json}; use sqlbuild_analysis::planner_scope::main::node_loads::node_loads; use sqlbuild_core::models::DiagnosticSeverity; +use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; use sqlbuild_discovery::project_validation::main::validate_orchestration_names::validate_orchestration_names; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ - compile_declarations, discover_declarations, + compile_declarations, python_declarations, }; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; @@ -27,9 +28,7 @@ use crate::bindings::_helpers::discovery::project_validation::validation_error; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; -use crate::bindings::_helpers::native_host::orchestration_declarations::{ - collect_declarations, orchestration_names, -}; +use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; @@ -40,9 +39,6 @@ use crate::bindings::_helpers::records::fields::field; /// The Python node kind whose functions load sources. const LOADER_KIND: &str = "loader"; -/// Where declarations passed to the snapshot directly are reported as declared. -const DECLARATIONS_ARGUMENT: &str = "declarations argument"; - /// Compile catalog nodes and identities in the location, never in the server. #[pyfunction] #[pyo3(signature = (project, metadata_json, project_dir, **options))] @@ -56,28 +52,22 @@ pub(crate) fn build_orchestration_snapshot<'py>( project.py(), "build_orchestration_snapshot", options, - (&["discovered_inputs"], &["declarations"]), + (&["discovered_inputs"], &[]), )?; - let declarations = arguments.value("declarations")?; orchestration_snapshot( project, metadata_json, project_dir, - ( - arguments.value("discovered_inputs")?, - (!declarations.is_none()) - .then(|| declarations.extract()) - .transpose()?, - ), + (arguments.value("discovered_inputs")?, true), ) } -/// Compile the snapshot from the project, its discovery, and the declarations passed directly, if any. +/// Compile the snapshot from the project, its discovery and, when `declared`, its orchestration declarations. pub(crate) fn orchestration_snapshot<'py>( project: PyRef<'py, NativeProjectView>, metadata_json: &str, project_dir: PathBuf, - (discovered_inputs, declarations): (Bound<'py, PyAny>, Option>>), + (discovered_inputs, declared): (Bound<'py, PyAny>, bool), ) -> PyResult { let py = project.py(); let mut input: SnapshotInput = serde_json::from_str(metadata_json) @@ -121,31 +111,23 @@ pub(crate) fn orchestration_snapshot<'py>( if let Some(providers) = providers_document(&discovered_inputs, &input.nodes)? { input.documents.insert("providers".into(), providers); } - let declarations = match declarations { - Some(declarations) => declarations - .into_iter() - .map(|declaration| (declaration, DECLARATIONS_ARGUMENT.to_owned())) - .collect(), - None => discover_declarations(py, &project_dir)?, - }; let graph = project.lineage_graph(py)?; - if !declarations.is_empty() { + let inventory = if declared { + orchestration_declarations(project_dir.clone().into_pyobject(py)?.as_any())? + } else { + OrchestrationInventory::default() + }; + if !inventory.is_empty() { let mut facts = project_facts(py, &discovered_inputs, named_node_rows(&discovered_inputs)?)?; - facts.orchestration = orchestration_names(py, &declarations)?; + facts.orchestration = inventory.names(); validate_orchestration_names(&facts).map_err(|failure| validation_error(py, failure))?; - let authored = collect_declarations( - py, - declarations - .into_iter() - .map(|(declaration, _)| declaration) - .collect(), - )?; + let declarations = python_declarations(py, &project_dir, &inventory)?; input = compile_declarations( py, input, (&project_dir, graph.bind(py).get().graph(), &selected), - authored, + (&inventory, declarations), )?; } input = compile_project_roles( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index d0ef09dcb..2f12728a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -3,7 +3,6 @@ /// The calendar's evaluation window is supplied by the host, not a provider. pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; -pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; pub(crate) const SOURCE_NEW_DATA_CONDITION: &str = "source_new_data"; pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; @@ -14,15 +13,9 @@ pub(crate) const AUTHORING_MODULE: &str = "sqlbuild.orchestration_declarations.m /// Frames of these modules construct declarations on the author's behalf. pub(crate) const CONSTRUCTING_MODULES: [&str; 2] = [AUTHORING_MODULE, "dataclasses"]; -/// Kinds named in the global unique-name check. -pub(crate) const JOB_LABEL: &str = "job"; -pub(crate) const GROUP_LABEL: &str = "job group"; -pub(crate) const SENSOR_LABEL: &str = "sensor"; -pub(crate) const CALENDAR_LABEL: &str = "calendar schedule"; +/// The authoring classes publication imports calendar and sensor modules for. +pub(crate) const CALENDAR_CLASS: &str = "Calendar"; +pub(crate) const SENSOR_CLASS: &str = "Sensor"; /// Pages one bounded state-history SELECT streams before the scan continues from its cursor. pub(crate) const HISTORY_STREAM_PAGES: usize = 50; -/// Inclusive bounds on a job's `on_failure` retries. -pub(crate) const MAX_RETRIES: (u64, u64) = (1, 10); -/// Inclusive bounds on a job's `on_failure` retry delay, in seconds: up to 24 hours. -pub(crate) const MAX_RETRY_DELAY: (u64, u64) = (0, 86_400); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index ae92c1946..f91523bf0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -1,79 +1,29 @@ //! The public authoring records enter native compilation here, never scheduler execution. -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::collections::HashSet; use std::path::{Component, Path}; -use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use pyo3::{FromPyObject, pyfunction}; use serde_json::Value; -use sqlbuild_analysis::graph::models::JobSelection; -use sqlbuild_discovery::project_validation::models::OrchestrationNameFact; +use sqlbuild_discovery::orchestration_files::models::{self as files, RunFile}; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::declarations::models::{DeclaredAt, OnFailure, Overlap, RunResources}; use crate::bindings::_helpers::native_host::constants::{ - AUTHORING_MODULE, CALENDAR_LABEL, CONSTRUCTING_MODULES, GROUP_LABEL, JOB_LABEL, MAX_RETRIES, - MAX_RETRY_DELAY, RUN_EVENT_CONDITION, SENSOR_LABEL, + AUTHORING_MODULE, CALENDAR_CLASS, CONSTRUCTING_MODULES, SENSOR_CLASS, }; use crate::bindings::_helpers::native_host::host::entry; -#[derive(FromPyObject)] -pub(crate) struct AuthoredJob { - pub(crate) name: String, - pub(crate) description: String, - pub(crate) select: String, - pub(crate) exclude: String, - #[pyo3(from_py_with = authored_setting)] - pub(crate) settings: Value, - #[pyo3(from_py_with = authored_setting)] - pub(crate) concurrency: Value, - pub(crate) concurrency_groups: Vec, - pub(crate) schedule: Option>, - pub(crate) targets: Vec, - #[pyo3(attribute("_declared_at"))] - pub(crate) declared_at: Option<(String, u32)>, - pub(crate) owner: Option, - pub(crate) overlap: String, - pub(crate) on_failure: Option>, - pub(crate) resources: Option>, -} - -#[derive(FromPyObject, PartialEq, Eq, PartialOrd, Ord)] -pub(crate) struct AuthoredJobReference { - pub(crate) name: String, -} - -#[derive(FromPyObject)] -pub(crate) struct AuthoredGroup { - pub(crate) name: String, - pub(crate) description: String, - pub(crate) jobs: Vec, - pub(crate) concurrency_groups: Vec, - pub(crate) schedule: Option>, - pub(crate) targets: Vec, - #[pyo3(attribute("_declared_at"))] - pub(crate) declared_at: Option<(String, u32)>, - pub(crate) owner: Option, - pub(crate) overlap: String, - pub(crate) on_failure: Option>, -} - -/// A job's or group's owner, overlap and failure policy, validated where they enter native code. -pub(crate) struct RunControls { - pub(crate) owner: Option, - pub(crate) overlap: Overlap, - pub(crate) on_failure: Option, -} - #[derive(FromPyObject)] pub(crate) struct AuthoredSensorCondition { pub(crate) kind: String, #[pyo3(default)] pub(crate) kinds: Vec, #[pyo3(default)] - pub(crate) job: Option, + pub(crate) job: Option, #[pyo3(default)] pub(crate) executed_successfully: Option>, #[pyo3(default)] @@ -87,8 +37,8 @@ pub(crate) struct AuthoredSensor { pub(crate) name: String, pub(crate) description: String, pub(crate) when: Option, - pub(crate) function: Option>, - pub(crate) run: AuthoredJobReference, + pub(crate) function: Option>, + pub(crate) run: String, #[pyo3(from_py_with = authored_setting)] pub(crate) minimum_interval_seconds: Value, pub(crate) default_status: String, @@ -97,11 +47,11 @@ pub(crate) struct AuthoredSensor { pub(crate) declared_at: Option<(String, u32)>, } -pub(crate) enum AuthoredDeclaration { - Job(AuthoredJob), - Group(AuthoredGroup), - Calendar(Py), - Sensor(AuthoredSensor), +/// A job's or group's owner, overlap and failure policy, as publication records them. +pub(crate) struct RunControls { + pub(crate) owner: Option, + pub(crate) overlap: Overlap, + pub(crate) on_failure: Option, } /// The file and line of the author's code constructing a declaration; `None` for code without a source file. @@ -165,70 +115,28 @@ pub(crate) fn authored_setting(value: &Bound<'_, PyAny>) -> PyResult, &str, Option<&Bound<'_, PyAny>>), -) -> PyResult { - if owner - .as_deref() - .is_some_and(|owner| owner.trim().is_empty()) - { - return Err(PyValueError::new_err(format!( - "job {name} owner must not be blank" - ))); +/// The controls of a TOML job or group; discovery has already checked the owner and the retry bounds. +pub(crate) fn run_controls(run: &RunFile) -> RunControls { + RunControls { + owner: run.owner.clone(), + overlap: match run.overlap { + files::Overlap::Queue => Overlap::Queue, + files::Overlap::Skip => Overlap::Skip, + }, + on_failure: run.on_failure.map(|policy| OnFailure { + retries: policy.retries, + retry_delay_seconds: policy.retry_delay_seconds, + }), } - let overlap = match overlap { - "queue" => Overlap::Queue, - "skip" => Overlap::Skip, - other => { - return Err(PyValueError::new_err(format!( - "job {name} overlap must be queue or skip, not {other}" - ))); - } - }; - let on_failure = if let Some(policy) = policy { - let bounded = |field: &str, (low, high): (u64, u64)| -> PyResult { - authored_setting(&policy.getattr(field)?)? - .as_u64() - .filter(|value| (low..=high).contains(value)) - .ok_or_else(|| { - PyValueError::new_err(format!( - "job {name} on_failure {field} must be a whole number from {low} to {high}" - )) - }) - }; - Some(OnFailure { - retries: u8::try_from(bounded("retries", MAX_RETRIES)?)?, - retry_delay_seconds: u32::try_from(bounded("retry_delay_seconds", MAX_RETRY_DELAY)?)?, - }) - } else { - None - }; - Ok(RunControls { - owner, - overlap, - on_failure, - }) } -/// Refuse a memory size Docker cannot read and CPUs or a process limit that is not positive. -pub(crate) fn job_resources( +/// Refuse an unknown limit, a memory size Docker cannot read, and CPUs or a process limit that is not positive. +pub(crate) fn run_resources( name: &str, - resources: Option<&Bound<'_, PyAny>>, -) -> PyResult> { - let Some(resources) = resources else { - return Ok(None); - }; - let mut written = serde_json::Map::new(); - for field in ["memory", "cpus", "pids"] { - let value = authored_setting(&resources.getattr(field)?)?; - if !value.is_null() { - written.insert(field.into(), value); - } - } + written: &serde_json::Map, +) -> PyResult { let invalid = |reason: String| PyValueError::new_err(format!("job {name} resources: {reason}")); - let resources: RunResources = serde_json::from_value(Value::Object(written)) + let resources: RunResources = serde_json::from_value(Value::Object(written.clone())) .map_err(|error| invalid(error.to_string()))?; if resources .cpus @@ -239,11 +147,11 @@ pub(crate) fn job_resources( if resources.pids == Some(0) { return Err(invalid("pids must be a positive whole number".into())); } - Ok(Some(resources)) + Ok(resources) } -/// Only module execution crosses into Python; discovery and collection stay native. -pub(crate) fn module_jobs<'py>( +/// The calendars and sensors one module defines; only module execution crosses into Python. +pub(crate) fn module_declarations<'py>( py: Python<'py>, file_path: &Path, project_dir: &Path, @@ -252,281 +160,27 @@ pub(crate) fn module_jobs<'py>( options.set_item("file_path", file_path)?; options.set_item("project_dir", project_dir)?; let module = entry(py, UserCodeHost::LoadOrchestrationModule)?.call((), Some(&options))?; - let job_class = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("Job")?; - let group_class = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("JobGroup")?; - let calendar_class = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("CalendarSchedule")?; - let sensor_class = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("Sensor")?; + let authoring = py.import(AUTHORING_MODULE)?; + let classes = [ + authoring.getattr(CALENDAR_CLASS)?, + authoring.getattr(SENSOR_CLASS)?, + ]; let attributes = module.getattr("__dict__")?.cast_into::()?; let mut pending: Vec> = attributes.values().iter().collect(); let mut seen: HashSet = HashSet::new(); - let mut jobs: Vec> = Vec::new(); + let mut found: Vec> = Vec::new(); while let Some(value) = pending.pop() { if !seen.insert(value.as_ptr() as usize) { continue; } - if value.is_instance(&job_class)? - || value.is_instance(&group_class)? - || value.is_instance(&calendar_class)? - || value.is_instance(&sensor_class)? + if classes + .iter() + .any(|class| value.is_instance(class).unwrap_or(false)) { - jobs.push(value); + found.push(value); } else if value.is_instance_of::() || value.is_instance_of::() { pending.extend(value.try_iter()?.collect::>>()?); } } - Ok(jobs) -} - -pub(crate) fn collect_declarations<'py>( - py: Python<'py>, - declarations: Vec>, -) -> PyResult> { - let job = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("Job")?; - let group = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("JobGroup")?; - let calendar = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("CalendarSchedule")?; - let sensor = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("Sensor")?; - let mut roots: HashSet = HashSet::new(); - for declaration in &declarations { - if !roots.insert(declaration.as_ptr() as usize) { - if declaration.is_instance(&calendar)? { - return Err(PyValueError::new_err("duplicate calendar declaration")); - } - return Err(PyValueError::new_err(format!( - "duplicate job {}", - declaration.getattr("name")?.extract::()? - ))); - } - } - let mut pending = declarations; - let mut seen: HashSet = HashSet::new(); - let mut collected: Vec = Vec::new(); - while let Some(declaration) = pending.pop() { - if !seen.insert(declaration.as_ptr() as usize) { - continue; - } - let authored = if declaration.is_instance(&sensor)? { - let target = declaration.getattr("run")?; - if !target.is_instance(&job)? && !target.is_instance(&group)? { - return Err(PyTypeError::new_err( - "sensor run must be a Job or JobGroup object", - )); - } - pending.push(target); - let condition = declaration.getattr("when")?; - if !condition.is_none() - && condition.getattr("kind")?.extract::()? == RUN_EVENT_CONDITION - { - let source = condition.getattr("job")?; - if !source.is_instance(&job)? && !source.is_instance(&group)? { - return Err(PyTypeError::new_err( - "run_event job must be a Job or JobGroup object", - )); - } - pending.push(source); - } - AuthoredDeclaration::Sensor(declaration.extract()?) - } else if declaration.is_instance(&calendar)? { - let target = declaration.getattr("job")?; - if !target.is_instance(&job)? && !target.is_instance(&group)? { - return Err(PyTypeError::new_err( - "calendar job must be a Job or JobGroup object", - )); - } - pending.push(target); - AuthoredDeclaration::Calendar(declaration.unbind()) - } else if declaration.is_instance(&group)? { - for member in declaration.getattr("jobs")?.try_iter()? { - let member = member?; - if !member.is_instance(&job)? { - return Err(PyTypeError::new_err("group members must be Job objects")); - } - pending.push(member); - } - AuthoredDeclaration::Group(declaration.extract()?) - } else if !declaration.is_instance(&job)? { - return Err(PyTypeError::new_err( - "snapshot declarations must be Job, JobGroup, CalendarSchedule or Sensor objects", - )); - } else { - AuthoredDeclaration::Job(declaration.extract()?) - }; - collected.push(authored); - } - Ok(collected) -} - -/// Every declaration reachable from `roots`, once each, with its kind and the module of the root that reaches it first. -pub(crate) fn reachable_declarations<'py>( - py: Python<'py>, - roots: &[(Bound<'py, PyAny>, String)], -) -> PyResult, &'static str, String)>> { - let authoring = py.import(AUTHORING_MODULE)?; - let classes = [ - (authoring.getattr("Job")?, JOB_LABEL), - (authoring.getattr("JobGroup")?, GROUP_LABEL), - (authoring.getattr("Sensor")?, SENSOR_LABEL), - (authoring.getattr("CalendarSchedule")?, CALENDAR_LABEL), - ]; - let mut seen: HashSet = HashSet::new(); - let mut reached: Vec<(Bound<'py, PyAny>, &'static str, String)> = Vec::new(); - for (root, path) in roots { - let mut pending: Vec> = vec![root.clone()]; - while let Some(declaration) = pending.pop() { - if !seen.insert(declaration.as_ptr() as usize) { - continue; - } - let Some(kind) = classes - .iter() - .find(|(class, _)| declaration.is_instance(class).unwrap_or(false)) - .map(|(_, kind)| *kind) - else { - continue; - }; - pending.extend(nested_declarations(&declaration, kind)?); - reached.push((declaration, kind, path.clone())); - } - } - Ok(reached) -} - -/// Every named declaration reachable from `roots`, labelled with the module of the root that reaches it first. -pub(crate) fn orchestration_names( - py: Python<'_>, - roots: &[(Bound<'_, PyAny>, String)], -) -> PyResult> { - let mut names: Vec = Vec::new(); - for (declaration, kind, path) in reachable_declarations(py, roots)? { - let name: Option = if kind == CALENDAR_LABEL { - let function = declaration.getattr("function")?; - (!function.is_none()) - .then(|| function.getattr("__name__")?.extract()) - .transpose()? - } else { - Some(declaration.getattr("name")?.extract()?) - }; - names.extend(name.map(|name| OrchestrationNameFact { name, kind, path })); - } - Ok(names) -} - -/// The declarations one declaration refers to: a group's jobs, a calendar's job, a sensor's job and run-event job. -fn nested_declarations<'py>( - declaration: &Bound<'py, PyAny>, - kind: &str, -) -> PyResult>> { - Ok(match kind { - GROUP_LABEL => declaration - .getattr("jobs")? - .try_iter()? - .collect::>>()?, - CALENDAR_LABEL => vec![declaration.getattr("job")?], - SENSOR_LABEL => { - let condition = declaration.getattr("when")?; - let mut nested = vec![declaration.getattr("run")?]; - if !condition.is_none() - && condition.getattr("kind")?.extract::()? == RUN_EVENT_CONDITION - { - nested.push(condition.getattr("job")?); - } - nested - } - _ => Vec::new(), - }) -} - -/// Each reachable job's own selector tokens and each group's member names for `target`, as publication reads them. -pub(crate) fn job_selections( - py: Python<'_>, - roots: &[(Bound<'_, PyAny>, String)], - target: Option<&str>, -) -> PyResult> { - let mut catalog: BTreeMap = BTreeMap::new(); - for (declaration, kind, _) in reachable_declarations(py, roots)? { - if kind != JOB_LABEL && kind != GROUP_LABEL { - continue; - } - if !declared_for(&declaration, target)? { - continue; - } - let tokens = |field: &str| -> PyResult> { - Ok(declaration - .getattr(field)? - .extract::()? - .split_whitespace() - .map(str::to_owned) - .collect()) - }; - let selection: JobSelection = match kind { - JOB_LABEL => JobSelection::Job { - select: tokens("select")?, - exclude: tokens("exclude")?, - }, - GROUP_LABEL => JobSelection::Group { - members: declaration - .getattr("jobs")? - .try_iter()? - .map(|job| job?.getattr("name")?.extract()) - .collect::>()?, - }, - _ => continue, - }; - let name: String = declaration.getattr("name")?.extract()?; - if catalog.insert(name.clone(), selection).is_some() { - return Err(PyValueError::new_err(format!( - "job name '{name}' is declared twice" - ))); - } - } - Ok(catalog) -} - -/// Whether a declaration applies to `target`: it names no targets, or names this one. -fn declared_for(declaration: &Bound<'_, PyAny>, target: Option<&str>) -> PyResult { - let targets: Vec = declaration.getattr("targets")?.extract()?; - Ok( - targets.is_empty() - || target.is_some_and(|target| targets.iter().any(|name| name == target)), - ) -} - -/// The jobs and groups that a schedule, calendar schedule or sensor declared for `target` starts. -pub(crate) fn triggered_job_names( - py: Python<'_>, - roots: &[(Bound<'_, PyAny>, String)], - target: Option<&str>, -) -> PyResult> { - let mut names: BTreeSet = BTreeSet::new(); - for (declaration, kind, _) in reachable_declarations(py, roots)? { - if !declared_for(&declaration, target)? { - continue; - } - let started: Option> = match kind { - JOB_LABEL | GROUP_LABEL => { - (!declaration.getattr("schedule")?.is_none()).then(|| declaration.clone()) - } - CALENDAR_LABEL => Some(declaration.getattr("job")?), - SENSOR_LABEL => Some(declaration.getattr("run")?), - _ => None, - }; - if let Some(job) = started { - names.insert(job.getattr("name")?.extract()?); - } - } - Ok(names) + Ok(found) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index 8a01736e9..da32ffc25 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -3,6 +3,7 @@ use std::path::Path; use pyo3::exceptions::PyValueError; +use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use sqlbuild_host::entries::models::UserCodeHost; @@ -11,7 +12,8 @@ use sqlbuild_orchestration::sensors::models::{ }; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; -use crate::bindings::_helpers::compiled_project::snapshot_declarations::discover_declarations; +use crate::bindings::_helpers::compiled_project::snapshot_declarations::python_declarations; +use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; use crate::bindings::_helpers::native_host::constants::SENSOR_CONTEXT_PARAMETER; @@ -35,7 +37,8 @@ pub(crate) fn evaluate_sensor( let authoring = py.import("sqlbuild.orchestration_declarations.models.authoring")?; let sensor_class = authoring.getattr("Sensor")?; let mut matches: Vec> = Vec::new(); - for (declaration, _) in discover_declarations(py, project_dir)? { + let inventory = orchestration_declarations(project_dir.into_pyobject(py)?.as_any())?; + for declaration in python_declarations(py, project_dir, &inventory)? { if declaration.is_instance(&sensor_class)? && declaration.getattr("name")?.extract::()? == *name { @@ -50,11 +53,7 @@ pub(crate) fn evaluate_sensor( let targets: Vec = sensor.getattr("targets")?.extract()?; if !sensor.getattr("when")?.is_none() || (!targets.is_empty() && !targets.contains(&evaluation.target)) - || sensor - .getattr("run")? - .getattr("name")? - .extract::()? - != evaluation.definition.job_key + || sensor.getattr("run")?.extract::()? != evaluation.definition.job_key { return Err(PyValueError::new_err("custom sensor definition changed")); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 9faf31f8e..cd03b0e3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -1,13 +1,12 @@ //! Authored trigger controls and bounded calendars evaluated only in the location interpreter. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use chrono::{DateTime, TimeDelta, Timelike, Utc}; use chrono_tz::Tz; -use pyo3::FromPyObject; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use serde_json::{Value, json}; use sqlbuild_analysis::graph::models::ProjectGraph; @@ -27,53 +26,17 @@ use crate::bindings::_helpers::native_host::constants::{ }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - AuthoredSensor, authored_setting, declared_at, + AuthoredSensor, declared_at, }; use crate::bindings::_helpers::native_host::orchestration_environment::{ callback_code_hash, orchestration_code_hash, }; -#[derive(FromPyObject)] -struct AuthoredControls { - #[pyo3(from_py_with = authored_setting)] - starts_at: Value, - default_status: String, -} - -fn compile_trigger( - authored: &Bound<'_, PyAny>, - job_key: &str, - description: &str, - schedule: Schedule, -) -> PyResult { - let controls = authored.extract::()?; - Ok(TriggerDefinition { - description: description.into(), - job_key: job_key.into(), - schedule, - starts_at: serde_json::from_value(controls.starts_at) - .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, - default_status: serde_json::from_value(Value::String(controls.default_status)) - .map_err(|error| PyValueError::new_err(format!("default_status: {error}")))?, - targets: BTreeSet::new(), - declared_at: None, - }) -} - -pub(crate) fn compile_cron( - authored: &Bound<'_, PyAny>, - name: &str, - description: &str, -) -> PyResult { - compile_trigger( - authored, - name, - description, - Schedule::Cron { - expression: authored.getattr("cron")?.extract()?, - timezone: authored.getattr("timezone")?.extract()?, - }, - ) +/// A job's or group's schedule on a named calendar, as its trigger before the calendar's times fill it. +pub(crate) struct CalendarRequest { + pub(crate) calendar: String, + pub(crate) timezone: String, + pub(crate) trigger: TriggerDefinition, } fn horizon(value: &str) -> PyResult { @@ -105,13 +68,12 @@ fn horizon(value: &str) -> PyResult { } fn calendar_times( - authored: &Bound<'_, PyAny>, + (authored, timezone): (&Bound<'_, PyAny>, &str), session: &Bound<'_, PyAny>, providers: &Bound<'_, PyDict>, start: DateTime, ) -> PyResult { let py = authored.py(); - let timezone: String = authored.getattr("timezone")?.extract()?; timezone .parse::() .map_err(|_| PyValueError::new_err("calendar timezone must be an IANA timezone"))?; @@ -184,42 +146,14 @@ fn calendar_times( }) } -/// The name and description of each calendar that decorates a function. -pub(crate) fn calendar_descriptions( - py: Python<'_>, - calendars: &[Py], -) -> PyResult> { - let mut described: Vec<(String, String)> = Vec::new(); - for calendar in calendars { - let calendar: &Bound<'_, PyAny> = calendar.bind(py); - let function: Bound<'_, PyAny> = calendar.getattr("function")?; - if !function.is_none() { - described.push(( - function.getattr("__name__")?.extract()?, - calendar.getattr("description")?.extract()?, - )); - } - } - Ok(described) -} - +/// Each calendar request's trigger, with each (calendar, timezone) pair evaluated once. pub(crate) fn compile_calendars( py: Python<'_>, - (project_dir, target): (&Path, &str), - authored: Vec>, + project_dir: &Path, + (calendars, requests): (&BTreeMap>, Vec), mut definitions: Definitions, ) -> PyResult { - let mut calendars: Vec> = Vec::new(); - for calendar in authored { - let targets = calendar - .bind(py) - .getattr("targets")? - .extract::>()?; - if targets.is_empty() || targets.iter().any(|value| value == target) { - calendars.push(calendar); - } - } - if calendars.is_empty() { + if requests.is_empty() { return Ok(definitions); } let providers = @@ -237,33 +171,37 @@ pub(crate) fn compile_calendars( .with_nanosecond(now.timestamp_subsec_micros() * 1_000) .ok_or_else(|| PyValueError::new_err("calendar clock is out of range"))?; within(&session, |_| { - for calendar in calendars { - let calendar = calendar.bind(py); - let name: String = calendar - .getattr("function")? - .getattr("__name__")? - .extract()?; - let job_key: String = calendar.getattr("job")?.getattr("name")?.extract()?; - let description: String = calendar.getattr("description")?.extract()?; - if description.trim().is_empty() - || (!definitions.jobs.contains_key(&job_key) - && !definitions.groups.contains_key(&job_key)) - { - return Err(PyValueError::new_err(format!( - "calendar {name} needs a description and a published job" - ))); - } - let schedule = calendar_times(calendar, &session, &by_name, start)?; - let mut trigger = compile_trigger(calendar, &job_key, &description, schedule)?; + let mut evaluated: BTreeMap<(String, String), Schedule> = BTreeMap::new(); + for request in requests { + let key = (request.calendar.clone(), request.timezone.clone()); + let schedule = match evaluated.get(&key) { + Some(schedule) => schedule.clone(), + None => { + let calendar = calendars.get(&request.calendar).ok_or_else(|| { + PyValueError::new_err(format!( + "calendar '{}' is not declared by a calendar module", + request.calendar + )) + })?; + let schedule = calendar_times( + (calendar, &request.timezone), + &session, + &by_name, + start, + ) + .map_err(|error| { + PyValueError::new_err(format!("calendar '{}': {error}", request.calendar)) + })?; + evaluated.insert(key, schedule.clone()); + schedule + } + }; + let mut trigger = TriggerDefinition { + schedule, + ..request.trigger + }; trigger.starts_at.get_or_insert(start); - trigger.targets = calendar - .getattr("targets")? - .extract::>()? - .into_iter() - .collect(); - trigger.declared_at = - declared_at(project_dir, calendar.getattr("_declared_at")?.extract()?)?; - definitions.triggers.insert(name, trigger); + definitions.triggers.insert(trigger.job_key.clone(), trigger); } Ok(definitions) }) @@ -294,7 +232,7 @@ pub(crate) fn compile_sensors( condition["kinds"] = json!(when.kinds); } if let Some(job) = when.job { - condition["job_key"] = json!(job.name); + condition["job_key"] = json!(job); } if let Some(select) = when.select { let selectors: Vec = @@ -317,7 +255,7 @@ pub(crate) fn compile_sensors( }; let definition: SensorDefinition = serde_json::from_value(json!({ "description": sensor.description, - "job_key": sensor.run.name, + "job_key": sensor.run, "when": condition, "minimum_interval_seconds": sensor.minimum_interval_seconds, "default_status": sensor.default_status, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index c72819394..622f8e1e8 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1777,7 +1777,6 @@ def build_orchestration_snapshot( project_dir: Path, *, discovered_inputs: DiscoveredProjectInputs, - declarations: Sequence[object] | None = None, ) -> NativeOrchestrationSnapshot: ... def run_location_command( *, diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 1b042a5e2..af83e60b2 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -1,24 +1,21 @@ -"""Public authoring and snapshot compilation API for orchestration declarations.""" +"""Public authoring and snapshot compilation API for orchestration declarations. + +Jobs, job groups and their schedules are TOML files under `orchestration/jobs/` and +`orchestration/job_groups/`; calendars and sensors are declared here. +""" from sqlbuild.orchestration_declarations.main.compile_snapshot import compile_snapshot -from sqlbuild.orchestration_declarations.models.authoring import ( - CalendarSchedule as calendar_schedule, -) +from sqlbuild.orchestration_declarations.models.authoring import Calendar as calendar from sqlbuild.orchestration_declarations.models.authoring import ( CalendarWindow, RunRequest, SkipReason, ) -from sqlbuild.orchestration_declarations.models.authoring import Job as job -from sqlbuild.orchestration_declarations.models.authoring import JobGroup as job_group from sqlbuild.orchestration_declarations.models.authoring import NodeEventCondition as node_event -from sqlbuild.orchestration_declarations.models.authoring import OnFailure as on_failure from sqlbuild.orchestration_declarations.models.authoring import ( PlatformEventCondition as platform_event, ) -from sqlbuild.orchestration_declarations.models.authoring import Resources as resources from sqlbuild.orchestration_declarations.models.authoring import RunEventCondition as run_event -from sqlbuild.orchestration_declarations.models.authoring import Schedule as schedule from sqlbuild.orchestration_declarations.models.authoring import Sensor as sensor from sqlbuild.orchestration_declarations.models.authoring import ( SourceNewDataCondition as source_new_data, @@ -37,16 +34,11 @@ "RunRequest", "SensorContext", "SkipReason", - "calendar_schedule", + "calendar", "compile_snapshot", - "job", - "job_group", "node_event", - "on_failure", "platform_event", - "resources", "run_event", - "schedule", "sensor", "source_new_data", "source_stale", diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py index fb42de5d9..f901b134b 100644 --- a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -1,8 +1,7 @@ -"""Delegate authoring-record compilation to the native retained snapshot owner.""" +"""Delegate snapshot compilation to the native retained snapshot owner.""" from __future__ import annotations -from collections.abc import Sequence from pathlib import Path from sqlbuild._native import ( @@ -11,12 +10,6 @@ build_orchestration_snapshot, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration_declarations.models.authoring import ( - CalendarSchedule, - Job, - JobGroup, - Sensor, -) def compile_snapshot( @@ -25,13 +18,11 @@ def compile_snapshot( metadata_json: str, project_dir: Path, discovered_inputs: DiscoveredProjectInputs, - declarations: Sequence[Job | JobGroup | CalendarSchedule | Sensor] | None = None, ) -> NativeOrchestrationSnapshot: - """Compile authoring records and retained project facts into a data-only snapshot.""" + """Compile the orchestration declarations and retained facts into a data-only snapshot.""" return build_orchestration_snapshot( project, metadata_json, project_dir, discovered_inputs=discovered_inputs, - declarations=declarations, ) diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index 0bf9122d9..79fa6a044 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -11,75 +11,6 @@ from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind -@dataclass(frozen=True, kw_only=True) -class Schedule: - """Cron with an RFC 3339 start; missed times collapse into one run of the latest.""" - - cron: str - timezone: str - starts_at: str | None = None - default_status: Literal["running", "paused"] = "running" - - -@dataclass(frozen=True, kw_only=True) -class OnFailure: - """Rerun a failed or partial run from failure, `retries` times, `retry_delay_seconds` apart.""" - - retries: int - retry_delay_seconds: int = 0 - - -@dataclass(frozen=True, kw_only=True) -class Resources: - """Worker container limits for a job; memory is bytes or a size such as "512m" or "2g".""" - - memory: int | str | None = None - cpus: float | None = None - pids: int | None = None - - -@dataclass(frozen=True, eq=False) -class Job: - """A named selection to execute, optionally on a wall-clock schedule.""" - - name: str - description: str = field(kw_only=True) - select: str = field(kw_only=True) - exclude: str = field(default="", kw_only=True) - settings: dict[str, object] = field(default_factory=dict, kw_only=True) - concurrency: int | None = field(default=None, kw_only=True) - concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) - schedule: Schedule | None = field(default=None, kw_only=True) - targets: tuple[str, ...] = field(default=(), kw_only=True) - owner: str | None = field(default=None, kw_only=True) - overlap: Literal["queue", "skip"] = field(default="queue", kw_only=True) - on_failure: OnFailure | None = field(default=None, kw_only=True) - resources: Resources | None = field(default=None, kw_only=True) - _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) - - def __post_init__(self) -> None: - object.__setattr__(self, "_declared_at", declaration_site()) - - -@dataclass(frozen=True, eq=False) -class JobGroup: - """Jobs ordered by the lineage between the nodes they build.""" - - name: str - description: str = field(kw_only=True) - jobs: Sequence[Job] = field(kw_only=True) - concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) - schedule: Schedule | None = field(default=None, kw_only=True) - targets: tuple[str, ...] = field(default=(), kw_only=True) - owner: str | None = field(default=None, kw_only=True) - overlap: Literal["queue", "skip"] = field(default="queue", kw_only=True) - on_failure: OnFailure | None = field(default=None, kw_only=True) - _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) - - def __post_init__(self) -> None: - object.__setattr__(self, "_declared_at", declaration_site()) - - @dataclass(frozen=True, kw_only=True) class CalendarWindow: """Timezone-aware, half-open interval for an upcoming calendar evaluation.""" @@ -89,23 +20,22 @@ class CalendarWindow: @dataclass(frozen=True, kw_only=True, eq=False) -class CalendarSchedule: - """Decorate a location callback returning upcoming timezone-aware datetimes.""" +class Calendar: + """Decorate a location callback returning upcoming timezone-aware datetimes. + + Jobs and job groups name the calendar in their TOML `schedule`, which also sets the timezone. + """ - job: Job | JobGroup + name: str description: str horizon: str - timezone: str - starts_at: str | None = None - default_status: Literal["running", "paused"] = "running" - targets: tuple[str, ...] = () function: Callable[..., Iterable[datetime]] | None = field(default=None, repr=False) _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) def __post_init__(self) -> None: object.__setattr__(self, "_declared_at", declaration_site()) - def __call__(self, function: Callable[..., Iterable[datetime]]) -> CalendarSchedule: + def __call__(self, function: Callable[..., Iterable[datetime]]) -> Calendar: return replace(self, function=function) @@ -119,9 +49,12 @@ class PlatformEventCondition: @dataclass(frozen=True, kw_only=True) class RunEventCondition: - """Match scoped terminal outcomes, optionally requiring genuinely executed successful nodes.""" + """Match scoped terminal outcomes of the named job or job group. + + `executed_successfully` optionally requires nodes that genuinely executed and succeeded. + """ - job: Job | JobGroup + job: str kinds: Sequence[RunEventKind] executed_successfully: Sequence[str] | None = None kind: Literal["run_event"] = field(default="run_event", init=False) @@ -167,7 +100,7 @@ class Sensor: | SourceStaleCondition | None ) = field(default=None, kw_only=True) - run: Job | JobGroup = field(kw_only=True) + run: str = field(kw_only=True) minimum_interval_seconds: int = field(default=30, kw_only=True) default_status: Literal["running", "paused"] = field(default="running", kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) diff --git a/src/sqlbuild/orchestration_declarations/models/legacy_authoring.py b/src/sqlbuild/orchestration_declarations/models/legacy_authoring.py new file mode 100644 index 000000000..71c8bd428 --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/models/legacy_authoring.py @@ -0,0 +1,132 @@ +"""Removed authoring records, rebuilt only while `sqb migrate orchestration` imports a project.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Sequence +from dataclasses import dataclass, field, replace +from datetime import datetime +from typing import Literal + +from sqlbuild.orchestration_declarations.models.authoring import ( + NodeEventCondition, + PlatformEventCondition, + RunRequest, + SkipReason, + SourceNewDataCondition, + SourceStaleCondition, +) +from sqlbuild.orchestration_declarations.types import RunEventKind + + +@dataclass(frozen=True, kw_only=True) +class Schedule: + """Cron with an RFC 3339 start; missed times collapse into one run of the latest.""" + + cron: str + timezone: str + starts_at: str | None = None + default_status: Literal["running", "paused"] = "running" + + +@dataclass(frozen=True, kw_only=True) +class OnFailure: + """Rerun a failed or partial run from failure, `retries` times, `retry_delay_seconds` apart.""" + + retries: int + retry_delay_seconds: int = 0 + + +@dataclass(frozen=True, kw_only=True) +class Resources: + """Worker container limits for a job; memory is bytes or a size such as "512m" or "2g".""" + + memory: int | str | None = None + cpus: float | None = None + pids: int | None = None + + +@dataclass(frozen=True, eq=False) +class Job: + """A named selection to execute, optionally on a wall-clock schedule.""" + + name: str + description: str = field(kw_only=True) + select: str = field(kw_only=True) + exclude: str = field(default="", kw_only=True) + settings: dict[str, object] = field(default_factory=dict, kw_only=True) + concurrency: int | None = field(default=None, kw_only=True) + concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) + schedule: Schedule | None = field(default=None, kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) + owner: str | None = field(default=None, kw_only=True) + overlap: Literal["queue", "skip"] = field(default="queue", kw_only=True) + on_failure: OnFailure | None = field(default=None, kw_only=True) + resources: Resources | None = field(default=None, kw_only=True) + + +@dataclass(frozen=True, eq=False) +class JobGroup: + """Jobs ordered by the lineage between the nodes they build.""" + + name: str + description: str = field(kw_only=True) + jobs: Sequence[Job] = field(kw_only=True) + concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) + schedule: Schedule | None = field(default=None, kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) + owner: str | None = field(default=None, kw_only=True) + overlap: Literal["queue", "skip"] = field(default="queue", kw_only=True) + on_failure: OnFailure | None = field(default=None, kw_only=True) + + +@dataclass(frozen=True, kw_only=True, eq=False) +class CalendarSchedule: + """Decorate a location callback returning upcoming timezone-aware datetimes.""" + + job: Job | JobGroup + description: str + horizon: str + timezone: str + starts_at: str | None = None + default_status: Literal["running", "paused"] = "running" + targets: tuple[str, ...] = () + function: Callable[..., Iterable[datetime]] | None = field(default=None, repr=False) + + def __call__(self, function: Callable[..., Iterable[datetime]]) -> CalendarSchedule: + return replace(self, function=function) + + +@dataclass(frozen=True, kw_only=True) +class RunEventCondition: + """Match scoped terminal outcomes, optionally requiring genuinely executed successful nodes.""" + + job: Job | JobGroup + kinds: Sequence[RunEventKind] + executed_successfully: Sequence[str] | None = None + kind: Literal["run_event"] = field(default="run_event", init=False) + + +@dataclass(frozen=True, eq=False) +class Sensor: + """Start a job for each matching event; event identity deduplicates replayed requests.""" + + name: str + description: str = field(kw_only=True) + when: ( + PlatformEventCondition + | RunEventCondition + | NodeEventCondition + | SourceNewDataCondition + | SourceStaleCondition + | None + ) = field(default=None, kw_only=True) + run: Job | JobGroup = field(kw_only=True) + minimum_interval_seconds: int = field(default=30, kw_only=True) + default_status: Literal["running", "paused"] = field(default="running", kw_only=True) + targets: tuple[str, ...] = field(default=(), kw_only=True) + function: Callable[..., Iterable[RunRequest] | SkipReason] | None = field( + default=None, repr=False, kw_only=True + ) + + def __call__(self, function: Callable[..., Iterable[RunRequest] | SkipReason]) -> Sensor: + return replace(self, function=function) From d38d482d9bcb7c00a5bf871cc648f3e312017d35 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:10:33 +0000 Subject: [PATCH 20/34] feat: read orchestration/renames.toml, check it at compile and publish it with the snapshot The renames file maps old job, group, sensor and trigger names to new ones. Compile refuses a new name that is not declared, an old name that still is, a chain, and two names mapped onto one, at the file position. Publication carries the map as the snapshot's renames document. Sensor targets written as literals are read statically for the job catalog. --- .../_helpers/python_modules.rs | 4 +- .../_helpers/references.rs | 76 +++++++++++++++++++ .../src/orchestration_files/constants.rs | 4 + .../src/orchestration_files/main/mod.rs | 1 + .../main/orchestration_inventory.rs | 14 +++- .../main/read_renames_file.rs | 61 +++++++++++++++ .../src/orchestration_files/models.rs | 36 +++++++++ .../tests/test_orchestration_inventory.rs | 64 +++++++++++++++- .../compiled_project/snapshot_declarations.rs | 19 ++++- .../_helpers/location_command/snapshot.rs | 2 +- .../native_host/orchestration_sensors.rs | 4 +- .../native_host/orchestration_triggers.rs | 21 ++--- 12 files changed, 284 insertions(+), 22 deletions(-) create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/main/read_renames_file.rs diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs index c650e520f..cd45afe62 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs @@ -139,9 +139,7 @@ pub(crate) fn module_declaration( let job_references = declaration .job_references .into_iter() - .map(|(argument, value)| { - Ok((argument, literal(value, &format!("sensor {argument} job"))?)) - }) + .map(|(argument, value)| Ok((argument, literal(value, &format!("sensor {argument} job"))?))) .collect::, _>>()?; Ok(Some(NamedModule { name: name.value, diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs index acd076023..9adb93c1b 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs @@ -151,3 +151,79 @@ pub(crate) fn check_references(inventory: &OrchestrationInventory) -> Result<(), } Ok(()) } + +/// Each renames-file section maps old names that no longer exist onto names declared now. +pub(crate) fn check_renames(inventory: &OrchestrationInventory) -> Result<(), ReferenceError> { + let Some((path, renames)) = &inventory.renames else { + return Ok(()); + }; + let runs: BTreeSet<&str> = inventory + .jobs + .iter() + .map(|(_, job)| &job.run) + .chain(inventory.groups.iter().map(|(_, group)| &group.run)) + .map(|run| run.name.value.as_str()) + .collect(); + let scheduled: BTreeSet<&str> = inventory + .jobs + .iter() + .map(|(_, job)| &job.run) + .chain(inventory.groups.iter().map(|(_, group)| &group.run)) + .filter(|run| run.schedule.is_some()) + .map(|run| run.name.value.as_str()) + .collect(); + let sensors: BTreeSet<&str> = inventory + .sensors + .iter() + .map(|module| module.name.as_str()) + .collect(); + let names = inventory.names(); + let declared: BTreeSet<&str> = names.iter().map(|fact| fact.name.as_str()).collect(); + for (section, pairs, known, wanted) in [ + ("jobs", &renames.jobs, &runs, "a declared job or job group"), + ("sensors", &renames.sensors, &sensors, "a declared sensor"), + ( + "triggers", + &renames.triggers, + &scheduled, + "a job or job group with a schedule", + ), + ] { + for (old, new) in pairs { + if !known.contains(new.value.as_str()) { + return Err(( + path.clone(), + at( + new.position, + format!( + "[{section}] renames '{}' to '{}', which is not {wanted}", + old.value, new.value + ), + None, + ), + )); + } + // A trigger's old name only has to be free among triggers: it was a calendar function's name. + let taken: &BTreeSet<&str> = if section == "triggers" { + &scheduled + } else { + &declared + }; + let old_declared: bool = taken.contains(old.value.as_str()); + if old_declared { + return Err(( + path.clone(), + at( + old.position, + format!( + "[{section}] renames '{}', which is still declared", + old.value + ), + Some("remove the pair, or finish the rename in the declaration".to_owned()), + ), + )); + } + } + } + Ok(()) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs index 2d256b1e7..d010b1071 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs @@ -55,3 +55,7 @@ pub(crate) const BYTECODE_DIRECTORY: &str = "__pycache__"; /// The prefix of hidden files and folders, and of private support modules. pub(crate) const HIDDEN_PREFIX: char = '.'; pub(crate) const PRIVATE_PREFIX: char = '_'; +/// The file `sqb migrate orchestration` writes in `orchestration/`, mapping old names to new. +pub const RENAMES_FILE: &str = "renames.toml"; +/// The sections of the renames file: jobs and groups, sensors, and schedule triggers. +pub const RENAME_SECTIONS: [&str; 3] = ["jobs", "sensors", "triggers"]; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs index 619ee67d4..38f0a2c48 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs @@ -1,3 +1,4 @@ pub mod orchestration_inventory; pub mod read_group_file; pub mod read_job_file; +pub mod read_renames_file; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs index 2d2936444..094309be1 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs @@ -6,13 +6,14 @@ use crate::_helpers::reading::read_authored_text; use crate::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; use crate::models::{DiscoveryFailure, FailureKind}; use crate::orchestration_files::_helpers::python_modules::module_declaration; -use crate::orchestration_files::_helpers::references::check_references; +use crate::orchestration_files::_helpers::references::{check_references, check_renames}; use crate::orchestration_files::constants::{ BYTECODE_DIRECTORY, CALENDARS_DIRECTORY, GROUPS_DIRECTORY, HIDDEN_PREFIX, JOBS_DIRECTORY, - SENSORS_DIRECTORY, TOML_FILE_SUFFIX, + RENAMES_FILE, SENSORS_DIRECTORY, TOML_FILE_SUFFIX, }; use crate::orchestration_files::main::read_group_file::read_group_file; use crate::orchestration_files::main::read_job_file::read_job_file; +use crate::orchestration_files::main::read_renames_file::read_renames_file; use crate::orchestration_files::models::{ FileError, FilePosition, OrchestrationFailure, OrchestrationInventory, }; @@ -112,10 +113,16 @@ pub fn orchestration_inventory( if ignored(&entry) { continue; } + if !entry.is_dir && entry.name == RENAMES_FILE { + let renames = + read_renames_file(&read(tree, &path)?).map_err(|error| failure(&path, error))?; + inventory.renames = Some((path, renames)); + continue; + } if !entry.is_dir { return Err(layout( &path, - "files cannot sit directly in orchestration/".to_owned(), + format!("files cannot sit directly in orchestration/, except {RENAMES_FILE}"), Some(format!( "move it into one of orchestration/{}/", ROLE_DIRECTORIES.join("/, orchestration/") @@ -161,5 +168,6 @@ pub fn orchestration_inventory( } } check_references(&inventory).map_err(|(path, error)| failure(&path, error))?; + check_renames(&inventory).map_err(|(path, error)| failure(&path, error))?; Ok(inventory) } diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/read_renames_file.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/read_renames_file.rs new file mode 100644 index 000000000..94e757ade --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/read_renames_file.rs @@ -0,0 +1,61 @@ +//! Read the renames file `sqb migrate orchestration` writes. + +use std::collections::BTreeMap; + +use toml::Spanned; + +use crate::orchestration_files::_helpers::reading::{check_keys, error_at, nonblank, toml_error}; +use crate::orchestration_files::constants::RENAME_SECTIONS; +use crate::orchestration_files::models::{FileError, RenameMap, RenamePairs}; + +/// Each section's old and new names as written, or the first problem with the file. +pub fn read_renames_file(text: &str) -> Result { + check_keys(text, &RENAME_SECTIONS, "renames file")?; + let sections: BTreeMap, Spanned>> = + toml::from_str(text).map_err(|error| toml_error(text, &error))?; + let mut map = RenameMap::default(); + for (section, pairs) in sections { + let mut read: RenamePairs = Vec::new(); + let mut targets: BTreeMap = BTreeMap::new(); + let mut pairs: Vec<(Spanned, Spanned)> = pairs.into_iter().collect(); + pairs.sort_by_key(|(old, _)| old.span().start); + for (old, new) in pairs { + let old = nonblank(text, old, "old name")?; + let new_start: usize = new.span().start; + let new = nonblank(text, new, "new name")?; + if old.value == new.value { + return Err(error_at( + text, + new_start, + format!("'{}' is renamed to itself", old.value), + )); + } + if targets.insert(new.value.clone(), read.len()).is_some() { + return Err(error_at( + text, + new_start, + format!("two names are renamed to '{}'", new.value), + )); + } + read.push((old, new)); + } + for (_, new) in &read { + if let Some((old, _)) = read.iter().find(|(old, _)| old.value == new.value) { + return Err(FileError { + message: format!( + "'{}' is both a new name and an old name; renames cannot chain", + old.value + ), + position: old.position, + help: Some("map each old name straight to its final name".to_owned()), + }); + } + } + match section.as_str() { + "jobs" => map.jobs = read, + "sensors" => map.sensors = read, + _ => map.triggers = read, + } + } + Ok(map) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/models.rs b/crates/sqlbuild-discovery/src/orchestration_files/models.rs index 95f4b3990..0343ec63c 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/models.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/models.rs @@ -115,6 +115,40 @@ pub struct NamedModule { pub targets: Option>, } +/// One section of the renames file: each old name and the name it became, where each is written. +pub type RenamePairs = Vec<(Positioned, Positioned)>; + +/// The renames file: old names of jobs and groups, sensors, and schedule triggers, and their new names. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RenameMap { + pub jobs: RenamePairs, + pub sensors: RenamePairs, + pub triggers: RenamePairs, +} + +impl RenameMap { + /// The map as the snapshot document the server applies: `{section: {old: new}}`. + #[must_use] + pub fn document(&self) -> serde_json::Value { + let section = |pairs: &RenamePairs| { + pairs + .iter() + .map(|(old, new)| { + ( + old.value.clone(), + serde_json::Value::String(new.value.clone()), + ) + }) + .collect::>() + }; + serde_json::json!({ + "jobs": section(&self.jobs), + "sensors": section(&self.sensors), + "triggers": section(&self.triggers), + }) + } +} + /// Every orchestration declaration in a project, checked for layout, names and references. #[derive(Clone, Debug, Default, PartialEq)] pub struct OrchestrationInventory { @@ -124,6 +158,8 @@ pub struct OrchestrationInventory { pub sensors: Vec, /// Every Python module under `calendars/` and `sensors/`, which publication imports. pub modules: Vec, + /// The renames file and its path, when the project has one. + pub renames: Option<(String, RenameMap)>, } impl OrchestrationInventory { diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs index 1df3aa263..cffb1fbb3 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs @@ -116,7 +116,7 @@ fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_ cases.push(InventoryErrorCase { description: "loose file", files: with("orchestration/jobs.py", "x = 1\n"), - expected_message: "orchestration/jobs.py:1:1: files cannot sit directly in orchestration/", + expected_message: "orchestration/jobs.py:1:1: files cannot sit directly in orchestration/, except renames.toml", expected_help: Some("move it into one of orchestration/jobs/, orchestration/job_groups/, orchestration/calendars/, orchestration/sensors/"), }); cases.push(InventoryErrorCase { @@ -188,6 +188,47 @@ fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_ expected_message: "orchestration/sensors/inventory/stock.py:1:36: `job` was removed from sqlbuild.orchestration; jobs, job groups and schedules are declared in TOML", expected_help: Some("run `sqb migrate orchestration` to move Python jobs, job groups and schedules into TOML"), }); + let renamed = |text: &'static str| { + let mut files = valid(); + files.push(("orchestration/renames.toml", text)); + files + }; + cases.push(InventoryErrorCase { + description: "rename to an undeclared job", + files: renamed("[jobs]\norders_export = \"orders_exports_job\"\n"), + expected_message: "orchestration/renames.toml:2:17: [jobs] renames 'orders_export' to 'orders_exports_job', which is not a declared job or job group", + expected_help: None, + }); + cases.push(InventoryErrorCase { + description: "old name still declared", + files: renamed("[jobs]\norders_export_job = \"orders_nightly_job_group\"\n"), + expected_message: "orchestration/renames.toml:2:1: [jobs] renames 'orders_export_job', which is still declared", + expected_help: Some("remove the pair, or finish the rename in the declaration"), + }); + cases.push(InventoryErrorCase { + description: "chained renames", + files: renamed("[sensors]\nstock = \"stock_stale\"\nstock_stale = \"stock_stale_sensor\"\n"), + expected_message: "orchestration/renames.toml:3:1: 'stock_stale' is both a new name and an old name; renames cannot chain", + expected_help: Some("map each old name straight to its final name"), + }); + cases.push(InventoryErrorCase { + description: "two names onto one", + files: renamed("[jobs]\norders = \"orders_export_job\"\nexport = \"orders_export_job\"\n"), + expected_message: "orchestration/renames.toml:3:10: two names are renamed to 'orders_export_job'", + expected_help: None, + }); + cases.push(InventoryErrorCase { + description: "trigger onto an unscheduled run", + files: renamed("[triggers]\ntrading_days = \"orders_nightly_job_group\"\n"), + expected_message: "orchestration/renames.toml:2:16: [triggers] renames 'trading_days' to 'orders_nightly_job_group', which is not a job or job group with a schedule", + expected_help: None, + }); + cases.push(InventoryErrorCase { + description: "unknown renames section", + files: renamed("[groups]\nnightly = \"orders_nightly_job_group\"\n"), + expected_message: "orchestration/renames.toml:1:2: unknown renames file key 'groups'", + expected_help: Some("supported keys: jobs, sensors, triggers"), + }); for case in cases { let (actual, actual_help) = message(&case.files); assert_eq!(actual, case.expected_message, "{}", case.description); @@ -199,3 +240,24 @@ fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_ ); } } + +#[test] +fn given_renames_file_when_inventorying_then_reads_each_section_as_the_server_document() { + let mut files = valid(); + files.push(( + "orchestration/renames.toml", + "[jobs]\norders_export = \"orders_export_job\"\nnightly = \"orders_nightly_job_group\"\n\n[sensors]\nstock = \"stock_stale_sensor\"\n\n[triggers]\ntrading_days = \"orders_export_job\"\n", + )); + let directory = project(&files); + let found = inventory(directory.path()).expect("valid renames"); + let (path, renames) = found.renames.expect("renames read"); + assert_eq!(path, "orchestration/renames.toml"); + assert_eq!( + renames.document(), + serde_json::json!({ + "jobs": {"orders_export": "orders_export_job", "nightly": "orders_nightly_job_group"}, + "sensors": {"stock": "stock_stale_sensor"}, + "triggers": {"trading_days": "orders_export_job"}, + }) + ); +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index c031cf77f..1017eddfc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -21,6 +21,9 @@ use sqlbuild_orchestration::declarations::models::{ use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; + +/// The snapshot document carrying the renames file, which the server applies to stored state. +const RENAMES_DOCUMENT: &str = "renames"; use crate::bindings::_helpers::native_host::constants::{AUTHORING_MODULE, SENSOR_CLASS}; use crate::bindings::_helpers::native_host::orchestration_declarations::{ AuthoredSensor, module_declarations, run_controls, run_resources, @@ -142,7 +145,10 @@ pub(crate) fn compile_declarations( authored_sensors.push(declaration.extract()?); } else { let name: String = declaration.getattr("name")?.extract()?; - if authored_calendars.insert(name.clone(), declaration).is_some() { + if authored_calendars + .insert(name.clone(), declaration) + .is_some() + { return Err(PyValueError::new_err(format!( "calendar '{name}' is declared twice" ))); @@ -166,7 +172,11 @@ pub(crate) fn compile_declarations( .map(|sensor| ("sensor", sensor.name.clone(), sensor.description.clone())), ); for (name, calendar) in &authored_calendars { - described.push(("calendar", name.clone(), calendar.getattr("description")?.extract()?)); + described.push(( + "calendar", + name.clone(), + calendar.getattr("description")?.extract()?, + )); } for (kind, name, description) in described { let limit: i64 = format.max_description_lines; @@ -315,6 +325,11 @@ pub(crate) fn compile_declarations( document.map_err(|error| PyValueError::new_err(error.to_string()))?, ); } + if let Some((_, renames)) = &inventory.renames { + input + .documents + .insert(RENAMES_DOCUMENT.into(), renames.document()); + } snapshot_definitions(&input).map_err(|error| PyValueError::new_err(error.to_string()))?; Ok(input) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 99b9680f8..8a2aff78a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -23,12 +23,12 @@ use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; use crate::bindings::_helpers::compiled_project::snapshot_roles::compile_project_roles; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; use crate::bindings::_helpers::discovery::project_facts::{named_node_rows, project_facts}; use crate::bindings::_helpers::discovery::project_validation::validation_error; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; -use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index da32ffc25..edb486b35 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -2,8 +2,8 @@ use std::path::Path; -use pyo3::exceptions::PyValueError; use pyo3::IntoPyObject; +use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use sqlbuild_host::entries::models::UserCodeHost; @@ -13,9 +13,9 @@ use sqlbuild_orchestration::sensors::models::{ use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::compiled_project::snapshot_declarations::python_declarations; -use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; use crate::bindings::_helpers::native_host::constants::SENSOR_CONTEXT_PARAMETER; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_environment::{ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index cd03b0e3a..a76334202 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -183,15 +183,14 @@ pub(crate) fn compile_calendars( request.calendar )) })?; - let schedule = calendar_times( - (calendar, &request.timezone), - &session, - &by_name, - start, - ) - .map_err(|error| { - PyValueError::new_err(format!("calendar '{}': {error}", request.calendar)) - })?; + let schedule = + calendar_times((calendar, &request.timezone), &session, &by_name, start) + .map_err(|error| { + PyValueError::new_err(format!( + "calendar '{}': {error}", + request.calendar + )) + })?; evaluated.insert(key, schedule.clone()); schedule } @@ -201,7 +200,9 @@ pub(crate) fn compile_calendars( ..request.trigger }; trigger.starts_at.get_or_insert(start); - definitions.triggers.insert(trigger.job_key.clone(), trigger); + definitions + .triggers + .insert(trigger.job_key.clone(), trigger); } Ok(definitions) }) From cf178b54a490991c04f04631055aa6520735d673 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:19:58 +0000 Subject: [PATCH 21/34] feat: migrate Python jobs, job groups and schedules to TOML The migration imports each orchestration module once with the removed records restored, so jobs built in loops or from constants migrate too. It appends missing name suffixes and prints each rename, writes one TOML file per job and group, writes renames.toml so stored history follows the new names, deletes modules that held only jobs, and prints the exact calendar and sensor edits. --- .../_helpers/native_host/constants.rs | 6 +- .../main/migrate_orchestration.py | 435 ++++++++++++++++++ .../models/legacy_authoring.py | 17 + 3 files changed, 457 insertions(+), 1 deletion(-) create mode 100644 src/sqlbuild/orchestration_declarations/main/migrate_orchestration.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index 2f12728a8..e284bd940 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -10,8 +10,12 @@ pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; pub(crate) const SENSOR_CONTEXT_PARAMETER: &str = "ctx"; pub(crate) const AUTHORING_MODULE: &str = "sqlbuild.orchestration_declarations.models.authoring"; +/// The removed records `sqb migrate orchestration` rebuilds while it imports a project. +pub(crate) const LEGACY_AUTHORING_MODULE: &str = + "sqlbuild.orchestration_declarations.models.legacy_authoring"; /// Frames of these modules construct declarations on the author's behalf. -pub(crate) const CONSTRUCTING_MODULES: [&str; 2] = [AUTHORING_MODULE, "dataclasses"]; +pub(crate) const CONSTRUCTING_MODULES: [&str; 3] = + [AUTHORING_MODULE, LEGACY_AUTHORING_MODULE, "dataclasses"]; /// The authoring classes publication imports calendar and sensor modules for. pub(crate) const CALENDAR_CLASS: &str = "Calendar"; diff --git a/src/sqlbuild/orchestration_declarations/main/migrate_orchestration.py b/src/sqlbuild/orchestration_declarations/main/migrate_orchestration.py new file mode 100644 index 000000000..bf73166ab --- /dev/null +++ b/src/sqlbuild/orchestration_declarations/main/migrate_orchestration.py @@ -0,0 +1,435 @@ +"""`sqb migrate orchestration`: rewrite Python jobs, job groups and schedules as TOML, once. + +Jobs built in loops or from constants exist only once their module runs, so this imports each +orchestration module with the removed records restored, then writes what it found. +""" + +from __future__ import annotations + +import ast +import json +import re +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path + +from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( + load_orchestration_module, +) +from sqlbuild.orchestration_declarations.models import legacy_authoring as legacy + +_ROOT = "orchestration" +_RENAMES = "renames.toml" +_REMOVED = { + "job": legacy.Job, + "job_group": legacy.JobGroup, + "schedule": legacy.Schedule, + "calendar_schedule": legacy.CalendarSchedule, + "on_failure": legacy.OnFailure, + "resources": legacy.Resources, + "sensor": legacy.Sensor, + "run_event": legacy.RunEventCondition, +} +_JOB_PARTS = (legacy.Job, legacy.JobGroup, legacy.Schedule, legacy.OnFailure, legacy.Resources) +_BARE_KEY = re.compile(r"^[A-Za-z0-9_-]+$") + + +class MigrationError(Exception): + """A declaration the migration cannot write without a person deciding.""" + + +@dataclass +class _Found: + """The legacy declarations of a project and the module each was found in first.""" + + modules: dict[int, Path] = field(default_factory=dict) + jobs: list[legacy.Job] = field(default_factory=list) + groups: list[legacy.JobGroup] = field(default_factory=list) + calendars: list[legacy.CalendarSchedule] = field(default_factory=list) + sensors: list[legacy.Sensor] = field(default_factory=list) + + +@contextmanager +def _removed_api() -> Iterator[None]: + import sqlbuild.orchestration as public + + missing = object() + saved = {name: getattr(public, name, missing) for name in _REMOVED} + for name, value in _REMOVED.items(): + setattr(public, name, value) + try: + yield + finally: + for name, value in saved.items(): + if value is missing: + delattr(public, name) + else: + setattr(public, name, value) + + +def _collect(value: object, module: Path, found: _Found) -> None: + pending = [value] + visited: set[int] = set() + while pending: + item = pending.pop() + if isinstance(item, list | tuple): + if id(item) not in visited: + visited.add(id(item)) + pending.extend(item) + continue + kinds = ( + (legacy.Job, found.jobs), + (legacy.JobGroup, found.groups), + (legacy.CalendarSchedule, found.calendars), + (legacy.Sensor, found.sensors), + ) + for kind, bucket in kinds: + if isinstance(item, kind) and id(item) not in found.modules: + found.modules[id(item)] = module + bucket.append(item) + if isinstance(item, legacy.JobGroup): + pending.extend(item.jobs) + + +def _suffixed(name: str, suffix: str) -> str: + return name if name.endswith(suffix) and len(name) > len(suffix) else f"{name}{suffix}" + + +def _value(value: object) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int | float): + return repr(value) + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False) + if isinstance(value, list | tuple): + return "[" + ", ".join(_value(item) for item in value) + "]" + if isinstance(value, dict): + items = [f"{_key(key)} = {_value(item)}" for key, item in value.items() if item is not None] + return "{ " + ", ".join(items) + " }" if items else "{}" + raise MigrationError(f"cannot write {value!r} as TOML") + + +def _key(key: object) -> str: + text = str(key) + return text if _BARE_KEY.match(text) else json.dumps(text, ensure_ascii=False) + + +def _schedule( + run: legacy.Job | legacy.JobGroup, calendar: tuple[str, legacy.CalendarSchedule] | None +) -> dict[str, object] | None: + if run.schedule is not None and calendar is not None: + raise MigrationError( + f"'{run.name}' has both a cron schedule and the calendar schedule " + f"'{calendar[0]}'; a job or job group now has one schedule, so remove one first" + ) + written: dict[str, object] + source: legacy.Schedule | legacy.CalendarSchedule + if run.schedule is not None: + source = run.schedule + written = {"cron": source.cron} + elif calendar is not None: + source = calendar[1] + written = {"calendar": calendar[0]} + else: + return None + written["timezone"] = source.timezone + if source.starts_at is not None: + written["starts_at"] = source.starts_at + if source.default_status != "running": + written["default_status"] = source.default_status + return written + + +def _run_lines(run: legacy.Job | legacy.JobGroup, name: str) -> list[str]: + return [f"name = {_value(name)}", f"description = {_value(run.description)}"] + + +def _common( + run: legacy.Job | legacy.JobGroup, calendar: tuple[str, legacy.CalendarSchedule] | None +) -> tuple[list[str], list[str]]: + lines: list[str] = [] + if run.targets: + lines.append(f"targets = {_value(list(run.targets))}") + if run.owner is not None: + lines.append(f"owner = {_value(run.owner)}") + if run.overlap != "queue": + lines.append(f"overlap = {_value(run.overlap)}") + if run.concurrency_groups: + lines.append(f"concurrency_groups = {_value(list(run.concurrency_groups))}") + schedule = _schedule(run, calendar) + if schedule is not None: + lines.append(f"schedule = {_value(schedule)}") + tables: list[str] = [] + if run.on_failure is not None: + tables += ["", "[on_failure]", f"retries = {_value(run.on_failure.retries)}"] + if run.on_failure.retry_delay_seconds: + tables.append(f"retry_delay_seconds = {_value(run.on_failure.retry_delay_seconds)}") + return lines, tables + + +def _job_toml( + job: legacy.Job, name: str, calendar: tuple[str, legacy.CalendarSchedule] | None +) -> str: + lines = _run_lines(job, name) + lines.append(f"select = {_value(job.select)}") + if job.exclude.strip(): + lines.append(f"exclude = {_value(job.exclude)}") + common, tables = _common(job, calendar) + lines += common + tables + if job.resources is not None: + limits = { + key: value + for key, value in ( + ("memory", job.resources.memory), + ("cpus", job.resources.cpus), + ("pids", job.resources.pids), + ) + if value is not None + } + lines += ["", "[resources]"] + [f"{key} = {_value(value)}" for key, value in limits.items()] + settings = dict(job.settings) + if job.concurrency is not None: + if "concurrency" in settings: + raise MigrationError(f"job '{job.name}' declares concurrency twice") + settings["concurrency"] = job.concurrency + if settings: + lines += ["", "[settings]"] + [ + f"{_key(key)} = {_value(value)}" for key, value in settings.items() if value is not None + ] + return "\n".join(lines) + "\n" + + +def _group_toml( + group: legacy.JobGroup, + name: str, + members: list[str], + calendar: tuple[str, legacy.CalendarSchedule] | None, +) -> str: + lines = _run_lines(group, name) + lines.append(f"jobs = {_value(members)}") + common, tables = _common(group, calendar) + return "\n".join(lines + common + tables) + "\n" + + +def _job_parts_only(module: Path, project_dir: Path) -> bool: + """Whether a module holds nothing but imports, a docstring and job, group or schedule values.""" + tree = ast.parse(module.read_text(encoding="utf-8")) + loaded = sys.modules.get(_module_name(module, project_dir)) + for statement in tree.body: + if isinstance(statement, ast.Import | ast.ImportFrom): + continue + if isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Constant): + continue + targets = ( + statement.targets + if isinstance(statement, ast.Assign) + else [statement.target] + if isinstance(statement, ast.AnnAssign) + else None + ) + if targets is None or loaded is None: + return False + for target in targets: + if not isinstance(target, ast.Name): + return False + if target.id == "__all__": + continue + value = getattr(loaded, target.id, None) + values = value if isinstance(value, list | tuple) else [value] + if not values or not all(isinstance(item, _JOB_PARTS) for item in values): + return False + return True + + +def _module_name(module: Path, project_dir: Path) -> str: + return ".".join(module.relative_to(project_dir).with_suffix("").parts) + + +def _site(declaration: object, fallback: Path, project_dir: Path) -> str: + site = getattr(declaration, "_declared_at", None) + if site is None: + return fallback.relative_to(project_dir).as_posix() + path = Path(site[0]) + try: + path = path.relative_to(project_dir.resolve()) + except ValueError: + pass + return f"{path.as_posix()}:{site[1]}" + + +def migrate_orchestration(*, project_dir: Path, dry_run: bool) -> int: + """Write TOML for each Python job and job group, print every rename and manual edit.""" + try: + return _migrate(project_dir, dry_run) + except MigrationError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + +def _migrate(project_dir: Path, dry_run: bool) -> int: + root = project_dir / _ROOT + if not root.is_dir(): + print("No orchestration/ folder; nothing to migrate.") + return 0 + if (root / _RENAMES).exists(): + raise MigrationError( + f"{_ROOT}/{_RENAMES} already exists; publish every location with it, delete it, " + "then run the migration again" + ) + found = _Found() + modules = sorted( + path + for path in root.rglob("*.py") + if "__pycache__" not in path.parts and path.name != "__init__.py" + ) + with _removed_api(): + for module in modules: + loaded = load_orchestration_module(file_path=module, project_dir=project_dir) + for value in vars(loaded).values(): + _collect(value, module, found) + if not (found.jobs or found.groups or found.calendars or found.sensors): + print("No Python jobs, job groups or schedules found; nothing to migrate.") + return 0 + names: dict[int, str] = {} + renames: dict[str, dict[str, str]] = {"jobs": {}, "sensors": {}, "triggers": {}} + printed: list[str] = [] + for runs, suffix, kind, section in ( + (found.jobs, "_job", "job", "jobs"), + (found.groups, "_job_group", "job group", "jobs"), + (found.sensors, "_sensor", "sensor", "sensors"), + ): + for run in runs: + new = _suffixed(run.name, suffix) + names[id(run)] = new + if new != run.name: + renames[section][run.name] = new + printed.append(f"Renamed {kind} {run.name} -> {new}") + calendars: dict[int, tuple[str, legacy.CalendarSchedule]] = {} + for schedule in found.calendars: + if schedule.function is None: + raise MigrationError("a calendar_schedule does not decorate a function") + old = schedule.function.__name__ + new = _suffixed(old, "_calendar") + if id(schedule.job) in calendars: + raise MigrationError( + f"'{schedule.job.name}' has two calendar schedules; keep one before migrating" + ) + calendars[id(schedule.job)] = (new, schedule) + if new != old: + printed.append(f"Renamed calendar {old} -> {new}") + job_name = names.get(id(schedule.job), schedule.job.name) + if old != job_name: + renames["triggers"][old] = job_name + taken: set[str] = set() + for new in [*names.values(), *(name for name, _ in calendars.values())]: + if new in taken: + raise MigrationError(f"two declarations would both be named '{new}'") + taken.add(new) + writes: list[tuple[Path, str]] = [] + for job in found.jobs: + relative = found.modules[id(job)].relative_to(root).with_suffix("") + writes.append( + ( + root / "jobs" / relative / f"{names[id(job)]}.toml", + _job_toml(job, names[id(job)], calendars.get(id(job))), + ) + ) + for group in found.groups: + relative = found.modules[id(group)].relative_to(root).with_suffix("") + members = [names[id(member)] for member in group.jobs] + writes.append( + ( + root / "job_groups" / relative / f"{names[id(group)]}.toml", + _group_toml(group, names[id(group)], members, calendars.get(id(group))), + ) + ) + if any(renames.values()): + sections = [ + f"[{section}]\n" + + "".join(f"{_key(old)} = {_value(new)}\n" for old, new in pairs.items()) + for section, pairs in renames.items() + if pairs + ] + writes.append((root / _RENAMES, "\n".join(sections))) + for path, _ in writes: + if path.exists(): + raise MigrationError(f"{path.relative_to(project_dir).as_posix()} already exists") + holding_others = {found.modules[id(item)] for item in [*found.calendars, *found.sensors]} + deletions = [ + module + for module in sorted({found.modules[id(run)] for run in [*found.jobs, *found.groups]}) + if module not in holding_others and _job_parts_only(module, project_dir) + ] + manual = _manual_edits(found, (names, calendars), deletions, project_dir) + verb = "Would write" if dry_run else "Wrote" + for line in printed: + print(line) + for path, text in writes: + if not dry_run: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + print(f"{verb} {path.relative_to(project_dir).as_posix()}") + for module in deletions: + if not dry_run: + module.unlink() + deleted: str = module.relative_to(project_dir).as_posix() + print(f"{'Would delete' if dry_run else 'Deleted'} {deleted}") + if manual: + print("Edit by hand:") + for line in manual: + print(f" {line}") + if any(renames.values()): + print( + f"{_ROOT}/{_RENAMES} keeps run history, pause and sensor state under the new names. " + "Publish every location, then delete it." + ) + return 0 + + +def _manual_edits( + found: _Found, + named: tuple[dict[int, str], dict[int, tuple[str, legacy.CalendarSchedule]]], + deletions: list[Path], + project_dir: Path, +) -> list[str]: + """The edits a person makes: leftover job declarations, calendars and sensors.""" + names, calendars = named + manual: list[str] = [] + for module in sorted({found.modules[id(run)] for run in [*found.jobs, *found.groups]}): + if module not in deletions: + manual.append( + f"{module.relative_to(project_dir).as_posix()}: delete the job, job group and " + "schedule declarations; they are TOML now" + ) + for new, schedule in calendars.values(): + function_name: str = getattr(schedule.function, "__name__", "") + targets = ( + f" Its targets {list(schedule.targets)} are dropped: a calendar runs wherever its" + " job does." + if schedule.targets + else "" + ) + manual.append( + f"{_site(schedule, found.modules[id(schedule)], project_dir)}: replace the " + f"calendar_schedule(...) decorator of {function_name} with " + f"@calendar(name={_value(new)}, description={_value(schedule.description)}, " + f"horizon={_value(schedule.horizon)}), import calendar from sqlbuild.orchestration, " + f"and move it to {_ROOT}/calendars/ in a module of its own.{targets}" + ) + for sensor in found.sensors: + edits = [ + f"name={_value(names[id(sensor)])}", + f"run={_value(names.get(id(sensor.run), sensor.run.name))}", + ] + if isinstance(sensor.when, legacy.RunEventCondition): + watched = names.get(id(sensor.when.job), sensor.when.job.name) + edits.append(f"run_event(job={_value(watched)}, ...)") + manual.append( + f"{_site(sensor, found.modules[id(sensor)], project_dir)}: in sensor " + f"'{sensor.name}', write {', '.join(edits)}, drop the job imports, and move it to " + f"{_ROOT}/sensors/ in a module of its own" + ) + return manual diff --git a/src/sqlbuild/orchestration_declarations/models/legacy_authoring.py b/src/sqlbuild/orchestration_declarations/models/legacy_authoring.py index 71c8bd428..79f3dc103 100644 --- a/src/sqlbuild/orchestration_declarations/models/legacy_authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/legacy_authoring.py @@ -7,6 +7,7 @@ from datetime import datetime from typing import Literal +from sqlbuild._native import declaration_site from sqlbuild.orchestration_declarations.models.authoring import ( NodeEventCondition, PlatformEventCondition, @@ -62,6 +63,10 @@ class Job: overlap: Literal["queue", "skip"] = field(default="queue", kw_only=True) on_failure: OnFailure | None = field(default=None, kw_only=True) resources: Resources | None = field(default=None, kw_only=True) + _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "_declared_at", declaration_site()) @dataclass(frozen=True, eq=False) @@ -77,6 +82,10 @@ class JobGroup: owner: str | None = field(default=None, kw_only=True) overlap: Literal["queue", "skip"] = field(default="queue", kw_only=True) on_failure: OnFailure | None = field(default=None, kw_only=True) + _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "_declared_at", declaration_site()) @dataclass(frozen=True, kw_only=True, eq=False) @@ -91,6 +100,10 @@ class CalendarSchedule: default_status: Literal["running", "paused"] = "running" targets: tuple[str, ...] = () function: Callable[..., Iterable[datetime]] | None = field(default=None, repr=False) + _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "_declared_at", declaration_site()) def __call__(self, function: Callable[..., Iterable[datetime]]) -> CalendarSchedule: return replace(self, function=function) @@ -127,6 +140,10 @@ class Sensor: function: Callable[..., Iterable[RunRequest] | SkipReason] | None = field( default=None, repr=False, kw_only=True ) + _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "_declared_at", declaration_site()) def __call__(self, function: Callable[..., Iterable[RunRequest] | SkipReason]) -> Sensor: return replace(self, function=function) From 56fb62ae87913c59b8db4d9cb60a5e01f275da36 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:35:09 +0000 Subject: [PATCH 22/34] feat: add sqb migrate orchestration and point old orchestration layouts at it Wire the migrate command through the CLI table, dispatch and bindings. Loose Python files, unknown folders holding Python, and Python files in jobs/ or job_groups/ now fail with help to run the migration. A test keeps the removed job records reachable only from the migration. --- .../sqlbuild-cli/src/argv/command_table.json | 20 ++++++ crates/sqlbuild-cli/src/argv/constants.rs | 3 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 15 ++++ .../src/dispatch/main/command_dispatch.rs | 7 +- .../src/command_connection/constants.rs | 3 +- .../main/orchestration_inventory.rs | 27 ++++++-- .../tests/test_orchestration_inventory.rs | 18 +++++ .../src/bindings/_helpers/cli/dispatch.rs | 3 +- .../bindings/_helpers/records/constants.rs | 2 + .../_helpers/workspace_commands/commands.rs | 29 ++++++++ python-boundary.toml | 12 +++- src/sqlbuild/_native.pyi | 4 ++ .../orchestration_declarations/_test_types.py | 11 +++ .../test_migration_shim_boundary.py | 68 +++++++++++++++++++ 14 files changed, 209 insertions(+), 13 deletions(-) create mode 100644 tests/unit/src/sqlbuild/orchestration_declarations/_test_types.py create mode 100644 tests/unit/src/sqlbuild/orchestration_declarations/test_migration_shim_boundary.py diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index fdf96c060..90e0f91af 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -7782,6 +7782,21 @@ ] } }, + { + "name": "migrate", + "parser": { + "prog": "sqb migrate", + "groups": [{"title": "positional arguments"}, {"title": "options"}], + "actions": [ + {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, + {"kind": "store", "dest": "migrate_subject", "choices": ["orchestration"], "metavar": "SUBJECT", "help": "what to migrate: orchestration moves Python jobs, job groups and schedules into TOML", "group": 0}, + {"kind": "store_true", "options": ["--dry-run"], "dest": "dry_run", "nargs": 0, "const": true, "default": false, "help": "print the renames, files and edits without writing anything", "group": 1}, + {"kind": "store", "options": ["--project-dir", "--sqb-project-dir"], "dest": "project_dir", "default": {"suppress": true}, "help": "SQLBuild project directory (default: the current directory)", "group": 1}, + {"kind": "store_true", "options": ["--no-color"], "dest": "no_color", "nargs": 0, "const": true, "default": {"suppress": true}, "help": "disable colored output", "group": 1}, + {"kind": "store_true", "options": ["--debug"], "dest": "debug", "nargs": 0, "const": true, "default": {"suppress": true}, "help": "enable verbose output and mirror internal diagnostics to stderr", "group": 1} + ] + } + }, { "name": "skills", "parser": { @@ -8861,6 +8876,11 @@ "dest": "review", "metavar": "review", "help": "build what a branch changed in a temporary schema, test it, compare it with a target and report" + }, + { + "dest": "migrate", + "metavar": "migrate", + "help": "move Python jobs, job groups and schedules into TOML" } ] } diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 5d238abd5..94cfef2fd 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -73,7 +73,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 187] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 188] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -219,6 +219,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 187] = [ ("scenario_max_snapshot_total_rows", NamespaceDefault::None), ("scenario_max_snapshot_bytes", NamespaceDefault::None), ("scenario_max_snapshot_total_bytes", NamespaceDefault::None), + ("migrate_subject", NamespaceDefault::None), ("skills_global", NamespaceDefault::Bool(false)), ("skills_target", NamespaceDefault::EmptyList), ("skills_force", NamespaceDefault::Bool(false)), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 43487a369..8d44aef66 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -693,6 +693,21 @@ pub(crate) const SKILLS_ROUTE: Route = routed( }, ); +const MIGRATE_ARGUMENTS: [Field; 3] = [ + PROJECT_DIR, + attr("subject", "migrate_subject"), + attr("dry_run", "dry_run"), +]; +pub(crate) const MIGRATE_ROUTE: Route = routed( + "run_migrate", + "run_migrate_command", + &MIGRATE_ARGUMENTS, + CallShape { + slot_positional: 0, + implementation_keywords: true, + }, +); + const FORMAT_ARGUMENTS: [Field; 10] = [ PROJECT_DIR, field("select", Source::FormatSelect), diff --git a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs index bb91c6290..989ff767e 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs @@ -6,9 +6,9 @@ use crate::dispatch::_helpers::subcommands::{ use crate::dispatch::constants::{ AUDIT_ROUTE, BUILD_ROUTE, CHECK_ROUTE, CLONE_ROUTE, COMPILE_ROUTE, CONTRACT_ROUTE, COST_ROUTE, DAG_ROUTE, DEBUG_ROUTE, DIFF_ROUTE, FORMAT_ROUTE, FRESHNESS_ROUTE, INIT_ROUTE, JANITOR_ROUTE, - LINEAGE_ROUTE, LOAD_ROUTE, LOCATION_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, - RECALL_COMMAND_NAME, RECALL_ROUTE, REFACTOR_ROUTE, REVIEW_ROUTE, RULES_ROUTE, SCOPE_ROUTE, - SEED_ROUTE, SKILLS_ROUTE, TEST_ROUTE, + LINEAGE_ROUTE, LOAD_ROUTE, LOCATION_ROUTE, MIGRATE_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, + QUERY_ROUTE, RECALL_COMMAND_NAME, RECALL_ROUTE, REFACTOR_ROUTE, REVIEW_ROUTE, RULES_ROUTE, + SCOPE_ROUTE, SEED_ROUTE, SKILLS_ROUTE, TEST_ROUTE, }; use crate::dispatch::models::{CommandFacts, Dispatch, DispatchFailure, Route}; @@ -60,6 +60,7 @@ pub fn command_dispatch(facts: &CommandFacts) -> Dispatch { "init" => run(&INIT_ROUTE), "playground" => run(&PLAYGROUND_ROUTE), "skills" => run(&SKILLS_ROUTE), + "migrate" => run(&MIGRATE_ROUTE), "scenario" => scenario_dispatch(facts, subcommand), "format" => checked(&FORMAT_ROUTE, format_failure(facts)), "rules" => run(&RULES_ROUTE), diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index 7a68e32d8..3aa5921d6 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -24,7 +24,7 @@ pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; /// Two quotes around at least one character. pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3; /// The warehouse group each CLI command selects a target's default warehouse from. -pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 32] = [ +pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 33] = [ ("audit", Some("build")), ("build", Some("build")), ("check", Some("build")), @@ -51,6 +51,7 @@ pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 32] = [ ("init", None), ("lineage", None), ("location", None), + ("migrate", None), ("mv", None), ("playground", None), ("rename", None), diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs index 094309be1..4f4ed54d3 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs @@ -9,7 +9,7 @@ use crate::orchestration_files::_helpers::python_modules::module_declaration; use crate::orchestration_files::_helpers::references::{check_references, check_renames}; use crate::orchestration_files::constants::{ BYTECODE_DIRECTORY, CALENDARS_DIRECTORY, GROUPS_DIRECTORY, HIDDEN_PREFIX, JOBS_DIRECTORY, - RENAMES_FILE, SENSORS_DIRECTORY, TOML_FILE_SUFFIX, + MIGRATE_HELP, RENAMES_FILE, SENSORS_DIRECTORY, TOML_FILE_SUFFIX, }; use crate::orchestration_files::main::read_group_file::read_group_file; use crate::orchestration_files::main::read_job_file::read_job_file; @@ -84,10 +84,13 @@ fn role_files( continue; } if !path.ends_with(suffix) { + let python_job: bool = path.ends_with(PYTHON_FILE_SUFFIX) + && matches!(role, JOBS_DIRECTORY | GROUPS_DIRECTORY); return Err(layout( &path, format!("orchestration/{role}/ holds only {suffix} files"), Some(match role { + _ if python_job => MIGRATE_HELP.to_owned(), JOBS_DIRECTORY | GROUPS_DIRECTORY => { "jobs and job groups are TOML; calendars and sensors are Python in orchestration/calendars/ and orchestration/sensors/".to_owned() } @@ -123,17 +126,29 @@ pub fn orchestration_inventory( return Err(layout( &path, format!("files cannot sit directly in orchestration/, except {RENAMES_FILE}"), - Some(format!( - "move it into one of orchestration/{}/", - ROLE_DIRECTORIES.join("/, orchestration/") - )), + Some(if entry.name.ends_with(PYTHON_FILE_SUFFIX) { + MIGRATE_HELP.to_owned() + } else { + format!( + "move it into one of orchestration/{}/", + ROLE_DIRECTORIES.join("/, orchestration/") + ) + }), )); } if !ROLE_DIRECTORIES.contains(&entry.name.as_str()) { + let holds_python: bool = !rglob(tree, &path, |found| { + found.name.ends_with(PYTHON_FILE_SUFFIX) + })? + .is_empty(); return Err(layout( &path, format!("unknown orchestration folder '{}'", entry.name), - Some(format!("use one of: {}", ROLE_DIRECTORIES.join(", "))), + Some(if holds_python { + MIGRATE_HELP.to_owned() + } else { + format!("use one of: {}", ROLE_DIRECTORIES.join(", ")) + }), )); } } diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs index cffb1fbb3..c3203feb2 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs @@ -117,8 +117,20 @@ fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_ description: "loose file", files: with("orchestration/jobs.py", "x = 1\n"), expected_message: "orchestration/jobs.py:1:1: files cannot sit directly in orchestration/, except renames.toml", + expected_help: Some("run `sqb migrate orchestration` to move Python jobs, job groups and schedules into TOML"), + }); + cases.push(InventoryErrorCase { + description: "loose non-Python file", + files: with("orchestration/nightly.toml", EXPORT_JOB), + expected_message: "orchestration/nightly.toml:1:1: files cannot sit directly in orchestration/, except renames.toml", expected_help: Some("move it into one of orchestration/jobs/, orchestration/job_groups/, orchestration/calendars/, orchestration/sensors/"), }); + cases.push(InventoryErrorCase { + description: "unknown folder of Python modules", + files: with("orchestration/pipelines/daily/orders.py", "x = 1\n"), + expected_message: "orchestration/pipelines:1:1: unknown orchestration folder 'pipelines'", + expected_help: Some("run `sqb migrate orchestration` to move Python jobs, job groups and schedules into TOML"), + }); cases.push(InventoryErrorCase { description: "unknown folder", files: with("orchestration/schedules/nightly.toml", EXPORT_JOB), @@ -129,6 +141,12 @@ fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_ description: "python job", files: with("orchestration/jobs/orders/legacy.py", "x = 1\n"), expected_message: "orchestration/jobs/orders/legacy.py:1:1: orchestration/jobs/ holds only .toml files", + expected_help: Some("run `sqb migrate orchestration` to move Python jobs, job groups and schedules into TOML"), + }); + cases.push(InventoryErrorCase { + description: "non-TOML job file", + files: with("orchestration/jobs/orders/export.yaml", "name: orders_export_job\n"), + expected_message: "orchestration/jobs/orders/export.yaml:1:1: orchestration/jobs/ holds only .toml files", expected_help: Some("jobs and job groups are TOML; calendars and sensors are Python in orchestration/calendars/ and orchestration/sensors/"), }); cases.push(InventoryErrorCase { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 5f6e49b0b..973ba6a5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -40,7 +40,7 @@ use crate::bindings::_helpers::scopes::scope_command::run_scope_command; use crate::bindings::_helpers::seed_command::command::run_seed_command; use crate::bindings::_helpers::sql_test_command::test_command::run_test_command; use crate::bindings::_helpers::workspace_commands::commands::{ - run_playground_command, run_skills_command, + run_migrate_command, run_playground_command, run_skills_command, }; use pyo3::IntoPyObjectExt; use pyo3::exceptions::{PyAttributeError, PyOSError, PyTypeError, PyUnicodeError}; @@ -412,6 +412,7 @@ fn run_implementation<'py>( )?, "run_janitor_command" => run_janitor_command(py, &request()?)?, "run_lineage_command" => run_lineage_command(py, &request()?)?, + "run_migrate_command" => run_migrate_command(py, options())?, "run_load_command" => run_load_command(py, &request()?)?, "run_location_command" => run_location(py, options(), Some(recall))?, "run_plan_command" => run_plan_command(&request()?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index caae4c98c..1c4e50700 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -991,6 +991,7 @@ const CLI_ENTRYPOINT_HANDLERS: RecordSpec = RecordSpec { FieldSpec::new("run_recall", FieldDefault::None), FieldSpec::new("run_review", FieldDefault::None), FieldSpec::new("run_location", FieldDefault::None), + FieldSpec::new("run_migrate", FieldDefault::None), ], }; const COMMAND_WAREHOUSE_SCOPE: RecordSpec = RecordSpec { @@ -1589,6 +1590,7 @@ const CLI_COMMAND: EnumSpec = EnumSpec { ("RENAME", "rename"), ("MV", "mv"), ("RECALL", "recall"), + ("MIGRATE", "migrate"), ], }; const PLAYGROUND_TEMPLATE: EnumSpec = EnumSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs index 53c087827..c99dcf442 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs @@ -31,6 +31,10 @@ use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; const SKILLS_DIRECTORY: &str = ".agents/skills/sqlbuild"; const TEMPLATES_DIRECTORY: &str = "playground_templates"; +/// The one module `sqb migrate orchestration` runs; nothing else imports it or the removed records. +const MIGRATE_ORCHESTRATION_MODULE: &str = + "sqlbuild.orchestration_declarations.main.migrate_orchestration"; +const MIGRATE_OPTIONS: (&[&str], &[&str]) = (&["project_dir", "subject"], &["dry_run"]); const SKILLS_OPTIONS: (&[&str], &[&str]) = (&["project_dir"], &["global_install", "targets", "force"]); const UPDATE_OPTIONS: (&[&str], &[&str]) = ( @@ -146,6 +150,30 @@ fn load_packaged_skill_files(py: Python<'_>) -> PyResult, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "run_migrate_command", options, MIGRATE_OPTIONS)?; + let project_dir: Bound<'_, PyAny> = arguments.value("project_dir")?; + let project_dir: PathBuf = if project_dir.is_none() { + current_dir(py)? + } else { + project_dir.extract()? + }; + let migrate = py + .import(MIGRATE_ORCHESTRATION_MODULE)? + .getattr("migrate_orchestration")?; + let options = PyDict::new(py); + options.set_item("project_dir", project_dir)?; + options.set_item("dry_run", arguments.value_or_false("dry_run")?.is_truthy()?)?; + migrate.call((), Some(&options))?.extract() +} + /// `sqb skills`: install or update SQLBuild skill files. #[pyfunction] #[pyo3(signature = (**options))] @@ -217,5 +245,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(maintain_sqlbuild_skills, module)?)?; module.add_function(wrap_pyfunction!(load_packaged_skill_files, module)?)?; module.add_function(wrap_pyfunction!(run_skills_command, module)?)?; + module.add_function(wrap_pyfunction!(run_migrate_command, module)?)?; module.add_function(wrap_pyfunction!(run_playground_command, module)?) } diff --git a/python-boundary.toml b/python-boundary.toml index 7f8009a8b..c2dacfee7 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -20,7 +20,17 @@ note = "Public orchestration facade: re-exports only." [[stays]] path = "orchestration_declarations/models/authoring.py" rule = 3 -note = "Public job and schedule authoring records only; selection resolution, validation and scheduling stay native." +note = "Public calendar and sensor authoring records only; jobs, job groups and schedules are TOML read natively, and selection, validation and scheduling stay native." + +[[stays]] +path = "orchestration_declarations/models/legacy_authoring.py" +rule = 2 +note = "Approved temporary exception: the removed job, job group and schedule records, rebuilt only while `sqb migrate orchestration` imports a project, because jobs built in loops or from constants exist only once their module runs. Remove in the next minor release after the TOML orchestration release." + +[[stays]] +path = "orchestration_declarations/main/migrate_orchestration.py" +rule = 2 +note = "Approved temporary exception: `sqb migrate orchestration` imports the project's old orchestration modules once and writes TOML from what they built, because jobs built in loops or from constants exist only at runtime. Remove in the next minor release after the TOML orchestration release." [[stays]] path = "orchestration_declarations/models/sensor_context.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 622f8e1e8..ac8b31bf4 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2332,6 +2332,7 @@ def update_sqlbuild_skills( def maintain_sqlbuild_skills(*, project_dir: Path) -> SkillMaintenanceResult: ... def load_packaged_skill_files() -> dict[str, str]: ... def run_playground_command(request: PlaygroundCommandRequest) -> int: ... +def run_migrate_command(**options: object) -> int: ... def describe_connection_warehouse( *, discovered_inputs: DiscoveredProjectInputs, @@ -7433,6 +7434,7 @@ class CliEntrypointHandlers(NativeRecord): run_recall: Callable[..., int] | None run_review: Callable[..., int] | None run_location: Callable[..., int] | None + run_migrate: Callable[..., int] | None def __init__( self, run_compile: Callable[[CompileCommandRequest], int], @@ -7470,6 +7472,7 @@ class CliEntrypointHandlers(NativeRecord): run_recall: Callable[..., int] | None = None, run_review: Callable[..., int] | None = None, run_location: Callable[..., int] | None = None, + run_migrate: Callable[..., int] | None = None, ) -> None: ... class CommandWarehouseScope(NativeRecord): @@ -8207,6 +8210,7 @@ class CliCommand(StrEnum): RENAME = "rename" MV = "mv" RECALL = "recall" + MIGRATE = "migrate" class ScopeCommandHandler(Protocol): def __call__(self, *, request: ScopeCommandRequest) -> int: ... diff --git a/tests/unit/src/sqlbuild/orchestration_declarations/_test_types.py b/tests/unit/src/sqlbuild/orchestration_declarations/_test_types.py new file mode 100644 index 000000000..9c4c91d35 --- /dev/null +++ b/tests/unit/src/sqlbuild/orchestration_declarations/_test_types.py @@ -0,0 +1,11 @@ +"""Frozen cases for the orchestration migration shim's import boundary.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ShimImportCase: + description: str + module: str + allowed_python_importers: frozenset[str] + allowed_rust_references: frozenset[str] diff --git a/tests/unit/src/sqlbuild/orchestration_declarations/test_migration_shim_boundary.py b/tests/unit/src/sqlbuild/orchestration_declarations/test_migration_shim_boundary.py new file mode 100644 index 000000000..a66bdb966 --- /dev/null +++ b/tests/unit/src/sqlbuild/orchestration_declarations/test_migration_shim_boundary.py @@ -0,0 +1,68 @@ +"""Only `sqb migrate orchestration` reaches the removed Python job records.""" + +import ast +from pathlib import Path + +import pytest + +from tests.unit.src.sqlbuild.orchestration_declarations._test_types import ShimImportCase + +_LEGACY_AUTHORING: str = "sqlbuild.orchestration_declarations.models.legacy_authoring" +_MIGRATE: str = "sqlbuild.orchestration_declarations.main.migrate_orchestration" + + +def _python_references(path: Path) -> set[str]: + """Modules `path` imports, plus dotted strings it could hand to an import function.""" + found: set[str] = set() + for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): + if isinstance(node, ast.Import): + found.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module is not None: + found.add(node.module) + found.update(f"{node.module}.{alias.name}" for alias in node.names) + elif isinstance(node, ast.Constant) and isinstance(node.value, str): + found.add(node.value) + return found + + +@pytest.mark.parametrize( + "test_case", + [ + ShimImportCase( + description="the removed job records reach only the migration command", + module=_LEGACY_AUTHORING, + allowed_python_importers=frozenset( + {"src/sqlbuild/orchestration_declarations/main/migrate_orchestration.py"} + ), + # Names the module only to skip its frames when locating a declaration's line. + allowed_rust_references=frozenset( + {"crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs"} + ), + ), + ShimImportCase( + description="the migration runs only from `sqb migrate orchestration`", + module=_MIGRATE, + allowed_python_importers=frozenset(), + allowed_rust_references=frozenset( + {"crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs"} + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_migration_shim_when_scanning_sources_then_only_the_migrate_command_reaches_it( + test_case: ShimImportCase, +) -> None: + python_importers: set[str] = { + path.as_posix() + for path in Path("src/sqlbuild").rglob("*.py") + if test_case.module in _python_references(path) + } + rust_references: set[str] = { + path.as_posix() + for path in Path("crates").rglob("*.rs") + if test_case.module in path.read_text() + } + + assert python_importers == test_case.allowed_python_importers + assert rust_references == test_case.allowed_rust_references From 180fa611d7c0be905ff00ed4165450f1aa437f4b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:18:11 +0000 Subject: [PATCH 23/34] revert: drop orchestration migration tooling and renames Projects declare jobs, job groups and schedules in TOML directly, so the Python-to-TOML migrator, the renames document and its server-side rewrite are removed. Python modules that still declare jobs or schedules now point at the TOML layout and the orchestration docs. --- .../sqlbuild-cli/src/argv/command_table.json | 20 - crates/sqlbuild-cli/src/argv/constants.rs | 3 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 15 - .../src/dispatch/main/command_dispatch.rs | 7 +- .../src/command_connection/constants.rs | 3 +- .../_helpers/python_modules.rs | 4 +- .../_helpers/references.rs | 76 --- .../src/orchestration_files/constants.rs | 9 +- .../src/orchestration_files/main/mod.rs | 1 - .../main/orchestration_inventory.rs | 20 +- .../main/read_renames_file.rs | 61 --- .../src/orchestration_files/models.rs | 36 -- .../tests/test_orchestration_inventory.rs | 74 +-- .../src/bindings/_helpers/cli/dispatch.rs | 3 +- .../compiled_project/snapshot_declarations.rs | 8 - .../_helpers/native_host/constants.rs | 6 +- .../bindings/_helpers/records/constants.rs | 2 - .../_helpers/workspace_commands/commands.rs | 29 -- python-boundary.toml | 10 - src/sqlbuild/_native.pyi | 4 - .../main/migrate_orchestration.py | 435 ------------------ .../models/legacy_authoring.py | 149 ------ .../orchestration_declarations/_test_types.py | 11 - .../test_migration_shim_boundary.py | 68 --- 24 files changed, 23 insertions(+), 1031 deletions(-) delete mode 100644 crates/sqlbuild-discovery/src/orchestration_files/main/read_renames_file.rs delete mode 100644 src/sqlbuild/orchestration_declarations/main/migrate_orchestration.py delete mode 100644 src/sqlbuild/orchestration_declarations/models/legacy_authoring.py delete mode 100644 tests/unit/src/sqlbuild/orchestration_declarations/_test_types.py delete mode 100644 tests/unit/src/sqlbuild/orchestration_declarations/test_migration_shim_boundary.py diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 90e0f91af..fdf96c060 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -7782,21 +7782,6 @@ ] } }, - { - "name": "migrate", - "parser": { - "prog": "sqb migrate", - "groups": [{"title": "positional arguments"}, {"title": "options"}], - "actions": [ - {"kind": "help", "options": ["-h", "--help"], "dest": "help", "nargs": 0, "default": {"suppress": true}, "help": "show this help message and exit", "group": 1}, - {"kind": "store", "dest": "migrate_subject", "choices": ["orchestration"], "metavar": "SUBJECT", "help": "what to migrate: orchestration moves Python jobs, job groups and schedules into TOML", "group": 0}, - {"kind": "store_true", "options": ["--dry-run"], "dest": "dry_run", "nargs": 0, "const": true, "default": false, "help": "print the renames, files and edits without writing anything", "group": 1}, - {"kind": "store", "options": ["--project-dir", "--sqb-project-dir"], "dest": "project_dir", "default": {"suppress": true}, "help": "SQLBuild project directory (default: the current directory)", "group": 1}, - {"kind": "store_true", "options": ["--no-color"], "dest": "no_color", "nargs": 0, "const": true, "default": {"suppress": true}, "help": "disable colored output", "group": 1}, - {"kind": "store_true", "options": ["--debug"], "dest": "debug", "nargs": 0, "const": true, "default": {"suppress": true}, "help": "enable verbose output and mirror internal diagnostics to stderr", "group": 1} - ] - } - }, { "name": "skills", "parser": { @@ -8876,11 +8861,6 @@ "dest": "review", "metavar": "review", "help": "build what a branch changed in a temporary schema, test it, compare it with a target and report" - }, - { - "dest": "migrate", - "metavar": "migrate", - "help": "move Python jobs, job groups and schedules into TOML" } ] } diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 94cfef2fd..5d238abd5 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -73,7 +73,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 188] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 187] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -219,7 +219,6 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 188] = [ ("scenario_max_snapshot_total_rows", NamespaceDefault::None), ("scenario_max_snapshot_bytes", NamespaceDefault::None), ("scenario_max_snapshot_total_bytes", NamespaceDefault::None), - ("migrate_subject", NamespaceDefault::None), ("skills_global", NamespaceDefault::Bool(false)), ("skills_target", NamespaceDefault::EmptyList), ("skills_force", NamespaceDefault::Bool(false)), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 8d44aef66..43487a369 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -693,21 +693,6 @@ pub(crate) const SKILLS_ROUTE: Route = routed( }, ); -const MIGRATE_ARGUMENTS: [Field; 3] = [ - PROJECT_DIR, - attr("subject", "migrate_subject"), - attr("dry_run", "dry_run"), -]; -pub(crate) const MIGRATE_ROUTE: Route = routed( - "run_migrate", - "run_migrate_command", - &MIGRATE_ARGUMENTS, - CallShape { - slot_positional: 0, - implementation_keywords: true, - }, -); - const FORMAT_ARGUMENTS: [Field; 10] = [ PROJECT_DIR, field("select", Source::FormatSelect), diff --git a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs index 989ff767e..bb91c6290 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs @@ -6,9 +6,9 @@ use crate::dispatch::_helpers::subcommands::{ use crate::dispatch::constants::{ AUDIT_ROUTE, BUILD_ROUTE, CHECK_ROUTE, CLONE_ROUTE, COMPILE_ROUTE, CONTRACT_ROUTE, COST_ROUTE, DAG_ROUTE, DEBUG_ROUTE, DIFF_ROUTE, FORMAT_ROUTE, FRESHNESS_ROUTE, INIT_ROUTE, JANITOR_ROUTE, - LINEAGE_ROUTE, LOAD_ROUTE, LOCATION_ROUTE, MIGRATE_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, - QUERY_ROUTE, RECALL_COMMAND_NAME, RECALL_ROUTE, REFACTOR_ROUTE, REVIEW_ROUTE, RULES_ROUTE, - SCOPE_ROUTE, SEED_ROUTE, SKILLS_ROUTE, TEST_ROUTE, + LINEAGE_ROUTE, LOAD_ROUTE, LOCATION_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, + RECALL_COMMAND_NAME, RECALL_ROUTE, REFACTOR_ROUTE, REVIEW_ROUTE, RULES_ROUTE, SCOPE_ROUTE, + SEED_ROUTE, SKILLS_ROUTE, TEST_ROUTE, }; use crate::dispatch::models::{CommandFacts, Dispatch, DispatchFailure, Route}; @@ -60,7 +60,6 @@ pub fn command_dispatch(facts: &CommandFacts) -> Dispatch { "init" => run(&INIT_ROUTE), "playground" => run(&PLAYGROUND_ROUTE), "skills" => run(&SKILLS_ROUTE), - "migrate" => run(&MIGRATE_ROUTE), "scenario" => scenario_dispatch(facts, subcommand), "format" => checked(&FORMAT_ROUTE, format_failure(facts)), "rules" => run(&RULES_ROUTE), diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index 3aa5921d6..7a68e32d8 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -24,7 +24,7 @@ pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; /// Two quotes around at least one character. pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3; /// The warehouse group each CLI command selects a target's default warehouse from. -pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 33] = [ +pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 32] = [ ("audit", Some("build")), ("build", Some("build")), ("check", Some("build")), @@ -51,7 +51,6 @@ pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 33] = [ ("init", None), ("lineage", None), ("location", None), - ("migrate", None), ("mv", None), ("playground", None), ("rename", None), diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs index cd45afe62..4fc66ffad 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs @@ -6,7 +6,7 @@ use sqlbuild_pysource::orchestration_modules::models::{ }; use crate::orchestration_files::constants::{ - CALENDAR_KIND, MIGRATE_HELP, PRIVATE_PREFIX, SENSOR_KIND, + CALENDAR_KIND, PRIVATE_PREFIX, SENSOR_KIND, TOML_JOBS_HELP, }; use crate::orchestration_files::models::{FileError, FilePosition, NamedModule, Positioned}; @@ -82,7 +82,7 @@ pub(crate) fn module_declaration( format!( "`{name}` was removed from sqlbuild.orchestration; jobs, job groups and schedules are declared in TOML" ), - Some(MIGRATE_HELP.to_owned()), + Some(TOML_JOBS_HELP.to_owned()), )); } let (noun, suffix) = label(kind); diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs index 9adb93c1b..acd076023 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs @@ -151,79 +151,3 @@ pub(crate) fn check_references(inventory: &OrchestrationInventory) -> Result<(), } Ok(()) } - -/// Each renames-file section maps old names that no longer exist onto names declared now. -pub(crate) fn check_renames(inventory: &OrchestrationInventory) -> Result<(), ReferenceError> { - let Some((path, renames)) = &inventory.renames else { - return Ok(()); - }; - let runs: BTreeSet<&str> = inventory - .jobs - .iter() - .map(|(_, job)| &job.run) - .chain(inventory.groups.iter().map(|(_, group)| &group.run)) - .map(|run| run.name.value.as_str()) - .collect(); - let scheduled: BTreeSet<&str> = inventory - .jobs - .iter() - .map(|(_, job)| &job.run) - .chain(inventory.groups.iter().map(|(_, group)| &group.run)) - .filter(|run| run.schedule.is_some()) - .map(|run| run.name.value.as_str()) - .collect(); - let sensors: BTreeSet<&str> = inventory - .sensors - .iter() - .map(|module| module.name.as_str()) - .collect(); - let names = inventory.names(); - let declared: BTreeSet<&str> = names.iter().map(|fact| fact.name.as_str()).collect(); - for (section, pairs, known, wanted) in [ - ("jobs", &renames.jobs, &runs, "a declared job or job group"), - ("sensors", &renames.sensors, &sensors, "a declared sensor"), - ( - "triggers", - &renames.triggers, - &scheduled, - "a job or job group with a schedule", - ), - ] { - for (old, new) in pairs { - if !known.contains(new.value.as_str()) { - return Err(( - path.clone(), - at( - new.position, - format!( - "[{section}] renames '{}' to '{}', which is not {wanted}", - old.value, new.value - ), - None, - ), - )); - } - // A trigger's old name only has to be free among triggers: it was a calendar function's name. - let taken: &BTreeSet<&str> = if section == "triggers" { - &scheduled - } else { - &declared - }; - let old_declared: bool = taken.contains(old.value.as_str()); - if old_declared { - return Err(( - path.clone(), - at( - old.position, - format!( - "[{section}] renames '{}', which is still declared", - old.value - ), - Some("remove the pair, or finish the rename in the declaration".to_owned()), - ), - )); - } - } - } - Ok(()) -} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs index d010b1071..078def0d3 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs @@ -47,15 +47,10 @@ pub const JOB_KIND: (&str, &str) = ("job", "_job"); pub const GROUP_KIND: (&str, &str) = ("job group", "_job_group"); pub const CALENDAR_KIND: (&str, &str) = ("calendar", "_calendar"); pub const SENSOR_KIND: (&str, &str) = ("sensor", "_sensor"); -/// What to run when a project still declares orchestration the old way. -pub const MIGRATE_HELP: &str = - "run `sqb migrate orchestration` to move Python jobs, job groups and schedules into TOML"; +/// Where jobs, job groups and schedules are declared now, for projects that still use Python. +pub const TOML_JOBS_HELP: &str = "jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/"; /// The folder Python writes bytecode to, which declarations never live in. pub(crate) const BYTECODE_DIRECTORY: &str = "__pycache__"; /// The prefix of hidden files and folders, and of private support modules. pub(crate) const HIDDEN_PREFIX: char = '.'; pub(crate) const PRIVATE_PREFIX: char = '_'; -/// The file `sqb migrate orchestration` writes in `orchestration/`, mapping old names to new. -pub const RENAMES_FILE: &str = "renames.toml"; -/// The sections of the renames file: jobs and groups, sensors, and schedule triggers. -pub const RENAME_SECTIONS: [&str; 3] = ["jobs", "sensors", "triggers"]; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs index 38f0a2c48..619ee67d4 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs @@ -1,4 +1,3 @@ pub mod orchestration_inventory; pub mod read_group_file; pub mod read_job_file; -pub mod read_renames_file; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs index 4f4ed54d3..280112973 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs @@ -6,14 +6,13 @@ use crate::_helpers::reading::read_authored_text; use crate::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; use crate::models::{DiscoveryFailure, FailureKind}; use crate::orchestration_files::_helpers::python_modules::module_declaration; -use crate::orchestration_files::_helpers::references::{check_references, check_renames}; +use crate::orchestration_files::_helpers::references::check_references; use crate::orchestration_files::constants::{ BYTECODE_DIRECTORY, CALENDARS_DIRECTORY, GROUPS_DIRECTORY, HIDDEN_PREFIX, JOBS_DIRECTORY, - MIGRATE_HELP, RENAMES_FILE, SENSORS_DIRECTORY, TOML_FILE_SUFFIX, + SENSORS_DIRECTORY, TOML_FILE_SUFFIX, TOML_JOBS_HELP, }; use crate::orchestration_files::main::read_group_file::read_group_file; use crate::orchestration_files::main::read_job_file::read_job_file; -use crate::orchestration_files::main::read_renames_file::read_renames_file; use crate::orchestration_files::models::{ FileError, FilePosition, OrchestrationFailure, OrchestrationInventory, }; @@ -90,7 +89,7 @@ fn role_files( &path, format!("orchestration/{role}/ holds only {suffix} files"), Some(match role { - _ if python_job => MIGRATE_HELP.to_owned(), + _ if python_job => TOML_JOBS_HELP.to_owned(), JOBS_DIRECTORY | GROUPS_DIRECTORY => { "jobs and job groups are TOML; calendars and sensors are Python in orchestration/calendars/ and orchestration/sensors/".to_owned() } @@ -116,18 +115,12 @@ pub fn orchestration_inventory( if ignored(&entry) { continue; } - if !entry.is_dir && entry.name == RENAMES_FILE { - let renames = - read_renames_file(&read(tree, &path)?).map_err(|error| failure(&path, error))?; - inventory.renames = Some((path, renames)); - continue; - } if !entry.is_dir { return Err(layout( &path, - format!("files cannot sit directly in orchestration/, except {RENAMES_FILE}"), + "files cannot sit directly in orchestration/".to_owned(), Some(if entry.name.ends_with(PYTHON_FILE_SUFFIX) { - MIGRATE_HELP.to_owned() + TOML_JOBS_HELP.to_owned() } else { format!( "move it into one of orchestration/{}/", @@ -145,7 +138,7 @@ pub fn orchestration_inventory( &path, format!("unknown orchestration folder '{}'", entry.name), Some(if holds_python { - MIGRATE_HELP.to_owned() + TOML_JOBS_HELP.to_owned() } else { format!("use one of: {}", ROLE_DIRECTORIES.join(", ")) }), @@ -183,6 +176,5 @@ pub fn orchestration_inventory( } } check_references(&inventory).map_err(|(path, error)| failure(&path, error))?; - check_renames(&inventory).map_err(|(path, error)| failure(&path, error))?; Ok(inventory) } diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/read_renames_file.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/read_renames_file.rs deleted file mode 100644 index 94e757ade..000000000 --- a/crates/sqlbuild-discovery/src/orchestration_files/main/read_renames_file.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! Read the renames file `sqb migrate orchestration` writes. - -use std::collections::BTreeMap; - -use toml::Spanned; - -use crate::orchestration_files::_helpers::reading::{check_keys, error_at, nonblank, toml_error}; -use crate::orchestration_files::constants::RENAME_SECTIONS; -use crate::orchestration_files::models::{FileError, RenameMap, RenamePairs}; - -/// Each section's old and new names as written, or the first problem with the file. -pub fn read_renames_file(text: &str) -> Result { - check_keys(text, &RENAME_SECTIONS, "renames file")?; - let sections: BTreeMap, Spanned>> = - toml::from_str(text).map_err(|error| toml_error(text, &error))?; - let mut map = RenameMap::default(); - for (section, pairs) in sections { - let mut read: RenamePairs = Vec::new(); - let mut targets: BTreeMap = BTreeMap::new(); - let mut pairs: Vec<(Spanned, Spanned)> = pairs.into_iter().collect(); - pairs.sort_by_key(|(old, _)| old.span().start); - for (old, new) in pairs { - let old = nonblank(text, old, "old name")?; - let new_start: usize = new.span().start; - let new = nonblank(text, new, "new name")?; - if old.value == new.value { - return Err(error_at( - text, - new_start, - format!("'{}' is renamed to itself", old.value), - )); - } - if targets.insert(new.value.clone(), read.len()).is_some() { - return Err(error_at( - text, - new_start, - format!("two names are renamed to '{}'", new.value), - )); - } - read.push((old, new)); - } - for (_, new) in &read { - if let Some((old, _)) = read.iter().find(|(old, _)| old.value == new.value) { - return Err(FileError { - message: format!( - "'{}' is both a new name and an old name; renames cannot chain", - old.value - ), - position: old.position, - help: Some("map each old name straight to its final name".to_owned()), - }); - } - } - match section.as_str() { - "jobs" => map.jobs = read, - "sensors" => map.sensors = read, - _ => map.triggers = read, - } - } - Ok(map) -} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/models.rs b/crates/sqlbuild-discovery/src/orchestration_files/models.rs index 0343ec63c..95f4b3990 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/models.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/models.rs @@ -115,40 +115,6 @@ pub struct NamedModule { pub targets: Option>, } -/// One section of the renames file: each old name and the name it became, where each is written. -pub type RenamePairs = Vec<(Positioned, Positioned)>; - -/// The renames file: old names of jobs and groups, sensors, and schedule triggers, and their new names. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct RenameMap { - pub jobs: RenamePairs, - pub sensors: RenamePairs, - pub triggers: RenamePairs, -} - -impl RenameMap { - /// The map as the snapshot document the server applies: `{section: {old: new}}`. - #[must_use] - pub fn document(&self) -> serde_json::Value { - let section = |pairs: &RenamePairs| { - pairs - .iter() - .map(|(old, new)| { - ( - old.value.clone(), - serde_json::Value::String(new.value.clone()), - ) - }) - .collect::>() - }; - serde_json::json!({ - "jobs": section(&self.jobs), - "sensors": section(&self.sensors), - "triggers": section(&self.triggers), - }) - } -} - /// Every orchestration declaration in a project, checked for layout, names and references. #[derive(Clone, Debug, Default, PartialEq)] pub struct OrchestrationInventory { @@ -158,8 +124,6 @@ pub struct OrchestrationInventory { pub sensors: Vec, /// Every Python module under `calendars/` and `sensors/`, which publication imports. pub modules: Vec, - /// The renames file and its path, when the project has one. - pub renames: Option<(String, RenameMap)>, } impl OrchestrationInventory { diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs index c3203feb2..2c8cd3ab1 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs @@ -116,20 +116,20 @@ fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_ cases.push(InventoryErrorCase { description: "loose file", files: with("orchestration/jobs.py", "x = 1\n"), - expected_message: "orchestration/jobs.py:1:1: files cannot sit directly in orchestration/, except renames.toml", - expected_help: Some("run `sqb migrate orchestration` to move Python jobs, job groups and schedules into TOML"), + expected_message: "orchestration/jobs.py:1:1: files cannot sit directly in orchestration/", + expected_help: Some("jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/"), }); cases.push(InventoryErrorCase { description: "loose non-Python file", files: with("orchestration/nightly.toml", EXPORT_JOB), - expected_message: "orchestration/nightly.toml:1:1: files cannot sit directly in orchestration/, except renames.toml", + expected_message: "orchestration/nightly.toml:1:1: files cannot sit directly in orchestration/", expected_help: Some("move it into one of orchestration/jobs/, orchestration/job_groups/, orchestration/calendars/, orchestration/sensors/"), }); cases.push(InventoryErrorCase { description: "unknown folder of Python modules", files: with("orchestration/pipelines/daily/orders.py", "x = 1\n"), expected_message: "orchestration/pipelines:1:1: unknown orchestration folder 'pipelines'", - expected_help: Some("run `sqb migrate orchestration` to move Python jobs, job groups and schedules into TOML"), + expected_help: Some("jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/"), }); cases.push(InventoryErrorCase { description: "unknown folder", @@ -141,7 +141,7 @@ fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_ description: "python job", files: with("orchestration/jobs/orders/legacy.py", "x = 1\n"), expected_message: "orchestration/jobs/orders/legacy.py:1:1: orchestration/jobs/ holds only .toml files", - expected_help: Some("run `sqb migrate orchestration` to move Python jobs, job groups and schedules into TOML"), + expected_help: Some("jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/"), }); cases.push(InventoryErrorCase { description: "non-TOML job file", @@ -204,48 +204,7 @@ fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_ "from sqlbuild.orchestration import job, sensor\nstale = sensor(name=\"stock_stale_sensor\", description=\"d\", run=\"orders_export_job\")\n", ), expected_message: "orchestration/sensors/inventory/stock.py:1:36: `job` was removed from sqlbuild.orchestration; jobs, job groups and schedules are declared in TOML", - expected_help: Some("run `sqb migrate orchestration` to move Python jobs, job groups and schedules into TOML"), - }); - let renamed = |text: &'static str| { - let mut files = valid(); - files.push(("orchestration/renames.toml", text)); - files - }; - cases.push(InventoryErrorCase { - description: "rename to an undeclared job", - files: renamed("[jobs]\norders_export = \"orders_exports_job\"\n"), - expected_message: "orchestration/renames.toml:2:17: [jobs] renames 'orders_export' to 'orders_exports_job', which is not a declared job or job group", - expected_help: None, - }); - cases.push(InventoryErrorCase { - description: "old name still declared", - files: renamed("[jobs]\norders_export_job = \"orders_nightly_job_group\"\n"), - expected_message: "orchestration/renames.toml:2:1: [jobs] renames 'orders_export_job', which is still declared", - expected_help: Some("remove the pair, or finish the rename in the declaration"), - }); - cases.push(InventoryErrorCase { - description: "chained renames", - files: renamed("[sensors]\nstock = \"stock_stale\"\nstock_stale = \"stock_stale_sensor\"\n"), - expected_message: "orchestration/renames.toml:3:1: 'stock_stale' is both a new name and an old name; renames cannot chain", - expected_help: Some("map each old name straight to its final name"), - }); - cases.push(InventoryErrorCase { - description: "two names onto one", - files: renamed("[jobs]\norders = \"orders_export_job\"\nexport = \"orders_export_job\"\n"), - expected_message: "orchestration/renames.toml:3:10: two names are renamed to 'orders_export_job'", - expected_help: None, - }); - cases.push(InventoryErrorCase { - description: "trigger onto an unscheduled run", - files: renamed("[triggers]\ntrading_days = \"orders_nightly_job_group\"\n"), - expected_message: "orchestration/renames.toml:2:16: [triggers] renames 'trading_days' to 'orders_nightly_job_group', which is not a job or job group with a schedule", - expected_help: None, - }); - cases.push(InventoryErrorCase { - description: "unknown renames section", - files: renamed("[groups]\nnightly = \"orders_nightly_job_group\"\n"), - expected_message: "orchestration/renames.toml:1:2: unknown renames file key 'groups'", - expected_help: Some("supported keys: jobs, sensors, triggers"), + expected_help: Some("jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/"), }); for case in cases { let (actual, actual_help) = message(&case.files); @@ -258,24 +217,3 @@ fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_ ); } } - -#[test] -fn given_renames_file_when_inventorying_then_reads_each_section_as_the_server_document() { - let mut files = valid(); - files.push(( - "orchestration/renames.toml", - "[jobs]\norders_export = \"orders_export_job\"\nnightly = \"orders_nightly_job_group\"\n\n[sensors]\nstock = \"stock_stale_sensor\"\n\n[triggers]\ntrading_days = \"orders_export_job\"\n", - )); - let directory = project(&files); - let found = inventory(directory.path()).expect("valid renames"); - let (path, renames) = found.renames.expect("renames read"); - assert_eq!(path, "orchestration/renames.toml"); - assert_eq!( - renames.document(), - serde_json::json!({ - "jobs": {"orders_export": "orders_export_job", "nightly": "orders_nightly_job_group"}, - "sensors": {"stock": "stock_stale_sensor"}, - "triggers": {"trading_days": "orders_export_job"}, - }) - ); -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 973ba6a5a..5f6e49b0b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -40,7 +40,7 @@ use crate::bindings::_helpers::scopes::scope_command::run_scope_command; use crate::bindings::_helpers::seed_command::command::run_seed_command; use crate::bindings::_helpers::sql_test_command::test_command::run_test_command; use crate::bindings::_helpers::workspace_commands::commands::{ - run_migrate_command, run_playground_command, run_skills_command, + run_playground_command, run_skills_command, }; use pyo3::IntoPyObjectExt; use pyo3::exceptions::{PyAttributeError, PyOSError, PyTypeError, PyUnicodeError}; @@ -412,7 +412,6 @@ fn run_implementation<'py>( )?, "run_janitor_command" => run_janitor_command(py, &request()?)?, "run_lineage_command" => run_lineage_command(py, &request()?)?, - "run_migrate_command" => run_migrate_command(py, options())?, "run_load_command" => run_load_command(py, &request()?)?, "run_location_command" => run_location(py, options(), Some(recall))?, "run_plan_command" => run_plan_command(&request()?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 1017eddfc..cd3dbb253 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -21,9 +21,6 @@ use sqlbuild_orchestration::declarations::models::{ use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; - -/// The snapshot document carrying the renames file, which the server applies to stored state. -const RENAMES_DOCUMENT: &str = "renames"; use crate::bindings::_helpers::native_host::constants::{AUTHORING_MODULE, SENSOR_CLASS}; use crate::bindings::_helpers::native_host::orchestration_declarations::{ AuthoredSensor, module_declarations, run_controls, run_resources, @@ -325,11 +322,6 @@ pub(crate) fn compile_declarations( document.map_err(|error| PyValueError::new_err(error.to_string()))?, ); } - if let Some((_, renames)) = &inventory.renames { - input - .documents - .insert(RENAMES_DOCUMENT.into(), renames.document()); - } snapshot_definitions(&input).map_err(|error| PyValueError::new_err(error.to_string()))?; Ok(input) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index e284bd940..2f12728a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -10,12 +10,8 @@ pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; pub(crate) const SENSOR_CONTEXT_PARAMETER: &str = "ctx"; pub(crate) const AUTHORING_MODULE: &str = "sqlbuild.orchestration_declarations.models.authoring"; -/// The removed records `sqb migrate orchestration` rebuilds while it imports a project. -pub(crate) const LEGACY_AUTHORING_MODULE: &str = - "sqlbuild.orchestration_declarations.models.legacy_authoring"; /// Frames of these modules construct declarations on the author's behalf. -pub(crate) const CONSTRUCTING_MODULES: [&str; 3] = - [AUTHORING_MODULE, LEGACY_AUTHORING_MODULE, "dataclasses"]; +pub(crate) const CONSTRUCTING_MODULES: [&str; 2] = [AUTHORING_MODULE, "dataclasses"]; /// The authoring classes publication imports calendar and sensor modules for. pub(crate) const CALENDAR_CLASS: &str = "Calendar"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 1c4e50700..caae4c98c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -991,7 +991,6 @@ const CLI_ENTRYPOINT_HANDLERS: RecordSpec = RecordSpec { FieldSpec::new("run_recall", FieldDefault::None), FieldSpec::new("run_review", FieldDefault::None), FieldSpec::new("run_location", FieldDefault::None), - FieldSpec::new("run_migrate", FieldDefault::None), ], }; const COMMAND_WAREHOUSE_SCOPE: RecordSpec = RecordSpec { @@ -1590,7 +1589,6 @@ const CLI_COMMAND: EnumSpec = EnumSpec { ("RENAME", "rename"), ("MV", "mv"), ("RECALL", "recall"), - ("MIGRATE", "migrate"), ], }; const PLAYGROUND_TEMPLATE: EnumSpec = EnumSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs index c99dcf442..53c087827 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs @@ -31,10 +31,6 @@ use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; const SKILLS_DIRECTORY: &str = ".agents/skills/sqlbuild"; const TEMPLATES_DIRECTORY: &str = "playground_templates"; -/// The one module `sqb migrate orchestration` runs; nothing else imports it or the removed records. -const MIGRATE_ORCHESTRATION_MODULE: &str = - "sqlbuild.orchestration_declarations.main.migrate_orchestration"; -const MIGRATE_OPTIONS: (&[&str], &[&str]) = (&["project_dir", "subject"], &["dry_run"]); const SKILLS_OPTIONS: (&[&str], &[&str]) = (&["project_dir"], &["global_install", "targets", "force"]); const UPDATE_OPTIONS: (&[&str], &[&str]) = ( @@ -150,30 +146,6 @@ fn load_packaged_skill_files(py: Python<'_>) -> PyResult, - options: Option>, -) -> PyResult { - let arguments: KeywordArguments<'_> = - KeywordArguments::read(py, "run_migrate_command", options, MIGRATE_OPTIONS)?; - let project_dir: Bound<'_, PyAny> = arguments.value("project_dir")?; - let project_dir: PathBuf = if project_dir.is_none() { - current_dir(py)? - } else { - project_dir.extract()? - }; - let migrate = py - .import(MIGRATE_ORCHESTRATION_MODULE)? - .getattr("migrate_orchestration")?; - let options = PyDict::new(py); - options.set_item("project_dir", project_dir)?; - options.set_item("dry_run", arguments.value_or_false("dry_run")?.is_truthy()?)?; - migrate.call((), Some(&options))?.extract() -} - /// `sqb skills`: install or update SQLBuild skill files. #[pyfunction] #[pyo3(signature = (**options))] @@ -245,6 +217,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(maintain_sqlbuild_skills, module)?)?; module.add_function(wrap_pyfunction!(load_packaged_skill_files, module)?)?; module.add_function(wrap_pyfunction!(run_skills_command, module)?)?; - module.add_function(wrap_pyfunction!(run_migrate_command, module)?)?; module.add_function(wrap_pyfunction!(run_playground_command, module)?) } diff --git a/python-boundary.toml b/python-boundary.toml index c2dacfee7..5a89a1c58 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -22,16 +22,6 @@ path = "orchestration_declarations/models/authoring.py" rule = 3 note = "Public calendar and sensor authoring records only; jobs, job groups and schedules are TOML read natively, and selection, validation and scheduling stay native." -[[stays]] -path = "orchestration_declarations/models/legacy_authoring.py" -rule = 2 -note = "Approved temporary exception: the removed job, job group and schedule records, rebuilt only while `sqb migrate orchestration` imports a project, because jobs built in loops or from constants exist only once their module runs. Remove in the next minor release after the TOML orchestration release." - -[[stays]] -path = "orchestration_declarations/main/migrate_orchestration.py" -rule = 2 -note = "Approved temporary exception: `sqb migrate orchestration` imports the project's old orchestration modules once and writes TOML from what they built, because jobs built in loops or from constants exist only at runtime. Remove in the next minor release after the TOML orchestration release." - [[stays]] path = "orchestration_declarations/models/sensor_context.py" rule = 3 diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index ac8b31bf4..622f8e1e8 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2332,7 +2332,6 @@ def update_sqlbuild_skills( def maintain_sqlbuild_skills(*, project_dir: Path) -> SkillMaintenanceResult: ... def load_packaged_skill_files() -> dict[str, str]: ... def run_playground_command(request: PlaygroundCommandRequest) -> int: ... -def run_migrate_command(**options: object) -> int: ... def describe_connection_warehouse( *, discovered_inputs: DiscoveredProjectInputs, @@ -7434,7 +7433,6 @@ class CliEntrypointHandlers(NativeRecord): run_recall: Callable[..., int] | None run_review: Callable[..., int] | None run_location: Callable[..., int] | None - run_migrate: Callable[..., int] | None def __init__( self, run_compile: Callable[[CompileCommandRequest], int], @@ -7472,7 +7470,6 @@ class CliEntrypointHandlers(NativeRecord): run_recall: Callable[..., int] | None = None, run_review: Callable[..., int] | None = None, run_location: Callable[..., int] | None = None, - run_migrate: Callable[..., int] | None = None, ) -> None: ... class CommandWarehouseScope(NativeRecord): @@ -8210,7 +8207,6 @@ class CliCommand(StrEnum): RENAME = "rename" MV = "mv" RECALL = "recall" - MIGRATE = "migrate" class ScopeCommandHandler(Protocol): def __call__(self, *, request: ScopeCommandRequest) -> int: ... diff --git a/src/sqlbuild/orchestration_declarations/main/migrate_orchestration.py b/src/sqlbuild/orchestration_declarations/main/migrate_orchestration.py deleted file mode 100644 index bf73166ab..000000000 --- a/src/sqlbuild/orchestration_declarations/main/migrate_orchestration.py +++ /dev/null @@ -1,435 +0,0 @@ -"""`sqb migrate orchestration`: rewrite Python jobs, job groups and schedules as TOML, once. - -Jobs built in loops or from constants exist only once their module runs, so this imports each -orchestration module with the removed records restored, then writes what it found. -""" - -from __future__ import annotations - -import ast -import json -import re -import sys -from collections.abc import Iterator -from contextlib import contextmanager -from dataclasses import dataclass, field -from pathlib import Path - -from sqlbuild.compiler.discovery._helpers.filesystem.project_modules import ( - load_orchestration_module, -) -from sqlbuild.orchestration_declarations.models import legacy_authoring as legacy - -_ROOT = "orchestration" -_RENAMES = "renames.toml" -_REMOVED = { - "job": legacy.Job, - "job_group": legacy.JobGroup, - "schedule": legacy.Schedule, - "calendar_schedule": legacy.CalendarSchedule, - "on_failure": legacy.OnFailure, - "resources": legacy.Resources, - "sensor": legacy.Sensor, - "run_event": legacy.RunEventCondition, -} -_JOB_PARTS = (legacy.Job, legacy.JobGroup, legacy.Schedule, legacy.OnFailure, legacy.Resources) -_BARE_KEY = re.compile(r"^[A-Za-z0-9_-]+$") - - -class MigrationError(Exception): - """A declaration the migration cannot write without a person deciding.""" - - -@dataclass -class _Found: - """The legacy declarations of a project and the module each was found in first.""" - - modules: dict[int, Path] = field(default_factory=dict) - jobs: list[legacy.Job] = field(default_factory=list) - groups: list[legacy.JobGroup] = field(default_factory=list) - calendars: list[legacy.CalendarSchedule] = field(default_factory=list) - sensors: list[legacy.Sensor] = field(default_factory=list) - - -@contextmanager -def _removed_api() -> Iterator[None]: - import sqlbuild.orchestration as public - - missing = object() - saved = {name: getattr(public, name, missing) for name in _REMOVED} - for name, value in _REMOVED.items(): - setattr(public, name, value) - try: - yield - finally: - for name, value in saved.items(): - if value is missing: - delattr(public, name) - else: - setattr(public, name, value) - - -def _collect(value: object, module: Path, found: _Found) -> None: - pending = [value] - visited: set[int] = set() - while pending: - item = pending.pop() - if isinstance(item, list | tuple): - if id(item) not in visited: - visited.add(id(item)) - pending.extend(item) - continue - kinds = ( - (legacy.Job, found.jobs), - (legacy.JobGroup, found.groups), - (legacy.CalendarSchedule, found.calendars), - (legacy.Sensor, found.sensors), - ) - for kind, bucket in kinds: - if isinstance(item, kind) and id(item) not in found.modules: - found.modules[id(item)] = module - bucket.append(item) - if isinstance(item, legacy.JobGroup): - pending.extend(item.jobs) - - -def _suffixed(name: str, suffix: str) -> str: - return name if name.endswith(suffix) and len(name) > len(suffix) else f"{name}{suffix}" - - -def _value(value: object) -> str: - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, int | float): - return repr(value) - if isinstance(value, str): - return json.dumps(value, ensure_ascii=False) - if isinstance(value, list | tuple): - return "[" + ", ".join(_value(item) for item in value) + "]" - if isinstance(value, dict): - items = [f"{_key(key)} = {_value(item)}" for key, item in value.items() if item is not None] - return "{ " + ", ".join(items) + " }" if items else "{}" - raise MigrationError(f"cannot write {value!r} as TOML") - - -def _key(key: object) -> str: - text = str(key) - return text if _BARE_KEY.match(text) else json.dumps(text, ensure_ascii=False) - - -def _schedule( - run: legacy.Job | legacy.JobGroup, calendar: tuple[str, legacy.CalendarSchedule] | None -) -> dict[str, object] | None: - if run.schedule is not None and calendar is not None: - raise MigrationError( - f"'{run.name}' has both a cron schedule and the calendar schedule " - f"'{calendar[0]}'; a job or job group now has one schedule, so remove one first" - ) - written: dict[str, object] - source: legacy.Schedule | legacy.CalendarSchedule - if run.schedule is not None: - source = run.schedule - written = {"cron": source.cron} - elif calendar is not None: - source = calendar[1] - written = {"calendar": calendar[0]} - else: - return None - written["timezone"] = source.timezone - if source.starts_at is not None: - written["starts_at"] = source.starts_at - if source.default_status != "running": - written["default_status"] = source.default_status - return written - - -def _run_lines(run: legacy.Job | legacy.JobGroup, name: str) -> list[str]: - return [f"name = {_value(name)}", f"description = {_value(run.description)}"] - - -def _common( - run: legacy.Job | legacy.JobGroup, calendar: tuple[str, legacy.CalendarSchedule] | None -) -> tuple[list[str], list[str]]: - lines: list[str] = [] - if run.targets: - lines.append(f"targets = {_value(list(run.targets))}") - if run.owner is not None: - lines.append(f"owner = {_value(run.owner)}") - if run.overlap != "queue": - lines.append(f"overlap = {_value(run.overlap)}") - if run.concurrency_groups: - lines.append(f"concurrency_groups = {_value(list(run.concurrency_groups))}") - schedule = _schedule(run, calendar) - if schedule is not None: - lines.append(f"schedule = {_value(schedule)}") - tables: list[str] = [] - if run.on_failure is not None: - tables += ["", "[on_failure]", f"retries = {_value(run.on_failure.retries)}"] - if run.on_failure.retry_delay_seconds: - tables.append(f"retry_delay_seconds = {_value(run.on_failure.retry_delay_seconds)}") - return lines, tables - - -def _job_toml( - job: legacy.Job, name: str, calendar: tuple[str, legacy.CalendarSchedule] | None -) -> str: - lines = _run_lines(job, name) - lines.append(f"select = {_value(job.select)}") - if job.exclude.strip(): - lines.append(f"exclude = {_value(job.exclude)}") - common, tables = _common(job, calendar) - lines += common + tables - if job.resources is not None: - limits = { - key: value - for key, value in ( - ("memory", job.resources.memory), - ("cpus", job.resources.cpus), - ("pids", job.resources.pids), - ) - if value is not None - } - lines += ["", "[resources]"] + [f"{key} = {_value(value)}" for key, value in limits.items()] - settings = dict(job.settings) - if job.concurrency is not None: - if "concurrency" in settings: - raise MigrationError(f"job '{job.name}' declares concurrency twice") - settings["concurrency"] = job.concurrency - if settings: - lines += ["", "[settings]"] + [ - f"{_key(key)} = {_value(value)}" for key, value in settings.items() if value is not None - ] - return "\n".join(lines) + "\n" - - -def _group_toml( - group: legacy.JobGroup, - name: str, - members: list[str], - calendar: tuple[str, legacy.CalendarSchedule] | None, -) -> str: - lines = _run_lines(group, name) - lines.append(f"jobs = {_value(members)}") - common, tables = _common(group, calendar) - return "\n".join(lines + common + tables) + "\n" - - -def _job_parts_only(module: Path, project_dir: Path) -> bool: - """Whether a module holds nothing but imports, a docstring and job, group or schedule values.""" - tree = ast.parse(module.read_text(encoding="utf-8")) - loaded = sys.modules.get(_module_name(module, project_dir)) - for statement in tree.body: - if isinstance(statement, ast.Import | ast.ImportFrom): - continue - if isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Constant): - continue - targets = ( - statement.targets - if isinstance(statement, ast.Assign) - else [statement.target] - if isinstance(statement, ast.AnnAssign) - else None - ) - if targets is None or loaded is None: - return False - for target in targets: - if not isinstance(target, ast.Name): - return False - if target.id == "__all__": - continue - value = getattr(loaded, target.id, None) - values = value if isinstance(value, list | tuple) else [value] - if not values or not all(isinstance(item, _JOB_PARTS) for item in values): - return False - return True - - -def _module_name(module: Path, project_dir: Path) -> str: - return ".".join(module.relative_to(project_dir).with_suffix("").parts) - - -def _site(declaration: object, fallback: Path, project_dir: Path) -> str: - site = getattr(declaration, "_declared_at", None) - if site is None: - return fallback.relative_to(project_dir).as_posix() - path = Path(site[0]) - try: - path = path.relative_to(project_dir.resolve()) - except ValueError: - pass - return f"{path.as_posix()}:{site[1]}" - - -def migrate_orchestration(*, project_dir: Path, dry_run: bool) -> int: - """Write TOML for each Python job and job group, print every rename and manual edit.""" - try: - return _migrate(project_dir, dry_run) - except MigrationError as error: - print(f"error: {error}", file=sys.stderr) - return 1 - - -def _migrate(project_dir: Path, dry_run: bool) -> int: - root = project_dir / _ROOT - if not root.is_dir(): - print("No orchestration/ folder; nothing to migrate.") - return 0 - if (root / _RENAMES).exists(): - raise MigrationError( - f"{_ROOT}/{_RENAMES} already exists; publish every location with it, delete it, " - "then run the migration again" - ) - found = _Found() - modules = sorted( - path - for path in root.rglob("*.py") - if "__pycache__" not in path.parts and path.name != "__init__.py" - ) - with _removed_api(): - for module in modules: - loaded = load_orchestration_module(file_path=module, project_dir=project_dir) - for value in vars(loaded).values(): - _collect(value, module, found) - if not (found.jobs or found.groups or found.calendars or found.sensors): - print("No Python jobs, job groups or schedules found; nothing to migrate.") - return 0 - names: dict[int, str] = {} - renames: dict[str, dict[str, str]] = {"jobs": {}, "sensors": {}, "triggers": {}} - printed: list[str] = [] - for runs, suffix, kind, section in ( - (found.jobs, "_job", "job", "jobs"), - (found.groups, "_job_group", "job group", "jobs"), - (found.sensors, "_sensor", "sensor", "sensors"), - ): - for run in runs: - new = _suffixed(run.name, suffix) - names[id(run)] = new - if new != run.name: - renames[section][run.name] = new - printed.append(f"Renamed {kind} {run.name} -> {new}") - calendars: dict[int, tuple[str, legacy.CalendarSchedule]] = {} - for schedule in found.calendars: - if schedule.function is None: - raise MigrationError("a calendar_schedule does not decorate a function") - old = schedule.function.__name__ - new = _suffixed(old, "_calendar") - if id(schedule.job) in calendars: - raise MigrationError( - f"'{schedule.job.name}' has two calendar schedules; keep one before migrating" - ) - calendars[id(schedule.job)] = (new, schedule) - if new != old: - printed.append(f"Renamed calendar {old} -> {new}") - job_name = names.get(id(schedule.job), schedule.job.name) - if old != job_name: - renames["triggers"][old] = job_name - taken: set[str] = set() - for new in [*names.values(), *(name for name, _ in calendars.values())]: - if new in taken: - raise MigrationError(f"two declarations would both be named '{new}'") - taken.add(new) - writes: list[tuple[Path, str]] = [] - for job in found.jobs: - relative = found.modules[id(job)].relative_to(root).with_suffix("") - writes.append( - ( - root / "jobs" / relative / f"{names[id(job)]}.toml", - _job_toml(job, names[id(job)], calendars.get(id(job))), - ) - ) - for group in found.groups: - relative = found.modules[id(group)].relative_to(root).with_suffix("") - members = [names[id(member)] for member in group.jobs] - writes.append( - ( - root / "job_groups" / relative / f"{names[id(group)]}.toml", - _group_toml(group, names[id(group)], members, calendars.get(id(group))), - ) - ) - if any(renames.values()): - sections = [ - f"[{section}]\n" - + "".join(f"{_key(old)} = {_value(new)}\n" for old, new in pairs.items()) - for section, pairs in renames.items() - if pairs - ] - writes.append((root / _RENAMES, "\n".join(sections))) - for path, _ in writes: - if path.exists(): - raise MigrationError(f"{path.relative_to(project_dir).as_posix()} already exists") - holding_others = {found.modules[id(item)] for item in [*found.calendars, *found.sensors]} - deletions = [ - module - for module in sorted({found.modules[id(run)] for run in [*found.jobs, *found.groups]}) - if module not in holding_others and _job_parts_only(module, project_dir) - ] - manual = _manual_edits(found, (names, calendars), deletions, project_dir) - verb = "Would write" if dry_run else "Wrote" - for line in printed: - print(line) - for path, text in writes: - if not dry_run: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(text, encoding="utf-8") - print(f"{verb} {path.relative_to(project_dir).as_posix()}") - for module in deletions: - if not dry_run: - module.unlink() - deleted: str = module.relative_to(project_dir).as_posix() - print(f"{'Would delete' if dry_run else 'Deleted'} {deleted}") - if manual: - print("Edit by hand:") - for line in manual: - print(f" {line}") - if any(renames.values()): - print( - f"{_ROOT}/{_RENAMES} keeps run history, pause and sensor state under the new names. " - "Publish every location, then delete it." - ) - return 0 - - -def _manual_edits( - found: _Found, - named: tuple[dict[int, str], dict[int, tuple[str, legacy.CalendarSchedule]]], - deletions: list[Path], - project_dir: Path, -) -> list[str]: - """The edits a person makes: leftover job declarations, calendars and sensors.""" - names, calendars = named - manual: list[str] = [] - for module in sorted({found.modules[id(run)] for run in [*found.jobs, *found.groups]}): - if module not in deletions: - manual.append( - f"{module.relative_to(project_dir).as_posix()}: delete the job, job group and " - "schedule declarations; they are TOML now" - ) - for new, schedule in calendars.values(): - function_name: str = getattr(schedule.function, "__name__", "") - targets = ( - f" Its targets {list(schedule.targets)} are dropped: a calendar runs wherever its" - " job does." - if schedule.targets - else "" - ) - manual.append( - f"{_site(schedule, found.modules[id(schedule)], project_dir)}: replace the " - f"calendar_schedule(...) decorator of {function_name} with " - f"@calendar(name={_value(new)}, description={_value(schedule.description)}, " - f"horizon={_value(schedule.horizon)}), import calendar from sqlbuild.orchestration, " - f"and move it to {_ROOT}/calendars/ in a module of its own.{targets}" - ) - for sensor in found.sensors: - edits = [ - f"name={_value(names[id(sensor)])}", - f"run={_value(names.get(id(sensor.run), sensor.run.name))}", - ] - if isinstance(sensor.when, legacy.RunEventCondition): - watched = names.get(id(sensor.when.job), sensor.when.job.name) - edits.append(f"run_event(job={_value(watched)}, ...)") - manual.append( - f"{_site(sensor, found.modules[id(sensor)], project_dir)}: in sensor " - f"'{sensor.name}', write {', '.join(edits)}, drop the job imports, and move it to " - f"{_ROOT}/sensors/ in a module of its own" - ) - return manual diff --git a/src/sqlbuild/orchestration_declarations/models/legacy_authoring.py b/src/sqlbuild/orchestration_declarations/models/legacy_authoring.py deleted file mode 100644 index 79f3dc103..000000000 --- a/src/sqlbuild/orchestration_declarations/models/legacy_authoring.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Removed authoring records, rebuilt only while `sqb migrate orchestration` imports a project.""" - -from __future__ import annotations - -from collections.abc import Callable, Iterable, Sequence -from dataclasses import dataclass, field, replace -from datetime import datetime -from typing import Literal - -from sqlbuild._native import declaration_site -from sqlbuild.orchestration_declarations.models.authoring import ( - NodeEventCondition, - PlatformEventCondition, - RunRequest, - SkipReason, - SourceNewDataCondition, - SourceStaleCondition, -) -from sqlbuild.orchestration_declarations.types import RunEventKind - - -@dataclass(frozen=True, kw_only=True) -class Schedule: - """Cron with an RFC 3339 start; missed times collapse into one run of the latest.""" - - cron: str - timezone: str - starts_at: str | None = None - default_status: Literal["running", "paused"] = "running" - - -@dataclass(frozen=True, kw_only=True) -class OnFailure: - """Rerun a failed or partial run from failure, `retries` times, `retry_delay_seconds` apart.""" - - retries: int - retry_delay_seconds: int = 0 - - -@dataclass(frozen=True, kw_only=True) -class Resources: - """Worker container limits for a job; memory is bytes or a size such as "512m" or "2g".""" - - memory: int | str | None = None - cpus: float | None = None - pids: int | None = None - - -@dataclass(frozen=True, eq=False) -class Job: - """A named selection to execute, optionally on a wall-clock schedule.""" - - name: str - description: str = field(kw_only=True) - select: str = field(kw_only=True) - exclude: str = field(default="", kw_only=True) - settings: dict[str, object] = field(default_factory=dict, kw_only=True) - concurrency: int | None = field(default=None, kw_only=True) - concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) - schedule: Schedule | None = field(default=None, kw_only=True) - targets: tuple[str, ...] = field(default=(), kw_only=True) - owner: str | None = field(default=None, kw_only=True) - overlap: Literal["queue", "skip"] = field(default="queue", kw_only=True) - on_failure: OnFailure | None = field(default=None, kw_only=True) - resources: Resources | None = field(default=None, kw_only=True) - _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) - - def __post_init__(self) -> None: - object.__setattr__(self, "_declared_at", declaration_site()) - - -@dataclass(frozen=True, eq=False) -class JobGroup: - """Jobs ordered by the lineage between the nodes they build.""" - - name: str - description: str = field(kw_only=True) - jobs: Sequence[Job] = field(kw_only=True) - concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) - schedule: Schedule | None = field(default=None, kw_only=True) - targets: tuple[str, ...] = field(default=(), kw_only=True) - owner: str | None = field(default=None, kw_only=True) - overlap: Literal["queue", "skip"] = field(default="queue", kw_only=True) - on_failure: OnFailure | None = field(default=None, kw_only=True) - _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) - - def __post_init__(self) -> None: - object.__setattr__(self, "_declared_at", declaration_site()) - - -@dataclass(frozen=True, kw_only=True, eq=False) -class CalendarSchedule: - """Decorate a location callback returning upcoming timezone-aware datetimes.""" - - job: Job | JobGroup - description: str - horizon: str - timezone: str - starts_at: str | None = None - default_status: Literal["running", "paused"] = "running" - targets: tuple[str, ...] = () - function: Callable[..., Iterable[datetime]] | None = field(default=None, repr=False) - _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) - - def __post_init__(self) -> None: - object.__setattr__(self, "_declared_at", declaration_site()) - - def __call__(self, function: Callable[..., Iterable[datetime]]) -> CalendarSchedule: - return replace(self, function=function) - - -@dataclass(frozen=True, kw_only=True) -class RunEventCondition: - """Match scoped terminal outcomes, optionally requiring genuinely executed successful nodes.""" - - job: Job | JobGroup - kinds: Sequence[RunEventKind] - executed_successfully: Sequence[str] | None = None - kind: Literal["run_event"] = field(default="run_event", init=False) - - -@dataclass(frozen=True, eq=False) -class Sensor: - """Start a job for each matching event; event identity deduplicates replayed requests.""" - - name: str - description: str = field(kw_only=True) - when: ( - PlatformEventCondition - | RunEventCondition - | NodeEventCondition - | SourceNewDataCondition - | SourceStaleCondition - | None - ) = field(default=None, kw_only=True) - run: Job | JobGroup = field(kw_only=True) - minimum_interval_seconds: int = field(default=30, kw_only=True) - default_status: Literal["running", "paused"] = field(default="running", kw_only=True) - targets: tuple[str, ...] = field(default=(), kw_only=True) - function: Callable[..., Iterable[RunRequest] | SkipReason] | None = field( - default=None, repr=False, kw_only=True - ) - _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) - - def __post_init__(self) -> None: - object.__setattr__(self, "_declared_at", declaration_site()) - - def __call__(self, function: Callable[..., Iterable[RunRequest] | SkipReason]) -> Sensor: - return replace(self, function=function) diff --git a/tests/unit/src/sqlbuild/orchestration_declarations/_test_types.py b/tests/unit/src/sqlbuild/orchestration_declarations/_test_types.py deleted file mode 100644 index 9c4c91d35..000000000 --- a/tests/unit/src/sqlbuild/orchestration_declarations/_test_types.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Frozen cases for the orchestration migration shim's import boundary.""" - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class ShimImportCase: - description: str - module: str - allowed_python_importers: frozenset[str] - allowed_rust_references: frozenset[str] diff --git a/tests/unit/src/sqlbuild/orchestration_declarations/test_migration_shim_boundary.py b/tests/unit/src/sqlbuild/orchestration_declarations/test_migration_shim_boundary.py deleted file mode 100644 index a66bdb966..000000000 --- a/tests/unit/src/sqlbuild/orchestration_declarations/test_migration_shim_boundary.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Only `sqb migrate orchestration` reaches the removed Python job records.""" - -import ast -from pathlib import Path - -import pytest - -from tests.unit.src.sqlbuild.orchestration_declarations._test_types import ShimImportCase - -_LEGACY_AUTHORING: str = "sqlbuild.orchestration_declarations.models.legacy_authoring" -_MIGRATE: str = "sqlbuild.orchestration_declarations.main.migrate_orchestration" - - -def _python_references(path: Path) -> set[str]: - """Modules `path` imports, plus dotted strings it could hand to an import function.""" - found: set[str] = set() - for node in ast.walk(ast.parse(path.read_text(), filename=str(path))): - if isinstance(node, ast.Import): - found.update(alias.name for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.module is not None: - found.add(node.module) - found.update(f"{node.module}.{alias.name}" for alias in node.names) - elif isinstance(node, ast.Constant) and isinstance(node.value, str): - found.add(node.value) - return found - - -@pytest.mark.parametrize( - "test_case", - [ - ShimImportCase( - description="the removed job records reach only the migration command", - module=_LEGACY_AUTHORING, - allowed_python_importers=frozenset( - {"src/sqlbuild/orchestration_declarations/main/migrate_orchestration.py"} - ), - # Names the module only to skip its frames when locating a declaration's line. - allowed_rust_references=frozenset( - {"crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs"} - ), - ), - ShimImportCase( - description="the migration runs only from `sqb migrate orchestration`", - module=_MIGRATE, - allowed_python_importers=frozenset(), - allowed_rust_references=frozenset( - {"crates/sqlbuild-python/src/bindings/_helpers/workspace_commands/commands.rs"} - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_migration_shim_when_scanning_sources_then_only_the_migrate_command_reaches_it( - test_case: ShimImportCase, -) -> None: - python_importers: set[str] = { - path.as_posix() - for path in Path("src/sqlbuild").rglob("*.py") - if test_case.module in _python_references(path) - } - rust_references: set[str] = { - path.as_posix() - for path in Path("crates").rglob("*.rs") - if test_case.module in path.read_text() - } - - assert python_importers == test_case.allowed_python_importers - assert rust_references == test_case.allowed_rust_references From f140ba034a7ce5fc89af2da2b6ece7f1a4f41682 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:30:25 +0000 Subject: [PATCH 24/34] feat: rewrite TOML job selectors when models are renamed or moved sqb rename and sqb mv now update exact model names and whole-folder paths in orchestration/jobs and orchestration/job_groups selectors, keeping quote style. A folder that only some models leave is reported as a note, and multi-line selector strings are left for a manual edit. --- Cargo.lock | 1 + crates/sqlbuild-refactor/Cargo.toml | 1 + .../_helpers/planning/job_selectors.rs | 301 ++++++++++++++++++ .../src/refactoring/_helpers/planning/mod.rs | 1 + .../_helpers/planning/single_planning.rs | 12 +- .../src/refactoring/constants.rs | 9 + .../src/refactoring/main/plan_refactor.rs | 18 ++ .../src/refactoring/models.rs | 3 + .../src/refactoring/tests.rs | 2 + .../refactoring/tests/test_job_selectors.rs | 154 +++++++++ .../src/refactoring/tests/test_types.rs | 10 + .../cli/commands/main/refactor/_test_types.py | 11 + .../main/refactor/test_job_selectors.py | 84 +++++ 13 files changed, 606 insertions(+), 1 deletion(-) create mode 100644 crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs create mode 100644 crates/sqlbuild-refactor/src/refactoring/tests/test_job_selectors.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py diff --git a/Cargo.lock b/Cargo.lock index acde81235..9b58b61cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4457,6 +4457,7 @@ dependencies = [ "sqlbuild-pysource", "sqlbuild-sqltext", "tempfile", + "toml", ] [[package]] diff --git a/crates/sqlbuild-refactor/Cargo.toml b/crates/sqlbuild-refactor/Cargo.toml index 8d992e24b..b8462bc04 100644 --- a/crates/sqlbuild-refactor/Cargo.toml +++ b/crates/sqlbuild-refactor/Cargo.toml @@ -20,3 +20,4 @@ sqlbuild-model-config.workspace = true sqlbuild-pysource.workspace = true sqlbuild-sqltext.workspace = true tempfile.workspace = true +toml.workspace = true diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs new file mode 100644 index 000000000..d153692b8 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs @@ -0,0 +1,301 @@ +//! `select` and `exclude` in TOML job files, rewritten for renamed and moved models. + +use std::collections::BTreeMap; +use std::path::Path; + +use serde::Deserialize; +use sqlbuild_core::text::errors::TextDecodeError; +use toml::Spanned; + +use crate::refactoring::_helpers::edits::text_edits::{line_column, text_edit}; +use crate::refactoring::_helpers::files::paths::parent; +use crate::refactoring::_helpers::files::workspace::project_files_with_suffix; +use crate::refactoring::_helpers::scanning::chars::chars; +use crate::refactoring::constants::{ + JOB_FILES_DIRECTORY, MODEL_ROOT, PARTIAL_FOLDER_MOVE_REASON, TOML_SUFFIX, + UNREADABLE_JOB_REASON, UNWRITABLE_SELECTOR_REASON, +}; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::{EditKind, ManualLocation, ModelFacts, TextEdit}; + +/// The selector fields of a job file; publication checks everything else. +#[derive(Deserialize)] +struct JobSelectors { + select: Option>, + exclude: Option>, +} + +/// Selector edits by path, selectors to edit by hand or to check, and unreadable job files. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct SelectorParts { + pub(crate) edits: Vec<(String, TextEdit)>, + pub(crate) manual: Vec, + pub(crate) notes: Vec, + pub(crate) blocking: Vec, +} + +/// What the refactoring changes, as job selectors see it. +struct Changes { + /// Old model name to new, for each model whose name changes. + names: BTreeMap, + /// Old folder to new, for each folder every model below it leaves together. + folders: BTreeMap, + /// Folders some, but not all, of their models leave. + partial: BTreeMap>, +} + +/// Rewrite exact model names and wholly moved folders in every job's `select` and `exclude`. +/// +/// `renamed` holds each target model and its new name and path. Tags, globs, `job:` and +/// other kinds are left alone; a folder only some of its models leave becomes a note. +pub(crate) fn job_selector_parts( + project_dir: &Path, + models: &[ModelFacts], + renamed: &[(&ModelFacts, &str, &str)], +) -> Result { + let mut parts = SelectorParts::default(); + let jobs_dir = project_dir.join(JOB_FILES_DIRECTORY); + if !jobs_dir.is_dir() { + return Ok(parts); + } + let changes = changes(models, renamed); + for (relative, text) in project_files_with_suffix(&jobs_dir, TOML_SUFFIX)? { + let path = format!("{JOB_FILES_DIRECTORY}/{relative}"); + let contents = match text { + Ok(contents) => contents, + Err(TextDecodeError { .. }) => { + parts + .blocking + .push(location(path, None, UNREADABLE_JOB_REASON)); + continue; + } + }; + // A job file that does not parse fails compile with its own error. + let Ok(job) = toml::from_str::(&contents) else { + continue; + }; + let text = chars(&contents); + for field in [job.select, job.exclude].into_iter().flatten() { + let span = field.span(); + let (start, end) = ( + contents[..span.start].chars().count(), + contents[..span.end].chars().count(), + ); + let (line, column) = line_column(&text, start); + for (folder, moved) in &changes.partial { + if selected_folders(field.get_ref()).any(|selected| selected == *folder) { + let reason = PARTIAL_FOLDER_MOVE_REASON + .replace("{folder}", folder) + .replace("{models}", &moved.join(", ")); + parts + .notes + .push(location(path.clone(), Some((line, column)), &reason)); + } + } + let rewritten = rewrite(field.get_ref(), &changes); + if rewritten == *field.get_ref() { + continue; + } + match literal(&contents[span.start..span.end], &rewritten) { + Some(replacement) => parts.edits.push(( + path.clone(), + text_edit(&text, (start, end), replacement, EditKind::Selector), + )), + None => parts.manual.push(location( + path.clone(), + Some((line, column)), + &UNWRITABLE_SELECTOR_REASON.replace("{selector}", &rewritten), + )), + } + } + } + Ok(parts) +} + +fn location(path: String, at: Option<(usize, usize)>, reason: &str) -> ManualLocation { + ManualLocation { + path, + line: at.map(|(line, _)| line), + column: at.map(|(_, column)| column), + reason: reason.to_owned(), + } +} + +fn changes(models: &[ModelFacts], renamed: &[(&ModelFacts, &str, &str)]) -> Changes { + let names: BTreeMap = renamed + .iter() + .filter(|(model, new_name, _)| model.name != *new_name) + .map(|(model, new_name, _)| (model.name.clone(), (*new_name).to_owned())) + .collect(); + let destinations: BTreeMap<&str, String> = renamed + .iter() + .map(|(model, _, destination)| (model.path.as_str(), parent(destination))) + .collect(); + let mut folders = BTreeMap::new(); + let mut partial = BTreeMap::new(); + let moved_from = renamed + .iter() + .map(|(model, _, destination)| (parent(&model.path), parent(destination))) + .filter(|(from, to)| from != to) + .flat_map(|(from, _)| ancestors(&from)); + for folder in moved_from { + if folder == MODEL_ROOT || folders.contains_key(&folder) || partial.contains_key(&folder) { + continue; + } + let below: Vec<&ModelFacts> = models + .iter() + .filter(|model| within(&parent(&model.path), &folder)) + .collect(); + let moved_to = |model: &ModelFacts| -> Option { + let to = destinations.get(model.path.as_str())?; + let rest = &parent(&model.path)[folder.len()..]; + to.strip_suffix(rest).map(str::to_owned) + }; + let targets: Vec> = below.iter().map(|model| moved_to(model)).collect(); + let whole = targets + .first() + .cloned() + .flatten() + .filter(|to| *to != folder && !within(to, &folder)) + .filter(|to| targets.iter().all(|target| target.as_ref() == Some(to))); + match whole { + Some(to) => { + folders.insert(folder, to); + } + None => { + let leaving: Vec = below + .iter() + .filter(|model| { + destinations + .get(model.path.as_str()) + .is_some_and(|to| !within(to, &folder)) + }) + .map(|model| model.name.clone()) + .collect(); + if !leaving.is_empty() { + partial.insert(folder, leaving); + } + } + } + } + Changes { + names, + folders, + partial, + } +} + +/// `folder` and each folder above it, up to the model root. +fn ancestors(folder: &str) -> Vec { + let parts: Vec<&str> = folder.split('/').collect(); + (1..=parts.len()) + .map(|count| parts[..count].join("/")) + .collect() +} + +fn within(path: &str, folder: &str) -> bool { + path == folder || path.starts_with(&format!("{folder}/")) +} + +/// The folders a selector string names, normalized as path selectors match them. +fn selected_folders(selector: &str) -> impl Iterator + '_ { + selector + .split_whitespace() + .flat_map(|token| token.split(',')) + .filter_map(|part| folder_of(part.trim_matches('+')).map(|(_, folder)| folder)) +} + +/// The `path:` prefix (if any) and normalized folder of a folder selector part. +fn folder_of(core: &str) -> Option<(&str, String)> { + let (prefix, value) = match core.split_once(':') { + Some(("path", value)) => ("path:", value), + Some(_) => return None, + None if core.contains('/') && !core.contains('~') => ("", core), + None => return None, + }; + Some(( + prefix, + value.replace('\\', "/").trim_matches('/').to_owned(), + )) +} + +/// `selector` with renamed model names and moved folders replaced, whitespace kept. +fn rewrite(selector: &str, changes: &Changes) -> String { + let mut out = String::with_capacity(selector.len()); + let mut token = String::new(); + for character in selector.chars().chain(std::iter::once(' ')) { + if character.is_whitespace() { + if !token.is_empty() { + let parts: Vec = token + .split(',') + .map(|part| part_of(part, changes)) + .collect(); + out.push_str(&parts.join(",")); + token.clear(); + } + out.push(character); + } else { + token.push(character); + } + } + out.pop(); + out +} + +fn part_of(part: &str, changes: &Changes) -> String { + let core = part.trim_matches('+'); + if core.is_empty() { + return part.to_owned(); + } + let lead = &part[..part.len() - part.trim_start_matches('+').len()]; + let tail = &part[part.trim_end_matches('+').len()..]; + let name = |value: &str| -> String { + changes + .names + .get(value.trim()) + .map_or_else(|| value.to_owned(), |new| value.replace(value.trim(), new)) + }; + let core = if let Some((start, end)) = core.split_once('~') { + format!("{}~{}", name(start), name(end)) + } else if let Some((prefix, folder)) = folder_of(core) { + match changes.folders.get(&folder) { + Some(to) => { + let trailing = if core.ends_with('/') { "/" } else { "" }; + format!("{prefix}{to}{trailing}") + } + None => core.to_owned(), + } + } else if core.contains(':') { + core.to_owned() + } else { + name(core) + }; + format!("{lead}{core}{tail}") +} + +/// `value` written as a TOML string in the style of `raw`, or `None` for multi-line strings. +fn literal(raw: &str, value: &str) -> Option { + if raw.starts_with("'''") || raw.starts_with("\"\"\"") { + return None; + } + if raw.starts_with('\'') && !value.contains(['\'', '\n', '\r']) { + return Some(format!("'{value}'")); + } + let mut escaped = String::with_capacity(value.len() + 2); + escaped.push('"'); + for character in value.chars() { + match character { + '"' => escaped.push_str("\\\""), + '\\' => escaped.push_str("\\\\"), + '\n' => escaped.push_str("\\n"), + '\t' => escaped.push_str("\\t"), + '\r' => escaped.push_str("\\r"), + control if control.is_control() => { + escaped.push_str(&format!("\\u{:04X}", u32::from(control))); + } + other => escaped.push(other), + } + } + escaped.push('"'); + Some(escaped) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs index c363fc9d2..ea437df70 100644 --- a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod batch_planning; pub(crate) mod column_planning; pub(crate) mod column_references; pub(crate) mod declaration_moves; +pub(crate) mod job_selectors; pub(crate) mod model_planning; pub(crate) mod model_references; pub(crate) mod python_imports; diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/single_planning.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/single_planning.rs index 7bd5d7374..4485bc6da 100644 --- a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/single_planning.rs +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/single_planning.rs @@ -6,6 +6,7 @@ use crate::refactoring::_helpers::edits::text_edits::build_plan; use crate::refactoring::_helpers::planning::column_planning::{ column_rename_context, column_rename_parts, }; +use crate::refactoring::_helpers::planning::job_selectors::job_selector_parts; use crate::refactoring::_helpers::planning::model_planning::{model_parts, model_target}; use crate::refactoring::_helpers::planning::python_imports::python_import_parts; use crate::refactoring::_helpers::scanning::scan_context::ScanContext; @@ -43,11 +44,20 @@ pub(crate) fn plan_single( let mut moves = parts.moves.clone(); let destination = target.destination(); if destination != target.source_path { - moves.push((target.source_path.clone(), destination)); + moves.push((target.source_path.clone(), destination.clone())); } let imports = python_import_parts(Path::new(&facts.project_dir), &moves)?; parts.edits.extend(imports.edits); parts.blocking.extend(imports.blocking); + let selectors = job_selector_parts( + Path::new(&facts.project_dir), + &facts.models, + &[(target.model, &target.request.new_name, &destination)], + )?; + parts.edits.extend(selectors.edits); + parts.manual.extend(selectors.manual); + parts.notes.extend(selectors.notes); + parts.blocking.extend(selectors.blocking); Ok(build_plan( target.request, parts, diff --git a/crates/sqlbuild-refactor/src/refactoring/constants.rs b/crates/sqlbuild-refactor/src/refactoring/constants.rs index eea18227e..8f7ebeb91 100644 --- a/crates/sqlbuild-refactor/src/refactoring/constants.rs +++ b/crates/sqlbuild-refactor/src/refactoring/constants.rs @@ -101,3 +101,12 @@ pub const GENERATED_SOURCE_CONFLICT: &str = "source file is marked as generated" pub const SOURCE_BLOCK_CONFLICT: &str = "source declaration has no unambiguous YAML block"; /// The `ValueError` prefix of a `name_pattern` the native matcher rejects. pub const INVALID_NAME_PATTERN_PREFIX: &str = "invalid name_pattern: "; +/// Where job files live; their `select` and `exclude` follow renamed and moved models. +pub const JOB_FILES_DIRECTORY: &str = "orchestration/jobs"; +pub const TOML_SUFFIX: &str = ".toml"; +/// The folder path selectors start from. +pub const MODEL_ROOT: &str = "models"; +pub const UNREADABLE_JOB_REASON: &str = "this job file cannot be read as UTF-8, so its selectors cannot be updated; fix or remove it first"; +pub const PARTIAL_FOLDER_MOVE_REASON: &str = "this job selects folder '{folder}', and {models} moved out of it; check the job still selects what it should"; +pub const UNWRITABLE_SELECTOR_REASON: &str = + "this selector is a multi-line string; change it to '{selector}' by hand"; diff --git a/crates/sqlbuild-refactor/src/refactoring/main/plan_refactor.rs b/crates/sqlbuild-refactor/src/refactoring/main/plan_refactor.rs index b9e0b4971..dae4b7e82 100644 --- a/crates/sqlbuild-refactor/src/refactoring/main/plan_refactor.rs +++ b/crates/sqlbuild-refactor/src/refactoring/main/plan_refactor.rs @@ -4,6 +4,7 @@ use std::path::Path; use crate::refactoring::_helpers::edits::text_edits::build_plan; use crate::refactoring::_helpers::planning::batch_planning::batch_parts; +use crate::refactoring::_helpers::planning::job_selectors::job_selector_parts; use crate::refactoring::_helpers::planning::model_planning::{ModelTarget, model_target}; use crate::refactoring::_helpers::planning::python_imports::python_import_parts; use crate::refactoring::_helpers::planning::single_planning::plan_single; @@ -38,6 +39,23 @@ pub fn plan_refactor( let imports = python_import_parts(Path::new(&facts.project_dir), &moves)?; parts.edits.extend(imports.edits); parts.blocking.extend(imports.blocking); + let destinations: Vec = targets.iter().map(ModelTarget::destination).collect(); + let renamed: Vec<(&_, &str, &str)> = targets + .iter() + .zip(&destinations) + .map(|(target, destination)| { + ( + target.model, + target.request.new_name.as_str(), + destination.as_str(), + ) + }) + .collect(); + let selectors = job_selector_parts(Path::new(&facts.project_dir), &facts.models, &renamed)?; + parts.edits.extend(selectors.edits); + parts.manual.extend(selectors.manual); + parts.notes.extend(selectors.notes); + parts.blocking.extend(selectors.blocking); let batch: Vec = targets.into_iter().map(|target| target.request).collect(); let mut plan = build_plan( batch[0].clone(), diff --git a/crates/sqlbuild-refactor/src/refactoring/models.rs b/crates/sqlbuild-refactor/src/refactoring/models.rs index 1c54b51bc..08708b981 100644 --- a/crates/sqlbuild-refactor/src/refactoring/models.rs +++ b/crates/sqlbuild-refactor/src/refactoring/models.rs @@ -24,6 +24,8 @@ pub enum EditKind { Migration, /// A Python import of a moved module. Import, + /// A job's `select` or `exclude` naming a renamed or moved model. + Selector, } impl EditKind { @@ -36,6 +38,7 @@ impl EditKind { Self::Column => "column", Self::Migration => "migration", Self::Import => "import", + Self::Selector => "selector", } } } diff --git a/crates/sqlbuild-refactor/src/refactoring/tests.rs b/crates/sqlbuild-refactor/src/refactoring/tests.rs index 3c5fadaa1..a35f9949b 100644 --- a/crates/sqlbuild-refactor/src/refactoring/tests.rs +++ b/crates/sqlbuild-refactor/src/refactoring/tests.rs @@ -8,6 +8,8 @@ mod declaration_edits; mod header_edits; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_job_selectors.rs"] +mod job_selectors; #[path = "tests/test_planning.rs"] mod planning; #[path = "tests/test_sql_sites.rs"] diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_job_selectors.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_job_selectors.rs new file mode 100644 index 000000000..ce6104ae4 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_job_selectors.rs @@ -0,0 +1,154 @@ +use crate::refactoring::_helpers::edits::text_edits::apply_text_edits; +use crate::refactoring::_helpers::planning::job_selectors::job_selector_parts; +use crate::refactoring::_helpers::scanning::chars::chars; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::ModelFacts; +use crate::refactoring::tests::helpers::{model, write_files}; +use crate::refactoring::tests::test_types::JobSelectorTestCase; + +const JOB: &str = "orchestration/jobs/orders/refresh.toml"; + +/// A model the refactoring leaves alone keeps its name and path. +const STAYS: (&str, &str, &str, &str) = ("customers", "models/marts/customers.sql", "", ""); + +#[test] +fn given_job_selectors_when_models_are_renamed_or_moved_then_exact_parts_follow() +-> Result<(), RefactorError> { + let test_cases = [ + JobSelectorTestCase { + description: "exact names follow; lookalikes, globs, tags and other kinds stay", + models: &[ + ( + "orders", + "models/marts/orders.sql", + "sales", + "models/marts/sales.sql", + ), + STAYS, + ], + job: "select = \"+orders+ orders_x tag:orders source:orders job:orders orders* order?s orders,customers orders~customers +customers~orders+\"\n", + expected_job: "select = \"+sales+ orders_x tag:orders source:orders job:orders orders* order?s sales,customers sales~customers +customers~sales+\"\n", + expected_manual: &[], + }, + JobSelectorTestCase { + description: "literal and escaped strings keep their style", + models: &[ + ( + "orders", + "models/marts/orders.sql", + "sales", + "models/marts/sales.sql", + ), + STAYS, + ], + job: "select = 'orders'\nexclude = \"orders \\u0074ag:x\"\n", + expected_job: "select = 'sales'\nexclude = \"sales tag:x\"\n", + expected_manual: &[], + }, + JobSelectorTestCase { + description: "a folder every model leaves moves, nested folders included", + models: &[ + ( + "orders", + "models/staging/orders.sql", + "orders", + "models/core/orders.sql", + ), + ( + "lines", + "models/staging/sub/lines.sql", + "lines", + "models/core/sub/lines.sql", + ), + STAYS, + ], + job: "select = \"models/staging/ path:models/staging +models/staging/sub+ models/stagingx models\"\n", + expected_job: "select = \"models/core/ path:models/core +models/core/sub+ models/stagingx models\"\n", + expected_manual: &[], + }, + JobSelectorTestCase { + description: "a folder only some models leave stays and is reported", + models: &[ + ( + "orders", + "models/staging/orders.sql", + "orders", + "models/core/orders.sql", + ), + ("lines", "models/staging/lines.sql", "", ""), + ], + job: "name = \"x_job\"\nselect = \"models/staging orders\"\n", + expected_job: "name = \"x_job\"\nselect = \"models/staging orders\"\n", + expected_manual: &[ + "2:10 this job selects folder 'models/staging', and orders moved out of it; check the job still selects what it should", + ], + }, + JobSelectorTestCase { + description: "a multi-line selector is reported instead of rewritten", + models: &[( + "orders", + "models/marts/orders.sql", + "sales", + "models/marts/sales.sql", + )], + job: "select = \"\"\"\norders\"\"\"\n", + expected_job: "select = \"\"\"\norders\"\"\"\n", + expected_manual: &[ + "1:10 this selector is a multi-line string; change it to 'sales' by hand", + ], + }, + ]; + for test_case in test_cases { + let project = + tempfile::tempdir().map_err(|error| RefactorError::value(error.to_string()))?; + write_files(project.path(), &[(JOB, test_case.job)])?; + let models: Vec = test_case + .models + .iter() + .map(|(name, path, _, _)| ModelFacts { + path: (*path).to_owned(), + ..model(name, Some("table"), "analytics") + }) + .collect(); + let renamed: Vec<(&ModelFacts, &str, &str)> = test_case + .models + .iter() + .zip(&models) + .filter(|((_, _, new_name, _), _)| !new_name.is_empty()) + .map(|((_, _, new_name, new_path), facts)| (facts, *new_name, *new_path)) + .collect(); + let parts = job_selector_parts(project.path(), &models, &renamed)?; + let edits = parts + .edits + .into_iter() + .map(|(_, edit)| edit) + .collect::>(); + let manual: Vec = parts + .manual + .iter() + .chain(&parts.notes) + .map(|location| { + format!( + "{}:{} {}", + location.line.unwrap_or_default(), + location.column.unwrap_or_default(), + location.reason + ) + }) + .collect(); + assert_eq!( + apply_text_edits(&chars(test_case.job), &edits)? + .into_iter() + .collect::(), + test_case.expected_job, + "{}", + test_case.description + ); + assert_eq!( + manual, test_case.expected_manual, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs index 3a74e7187..7ac01f504 100644 --- a/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs @@ -257,3 +257,13 @@ pub(super) struct StagedCommitOrderTestCase { pub(super) changes: &'static [(&'static str, &'static str)], pub(super) expected_files: &'static [(&'static str, &'static str)], } + +/// Models as `(name, path, new name, new path)`, a job's selector lines and what they become. +pub(super) struct JobSelectorTestCase { + pub(super) description: &'static str, + pub(super) models: &'static [(&'static str, &'static str, &'static str, &'static str)], + pub(super) job: &'static str, + pub(super) expected_job: &'static str, + /// `line:column reason` of each selector a person must edit or check. + pub(super) expected_manual: &'static [&'static str], +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py index ab99b5aa0..22720cb3a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py @@ -299,3 +299,14 @@ class PythonNamedMoveE2ETestCase: destination: str expected_exit_code: int expected_model_path: str + + +@dataclass(frozen=True) +class JobSelectorRefactorE2ETestCase: + """A rename or move, the job file before and after it, and the job built afterwards.""" + + description: str + command: tuple[str, ...] + job: str + expected_job: str + expected_output: tuple[str, ...] = () diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py new file mode 100644 index 000000000..f7d82b1d4 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py @@ -0,0 +1,84 @@ +"""CLI e2e coverage for `sqb rename` and `sqb mv` rewriting TOML job selectors.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + JobSelectorRefactorE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import sqb, write_orders_project + +_JOB: str = "orchestration/jobs/orders/refresh.toml" +_UNRELATED: str = "orchestration/jobs/orders/totals.toml" +_UNRELATED_JOB: str = ( + 'name = "orders_totals_job"\ndescription = "Rebuild totals."\nselect = "customer_totals"\n' +) + + +def _job(select: str, exclude: str = "") -> str: + return f'name = "orders_refresh_job"\ndescription = "Refresh orders."\nselect = {select}\n' + ( + f"exclude = {exclude}\n" if exclude else "" + ) + + +@pytest.mark.parametrize( + "test_case", + [ + JobSelectorRefactorE2ETestCase( + description="a rename rewrites exact names and leaves globs and other kinds alone", + command=("rename", "stg_orders", "stg_orders_clean"), + job=_job( + '"+stg_orders+ fact_orders,+stg_orders stg_orders~customer_totals stg_ord*"', + "'stg_orders+,customer_totals'", + ), + expected_job=_job( + '"+stg_orders_clean+ fact_orders,+stg_orders_clean ' + 'stg_orders_clean~customer_totals stg_ord*"', + "'stg_orders_clean+,customer_totals'", + ), + expected_output=("selector", _JOB), + ), + JobSelectorRefactorE2ETestCase( + description="moving every model out of a folder moves its folder selectors", + command=("mv", "stg_orders", "models/core/"), + job=_job('"models/staging +path:models/staging/ stg_orders"'), + expected_job=_job('"models/core +path:models/core/ stg_orders"'), + expected_output=("selector", _JOB), + ), + JobSelectorRefactorE2ETestCase( + description="moving some models out of a folder leaves it and asks for a check", + command=("mv", "fact_orders", "models/finance/"), + job=_job('"+models/marts"'), + expected_job=_job('"+models/marts"'), + expected_output=( + "Notes", + f"{_JOB}:3:10", + "this job selects folder 'models/marts', and fact_orders moved out of it", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_job_selectors_when_refactoring_then_jobs_follow_and_still_build( + tmp_path: Path, test_case: JobSelectorRefactorE2ETestCase +) -> None: + project_dir: Path = write_orders_project( + tmp_path=tmp_path, files={_JOB: test_case.job, _UNRELATED: _UNRELATED_JOB} + ) + + refactored: subprocess.CompletedProcess[str] = sqb(project_dir, *test_case.command) + built: subprocess.CompletedProcess[str] = sqb( + project_dir, "build", "--select", "job:orders_refresh_job" + ) + + output: str = refactored.stdout + refactored.stderr + assert refactored.returncode == 0, output + for fragment in test_case.expected_output: + assert fragment in output, output + assert (project_dir / _JOB).read_text(encoding="utf-8") == test_case.expected_job + assert (project_dir / _UNRELATED).read_text(encoding="utf-8") == _UNRELATED_JOB + assert built.returncode == 0, built.stdout + built.stderr From ee5a026bd8c063f9af23523733401ff048b9c5d4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 12:17:19 +0000 Subject: [PATCH 25/34] feat!: declare jobs, job groups and schedules in TOML Jobs, job groups and their schedules are now TOML files under orchestration/jobs/ and orchestration/job_groups/. Sensors and calendars stay Python. Python job, job_group, schedule and calendar_schedule usage fails compilation with D017 and points to the orchestration docs. Convert DuckShop, its orchestration edge cases, the deploy example and the tests to TOML, and add the orchestration concepts page. --- .../orchestration/jobs.py | 15 - .../orchestration/jobs/adhoc_exports_job.toml | 3 + .../orchestration/jobs/finance_daily_job.toml | 4 + .../orchestration/jobs/staging_daily_job.toml | 5 + .../orchestration/jobs.py | 17 - .../orchestration/jobs/crm_daily_job.toml | 5 + .../orchestration/jobs/sales_daily_job.toml | 4 + .../orchestration/attached_checks.json | 4 +- .../orchestration/calendar_orders.py | 25 - .../calendars/fulfillment_calendar.py | 12 + .../jobs/order_reporting_job.toml | 5 + .../orchestration/changed_source_orders.py | 17 - .../jobs/orders_refresh_job.toml | 7 + .../sensors/on_source_orders_sensor.py | 11 + .../orchestration/conflicting_settings.py | 10 - .../jobs/refresh_orders_job.toml | 8 + .../container_resources_orders.py | 11 - .../jobs/annual_orders_job.toml | 9 + .../edge_cases/orchestration/custom_orders.py | 15 - .../custom_orders/jobs/annual_orders_job.toml | 3 + .../sensors/poll_orders_sensor.py | 16 + .../orchestration/development_orders.py | 8 - .../jobs/annual_orders_job.toml | 5 + .../edge_cases/orchestration/failed_import.py | 3 - .../failed_import/jobs/orders_job.toml | 3 + .../sensors/broken_orders_sensor.py | 12 + .../orchestration/invalid_settings.py | 10 - .../jobs/refresh_orders_job.toml | 7 + .../edge_cases/orchestration/loaded_orders.py | 14 - .../loaded_orders/jobs/annual_orders_job.toml | 4 + .../orchestration/local_recovery_orders.py | 11 - .../jobs/annual_orders_job.toml | 6 + .../orchestration/name_collision.py | 5 - .../name_collision/jobs/orders_job.toml | 4 + .../edge_cases/orchestration/nested_groups.py | 7 - .../job_groups/daily_orders_job_group.toml | 3 + .../job_groups/weekly_orders_job_group.toml | 4 + .../nested_groups/jobs/orders_job.toml | 3 + .../edge_cases/orchestration/node_orders.py | 12 - .../node_orders/jobs/inventory_job.toml | 3 + .../node_orders/jobs/source_orders_job.toml | 3 + .../node_orders/sensors/on_orders_sensor.py | 10 + .../orchestration/platform_orders.py | 30 - .../jobs/health_orders_job.toml | 4 + .../sensors/other_target_orders_sensor.py | 11 + .../sensors/overdue_orders_sensor.py | 22 + .../orchestration/python_node_jobs.py | 10 - .../jobs/customer_counts_job.toml | 4 + .../jobs/order_counts_job.toml | 3 + .../edge_cases/orchestration/run_orders.py | 16 - .../run_orders/jobs/inventory_job.toml | 3 + .../run_orders/jobs/source_orders_job.toml | 3 + .../run_orders/sensors/on_orders_sensor.py | 14 + .../scheduler_crash_recovery.json | 6 +- .../jobs/orders_refresh_job.toml | 6 + .../orchestration/shared_function_layers.py | 14 - .../job_groups/layered_orders_job_group.toml | 4 + .../jobs/marts_layer_job.toml | 3 + .../jobs/staging_layer_job.toml | 3 + .../jobs/orders_refresh_job.toml | 3 + .../sensors/on_source_orders_sensor.py} | 11 +- .../edge_cases/orchestration/stale_orders.py | 13 - .../stale_orders/jobs/orders_refresh_job.toml | 3 + .../sensors/on_stale_orders_sensor.py | 11 + .../orchestration/upstream_loaded_orders.py | 15 - .../jobs/annual_orders_job.toml | 4 + .../orchestration/upstream_source_orders.py | 10 - .../jobs/annual_orders_job.toml | 5 + .../orchestration/edge_cases/definitions.py | 95 -- .../edge_cases/edge_inventory_job_group.toml | 4 + .../edge_cases/edge_products_job_group.toml | 4 + .../edge_cases/edge_snapshot_job_group.toml | 3 + .../edge_snapshot_inventory_job.toml | 4 + .../edge_cases/edge_snapshot_orders_job.toml | 11 + .../edge_snapshot_products_job.toml | 4 + .../edge_cases/edge_group_outcomes_sensor.py | 15 + .../edge_cases/edge_job_outcomes_sensor.py | 11 + .../edge_cases/edge_platform_orders_sensor.py | 21 + .../jobs/refresh_orders_job.toml | 5 + deploy/project/orchestration/orders.py | 10 - .../sqlbuild/references/docs/CONTENTS.md | 1 + .../sqlbuild/references/docs/cli/build.md | 6 +- .../sqlbuild/references/docs/cli/rename.md | 1 + .../sqlbuild/references/docs/cli/server.md | 48 +- .../docs/concepts/models/configuration.md | 2 +- .../references/docs/concepts/orchestration.md | 226 +++++ .../docs/concepts/project-configuration.md | 2 +- .../docs/concepts/python-nodes/overview.md | 2 +- .../references/docs/concepts/selectors.md | 11 +- .../cli/commands/main/build/helpers.py | 7 +- .../main/build/test_changed_builds.py | 14 +- .../main/build/test_python_node_selection.py | 6 +- .../compile/test_python_project_layout.py | 36 + .../main/refactor/test_job_selectors.py | 17 +- .../sqlbuild/cli/commands/main/_test_types.py | 1 - .../cli/commands/main/test_job_selectors.py | 54 +- .../compiler/compiled_project/_test_types.py | 27 +- .../compiler/compiled_project/helpers.py | 30 +- .../publication/docker/compose/helpers.py | 4 +- .../compiled_project/publication/helpers.py | 67 +- .../publication/test_adhoc.py | 18 +- .../publication/test_backfill.py | 20 +- .../publication/test_calendar.py | 88 +- .../publication/test_changed.py | 2 +- .../publication/test_custom_sensor.py | 7 +- .../compiled_project/publication/test_dev.py | 28 +- .../publication/test_docker_worker.py | 30 +- .../publication/test_job_subsets.py | 11 +- .../publication/test_local_recovery.py | 5 +- .../publication/test_person_journey.py | 4 +- .../publication/test_plan_preview.py | 1 - .../publication/test_project_roles.py | 2 +- .../publication/test_publication.py | 59 +- .../publication/test_reload.py | 38 +- .../publication/test_rerun.py | 30 +- .../publication/test_source_sensor.py | 15 +- .../publication/test_statistics.py | 2 +- .../publication/test_warehouse_catalog.py | 9 +- .../publication/test_warehouse_metadata.py | 10 +- .../publication/test_worker.py | 380 ++++---- .../test_orchestration_snapshot.py | 852 ++++++++---------- website/src/content/docs/docs/cli/build.mdx | 6 +- website/src/content/docs/docs/cli/rename.mdx | 1 + website/src/content/docs/docs/cli/server.mdx | 48 +- .../docs/concepts/models/configuration.mdx | 2 +- .../docs/docs/concepts/orchestration.mdx | 213 +++++ .../docs/concepts/project-configuration.mdx | 2 +- .../docs/concepts/python-nodes/overview.mdx | 2 +- .../content/docs/docs/concepts/selectors.mdx | 11 +- website/src/sidebar.json | 3 +- 130 files changed, 1818 insertions(+), 1404 deletions(-) delete mode 100644 benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs.py create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/adhoc_exports_job.toml create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/finance_daily_job.toml create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/staging_daily_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs.py create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/crm_daily_job.toml create mode 100644 benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/sales_daily_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/calendar_orders/calendars/fulfillment_calendar.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/calendar_orders/jobs/order_reporting_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/changed_source_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/jobs/orders_refresh_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/sensors/on_source_orders_sensor.py delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/conflicting_settings/jobs/refresh_orders_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/container_resources_orders/jobs/annual_orders_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/custom_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/custom_orders/jobs/annual_orders_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/custom_orders/sensors/poll_orders_sensor.py delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/development_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/development_orders/jobs/annual_orders_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/failed_import.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/failed_import/jobs/orders_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/failed_import/sensors/broken_orders_sensor.py delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/invalid_settings/jobs/refresh_orders_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/loaded_orders/jobs/annual_orders_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders/jobs/annual_orders_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/name_collision.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/name_collision/jobs/orders_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/nested_groups.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/daily_orders_job_group.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/weekly_orders_job_group.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/nested_groups/jobs/orders_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/node_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/inventory_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/source_orders_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/node_orders/sensors/on_orders_sensor.py delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/platform_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/platform_orders/jobs/health_orders_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/other_target_orders_sensor.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/overdue_orders_sensor.py delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/customer_counts_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/order_counts_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/run_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/inventory_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/source_orders_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/run_orders/sensors/on_orders_sensor.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/selector_rewrite/jobs/orders_refresh_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/job_groups/layered_orders_job_group.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/marts_layer_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/staging_layer_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/source_orders/jobs/orders_refresh_job.toml rename benchmarks/duckshop/edge_cases/orchestration/{source_orders.py => source_orders/sensors/on_source_orders_sensor.py} (54%) delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/stale_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/stale_orders/jobs/orders_refresh_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/stale_orders/sensors/on_stale_orders_sensor.py delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders/jobs/annual_orders_job.toml delete mode 100644 benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders/jobs/annual_orders_job.toml delete mode 100644 benchmarks/duckshop/project/orchestration/edge_cases/definitions.py create mode 100644 benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_inventory_job_group.toml create mode 100644 benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_products_job_group.toml create mode 100644 benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_snapshot_job_group.toml create mode 100644 benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_inventory_job.toml create mode 100644 benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_orders_job.toml create mode 100644 benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_products_job.toml create mode 100644 benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_group_outcomes_sensor.py create mode 100644 benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_job_outcomes_sensor.py create mode 100644 benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_platform_orders_sensor.py create mode 100644 deploy/project/orchestration/jobs/refresh_orders_job.toml delete mode 100644 deploy/project/orchestration/orders.py create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md create mode 100644 website/src/content/docs/docs/concepts/orchestration.mdx diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs.py b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs.py deleted file mode 100644 index 24b96260d..000000000 --- a/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Two scheduled jobs split the orders lineage and share int_orders; a third job has no trigger.""" - -from sqlbuild.orchestration import job, schedule - -daily = schedule(cron="0 2 * * *", timezone="UTC") -staging = job( - "staging_daily", description="Stage orders.", select="stg_orders int_orders", schedule=daily -) -finance = job( - "finance_daily", - description="Build revenue reports.", - select="int_orders fct_revenue rpt_pnl", - schedule=daily, -) -exports = job("adhoc_exports", description="Export orders on request.", select="orders_export") diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/adhoc_exports_job.toml b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/adhoc_exports_job.toml new file mode 100644 index 000000000..e6c4a3963 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/adhoc_exports_job.toml @@ -0,0 +1,3 @@ +name = "adhoc_exports_job" +description = "Export orders on request." +select = "orders_export" diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/finance_daily_job.toml b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/finance_daily_job.toml new file mode 100644 index 000000000..5075f6d97 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/finance_daily_job.toml @@ -0,0 +1,4 @@ +name = "finance_daily_job" +description = "Build revenue reports." +select = "int_orders fct_revenue rpt_pnl" +schedule = { cron = "0 2 * * *", timezone = "UTC" } diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/staging_daily_job.toml b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/staging_daily_job.toml new file mode 100644 index 000000000..a53b42295 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/staging_daily_job.toml @@ -0,0 +1,5 @@ +# Two scheduled jobs split the orders lineage and share int_orders; a third job has no trigger. +name = "staging_daily_job" +description = "Stage orders." +select = "stg_orders int_orders" +schedule = { cron = "0 2 * * *", timezone = "UTC" } diff --git a/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs.py b/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs.py deleted file mode 100644 index be6161714..000000000 --- a/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Two scheduled jobs each own one staging model; customer_orders reads both, in crm_daily.""" - -from sqlbuild.orchestration import job, schedule - -daily = schedule(cron="0 2 * * *", timezone="UTC") -crm = job( - "crm_daily", - description="Refresh customers.", - select="stg_customers customer_orders", - schedule=daily, -) -sales = job( - "sales_daily", - description="Refresh sales.", - select="stg_orders customer_sales", - schedule=daily, -) diff --git a/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/crm_daily_job.toml b/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/crm_daily_job.toml new file mode 100644 index 000000000..534d1a0ab --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/crm_daily_job.toml @@ -0,0 +1,5 @@ +# Two scheduled jobs each own one staging model; customer_orders reads both, in crm_daily_job. +name = "crm_daily_job" +description = "Refresh customers." +select = "stg_customers customer_orders" +schedule = { cron = "0 2 * * *", timezone = "UTC" } diff --git a/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/sales_daily_job.toml b/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/sales_daily_job.toml new file mode 100644 index 000000000..66eb43b1a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/sales_daily_job.toml @@ -0,0 +1,4 @@ +name = "sales_daily_job" +description = "Refresh sales." +select = "stg_orders customer_sales" +schedule = { cron = "0 2 * * *", timezone = "UTC" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json index f111fc629..76c5dad56 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json +++ b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json @@ -3,6 +3,8 @@ "warned": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.warn(message='Review order totals.')\n", "failed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.fail(message='Order totals are invalid.')\n", "cancelled": "import signal\nfrom sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Heartbeat cancellation interrupts the driver without SIGTERM.')\ndef orders_valid(ctx):\n signal.signal(signal.SIGTERM, signal.SIG_IGN)\n ctx.query('WITH RECURSIVE pending(i) AS ((SELECT i FROM range(10000000000) AS seed(i) LIMIT 1) UNION ALL SELECT i FROM pending) SELECT sum(i) FROM pending')\n return ctx.pass_()\n", - "on_failure": "from sqlbuild.orchestration import NodeEventKind, job, node_event, schedule, sensor\nannual_orders = job('annual_orders', select='orders', description='Build orders.', schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\nrefresh_inventory = job('refresh_inventory', select='inventory', description='Refresh inventory.')\nfailed_orders = sensor('failed_orders', description='React to an actual orders check failure.', when=node_event(select='check:orders_valid', kinds=[NodeEventKind.FAILED]), run=refresh_inventory, minimum_interval_seconds=1)\n", + "on_failure_job": "name = \"annual_orders_job\"\ndescription = \"Build orders.\"\nselect = \"orders\"\nschedule = { cron = \"0 0 * * *\", timezone = \"UTC\", starts_at = \"2000-01-01T00:00:00Z\" }\n", + "on_failure_destination": "name = \"refresh_inventory_job\"\ndescription = \"Refresh inventory.\"\nselect = \"inventory\"\n", + "on_failure_sensor": "from sqlbuild.orchestration import NodeEventKind, node_event, sensor\n\nfailed_orders = sensor(\n 'failed_orders_sensor',\n description='React to an actual orders check failure.',\n when=node_event(select='check:orders_valid', kinds=[NodeEventKind.FAILED]),\n run='refresh_inventory_job',\n minimum_interval_seconds=1,\n)\n", "inventory": "MODEL (description 'Inventory after check failure.', materialized table);\nSELECT 59 AS amount;\n" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py deleted file mode 100644 index ce5a9b672..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Provider-backed fulfillment times evaluated in the code location, never by the scheduler.""" - -from sqlbuild.orchestration import calendar_schedule, job - -orders = job("order_reporting", description="Build order reporting.", select="orders") - - -@calendar_schedule( - job=orders, - description="Refresh orders at upcoming fulfillment times.", - horizon="8s", - timezone="America/New_York", -) -def fulfillment_calendar(window, fulfillment): - return fulfillment.upcoming(window) - - -# Declared for another target without decorating a function: development publication skips it. -production_calendar = calendar_schedule( - job=orders, - description="Production fulfillment times, not yet attached to a provider.", - horizon="8s", - timezone="UTC", - targets=("production",), -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/calendars/fulfillment_calendar.py b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/calendars/fulfillment_calendar.py new file mode 100644 index 000000000..ddb78ea4d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/calendars/fulfillment_calendar.py @@ -0,0 +1,12 @@ +"""Upcoming fulfillment times from the provider, evaluated in the code location.""" + +from sqlbuild.orchestration import calendar + + +@calendar( + name="fulfillment_calendar", + description="Refresh orders at upcoming fulfillment times.", + horizon="8s", +) +def fulfillment_calendar(window, fulfillment): + return fulfillment.upcoming(window) diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/jobs/order_reporting_job.toml b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/jobs/order_reporting_job.toml new file mode 100644 index 000000000..80fe02628 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/jobs/order_reporting_job.toml @@ -0,0 +1,5 @@ +# Provider-backed fulfillment times evaluated in the code location, never by the scheduler. +name = "order_reporting_job" +description = "Build order reporting." +select = "orders" +schedule = { calendar = "fulfillment_calendar", timezone = "America/New_York" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders.py b/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders.py deleted file mode 100644 index 377130a8b..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders.py +++ /dev/null @@ -1,17 +0,0 @@ -"""A sensor on new source data starts a job that builds only what changed, source data included.""" - -from sqlbuild.orchestration import job, sensor, source_new_data - -orders_refresh = job( - "orders_refresh", - description="Refresh orders when the source has new data.", - select="orders", - settings={"changed": True}, -) -on_source_orders = sensor( - "on_source_orders", - description="Refresh orders for each changed source version.", - when=source_new_data(select="source:raw_orders"), - run=orders_refresh, - minimum_interval_seconds=1, -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/jobs/orders_refresh_job.toml b/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/jobs/orders_refresh_job.toml new file mode 100644 index 000000000..188309855 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/jobs/orders_refresh_job.toml @@ -0,0 +1,7 @@ +# A sensor on new source data starts a job that builds only what changed, source data included. +name = "orders_refresh_job" +description = "Refresh orders when the source has new data." +select = "orders" + +[settings] +changed = true diff --git a/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/sensors/on_source_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/sensors/on_source_orders_sensor.py new file mode 100644 index 000000000..a8abd7568 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/sensors/on_source_orders_sensor.py @@ -0,0 +1,11 @@ +"""Start the changed-only refresh for each new source version.""" + +from sqlbuild.orchestration import sensor, source_new_data + +on_source_orders = sensor( + "on_source_orders_sensor", + description="Refresh orders for each changed source version.", + when=source_new_data(select="source:raw_orders"), + run="orders_refresh_job", + minimum_interval_seconds=1, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py deleted file mode 100644 index e864cf753..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Negative fixture: mutually exclusive build flags must fail before publication.""" - -from sqlbuild.orchestration import job - -orders = job( - "refresh_orders", - description="Refresh orders.", - select="orders", - settings={"load_sources": False, "reload": True}, -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings/jobs/refresh_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings/jobs/refresh_orders_job.toml new file mode 100644 index 000000000..82c136e70 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings/jobs/refresh_orders_job.toml @@ -0,0 +1,8 @@ +# Negative fixture: mutually exclusive build flags must fail before publication. +name = "refresh_orders_job" +description = "Refresh orders." +select = "orders" + +[settings] +load_sources = false +reload = true diff --git a/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py b/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py deleted file mode 100644 index b419a1f81..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py +++ /dev/null @@ -1,11 +0,0 @@ -"""A job asks for more memory than its worker may have and fewer CPUs than the launcher default.""" - -from sqlbuild.orchestration import job, resources, schedule - -annual_orders = job( - "annual_orders", - select="orders", - description="Build orders.", - resources=resources(memory="2g", cpus=0.5), - schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..0bae046ba --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders/jobs/annual_orders_job.toml @@ -0,0 +1,9 @@ +# A job asks for more memory than its worker may have and fewer CPUs than the launcher default. +name = "annual_orders_job" +description = "Build orders." +select = "orders" +schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" } + +[resources] +memory = "2g" +cpus = 0.5 diff --git a/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py b/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py deleted file mode 100644 index f448f2b85..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Polling callbacks return durable run keys and keep their cursor in the native transaction.""" - -from sqlbuild.orchestration import job, sensor - -orders = job("annual_orders", description="Build orders after source inspection.", select="orders") - - -@sensor( - "poll_orders", description="Poll fulfillment readiness.", run=orders, minimum_interval_seconds=1 -) -def poll_orders(ctx, fulfillment): - def poll(): - return fulfillment.poll(ctx) - - return poll() diff --git a/benchmarks/duckshop/edge_cases/orchestration/custom_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/custom_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..2f4d35d6c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/custom_orders/jobs/annual_orders_job.toml @@ -0,0 +1,3 @@ +name = "annual_orders_job" +description = "Build orders after source inspection." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/custom_orders/sensors/poll_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/custom_orders/sensors/poll_orders_sensor.py new file mode 100644 index 000000000..ad888268d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/custom_orders/sensors/poll_orders_sensor.py @@ -0,0 +1,16 @@ +"""Polling callbacks return durable run keys and keep their cursor in the native transaction.""" + +from sqlbuild.orchestration import sensor + + +@sensor( + "poll_orders_sensor", + description="Poll fulfillment readiness.", + run="annual_orders_job", + minimum_interval_seconds=1, +) +def poll_orders(ctx, fulfillment): + def poll(): + return fulfillment.poll(ctx) + + return poll() diff --git a/benchmarks/duckshop/edge_cases/orchestration/development_orders.py b/benchmarks/duckshop/edge_cases/orchestration/development_orders.py deleted file mode 100644 index 7306a2147..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/development_orders.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Development defaults pause code-enabled triggers without changing their definitions.""" - -from sqlbuild.orchestration import job, schedule - -annual = schedule(cron="0 0 1 1 *", timezone="UTC", starts_at="2099-01-01T00:00:00Z") -annual_orders = job( - "annual_orders", description="Build annual orders.", select="+orders", schedule=annual -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/development_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/development_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..c5d320cbe --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/development_orders/jobs/annual_orders_job.toml @@ -0,0 +1,5 @@ +# Development defaults pause code-enabled triggers without changing their definitions. +name = "annual_orders_job" +description = "Build annual orders." +select = "+orders" +schedule = { cron = "0 0 1 1 *", timezone = "UTC", starts_at = "2099-01-01T00:00:00Z" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/failed_import.py b/benchmarks/duckshop/edge_cases/orchestration/failed_import.py deleted file mode 100644 index 6483f0b2a..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/failed_import.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Negative fixture: a failed declaration import must fail every publication retry.""" - -raise RuntimeError("broken declaration") diff --git a/benchmarks/duckshop/edge_cases/orchestration/failed_import/jobs/orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/failed_import/jobs/orders_job.toml new file mode 100644 index 000000000..e2acd270d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/failed_import/jobs/orders_job.toml @@ -0,0 +1,3 @@ +name = "orders_job" +description = "Build orders." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/failed_import/sensors/broken_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/failed_import/sensors/broken_orders_sensor.py new file mode 100644 index 000000000..454fea133 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/failed_import/sensors/broken_orders_sensor.py @@ -0,0 +1,12 @@ +"""Negative fixture: a failed declaration import must fail every publication retry.""" + +from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor + +raise RuntimeError("broken declaration") + +broken_orders = sensor( + "broken_orders_sensor", + description="Never declared: the module fails first.", + when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), + run="orders_job", +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py deleted file mode 100644 index 621d03807..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Negative fixture: job settings must not override the frozen selection.""" - -from sqlbuild.orchestration import job - -orders = job( - "refresh_orders", - description="Refresh orders.", - select="orders", - settings={"select": "customers"}, -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/invalid_settings/jobs/refresh_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings/jobs/refresh_orders_job.toml new file mode 100644 index 000000000..c7dd89f19 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings/jobs/refresh_orders_job.toml @@ -0,0 +1,7 @@ +# Negative fixture: job settings must not override the frozen selection. +name = "refresh_orders_job" +description = "Refresh orders." +select = "orders" + +[settings] +select = "customers" diff --git a/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py deleted file mode 100644 index 2941037d7..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py +++ /dev/null @@ -1,14 +0,0 @@ -"""A tag-selected job whose model loads a loader chain and a hook-read source. - -The build loads raw_orders and the raw_customers loader it depends on, and raw_returns, which only -the model's pre-hook reads. The job's write set claims those three and not raw_inventory. -""" - -from sqlbuild.orchestration import job, schedule - -annual_orders = job( - "annual_orders", - description="Build the marts layer and the sources it loads.", - select="tag:marts", - schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/loaded_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..66b4cb5e0 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders/jobs/annual_orders_job.toml @@ -0,0 +1,4 @@ +name = "annual_orders_job" +description = "Build the marts layer and the sources it loads." +select = "tag:marts" +schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py deleted file mode 100644 index 6724f992a..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Keep a populated concurrency slot until recovered terminal publication commits.""" - -from sqlbuild.orchestration import job, schedule - -annual_orders = job( - "annual_orders", - description="Build orders under a shared concurrency limit.", - select="orders", - concurrency_groups=("orders_limit",), - schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..7f3e260da --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders/jobs/annual_orders_job.toml @@ -0,0 +1,6 @@ +# Keep a populated concurrency slot until recovered terminal publication commits. +name = "annual_orders_job" +description = "Build orders under a shared concurrency limit." +select = "orders" +concurrency_groups = ["orders_limit"] +schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/name_collision.py b/benchmarks/duckshop/edge_cases/orchestration/name_collision.py deleted file mode 100644 index fb0e024e9..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/name_collision.py +++ /dev/null @@ -1,5 +0,0 @@ -"""A job may not reuse a model's name: jobs share the project's one resource namespace.""" - -from sqlbuild.orchestration import job - -orders = job("orders", description="Build orders.", select="orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/name_collision/jobs/orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/name_collision/jobs/orders_job.toml new file mode 100644 index 000000000..48d401f69 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/name_collision/jobs/orders_job.toml @@ -0,0 +1,4 @@ +# A job may not reuse another resource's name: jobs share the project's one resource namespace. +name = "orders_job" +description = "Build orders." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py b/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py deleted file mode 100644 index 35727b209..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py +++ /dev/null @@ -1,7 +0,0 @@ -"""A group can contain jobs, but cannot contain another group.""" - -from sqlbuild.orchestration import job, job_group - -orders = job("orders_job", description="Build orders.", select="orders") -daily = job_group("daily_orders", description="Build daily orders.", jobs=[orders]) -weekly = job_group("weekly_orders", description="Unsupported nested membership.", jobs=[daily]) diff --git a/benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/daily_orders_job_group.toml b/benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/daily_orders_job_group.toml new file mode 100644 index 000000000..3b33d97cb --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/daily_orders_job_group.toml @@ -0,0 +1,3 @@ +name = "daily_orders_job_group" +description = "Build daily orders." +jobs = ["orders_job"] diff --git a/benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/weekly_orders_job_group.toml b/benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/weekly_orders_job_group.toml new file mode 100644 index 000000000..693cdd117 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/weekly_orders_job_group.toml @@ -0,0 +1,4 @@ +# A group can contain jobs, but cannot contain another group. +name = "weekly_orders_job_group" +description = "Unsupported nested membership." +jobs = ["daily_orders_job_group"] diff --git a/benchmarks/duckshop/edge_cases/orchestration/nested_groups/jobs/orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/nested_groups/jobs/orders_job.toml new file mode 100644 index 000000000..e2acd270d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/nested_groups/jobs/orders_job.toml @@ -0,0 +1,3 @@ +name = "orders_job" +description = "Build orders." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_orders.py b/benchmarks/duckshop/edge_cases/orchestration/node_orders.py deleted file mode 100644 index ac8f6cb8f..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/node_orders.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Trigger on each real orders rebuild, not on freshness-only success.""" - -from sqlbuild.orchestration import NodeEventKind, job, node_event, sensor - -source_orders = job("source_orders", description="Source orders.", select="orders") -inventory = job("inventory", description="Destination inventory.", select="stg_orders") -on_orders = sensor( - "on_orders", - description="Build inventory for every genuine orders rebuild.", - when=node_event(select="orders orders*", kinds=[NodeEventKind.SUCCESS]), - run=inventory, -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/inventory_job.toml b/benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/inventory_job.toml new file mode 100644 index 000000000..6d273e69a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/inventory_job.toml @@ -0,0 +1,3 @@ +name = "inventory_job" +description = "Destination inventory." +select = "stg_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/source_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/source_orders_job.toml new file mode 100644 index 000000000..aa4b407a4 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/source_orders_job.toml @@ -0,0 +1,3 @@ +name = "source_orders_job" +description = "Source orders." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_orders/sensors/on_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/node_orders/sensors/on_orders_sensor.py new file mode 100644 index 000000000..40d4d909f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_orders/sensors/on_orders_sensor.py @@ -0,0 +1,10 @@ +"""Trigger on each real orders rebuild, not on freshness-only success.""" + +from sqlbuild.orchestration import NodeEventKind, node_event, sensor + +on_orders = sensor( + "on_orders_sensor", + description="Build inventory for every genuine orders rebuild.", + when=node_event(select="orders orders*", kinds=[NodeEventKind.SUCCESS]), + run="inventory_job", +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py b/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py deleted file mode 100644 index 575d5ccaa..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Sensor-only roots retain referenced jobs while target filters and event enums stay native.""" - -from sqlbuild.orchestration import PlatformEventKind, job, platform_event, sensor - - -def definitions(): - refresh = job("health_orders", description="Refresh orders.", select="orders") - return [ - sensor( - "overdue_orders", - description="Catch up overdue order reporting.", - when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), - run=refresh, - minimum_interval_seconds=7, - default_status="paused", - targets=("development",), - ), - sensor( - "other_target_orders", - description="Refresh another target.", - when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), - run=refresh, - targets=("production",), - ), - ] - - -triggers = definitions() -alias = triggers[0] -triggers.append(triggers) diff --git a/benchmarks/duckshop/edge_cases/orchestration/platform_orders/jobs/health_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/platform_orders/jobs/health_orders_job.toml new file mode 100644 index 000000000..a74fd6136 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/platform_orders/jobs/health_orders_job.toml @@ -0,0 +1,4 @@ +# Sensor-only roots retain referenced jobs while target filters and event enums stay native. +name = "health_orders_job" +description = "Refresh orders." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/other_target_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/other_target_orders_sensor.py new file mode 100644 index 000000000..b3301c551 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/other_target_orders_sensor.py @@ -0,0 +1,11 @@ +"""A sensor declared for another target is left out of this target's snapshot.""" + +from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor + +other_target_orders = sensor( + "other_target_orders_sensor", + description="Refresh another target.", + when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), + run="health_orders_job", + targets=("production",), +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/overdue_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/overdue_orders_sensor.py new file mode 100644 index 000000000..6f2d75933 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/overdue_orders_sensor.py @@ -0,0 +1,22 @@ +"""A sensor returned from a factory, aliased and held in a cyclic container, is found once.""" + +from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor + + +def definitions(): + return [ + sensor( + "overdue_orders_sensor", + description="Catch up overdue order reporting.", + when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), + run="health_orders_job", + minimum_interval_seconds=7, + default_status="paused", + targets=("development",), + ) + ] + + +triggers = definitions() +alias = triggers[0] +triggers.append(triggers) diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py deleted file mode 100644 index d153e434f..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Jobs whose selections are Python nodes, which `--select job:` must select as published.""" - -from sqlbuild.orchestration import job - -customer_counts = job( - "customer_counts", - description="Count customers.", - select="task:count_customers asset:customers_extract", -) -order_counts = job("order_counts", description="Count orders.", select="+task:count_orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/customer_counts_job.toml b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/customer_counts_job.toml new file mode 100644 index 000000000..e552003d6 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/customer_counts_job.toml @@ -0,0 +1,4 @@ +# Jobs whose selections are Python nodes, which `--select job:` must select as published. +name = "customer_counts_job" +description = "Count customers." +select = "task:count_customers asset:customers_extract" diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/order_counts_job.toml b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/order_counts_job.toml new file mode 100644 index 000000000..3a0104dc8 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/order_counts_job.toml @@ -0,0 +1,3 @@ +name = "order_counts_job" +description = "Count orders." +select = "+task:count_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/run_orders.py b/benchmarks/duckshop/edge_cases/orchestration/run_orders.py deleted file mode 100644 index bf6d37618..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/run_orders.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Freeze source-job selectors without executing user code in the scheduler.""" - -from sqlbuild.orchestration import RunEventKind, job, run_event, sensor - -source_orders = job("source_orders", description="Source orders.", select="orders") -inventory = job("inventory", description="Destination inventory.", select="stg_orders") -on_orders = sensor( - "on_orders", - description="Build inventory only after orders actually succeeds.", - when=run_event( - job=source_orders, - kinds=[RunEventKind.COMPLETED], - executed_successfully=("orders",), - ), - run=inventory, -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/inventory_job.toml b/benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/inventory_job.toml new file mode 100644 index 000000000..6d273e69a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/inventory_job.toml @@ -0,0 +1,3 @@ +name = "inventory_job" +description = "Destination inventory." +select = "stg_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/source_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/source_orders_job.toml new file mode 100644 index 000000000..aa4b407a4 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/source_orders_job.toml @@ -0,0 +1,3 @@ +name = "source_orders_job" +description = "Source orders." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/run_orders/sensors/on_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/run_orders/sensors/on_orders_sensor.py new file mode 100644 index 000000000..d8fbf6746 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/run_orders/sensors/on_orders_sensor.py @@ -0,0 +1,14 @@ +"""Freeze source-job selectors without executing user code in the scheduler.""" + +from sqlbuild.orchestration import RunEventKind, run_event, sensor + +on_orders = sensor( + "on_orders_sensor", + description="Build inventory only after orders actually succeeds.", + when=run_event( + job="source_orders_job", + kinds=[RunEventKind.COMPLETED], + executed_successfully=("orders",), + ), + run="inventory_job", +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json b/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json index ab9c620d9..396777d00 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json +++ b/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json @@ -1,9 +1,9 @@ { "description": "Three minutely cron jobs start ten hours back with the default latest catch-up. Two installed schedulers catch up while being SIGKILLed at seeded random moments; every lapse of a killed leader's lease is simulated. After a final uninterrupted scheduler, each trigger has ticked its latest due minute exactly once and nothing off schedule, each tick requests or is covered by exactly one run, and every requested run has one request event.", "jobs": [ - "refresh_orders", - "refresh_customers", - "refresh_inventory" + "refresh_orders_job", + "refresh_customers_job", + "refresh_inventory_job" ], "backlog_minutes": 600, "rounds": 16, diff --git a/benchmarks/duckshop/edge_cases/orchestration/selector_rewrite/jobs/orders_refresh_job.toml b/benchmarks/duckshop/edge_cases/orchestration/selector_rewrite/jobs/orders_refresh_job.toml new file mode 100644 index 000000000..6ae2bd8d9 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/selector_rewrite/jobs/orders_refresh_job.toml @@ -0,0 +1,6 @@ +# Renaming stg_orders rewrites every exact name in these selectors, inside graph operators, unions, +# intersections and path endpoints, keeps each string's quote style, and leaves the glob alone. +name = "orders_refresh_job" +description = "Refresh orders." +select = "+stg_orders+ fact_orders,+stg_orders stg_orders~customer_totals stg_ord*" +exclude = 'stg_orders+,customer_totals' diff --git a/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py deleted file mode 100644 index 77a937120..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Layer jobs selected by tag share the function their models call, and still order by lineage. - -A tag selection brings in the functions its models call, so both jobs build the shared function. A -job that builds a node itself does not wait for that node's other builders, so the group runs the -marts layer after the staging layer instead of reporting a cycle. -""" - -from sqlbuild.orchestration import job, job_group - -staging = job("staging_layer", description="Build the staging layer.", select="tag:staging") -marts = job("marts_layer", description="Build the marts layer.", select="tag:marts") -layers = job_group( - "layered_orders", description="Build staging, then marts.", jobs=[marts, staging] -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/job_groups/layered_orders_job_group.toml b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/job_groups/layered_orders_job_group.toml new file mode 100644 index 000000000..fe34d0d60 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/job_groups/layered_orders_job_group.toml @@ -0,0 +1,4 @@ +# Layer jobs that share a function wait for each other only by lineage. +name = "layered_orders_job_group" +description = "Build staging, then marts." +jobs = ["marts_layer_job", "staging_layer_job"] diff --git a/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/marts_layer_job.toml b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/marts_layer_job.toml new file mode 100644 index 000000000..dccd96a30 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/marts_layer_job.toml @@ -0,0 +1,3 @@ +name = "marts_layer_job" +description = "Build the marts layer." +select = "tag:marts" diff --git a/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/staging_layer_job.toml b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/staging_layer_job.toml new file mode 100644 index 000000000..f26d0ab3e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/staging_layer_job.toml @@ -0,0 +1,3 @@ +name = "staging_layer_job" +description = "Build the staging layer." +select = "tag:staging" diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_orders/jobs/orders_refresh_job.toml b/benchmarks/duckshop/edge_cases/orchestration/source_orders/jobs/orders_refresh_job.toml new file mode 100644 index 000000000..7eabe333f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/source_orders/jobs/orders_refresh_job.toml @@ -0,0 +1,3 @@ +name = "orders_refresh_job" +description = "Refresh orders for a changed source version." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_orders.py b/benchmarks/duckshop/edge_cases/orchestration/source_orders/sensors/on_source_orders_sensor.py similarity index 54% rename from benchmarks/duckshop/edge_cases/orchestration/source_orders.py rename to benchmarks/duckshop/edge_cases/orchestration/source_orders/sensors/on_source_orders_sensor.py index c700b7d23..7ee39f8dd 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/source_orders.py +++ b/benchmarks/duckshop/edge_cases/orchestration/source_orders/sensors/on_source_orders_sensor.py @@ -1,16 +1,11 @@ """Consume changed warehouse source versions without treating a sync poll as new data.""" -from sqlbuild.orchestration import job, sensor, source_new_data +from sqlbuild.orchestration import sensor, source_new_data -orders_refresh = job( - "orders_refresh", - description="Refresh orders for a changed source version.", - select="orders", -) on_source_orders = sensor( - "on_source_orders", + "on_source_orders_sensor", description="Refresh orders for the first source version and each changed version.", when=source_new_data(select="source:raw_orders"), - run=orders_refresh, + run="orders_refresh_job", minimum_interval_seconds=1, ) diff --git a/benchmarks/duckshop/edge_cases/orchestration/stale_orders.py b/benchmarks/duckshop/edge_cases/orchestration/stale_orders.py deleted file mode 100644 index bf17c3428..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/stale_orders.py +++ /dev/null @@ -1,13 +0,0 @@ -from sqlbuild.orchestration import job, sensor, source_stale - -orders_refresh = job( - "orders_refresh", description="Refresh orders from stale timestamp data.", select="orders" -) - -on_stale_orders = sensor( - "on_stale_orders", - description="Refresh orders once for each timestamp version older than one minute.", - when=source_stale(select="source:raw_orders", after_seconds=60), - run=orders_refresh, - minimum_interval_seconds=1, -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/stale_orders/jobs/orders_refresh_job.toml b/benchmarks/duckshop/edge_cases/orchestration/stale_orders/jobs/orders_refresh_job.toml new file mode 100644 index 000000000..a574364e9 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/stale_orders/jobs/orders_refresh_job.toml @@ -0,0 +1,3 @@ +name = "orders_refresh_job" +description = "Refresh orders from stale timestamp data." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/stale_orders/sensors/on_stale_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/stale_orders/sensors/on_stale_orders_sensor.py new file mode 100644 index 000000000..01ad50030 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/stale_orders/sensors/on_stale_orders_sensor.py @@ -0,0 +1,11 @@ +"""Start one refresh for each timestamp source version older than a minute.""" + +from sqlbuild.orchestration import sensor, source_stale + +on_stale_orders = sensor( + "on_stale_orders_sensor", + description="Refresh orders once for each timestamp version older than one minute.", + when=source_stale(select="source:raw_orders", after_seconds=60), + run="orders_refresh_job", + minimum_interval_seconds=1, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py deleted file mode 100644 index 0723fa403..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py +++ /dev/null @@ -1,15 +0,0 @@ -"""A job that reaches a managed source through `+`, so its run selects the source's loader chain. - -`+orders` selects the model, its managed source raw_shipments, the raw_shipments loader and the -intermediate stage_shipments loader it depends on. The run records each loader as the source it -writes, which is the node its load reports under. -""" - -from sqlbuild.orchestration import job, schedule - -annual_orders = job( - "annual_orders", - description="Build orders and everything upstream of it, loaders included.", - select="+orders", - schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..81a343141 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders/jobs/annual_orders_job.toml @@ -0,0 +1,4 @@ +name = "annual_orders_job" +description = "Build orders and everything upstream of it, loaders included." +select = "+orders" +schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders.py b/benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders.py deleted file mode 100644 index 7fc6d7150..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Schedule a job whose selection reaches an upstream source that a build never executes.""" - -from sqlbuild.orchestration import job, schedule - -annual_orders = job( - "annual_orders", - description="Build orders together with the source it reads.", - select="+orders", - schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..c7e5aafa3 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders/jobs/annual_orders_job.toml @@ -0,0 +1,5 @@ +# Schedule a job whose selection reaches an upstream source that a build never executes. +name = "annual_orders_job" +description = "Build orders together with the source it reads." +select = "+orders" +schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" } diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py deleted file mode 100644 index 8c5069c1d..000000000 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Exercise alias deduplication and cyclic factory containers during native discovery.""" - -from sqlbuild.orchestration import ( - PlatformEventKind, - RunEventKind, - job, - job_group, - platform_event, - run_event, - schedule, - sensor, -) - -edge_orders = job( - "edge_snapshot_orders", - description="Refresh only the synthetic edge-case models.", - select="tag:edge_case", - settings={"run_tests": False, "vars": {"edge_delivery_batch": "1"}, "reload": True}, - concurrency=2, - schedule=schedule( - cron="15 2 * * *", - timezone="UTC", - starts_at="2026-07-03T01:17:23.456789+05:30", - default_status="paused", - ), -) -edge_alias = edge_orders -edge_factory_results = [[edge_orders]] -edge_factory_results.append(edge_factory_results) -edge_group = job_group( - "edge_snapshot_group", - description="Exercise group discovery through shared, cyclic declaration containers.", - jobs=[edge_orders], -) -edge_factory_results.append(edge_group) - -edge_inventory = job( - "edge_snapshot_inventory", - description="Exercise a child limit independent of its parent.", - select="tag:edge_case", - concurrency_groups=("edge_products_limit",), -) -edge_products = job( - "edge_snapshot_products", - description="Exercise the opposing parent and child acquisition order.", - select="tag:edge_case", - concurrency_groups=("edge_inventory_limit",), -) -edge_inventory_group = job_group( - "edge_inventory_group", - description="Complete inventory without reserving its child's product limit.", - jobs=[edge_inventory], - concurrency_groups=("edge_inventory_limit",), -) -edge_products_group = job_group( - "edge_products_group", - description="Wait for safe admission when both installation limits are one.", - jobs=[edge_products], - concurrency_groups=("edge_products_limit",), -) -edge_health = sensor( - "edge_platform_orders", - description="Catch up retained platform observations with a stopped group sensor.", - when=platform_event( - kinds=[ - PlatformEventKind.TICK_OVERDUE, - PlatformEventKind.LOCATION_BROKEN, - PlatformEventKind.LAUNCH_FAILED, - ] - ), - run=edge_group, - minimum_interval_seconds=7, - default_status="paused", -) -edge_factory_results.append(edge_health) - -edge_job_outcomes = sensor( - "edge_job_outcomes", - description="Keep warning and partial outcomes distinct from success.", - when=run_event(job=edge_orders, kinds=[RunEventKind.WARN, RunEventKind.PARTIAL]), - run=edge_inventory, - default_status="paused", -) -edge_group_outcomes = sensor( - "edge_group_outcomes", - description="Observe group completion including cancellation and presumed failure.", - when=run_event( - job=edge_group, - kinds=[RunEventKind.COMPLETED], - executed_successfully=("tag:edge_case",), - ), - run=edge_inventory, - default_status="paused", -) -edge_factory_results.extend([edge_job_outcomes, edge_group_outcomes]) diff --git a/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_inventory_job_group.toml b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_inventory_job_group.toml new file mode 100644 index 000000000..91d5adf5b --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_inventory_job_group.toml @@ -0,0 +1,4 @@ +name = "edge_inventory_job_group" +description = "Complete inventory without reserving its child's product limit." +jobs = ["edge_snapshot_inventory_job"] +concurrency_groups = ["edge_inventory_limit"] diff --git a/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_products_job_group.toml b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_products_job_group.toml new file mode 100644 index 000000000..8177a6402 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_products_job_group.toml @@ -0,0 +1,4 @@ +name = "edge_products_job_group" +description = "Wait for safe admission when both installation limits are one." +jobs = ["edge_snapshot_products_job"] +concurrency_groups = ["edge_products_limit"] diff --git a/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_snapshot_job_group.toml b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_snapshot_job_group.toml new file mode 100644 index 000000000..cae323412 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_snapshot_job_group.toml @@ -0,0 +1,3 @@ +name = "edge_snapshot_job_group" +description = "Run the edge-case refresh as a group that sensors start and observe." +jobs = ["edge_snapshot_orders_job"] diff --git a/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_inventory_job.toml b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_inventory_job.toml new file mode 100644 index 000000000..4b624fe69 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_inventory_job.toml @@ -0,0 +1,4 @@ +name = "edge_snapshot_inventory_job" +description = "Exercise a child limit independent of its parent." +select = "tag:edge_case" +concurrency_groups = ["edge_products_limit"] diff --git a/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_orders_job.toml b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_orders_job.toml new file mode 100644 index 000000000..85b15b929 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_orders_job.toml @@ -0,0 +1,11 @@ +# Refresh only the synthetic edge-case models, with nested settings and a paused offset-start schedule. +name = "edge_snapshot_orders_job" +description = "Refresh only the synthetic edge-case models." +select = "tag:edge_case" +schedule = { cron = "15 2 * * *", timezone = "UTC", starts_at = "2026-07-03T01:17:23.456789+05:30", default_status = "paused" } + +[settings] +run_tests = false +vars = { edge_delivery_batch = "1" } +reload = true +concurrency = 2 diff --git a/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_products_job.toml b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_products_job.toml new file mode 100644 index 000000000..68c81fe2e --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_products_job.toml @@ -0,0 +1,4 @@ +name = "edge_snapshot_products_job" +description = "Exercise the opposing parent and child acquisition order." +select = "tag:edge_case" +concurrency_groups = ["edge_inventory_limit"] diff --git a/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_group_outcomes_sensor.py b/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_group_outcomes_sensor.py new file mode 100644 index 000000000..a1f8a750d --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_group_outcomes_sensor.py @@ -0,0 +1,15 @@ +"""Observe group completion, including cancellation and presumed failure.""" + +from sqlbuild.orchestration import RunEventKind, run_event, sensor + +edge_group_outcomes = sensor( + "edge_group_outcomes_sensor", + description="Observe group completion including cancellation and presumed failure.", + when=run_event( + job="edge_snapshot_job_group", + kinds=[RunEventKind.COMPLETED], + executed_successfully=("tag:edge_case",), + ), + run="edge_snapshot_inventory_job", + default_status="paused", +) diff --git a/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_job_outcomes_sensor.py b/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_job_outcomes_sensor.py new file mode 100644 index 000000000..fc35482e5 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_job_outcomes_sensor.py @@ -0,0 +1,11 @@ +"""Keep warning and partial job outcomes distinct from success.""" + +from sqlbuild.orchestration import RunEventKind, run_event, sensor + +edge_job_outcomes = sensor( + "edge_job_outcomes_sensor", + description="Keep warning and partial outcomes distinct from success.", + when=run_event(job="edge_snapshot_orders_job", kinds=[RunEventKind.WARN, RunEventKind.PARTIAL]), + run="edge_snapshot_inventory_job", + default_status="paused", +) diff --git a/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_platform_orders_sensor.py b/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_platform_orders_sensor.py new file mode 100644 index 000000000..5275a4581 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_platform_orders_sensor.py @@ -0,0 +1,21 @@ +"""Exercise alias deduplication and cyclic containers while discovering a sensor module.""" + +from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor + +edge_health = sensor( + "edge_platform_orders_sensor", + description="Catch up retained platform observations with a stopped group sensor.", + when=platform_event( + kinds=[ + PlatformEventKind.TICK_OVERDUE, + PlatformEventKind.LOCATION_BROKEN, + PlatformEventKind.LAUNCH_FAILED, + ] + ), + run="edge_snapshot_job_group", + minimum_interval_seconds=7, + default_status="paused", +) +edge_alias = edge_health +edge_factory_results = [[edge_health], (edge_alias,)] +edge_factory_results.append(edge_factory_results) diff --git a/deploy/project/orchestration/jobs/refresh_orders_job.toml b/deploy/project/orchestration/jobs/refresh_orders_job.toml new file mode 100644 index 000000000..b3727eca8 --- /dev/null +++ b/deploy/project/orchestration/jobs/refresh_orders_job.toml @@ -0,0 +1,5 @@ +# A small scheduled project baked into the reference location image. +name = "refresh_orders_job" +description = "Refresh orders every minute." +select = "orders" +schedule = { cron = "* * * * *", timezone = "UTC" } diff --git a/deploy/project/orchestration/orders.py b/deploy/project/orchestration/orders.py deleted file mode 100644 index 802379465..000000000 --- a/deploy/project/orchestration/orders.py +++ /dev/null @@ -1,10 +0,0 @@ -"""A small scheduled project baked into the reference location image.""" - -from sqlbuild.orchestration import job, schedule - -orders = job( - "refresh_orders", - select="orders", - description="Refresh orders every minute.", - schedule=schedule(cron="* * * * *", timezone="UTC"), -) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 6b4a04a62..2ffdb2b67 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -57,6 +57,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [Source freshness](concepts/planning/source-freshness.md) (`concepts/planning/source-freshness`) - Observe external source changes and propagate them through planning. - [Selection and staleness](concepts/planning/selection-and-staleness.md) (`concepts/planning/selection-and-staleness`) - How selection interacts with change detection, and the stale warnings that stop silent partial rebuilds. - [Column Lineage](concepts/column-lineage.md) (`concepts/column-lineage`) - Trace individual columns through your SQL pipeline - understand where data comes from and where it goes. +- [Orchestration](concepts/orchestration.md) (`concepts/orchestration`) - Declare jobs, job groups and schedules in TOML, and calendars and sensors in Python, for a SQLBuild server to run. ## Refactor diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index f2ab366db..a716fb092 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -159,18 +159,18 @@ belongs to a scheduled job without the changed model. That job's next run refres model is held back: ``` - staging_daily (schedule) finance_daily (schedule) + staging_daily_job (schedule) finance_daily_job (schedule) stg_orders* ─▶ int_orders ✓ ──────────▶ fct_revenue ⏸ ─▶ rpt_pnl ⏸ │ └──▶ orders_export ✓ (in no scheduled job) ``` -- A job counts when a schedule, calendar schedule or sensor starts it, directly or through its +- A job counts when a schedule, calendar or sensor starts it, directly or through its group, for the target you build. A job nothing starts counts as no job. - A model held back by one job runs if another job that holds the change also contains it. - A model that reads a held-back model is held back too. Unchanged parents never hold a model back. - A job never holds back its own models while it runs: a scheduled run, or a build that selects the - job, such as `sqb build finance_daily --changed`. + job, such as `sqb build finance_daily_job --changed`. - With no scheduled jobs, everything downstream runs. `--reach all` builds everything downstream of a change. `--reach` without `--changed` is refused. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md index c80138c62..b936b25b5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md @@ -98,6 +98,7 @@ Renaming or moving a model rewrites: - `__ref__` and `__expected__` fixture CTEs in tests and scenarios; - `cursor_inputs` keys, `relationships (to ...)` audits, and `__ref("")` inside quoted SQL in `MODEL` headers and reusable `SCHEMA` declarations; - `relationships` audits (`to:` and `field:`) and `__ref("")` in source and seed YAML. +- the model's name and folder in job and job group `select` and `exclude` selectors; see [Rename models in jobs](../concepts/orchestration.md#rename-models-in-jobs). The model's own unit tests, the test files whose every `TEST` block targets only that model, follow it: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index e0beb2e32..8e63ef0d3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -371,18 +371,18 @@ Details: Declare who owns a job, whether scheduled requests wait for a running copy, and how often a failed run retries: -```python -from sqlbuild.orchestration import job, on_failure, schedule - -nightly_orders = job( - "nightly_orders", - description="Orders and their daily totals", - select="+order_totals", - owner="Orders team", - overlap="skip", - on_failure=on_failure(retries=2, retry_delay_seconds=600), - schedule=schedule(cron="0 2 * * *", timezone="UTC"), -) +```toml +# orchestration/jobs/nightly_orders_job.toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" +owner = "Orders team" +overlap = "skip" +schedule = { cron = "0 2 * * *", timezone = "UTC" } + +[on_failure] +retries = 2 +retry_delay_seconds = 600 ``` | Setting | Values | Default | Effect | @@ -410,8 +410,8 @@ Details: - A group retries at group level, with the group's own `on_failure`: each retry is one rerun of the group's failed and skipped nodes. A member job's `on_failure` applies only when that job runs on its own. -- An invalid value fails publication with the job's name, such as - `job nightly_orders overlap must be queue or skip, not coalesce`. +- An invalid value is a `D017` error at its line, such as + ``orchestration/jobs/nightly_orders_job.toml:6:11: unknown variant `coalesce`, expected `queue` or `skip` ``. ## Limit worker containers @@ -435,15 +435,15 @@ pids = 512 max_memory = "4g" ``` -```python -from sqlbuild.orchestration import job, resources - -nightly_orders = job( - "nightly_orders", - description="Orders and their daily totals", - select="+order_totals", - resources=resources(memory="3g", cpus=2), -) +```toml +# orchestration/jobs/nightly_orders_job.toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" + +[resources] +memory = "3g" +cpus = 2 ``` | Setting | `run` table | `resources` | Value | @@ -463,7 +463,7 @@ nightly_orders = job( `no-new-privileges`. - A job in a group uses its own `resources`. Local launchers ignore `resources`. - An invalid value fails publication with the job's name, such as - `job nightly_orders resources: cpus must be a positive number`. + `job nightly_orders_job resources: cpus must be a positive number`. ## Run ad-hoc commands diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index 0155b1385..243685b4d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -90,7 +90,7 @@ error[P010]: model 'orders_summary' has no description A description may take at most `[format] max_description_lines` lines (default `10`) once wrapped at the line width, the same limit `sqb format --check` applies to headers. A longer Python description, from a decorator or a docstring, is a `P027` compile error at its declaration. A job, -job group, sensor or calendar schedule with a longer description cannot publish. +job group, sensor or calendar with a longer description cannot publish. SQL tests, audits, macros, enums, constants, materializations and event or output sinks do not need a description. Descriptions do not change fingerprints, so adding or editing one does not rebuild a diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md new file mode 100644 index 000000000..85cf73fd4 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -0,0 +1,226 @@ + + +# Orchestration + +> Declare jobs, job groups and schedules in TOML, and calendars and sensors in Python, for a SQLBuild server to run. + +Online: https://sqlbuild.com/docs/concepts/orchestration/ + +## Contents + +- Project layout +- Declare a job +- Group jobs +- Schedule a job +- Schedule from a calendar +- Start a job from a sensor +- Rename models in jobs +- Errors + +A job names the nodes a server builds together, and a schedule says when. Write each job as a TOML +file under `orchestration/jobs/`; `sqb compile` checks it and a code location publishes it. + +```toml +# orchestration/jobs/nightly_orders_job.toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" +schedule = { cron = "0 2 * * *", timezone = "UTC" } +``` + +```bash +sqb build nightly_orders_job # build what the job builds, locally +``` + +## Project layout + +| Folder | Holds | Name suffix | +| --- | --- | --- | +| `orchestration/jobs/` | One job per `.toml` file | `_job` | +| `orchestration/job_groups/` | One job group per `.toml` file | `_job_group` | +| `orchestration/calendars/` | One calendar per `.py` module | `_calendar` | +| `orchestration/sensors/` | One sensor per `.py` module | `_sensor` | + +- Subfolders are allowed, such as `orchestration/jobs/finance/revenue_job.toml`. +- A name must end with its folder's suffix. Names are unique across the whole project, with models, + sources and every other resource. +- A Python module whose file name starts with `_` is a private helper and declares nothing. +- Any other file under `orchestration/`, or a folder not in the table, is a `D017` error. + +## Declare a job + +A job builds its `select` minus its `exclude`, with the same [selectors](selectors.md) +as `sqb build`. + +```toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" +exclude = "tag:deprecated" +targets = ["production"] +owner = "Orders team" +overlap = "skip" +concurrency_groups = ["warehouse_heavy"] +schedule = { cron = "0 2 * * *", timezone = "Europe/London" } + +[on_failure] +retries = 2 +retry_delay_seconds = 600 + +[settings] +concurrency = 4 +vars = { batch_size = 500 } + +[resources] +memory = "3g" +cpus = 2 +``` + +| Key | Required | Value | +| --- | --- | --- | +| `name` | Yes | The job's name, ending in `_job` | +| `description` | Yes | Shown on the job's page | +| `select` | Yes | The nodes the job builds | +| `exclude` | No | Nodes removed from `select` | +| `targets` | No | The targets the job exists in; all targets when omitted | +| `owner` | No | Shown on the job's page and in the job list | +| `overlap` | No | `queue` (default) or `skip`; see [Skip overlapping runs](../cli/server.md#skip-overlapping-runs-and-retry-failures) | +| `concurrency_groups` | No | Server concurrency limits the job's runs share | +| `schedule` | No | When the server starts the job; see below | +| `[on_failure]` | No | `retries` from 1 to 10, `retry_delay_seconds` from 0 to 86400 | +| `[settings]` | No | Build options for each run, such as `concurrency`, `vars`, `full_refresh` or `changed` | +| `[resources]` | No | Docker worker limits; see [Limit worker containers](../cli/server.md#limit-worker-containers) | + +A job's `[settings]` cannot set `select` or `exclude`: the job's own selection is the run's +selection. + +## Group jobs + +A job group runs its jobs as one run. A job waits for another job in the group when it reads a node +that job builds. + +```toml +# orchestration/job_groups/daily_orders_job_group.toml +name = "daily_orders_job_group" +description = "Stage orders, then build the marts" +jobs = ["staging_orders_job", "orders_marts_job"] +schedule = { cron = "0 3 * * *", timezone = "UTC" } +``` + +A group takes `name`, `description` and `jobs`, and the same `targets`, `owner`, `overlap`, +`concurrency_groups`, `schedule` and `[on_failure]` as a job. Its members are jobs, never other +groups. + +## Schedule a job + +`schedule` takes a cron expression or the name of a calendar, and always a timezone. + +| Key | Value | +| --- | --- | +| `cron` | A five-field cron expression | +| `calendar` | The name of a calendar in `orchestration/calendars/` | +| `timezone` | An IANA timezone, such as `"UTC"` or `"America/New_York"` | +| `starts_at` | An RFC 3339 time with an offset; no ticks before it | +| `default_status` | `running` (default) or `paused` when the schedule is first published | + +```toml +schedule = { cron = "15 2 * * *", timezone = "Europe/London", starts_at = "2026-07-03T00:00:00Z" } +``` + +## Schedule from a calendar + +A calendar returns the upcoming times a job runs, such as fulfillment cut-offs that come from +another system. The code location evaluates it when it publishes and again before its times run +out. + +```python +# orchestration/calendars/fulfillment_calendar.py +from sqlbuild.orchestration import calendar + +@calendar(name="fulfillment_calendar", description="Upcoming fulfillment times", horizon="1d") +def fulfillment_calendar(window, fulfillment): + return fulfillment.upcoming(window) +``` + +```toml +schedule = { calendar = "fulfillment_calendar", timezone = "America/New_York" } +``` + +The function receives a `window` with timezone-aware `start` and `end`, in the job's timezone, and +returns timezone-aware datetimes inside it. Other parameters name +[providers](python-nodes/providers.md). `horizon` sets how far ahead the window reaches. + +## Start a job from a sensor + +A sensor starts a job or job group when something happens, named by `run`. + +```python +# orchestration/sensors/on_new_orders_sensor.py +from sqlbuild.orchestration import sensor, source_new_data + +on_new_orders = sensor( + "on_new_orders_sensor", + description="Refresh orders for each new source version", + when=source_new_data(select="source:raw_orders"), + run="orders_refresh_job", + minimum_interval_seconds=60, +) +``` + +| Condition | Starts a run for | +| --- | --- | +| `platform_event(kinds=[...])` | Server events, such as an overdue tick | +| `run_event(job="...", kinds=[...])` | Outcomes of another job or group, optionally only when `executed_successfully` nodes ran and succeeded | +| `node_event(select="...", kinds=[...])` | Each success, failure or recovery of the selected nodes in any run | +| `source_new_data(select="...")` | Each new version of the selected sources | +| `source_stale(select="...", after_seconds=...)` | Each source version older than `after_seconds` | + +A sensor without `when` decorates a function that polls and returns `RunRequest(run_key=...)` +values or a `SkipReason`. A sensor also takes `minimum_interval_seconds` (default 30), +`default_status` and `targets`. + +## Rename models in jobs + +`sqb rename` and `sqb mv` update job and job group selectors with the models they rename or move: + +```bash +sqb rename stg_orders stg_orders_clean +``` + +```text +Rename model stg_orders -> stg_orders_clean +├── models/marts/fact_orders.sql +│ └── 10:13 reference __ref("stg_orders") -> __ref("stg_orders_clean") +├── models/staging/stg_orders_clean.sql (moved from models/staging/stg_orders.sql) +│ └── 1:8 migration + migrate_from stg_orders +└── orchestration/jobs/orders_refresh_job.toml + └── 3:10 selector "+stg_orders+ fact_orders" -> "+stg_orders_clean+ fact_orders" +``` + +- Exact names are rewritten, including inside graph operators such as `+stg_orders+`, `,` and `~`. + Tags, globs and `job:` selectors stay as written. +- A folder selector, such as `models/staging` or `path:models/staging/`, moves when every model + under it moves to the same folder. When only some leave, the command lists a note at the + selector to check by hand. +- A selector in a multi-line TOML string is reported for a manual edit and blocks the refactor. + +## Errors + +Every problem with an orchestration file is a `D017` error at its file, line and column, reported by +`sqb compile` and every other command that reads the project. + +| Error | Fix | +| --- | --- | +| ``orchestration/jobs/nightly_orders_job.toml:4:11: unknown variant `coalesce`, expected `queue` or `skip` `` | Use one of the listed values | +| `files cannot sit directly in orchestration/` | Move jobs and groups to TOML files in `orchestration/jobs/` or `orchestration/job_groups/` | +| `` `job` was removed from sqlbuild.orchestration`` | Declare the job in TOML and name it in the sensor's `run` | +| `declares a second sensor; each module declares exactly one` | Split the module | + +Details: + +- `sqb build` and the other commands read job and group files without running project Python, so + a job selector never imports a calendar or sensor module. +- A job or group declared with `targets` is selectable only when the command's target is one of + them, as on the server. +- A job's resources are checked when a location publishes, such as + `job nightly_orders_job resources: cpus must be a positive number`. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 1bfea05be..462f0184a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -738,7 +738,7 @@ Code that extends SQLBuild itself keeps its own root: | `hooks/python/` | Python model lifecycle hooks | | `functions/python/` | Python UDFs | | `macros/python/`, `enums/`, `constants/` and scoped `_macros/python/`, `_enums/`, `_constants/` | Python macros and declarations | -| `orchestration/` | Jobs, job groups, sensors and calendar schedules. SQLBuild imports these modules only when a selector names a job or an unknown name, and when a location publishes. See [Job selectors](selectors.md#job) | +| `orchestration/` | Jobs and job groups as TOML in `jobs/` and `job_groups/`; calendars and sensors as Python in `calendars/` and `sensors/`. See [Orchestration](orchestration.md) | Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create an unofficial project structure. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index c4c2db12e..cb06ba474 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -162,7 +162,7 @@ sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` -Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendar schedules. +Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendars. ## Lifecycle: run, build, check diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index 09e66562e..9ba80be23 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -82,13 +82,13 @@ sqb build --select source:raw__orders ### Job -Select the SQL and Python nodes a job or job group declared under `orchestration/` builds. Job names are globally +Select the SQL and Python nodes a [job or job group](orchestration.md) builds. Job names are globally unique, so a bare name works; `job:` is the explicit form and accepts glob patterns: ```bash -sqb build nightly_orders -sqb build --select job:nightly_orders --exclude order_totals -sqb build --select "job:nightly_orders,tag:finance" +sqb build nightly_orders_job +sqb build --select job:nightly_orders_job --exclude order_totals +sqb build --select "job:nightly_orders_job,tag:finance" ``` A job selects its own `select` minus its own `exclude`. A job group selects the union of its jobs. @@ -97,8 +97,7 @@ them, as on the server. Job selectors compose with `+`, `,` and `--exclude` like any other selector. A job's own selection cannot name another job, and path-between selectors cannot start or end at a job. -SQLBuild imports the modules under `orchestration/` only when a selector names a job or a name no -model, source, seed or function has. +SQLBuild reads job and job group files without running project Python. ### Unit test diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 8140e645d..62caae701 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -26,6 +26,7 @@ query_duckdb, run_sqb, ) +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import duckshop_orchestration def run_replay_command( @@ -916,9 +917,7 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: 'MODEL (description "Test model order_summary.", ' 'materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' ), - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py" - ).read_text(), + **duckshop_orchestration("python_node_jobs"), "python/tasks/reports.py": ( "from python.tasks.counts import count_orders\nfrom sqlbuild.tasks import task\n\n\n" "@task(depends_on=count_orders)\n" @@ -1377,7 +1376,7 @@ def prepare_changed_project(*, tmp_path: Path, warehouse: ChangedWarehouse) -> P def add_changed_overlay(project_dir: Path, fixture: Path) -> None: """Overlay one of the DuckShop changed-builds cases on the base case.""" - for path in (*fixture.rglob("*.sql"), *fixture.rglob("*.py")): + for path in (*fixture.rglob("*.sql"), *fixture.rglob("*.py"), *fixture.rglob("*.toml")): destination: Path = project_dir / path.relative_to(fixture) destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(path.read_text(encoding="utf-8"), encoding="utf-8") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py index 54c79457f..20fedd7e7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py @@ -231,7 +231,7 @@ def test_given_a_build_killed_mid_write_when_building_with_changed_then_every_in "Changed (1)", "Downstream of changes (3)", "Held back (3)", - "job finance_daily", + "job finance_daily_job", "reads fct_revenue, which is held back", ), expected_job_run=frozenset({"fct_revenue", "rpt_pnl"}), @@ -258,7 +258,7 @@ def test_given_scheduled_jobs_when_building_with_changed_then_a_change_stops_at_ redefined, build_output = changed_build_output(project_dir, ("--changed",)) held: frozenset[str] = run_changed_build(project_dir, ("--changed",)) job_run: frozenset[str] = run_changed_build( - project_dir, ("--changed", "--select", "finance_daily") + project_dir, ("--changed", "--select", "finance_daily_job") ) reach_all: frozenset[str] = run_changed_build(project_dir, ("--changed", "--reach", "all")) refused: subprocess.CompletedProcess[str] = run_sqb( @@ -268,7 +268,7 @@ def test_given_scheduled_jobs_when_building_with_changed_then_a_change_stops_at_ assert first == REACH_MODELS assert new_data == frozenset(), "new source data is not a change by default" assert redefined == test_case.expected_redefined, ( - "the change stops at finance_daily; a job without a trigger and a model in no job run" + "the change stops at finance_daily_job; a job without a trigger and a model in no job run" ) assert test_case.expected_summary in build_output assert plan.returncode == 0, plan.stdout + plan.stderr @@ -309,13 +309,15 @@ def test_given_two_jobs_each_holding_a_redefined_upstream_when_building_changed_ sales_staged: frozenset[str] = run_changed_build( project_dir, ("--changed", "--select", "stg_orders") ) - crm_run: frozenset[str] = run_changed_build(project_dir, ("--changed", "--select", "crm_daily")) + crm_run: frozenset[str] = run_changed_build( + project_dir, ("--changed", "--select", "crm_daily_job") + ) downstream: frozenset[str] = run_changed_build(project_dir, ("--changed",)) assert first == {"stg_customers", "stg_orders", "customer_orders", "customer_sales"} assert sales_staged == {"stg_orders"} assert crm_run == test_case.expected_crm_run assert downstream == test_case.expected_downstream, ( - "sales_daily holds the stg_orders change that reached customer_orders, so customer_sales " - "runs even though crm_daily holds the stg_customers change too" + "sales_daily_job holds the stg_orders change that reached customer_orders, so customer_sales " + "runs even though crm_daily_job holds the stg_customers change too" ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py index 80803f98f..9f94302b1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py @@ -106,7 +106,7 @@ ), PythonNodeSelectionE2ETestCase( description="a job selecting only Python nodes builds them", - select=("job:customer_counts",), + select=("job:customer_counts_job",), expected_exit_code=0, expected_output_patterns=(r"count_customers\s+OK", r"customers_extract\s+OK"), expected_row_counts={"raw_customers": 3}, @@ -114,7 +114,7 @@ ), PythonNodeSelectionE2ETestCase( description="a job named without job: builds its Python nodes", - select=("customer_counts",), + select=("customer_counts_job",), expected_exit_code=0, expected_output_patterns=(r"count_customers\s+OK", r"customers_extract\s+OK"), expected_row_counts={"raw_customers": 3}, @@ -122,7 +122,7 @@ ), PythonNodeSelectionE2ETestCase( description="a job selecting a task with its upstream model builds both", - select=("job:order_counts",), + select=("job:order_counts_job",), expected_exit_code=0, expected_output_patterns=(r"count_orders\s+OK",), expected_row_counts={"orders": 2}, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py index 6fc4af0c6..ed0930ad4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py @@ -32,6 +32,42 @@ "Move pipeline Python nodes and the helper modules they import under python/", ), ), + PythonProjectLayoutCompileTestCase( + description="Python jobs left in orchestration/ point to the TOML layout", + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "nightly = job('nightly', description='Orders.', select='orders')\n" + ), + }, + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs.py", + "files cannot sit directly in orchestration/", + "jobs, job groups and schedules are TOML files in orchestration/jobs/", + "https://sqlbuild.com/docs/concepts/orchestration/", + ), + ), + PythonProjectLayoutCompileTestCase( + description="a sensor importing the removed job API points to the TOML layout", + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "orchestration/sensors/orders_sensor.py": ( + "from sqlbuild.orchestration import job, sensor\n" + "orders = sensor('orders_sensor', description='Orders.', " + "run=job('nightly', description='Orders.', select='orders'))\n" + ), + }, + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/sensors/orders_sensor.py:1:", + "`job` was removed from sqlbuild.orchestration", + "https://sqlbuild.com/docs/concepts/orchestration/", + ), + ), PythonProjectLayoutCompileTestCase( description="removed tasks Python root", repo_files={ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py index f7d82b1d4..de4e65ace 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py @@ -14,6 +14,9 @@ _JOB: str = "orchestration/jobs/orders/refresh.toml" _UNRELATED: str = "orchestration/jobs/orders/totals.toml" +_SELECTOR_REWRITE: str = Path( + "benchmarks/duckshop/edge_cases/orchestration/selector_rewrite/jobs/orders_refresh_job.toml" +).read_text() _UNRELATED_JOB: str = ( 'name = "orders_totals_job"\ndescription = "Rebuild totals."\nselect = "customer_totals"\n' ) @@ -31,15 +34,11 @@ def _job(select: str, exclude: str = "") -> str: JobSelectorRefactorE2ETestCase( description="a rename rewrites exact names and leaves globs and other kinds alone", command=("rename", "stg_orders", "stg_orders_clean"), - job=_job( - '"+stg_orders+ fact_orders,+stg_orders stg_orders~customer_totals stg_ord*"', - "'stg_orders+,customer_totals'", - ), - expected_job=_job( - '"+stg_orders_clean+ fact_orders,+stg_orders_clean ' - 'stg_orders_clean~customer_totals stg_ord*"', - "'stg_orders_clean+,customer_totals'", - ), + job=_SELECTOR_REWRITE, + expected_job=_SELECTOR_REWRITE.replace( + "+stg_orders+ fact_orders,+stg_orders stg_orders~customer_totals", + "+stg_orders_clean+ fact_orders,+stg_orders_clean stg_orders_clean~customer_totals", + ).replace("'stg_orders+,customer_totals'", "'stg_orders_clean+,customer_totals'"), expected_output=("selector", _JOB), ), JobSelectorRefactorE2ETestCase( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index a0984474b..c68d9cbad 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1381,5 +1381,4 @@ class JobSelectorBuildTestCase: build_args: tuple[str, ...] expected_exit_code: int expected_tables: frozenset[str] - expected_jobs_imported: bool expected_fragment: str = "" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py index 6b1d2d8bf..bf12cbf56 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py @@ -26,14 +26,20 @@ "models/customers.sql": ( "MODEL (description 'Customers.', materialized table);\nSELECT 7 AS customer_id\n" ), - "orchestration/jobs.py": ( - "from pathlib import Path\n" - "from sqlbuild.orchestration import job, job_group\n" - "Path(__file__).parent.parent.joinpath('jobs_imported').write_text('yes')\n" - "nightly = job('nightly', description='Orders.', select='+orders')\n" - "crm = job('crm', description='Customers.', select='customers')\n" - "daily = job_group('daily', description='Everything.', jobs=[nightly, crm])\n" - "restock = job('restock', description='Prod only.', select='customers', targets=['prod'])\n" + "orchestration/jobs/nightly_job.toml": ( + 'name = "nightly_job"\ndescription = "Orders."\nselect = "+orders"\n' + ), + "orchestration/jobs/crm_job.toml": ( + 'name = "crm_job"\ndescription = "Customers."\nselect = "customers"\n' + ), + "orchestration/jobs/restock_job.toml": ( + 'name = "restock_job"\n' + 'description = "Prod only."\n' + 'select = "customers"\n' + 'targets = ["prod"]\n' + ), + "orchestration/job_groups/daily_job_group.toml": ( + 'name = "daily_job_group"\ndescription = "Everything."\njobs = ["nightly_job", "crm_job"]\n' ), } @@ -43,67 +49,52 @@ [ JobSelectorBuildTestCase( description="a job prefix builds the job's nodes", - build_args=("--select", "job:nightly"), + build_args=("--select", "job:nightly_job"), expected_exit_code=0, expected_tables=frozenset({"orders", "stg_orders"}), - expected_jobs_imported=True, ), JobSelectorBuildTestCase( description="a bare job name needs no prefix", - build_args=("--select", "nightly"), + build_args=("--select", "nightly_job"), expected_exit_code=0, expected_tables=frozenset({"orders", "stg_orders"}), - expected_jobs_imported=True, ), JobSelectorBuildTestCase( description="a job group builds its jobs' nodes", - build_args=("--select", "job:daily"), + build_args=("--select", "job:daily_job_group"), expected_exit_code=0, expected_tables=frozenset({"customers", "orders", "stg_orders"}), - expected_jobs_imported=True, ), JobSelectorBuildTestCase( description="a job intersects with a tag", - build_args=("--select", "job:daily,tag:staging"), + build_args=("--select", "job:daily_job_group,tag:staging"), expected_exit_code=0, expected_tables=frozenset({"stg_orders"}), - expected_jobs_imported=True, ), JobSelectorBuildTestCase( description="a job can be excluded from a group", - build_args=("--select", "job:daily", "--exclude", "job:crm"), + build_args=("--select", "job:daily_job_group", "--exclude", "job:crm_job"), expected_exit_code=0, expected_tables=frozenset({"orders", "stg_orders"}), - expected_jobs_imported=True, ), JobSelectorBuildTestCase( description="a job declared for another target is not selectable", - build_args=("--select", "job:restock"), + build_args=("--select", "job:restock_job"), expected_exit_code=1, expected_tables=frozenset(), - expected_jobs_imported=True, - expected_fragment="unknown job name 'restock'", + expected_fragment="unknown job name 'restock_job'", ), JobSelectorBuildTestCase( description="a job declared for the selected target is selectable", - build_args=("--target", "prod", "--select", "job:restock"), + build_args=("--target", "prod", "--select", "job:restock_job"), expected_exit_code=0, expected_tables=frozenset({"customers"}), - expected_jobs_imported=True, - ), - JobSelectorBuildTestCase( - description="model selectors never import orchestration modules", - build_args=("--select", "+orders"), - expected_exit_code=0, - expected_tables=frozenset({"orders", "stg_orders"}), - expected_jobs_imported=False, ), JobSelectorBuildTestCase( description="an unknown job suggests a known one", build_args=("--select", "job:nightyl"), expected_exit_code=1, expected_tables=frozenset(), - expected_jobs_imported=True, expected_fragment="unknown job name 'nightyl'", ), ], @@ -126,7 +117,6 @@ def test_given_declared_jobs_when_building_by_job_then_only_their_nodes_build( assert exit_code == test_case.expected_exit_code, captured assert test_case.expected_fragment in captured - assert (tmp_path / "jobs_imported").exists() == test_case.expected_jobs_imported with duckdb.connect(str(tmp_path / "demo.duckdb")) as warehouse: rows: list[tuple[str]] = warehouse.sql( "SELECT table_name FROM duckdb_tables() " diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index fa3f650f9..406627370 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -3,17 +3,8 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.orchestration_declarations.models.authoring import ( - CalendarSchedule, - Job, - JobGroup, - OnFailure, - Schedule, - Sensor, -) @dataclass(frozen=True) @@ -300,7 +291,8 @@ class PythonCheckJobRejectionTestCase: @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str - schedule: Schedule + schedule: str + expected_timezone: str expected_selection: tuple[str, ...] expected_controls: dict[str, object] = field( default_factory=lambda: { @@ -313,18 +305,10 @@ class SnapshotScheduleTestCase: @dataclass(frozen=True) class SnapshotDeclarationRejectionTestCase: description: str - job_description: str = "Orders." - select: str = "orders" - cron: str = "0 2 * * *" - timezone: str = "UTC" - settings: dict[str, object] = field(default_factory=dict) - concurrency: int | None = None + expected_message: str + fields: dict[str, str] = field(default_factory=dict) + tables: str = "" copies: int = 1 - expected_message: str = "" - schedule: Schedule | None = None - owner: str | None = None - overlap: Any = "queue" - on_failure: OnFailure | None = None @dataclass(frozen=True) @@ -332,7 +316,6 @@ class SnapshotDiscoveryTestCase: description: str files: dict[str, str] expected_jobs: tuple[str, ...] - declarations: tuple[Job | JobGroup | CalendarSchedule | Sensor, ...] | None = None expected_groups: dict[str, dict[str, list[str]]] = field(default_factory=dict) expected_sensors: dict[str, dict[str, object]] = field(default_factory=dict) expected_schedules: tuple[str, ...] = () diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index d114213a6..897269da8 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -46,7 +46,6 @@ CompiledProjectView, ) from sqlbuild.orchestration import compile_snapshot -from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files if TYPE_CHECKING: @@ -429,6 +428,16 @@ def expected_summary(name: str) -> dict[str, object]: WAFFLE_SHOP_NATIVE_MODELS: int = 13 +def duckshop_orchestration(fixture: str) -> dict[str, str]: + """A DuckShop orchestration fixture tree as project files below `orchestration/`.""" + root: Path = Path("benchmarks/duckshop/edge_cases/orchestration") / fixture + return { + f"orchestration/{path.relative_to(root).as_posix()}": path.read_text() + for path in sorted(root.rglob("*")) + if path.is_file() + } + + def copy_project(*, fixtures: tuple[str, ...], files: dict[str, str], project_dir: Path) -> Path: """Copy each fixture into `project_dir`, then write `files` over them.""" @@ -754,10 +763,22 @@ def published_texts(test_case: SnapshotExpressionRangeTestCase, tmp_path: Path) def python_pipeline_snapshot( - tmp_path: Path, declarations: list[Job | JobGroup] + tmp_path: Path, jobs: dict[str, str], groups: dict[str, tuple[str, ...]] | None = None ) -> NativeOrchestrationSnapshot: - """Publish the Python pipeline project with the given job declarations.""" - write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES) + """Publish the Python pipeline project with TOML jobs, by name and selector, and groups.""" + files: dict[str, str] = { + f"orchestration/jobs/{name}.toml": ( + f'name = {json.dumps(name)}\ndescription = "Orders."\nselect = {json.dumps(select)}\n' + ) + for name, select in jobs.items() + } | { + f"orchestration/job_groups/{name}.toml": ( + f'name = {json.dumps(name)}\ndescription = "Nightly orders."\n' + f"jobs = {json.dumps(list(members))}\n" + ) + for name, members in (groups or {}).items() + } + write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES | files) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered, @@ -768,6 +789,5 @@ def python_pipeline_snapshot( project=project, metadata_json=json.dumps(METADATA), project_dir=tmp_path, - declarations=declarations, discovered_inputs=discovered, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py index b8121c3fa..3117d3bfc 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py @@ -202,8 +202,8 @@ def build_projects(self) -> list[str]: for version, sql in cases.items(): project: Path = self.directory / version (project / "models").mkdir(parents=True) - (project / "orchestration").mkdir() - for relative in ("sqlbuild_project.toml", "orchestration/orders.py"): + (project / "orchestration/jobs").mkdir(parents=True) + for relative in ("sqlbuild_project.toml", "orchestration/jobs/refresh_orders_job.toml"): shutil.copyfile(Path("deploy/project") / relative, project / relative) (project / "models/orders.sql").write_text( "MODEL (description 'Example orders.', materialized table);\n" + sql + "\n" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index d29d97e19..4d0a0e168 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -61,6 +61,7 @@ from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec from sqlbuild.microbatches.constants import MICROBATCH_COLUMNS from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchWriteResult +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import duckshop_orchestration from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, ScheduledWorker, @@ -123,7 +124,13 @@ def open_legacy_warehouse(project: Path, adapter_name: str, connection_key: str) f'[connections.local]\n{connection_key} = "warehouse.db"\n' '[targets.development]\nschema = "main"\n' ), - "orchestration/orders.py": 'raise AssertionError("unexpected declaration import")\n', + "orchestration/calendars/unexpected_calendar.py": ( + "from sqlbuild.orchestration import calendar\n" + 'raise AssertionError("unexpected declaration import")\n' + "@calendar(name='unexpected_calendar', description='Never imported.', " + "horizon='1d')\n" + "def unexpected(window):\n return []\n" + ), }, ) adapter: Any = resolve_adapter(adapter_name=adapter_name) @@ -802,17 +809,26 @@ def scheduled_worker( "models/orders.sql": ( "MODEL (description 'Orders.', materialized table);\n" + test_case.model_sql + "\n" ), - "orchestration/orders.py": ( - "import os\nfrom sqlbuild.orchestration import job, schedule\n" + "python/environment_guard.py": ( + "import os\n" "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" " assert 'SQLBUILD_PLATFORM_WEBHOOK_URL' not in os.environ\n" " assert 'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL' not in os.environ\n" - 'annual_orders = job("annual_orders", select="orders", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' ), } + | ( + {} + if any(path.startswith("orchestration/jobs/") for path in test_case.files) + else { + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", ' + 'starts_at = "2000-01-01T00:00:00Z" }\n' + ), + } + ) | test_case.files, ) test_case.warehouse_setup(project / "warehouse.duckdb") @@ -907,7 +923,7 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - assert run["node_counts_json"] == ( test_case.expected_node_counts or {test_case.expected_status: test_case.expected_node_count} ), log.read_text() - assert run["job_key"] == "annual_orders" + assert run["job_key"] == "annual_orders_job" runs_url: str = f"{api.url}/api/locations/{api.location_id}/targets/development/runs" headers: dict[str, str] = {"Authorization": f"Bearer {api.bearer}"} with urlopen( @@ -1250,11 +1266,13 @@ def scheduler_crash_history(directory: Path, fixture: dict[str, Any]) -> dict[st start: datetime = datetime.now(UTC).replace(second=0, microsecond=0) - timedelta( minutes=fixture["backlog_minutes"] ) - declarations: str = "".join( - f'{name} = job("{name}", select="orders", description="Build {name}.", ' - f'schedule=schedule(cron="* * * * *", timezone="UTC"))\n' + jobs: dict[str, str] = { + f"orchestration/jobs/{name}.toml": ( + f'name = "{name}"\ndescription = "Build {name}."\nselect = "orders"\n' + 'schedule = { cron = "* * * * *", timezone = "UTC" }\n' + ) for name in fixture["jobs"] - ) + } write_files( project_dir=project, files={ @@ -1266,9 +1284,8 @@ def scheduler_crash_history(directory: Path, fixture: dict[str, Any]) -> dict[st "models/orders.sql": ( "MODEL (description 'Orders.', materialized table);\nSELECT 1 AS order_id\n" ), - "orchestration/orders.py": "from sqlbuild.orchestration import job, schedule\n" - + declarations, - }, + } + | jobs, ) executable: str = str(Path(sys.executable).with_name("sqb")) control_store: Path = directory / "control/.sqlbuild/control.sqlite3" @@ -1452,9 +1469,10 @@ def tagged_model(name: str, tags: str) -> str: CHANGED_FIXTURE: Path = Path("benchmarks/duckshop/edge_cases/changed_builds") CHANGED_JOB: str = ( - "from sqlbuild.orchestration import job\n" - "changed_orders = job('changed_orders', description='Build what changed.', " - "select='stg_orders stg_customers customer_orders', settings={'changed': True})\n" + 'name = "changed_orders_job"\n' + 'description = "Build what changed."\n' + 'select = "stg_orders stg_customers customer_orders"\n' + "\n[settings]\nchanged = true\n" ) @@ -1475,7 +1493,10 @@ def changed_job_files(project_toml: str) -> dict[str, str]: return { str(path.relative_to(CHANGED_FIXTURE)): path.read_text() for path in sorted(CHANGED_FIXTURE.glob("*/*")) - } | {"orchestration/orders.py": CHANGED_JOB, "sqlbuild_project.toml": project_toml} + } | { + "orchestration/jobs/changed_orders_job.toml": CHANGED_JOB, + "sqlbuild_project.toml": project_toml, + } def request_job_run(worker: ScheduledWorker, job: str) -> dict[str, Any]: @@ -1600,9 +1621,7 @@ def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: "models/orders.sql": Path( "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.sql" ).read_text(), - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" - ).read_text(), + **duckshop_orchestration("loaded_orders"), } @@ -1616,9 +1635,7 @@ def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: "models/orders.sql": Path( "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql" ).read_text(), - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" - ).read_text(), + **duckshop_orchestration("loaded_orders"), } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index b7247fd0a..27f176402 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -60,7 +60,7 @@ expected_customers_tables=0, # DuckDB has no session warehouse, so the CLI refuses the job's warehouse: the # published job reached the CLI instead of being refused by the worker. - job_settings="settings={'warehouse': 'ANALYST_XS'}, ", + job_settings='\n[settings]\nwarehouse = "ANALYST_XS"\n', expected_scheduled_status="failed", expected_worker_log="error[C260]: --warehouse ANALYST_XS was given", ), @@ -82,12 +82,12 @@ def test_given_an_ad_hoc_command_when_checked_and_launched_then_the_worker_runs_ ) } | { - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", select="orders", ' - f'description="Build orders.", {test_case.job_settings}' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", ' + 'starts_at = "2000-01-01T00:00:00Z" }\n' + test_case.job_settings ) }, ) @@ -98,7 +98,7 @@ def test_given_an_ad_hoc_command_when_checked_and_launched_then_the_worker_runs_ server_arguments=("--password-origin", "https://control.example.com"), ) as worker: scheduled: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert scheduled["status"] == test_case.expected_scheduled_status, worker.log.read_text() assert scheduled["status_reason"] != WAREHOUSE_REFUSAL, "published jobs are not refused" @@ -332,7 +332,7 @@ def test_given_an_ad_hoc_verb_when_the_checkout_changes_after_verification_then_ worker_case, server_arguments=("--password-origin", "https://control.example.com"), ) as worker: - _ = wait_for_run(worker.api, worker.process, worker.log, job_key="annual_orders") + _ = wait_for_run(worker.api, worker.process, worker.log, job_key="annual_orders_job") admin: dict[str, Any] = provision_person( tmp_path / "control", "create-user", "orders-admin" ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py index 512db9188..51988adfa 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py @@ -73,11 +73,19 @@ def test_given_explicit_backfill_bounds_when_running_and_rerunning_a_group_then_ files=test_case.files | { "sources/raw_orders.yml": "sources:\n - name: raw_orders\n description: Synthetic order windows.\n schema: main\n table: raw_orders\n", - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, job_group\n" - "orders = job('windowed_orders', description='Windowed orders.', select='orders', " - "settings={'start_cursor': '0', 'end_cursor': '1'})\n" - "annual_orders = job_group('annual_orders', description='Windowed group.', jobs=[orders])\n" + "orchestration/jobs/windowed_orders_job.toml": ( + 'name = "windowed_orders_job"\n' + 'description = "Windowed orders."\n' + 'select = "orders"\n' + "\n" + "[settings]\n" + 'start_cursor = "0"\n' + 'end_cursor = "1"\n' + ), + "orchestration/job_groups/annual_orders_job_group.toml": ( + 'name = "annual_orders_job_group"\n' + 'description = "Windowed group."\n' + 'jobs = ["windowed_orders_job"]\n' ), }, ) @@ -95,7 +103,7 @@ def test_given_explicit_backfill_bounds_when_running_and_rerunning_a_group_then_ "SQLBUILD_SCHEDULER_TEST_STORE", "request-run", worker.api.location_id, - "annual_orders", + "annual_orders_job_group", "--target", "development", "--run-id", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index c5325d6f5..799952b08 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -13,7 +13,10 @@ import duckdb import pytest -from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + TYPE_PROOF_FILES, + duckshop_orchestration, +) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( CalendarRefreshTestCase, CalendarRejectionTestCase, @@ -71,11 +74,16 @@ def test_given_invalid_calendar_results_when_publishing_through_the_cli_then_val "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + '\n[targets.development]\nschema = "main"\n', - "orchestration/orders.py": ( + "orchestration/jobs/order_reporting_job.toml": ( + 'name = "order_reporting_job"\n' + 'description = "Order reporting."\n' + 'select = "orders"\n' + f'schedule = {{ calendar = "deliveries_calendar", timezone = {json.dumps(test_case.timezone)} }}\n' + ), + "orchestration/calendars/deliveries_calendar.py": ( "from datetime import datetime, timedelta\nfrom itertools import repeat\n" - "from sqlbuild.orchestration import calendar_schedule, job\n" - "orders = job('order_reporting', description='Order reporting.', select='orders')\n" - f"@calendar_schedule(job=orders, description='Delivery times.', horizon={test_case.horizon!r}, timezone={test_case.timezone!r})\n" + "from sqlbuild.orchestration import calendar\n" + f"@calendar(name='deliveries_calendar', description='Delivery times.', horizon={test_case.horizon!r})\n" f"def deliveries(window):\n return {test_case.expression}\n" ), }, @@ -134,18 +142,23 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + '\n[targets.development]\nschema = "main"\n', - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py" - ) - .read_text() - .replace("America/New_York", test_case.expected_timezone) - + ( - "\n@calendar_schedule(job=orders, description='Other target.', horizon='8s', timezone='UTC', targets=('production',))\n" + **{ + path: text.replace("America/New_York", test_case.expected_timezone) + for path, text in duckshop_orchestration("calendar_orders").items() + }, + "orchestration/jobs/production_reporting_job.toml": ( + 'name = "production_reporting_job"\n' + 'description = "Other target."\n' + 'select = "orders"\n' + 'targets = ["production"]\n' + 'schedule = { calendar = "excluded_calendar", timezone = "UTC" }\n' + ), + "orchestration/calendars/excluded_calendar.py": ( + "from sqlbuild.orchestration import calendar\n" + "@calendar(name='excluded_calendar', description='Other target.', horizon='8s')\n" "def excluded_calendar(window):\n raise ValueError('excluded calendar must not evaluate')\n" ), - "orchestration/platform_orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" - ).read_text(), + **duckshop_orchestration("platform_orders"), "providers/fulfillment.py": ( "import json, os\nfrom pathlib import Path\nfrom datetime import timedelta, timezone\n" "from sqlbuild.provider.classes.provider import Provider\n" @@ -189,23 +202,23 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr assert first["nodes"] == second["nodes"] assert second["changes"] == [] assert first["documents"]["triggers"].keys() == { - "fulfillment_calendar", - "overdue_orders", + "order_reporting_job", + "overdue_orders_sensor", } - assert first["documents"]["triggers"]["fulfillment_calendar"]["declared_at"] == { - "path": "orchestration/orders.py", - "line": 8, + assert first["documents"]["triggers"]["order_reporting_job"]["declared_at"] == { + "path": "orchestration/jobs/order_reporting_job.toml", + "line": 2, } with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as control: stored, next_due = control.execute( "SELECT definition_json, next_due_at FROM sqb_triggers JOIN sqb_trigger_state USING (trigger_id) WHERE kind = 'schedule'" ).fetchone() - original_times: list[str] = first["documents"]["triggers"]["fulfillment_calendar"][ + original_times: list[str] = first["documents"]["triggers"]["order_reporting_job"][ "schedule" ]["times"] assert json.loads(stored)["schedule"]["times"] == sorted( original_times - + second["documents"]["triggers"]["fulfillment_calendar"]["schedule"]["times"] + + second["documents"]["triggers"]["order_reporting_job"]["schedule"]["times"] ) assert datetime.fromisoformat(next_due) == datetime.fromisoformat(original_times[0]) observations: list[dict[str, Any]] = [ @@ -220,9 +233,9 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr start: datetime = datetime.fromisoformat(evaluation["start"]) end: datetime = datetime.fromisoformat(evaluation["end"]) assert end - start == timedelta(seconds=8) - schedule: dict[str, Any] = snapshot["documents"]["triggers"][ - "fulfillment_calendar" - ]["schedule"] + schedule: dict[str, Any] = snapshot["documents"]["triggers"]["order_reporting_job"][ + "schedule" + ] assert schedule["kind"] == "calendar" assert [datetime.fromisoformat(value) for value in schedule["times"]] == [ start + timedelta(seconds=offset) for offset in [0, 3] @@ -246,8 +259,7 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr third: dict[str, Any] = wait_for_generation(api, "development", 3, process, log) assert "CAST(19 AS INTEGER)" in third["nodes"]["model:stg_orders"]["raw_sql"] assert ( - len(third["documents"]["triggers"]["fulfillment_calendar"]["schedule"]["times"]) - == 1 + len(third["documents"]["triggers"]["order_reporting_job"]["schedule"]["times"]) == 1 ) assert [ json.loads(line)["generation"] for line in publications.read_text().splitlines() @@ -277,15 +289,27 @@ def test_given_a_published_calendar_when_a_real_worker_compiles_then_it_does_not test_case, files=test_case.files | { - "orchestration/orders.py": ( - "from pathlib import Path\nfrom sqlbuild.orchestration import calendar_schedule, job, schedule\n" - "orders = job('order_reporting', description='Order reporting.', select='orders', schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" - "@calendar_schedule(job=orders, description='Upcoming deliveries.', horizon='1d', timezone='UTC')\n" + "orchestration/jobs/order_reporting_job.toml": ( + 'name = "order_reporting_job"\n' + 'description = "Order reporting."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", ' + 'starts_at = "2000-01-01T00:00:00Z" }\n' + ), + "orchestration/jobs/deliveries_job.toml": ( + 'name = "deliveries_job"\n' + 'description = "Upcoming deliveries."\n' + 'select = "orders"\n' + 'schedule = { calendar = "deliveries_calendar", timezone = "UTC" }\n' + ), + "orchestration/calendars/deliveries_calendar.py": ( + "from pathlib import Path\nfrom sqlbuild.orchestration import calendar\n" + "@calendar(name='deliveries_calendar', description='Upcoming deliveries.', horizon='1d')\n" "def deliveries(window):\n" f" marker = Path({str(evaluated)!r})\n" " if marker.exists():\n raise ValueError('calendar evaluated inside a run worker')\n" " marker.write_text('evaluated')\n return []\n" - ) + ), }, ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py index c6a00930b..68695dd67 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py @@ -95,7 +95,7 @@ def test_given_a_changed_job_when_inputs_change_between_runs_then_only_their_con time.sleep(ORDER_SKEW_SECONDS) for sql in mutation: test_case.execute(warehouse, sql) - run: dict[str, Any] = request_job_run(worker, "changed_orders") + run: dict[str, Any] = request_job_run(worker, "changed_orders_job") assert run["status"] == "success", worker.log.read_text() started: set[tuple[Any, ...]] = set( test_case.execute( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py index e6da7b8e1..2b651b832 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py @@ -11,6 +11,7 @@ import duckdb import pytest +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import duckshop_orchestration from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( WorkerExecutionTestCase, ) @@ -43,9 +44,7 @@ [(83,)], expected_origin="sensor", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/custom_orders.py" - ).read_text(), + **duckshop_orchestration("custom_orders"), "providers/fulfillment.py": ( "import json, os, time\nfrom pathlib import Path\n" "from sqlbuild.orchestration import RunRequest, SkipReason\n" @@ -168,7 +167,7 @@ def test_given_a_custom_polling_sensor_when_a_callback_fails_or_times_out_then_a os.kill(timed_call["pid"], 0) data.write_text(json.dumps({"mode": "ready"})) destination: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert destination["status"] == test_case.expected_status, log.read_text() for browser in [admin, session]: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py index 5c85fca19..69a5bcc7f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py @@ -17,7 +17,10 @@ import duckdb import pytest -from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + TYPE_PROOF_FILES, + duckshop_orchestration, +) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( DevelopmentTestCase, SnapshotApi, @@ -68,11 +71,12 @@ def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paus ) -> None: project: Path = tmp_path / "project" pidfile: Path = tmp_path / "interpreter.pid" - declaration: str = ( - "import os\nfrom pathlib import Path\n" + probe: str = ( + "import os\nfrom pathlib import Path\nfrom sqlbuild.orchestration import calendar\n" "assert 'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL' not in os.environ\n" f"Path({str(pidfile)!r}).write_text(str(os.getpid()))\n" - + Path("benchmarks/duckshop/edge_cases/orchestration/development_orders.py").read_text() + "@calendar(name='interpreter_probe_calendar', description='Probe.', horizon='1d')\n" + "def interpreter_probe(window):\n return []\n" ) write_files( project_dir=project, @@ -86,7 +90,8 @@ def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paus "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( "SELECT order_id, amount", "SELECT order_id, CAST(amount AS BIGINT) AS amount" ), - "orchestration/orders.py": declaration, + **duckshop_orchestration("development_orders"), + "orchestration/calendars/interpreter_probe_calendar.py": probe, }, ) executable: str = str(Path(sys.executable).with_name("sqb")) @@ -137,7 +142,7 @@ def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paus "admin", "request-run", identity, - "annual_orders", + "annual_orders_job", "--target", test_case.target, "--run-id", @@ -182,9 +187,12 @@ def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paus with urlopen(f"{scope}/changes", timeout=5) as stream: assert stream.readline() == b"event: changed\n" assert json.loads(stream.readline().removeprefix(b"data: ")) == {"topic": "all"} - _ = (project / "orchestration/orders.py").write_text( - declaration - + '\nadditional_orders = job("additional_orders", description="Additional orders.", select="orders", schedule=annual)\n' + _ = (project / "orchestration/jobs/additional_orders_job.toml").write_text( + 'name = "additional_orders_job"\n' + 'description = "Additional orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 1 1 *", timezone = "UTC", ' + 'starts_at = "2099-01-01T00:00:00Z" }\n' ) assert f"snapshot:{identity}" in islice( ( @@ -198,7 +206,7 @@ def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paus statuses: dict[str, str] = { entry["job_key"]: entry["status"] for entry in json.load(response) } - assert statuses == {"annual_orders": "paused", "additional_orders": "running"} + assert statuses == {"annual_orders_job": "paused", "additional_orders_job": "running"} stop_location(restarted, int(pidfile.read_text()), log) with sqlite3.connect(control) as connection: assert connection.execute("SELECT count(*) FROM sqb_development_locations").fetchone() == ( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py index 69792e82d..91e07f168 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -13,6 +13,7 @@ import pytest +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import duckshop_orchestration from tests.integration.src.sqlbuild.compiler.compiled_project.publication import helpers from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ScheduledWorker, @@ -50,18 +51,11 @@ [(37,)], launcher="docker", files={ - "orchestration/orders.py": ( - "import os\nfrom sqlbuild.orchestration import job, schedule\n" - "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" - " assert os.getenv('SQLBUILD_WORKER_TEST_VALUE') == 'orders-context'\n" - " for name in ('SQLBUILD_LOCATION_TOKEN', 'SQLBUILD_CONTROL_DATABASE_URL', " - "'SQLBUILD_SCHEDULER_TEST_STORE', 'SQLBUILD_PLATFORM_WEBHOOK_URL', " - "'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL'):\n" - " assert name not in os.environ, name\n" - 'annual_orders = job("annual_orders", select="orders", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ) }, ), @@ -72,11 +66,7 @@ "success", [(41,)], launcher="docker", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py" - ).read_text() - }, + files={**duckshop_orchestration("container_resources_orders")}, launcher_settings={ "run": {"memory": "768m", "cpus": 2, "pids": 256, "max_memory": "1g"} }, @@ -89,11 +79,7 @@ "success", [(43,)], launcher="docker", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py" - ).read_text() - }, + files={**duckshop_orchestration("container_resources_orders")}, expected_host_config={"Memory": 768 << 20, "NanoCpus": 500_000_000}, ), WorkerExecutionTestCase( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py index 79e8367e7..7af8561cc 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py @@ -32,9 +32,10 @@ files={ "models/customers.sql": "MODEL (description 'Customers.', materialized table);\nSELECT 7 AS customer_id\n", "models/products.sql": "MODEL (description 'Products.', materialized table);\nSELECT 3 AS product_id\n", - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job\n" - "nightly = job('nightly', description='Orders and customers.', select='orders customers')\n" + "orchestration/jobs/nightly_job.toml": ( + 'name = "nightly_job"\n' + 'description = "Orders and customers."\n' + 'select = "orders customers"\n' ), }, ) @@ -53,7 +54,7 @@ def test_given_a_published_job_when_an_operator_excludes_a_model_then_only_the_r "SQLBUILD_SCHEDULER_TEST_STORE", "request-run", worker.api.location_id, - "nightly", + "nightly_job", "--target", "development", "--reason", @@ -82,7 +83,7 @@ def test_given_a_published_job_when_an_operator_excludes_a_model_then_only_the_r worker.api, worker.process, worker.log, run_id=run_id ) assert completed["status"] == test_case.expected_status, worker.log.read_text() - assert completed["job_key"] == "nightly" + assert completed["job_key"] == "nightly_job" with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: assert warehouse.sql( "SELECT table_name FROM duckdb_tables() " diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py index 73c839748..584d7c405 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py @@ -13,6 +13,7 @@ import pytest +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import duckshop_orchestration from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ScheduledWorker, WorkerExecutionTestCase, @@ -113,9 +114,7 @@ def test_given_a_running_local_worker_when_scheduler_dies_then_restart_adopts_an launcher="uncooperative", concurrency_limits={"orders_limit": 1}, files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py" - ).read_text(), + **duckshop_orchestration("local_recovery_orders"), }, store_setup=Path( "benchmarks/duckshop/edge_cases/orchestration/local_recovery_publication_fault.sql" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py index 780abc474..5a452a73a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py @@ -93,7 +93,7 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi "run_id": str(uuid4()), "location_id": worker.api.location_id, "target": "development", - "job_key": "annual_orders", + "job_key": "annual_orders_job", "reason": "Refresh order totals", "priority": 0, } @@ -349,7 +349,7 @@ def test_given_person_run_controls_when_scopes_change_then_frozen_execution_and_ "run_id": str(uuid4()), "location_id": worker.api.location_id, "target": "development", - "job_key": "annual_orders", + "job_key": "annual_orders_job", "reason": "Build order totals", "priority": 0, } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py index e79a6f48f..349e372f5 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py @@ -43,7 +43,6 @@ 'name = "orders"\nadapter = "duckdb"\n[connections.local]\ndatabase = "warehouse.db"\n' '[targets.development]\nschema = "main"\n' ), - "orchestration/orders.py": "", "seeds/raw_orders.csv": "id,ordered_at\n1,2026-01-01 00:00:00\n2,2026-01-01 01:00:00\n", "seeds/raw_orders.yml": dedent( """ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py index 0215796c8..47eebd80a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py @@ -167,7 +167,7 @@ def test_given_a_job_loading_an_intermediate_loader_when_a_role_runs_it_then_the "run_id": str(uuid4()), "location_id": api.location_id, "target": "development", - "job_key": "annual_orders", + "job_key": "annual_orders_job", "reason": "Refresh shipments", "priority": 0, } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index bc538509f..5de785eb7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -62,14 +62,20 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical test_case: SnapshotPublicationTestCase, tmp_path: Path ) -> None: project_dir: Path = tmp_path / "project" - files: dict[str, str] = TYPE_PROOF_FILES | { - "orchestration/edge_cases/definitions.py": Path( - "benchmarks/duckshop/project/orchestration/edge_cases/definitions.py" - ).read_text(), - "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( - "MODEL (", "MODEL (tags ['edge_case'],", 1 - ), - } + duckshop: Path = Path("benchmarks/duckshop/project") + files: dict[str, str] = ( + TYPE_PROOF_FILES + | { + path.relative_to(duckshop).as_posix(): path.read_text() + for path in (duckshop / "orchestration").rglob("*") + if path.is_file() + } + | { + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "MODEL (", "MODEL (tags ['edge_case'],", 1 + ), + } + ) write_files(project_dir=project_dir, files=files) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) project: CompiledProjectView = build_compiled_project( @@ -130,34 +136,36 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical by_name: dict[str, dict[str, Any]] = {trigger["name"]: trigger for trigger in triggers} assert len(triggers) == 4 assert set(by_name) == { - "edge_snapshot_orders", - "edge_platform_orders", - "edge_job_outcomes", - "edge_group_outcomes", + "edge_snapshot_orders_job", + "edge_platform_orders_sensor", + "edge_job_outcomes_sensor", + "edge_group_outcomes_sensor", } - scheduled: dict[str, Any] = by_name["edge_snapshot_orders"] + scheduled: dict[str, Any] = by_name["edge_snapshot_orders_job"] assert scheduled["status"] == "paused" assert scheduled["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" assert scheduled["definition_json"]["default_status"] == "paused" - assert by_name["edge_platform_orders"]["status"] == "paused" - assert by_name["edge_platform_orders"]["definition_json"]["default_status"] == "paused" - assert by_name["edge_platform_orders"]["definition_json"]["when"] == { + assert by_name["edge_platform_orders_sensor"]["status"] == "paused" + assert ( + by_name["edge_platform_orders_sensor"]["definition_json"]["default_status"] == "paused" + ) + assert by_name["edge_platform_orders_sensor"]["definition_json"]["when"] == { "kind": "platform_event", "kinds": ["tick_overdue", "location_broken", "launch_failed"], } - assert by_name["edge_job_outcomes"]["definition_json"]["when"] == { + assert by_name["edge_job_outcomes_sensor"]["definition_json"]["when"] == { "kind": "run_event", - "job_key": "edge_snapshot_orders", + "job_key": "edge_snapshot_orders_job", "kinds": ["warn", "partial"], } - assert by_name["edge_group_outcomes"]["definition_json"]["when"] == { + assert by_name["edge_group_outcomes_sensor"]["definition_json"]["when"] == { "kind": "run_event", - "job_key": "edge_snapshot_group", + "job_key": "edge_snapshot_job_group", "kinds": ["completed"], "executed_successfully": ["model:orders"], } - assert by_name["edge_job_outcomes"]["status"] == "paused" - assert by_name["edge_group_outcomes"]["status"] == "paused" + assert by_name["edge_job_outcomes_sensor"]["status"] == "paused" + assert by_name["edge_group_outcomes_sensor"]["status"] == "paused" @pytest.mark.parametrize( @@ -189,9 +197,8 @@ def test_given_a_code_location_when_publishing_from_cli_then_compilation_isolate "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job\n" - 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' + "orchestration/jobs/daily_orders_job.toml": ( + 'name = "daily_orders_job"\ndescription = "Build orders."\nselect = "orders"\n' ), } write_files(project_dir=project_dir, files=files) @@ -238,7 +245,7 @@ def test_given_a_code_location_when_publishing_from_cli_then_compilation_isolate assert "Location snapshot published." in first_output.err initial: dict[str, Any] = current_snapshot(api, test_case.target) assert frozenset(initial["nodes"]) == test_case.expected_nodes - assert initial["documents"]["jobs"]["daily_orders"]["selection"] == ["model:orders"] + assert initial["documents"]["jobs"]["daily_orders_job"]["selection"] == ["model:orders"] assert not (project_dir / ".sqlbuild/control.sqlite3").exists() assert not (project_dir / "warehouse.duckdb").exists() with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as connection: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py index 84c65833b..e2562c2f3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py @@ -58,18 +58,25 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ ) -> None: project: Path = tmp_path / "project" compiler_pid: Path = tmp_path / "compiler.pid" - declaration: str = ( - "from sqlbuild.orchestration import job\n" - "import os\nfrom pathlib import Path\n" - f"Path({str(compiler_pid)!r}).write_text(str(os.getpid()))\n" - 'print("Orders declaration loaded.")\n' - 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' - ) + calendar: str = "orchestration/calendars/compiler_probe_calendar.py" + orchestration: dict[str, str] = { + calendar: ( + "from sqlbuild.orchestration import calendar\n" + "import os\nfrom pathlib import Path\n" + f"Path({str(compiler_pid)!r}).write_text(str(os.getpid()))\n" + 'print("Orders declaration loaded.")\n' + "@calendar(name='compiler_probe_calendar', description='Probe.', horizon='1d')\n" + "def compiler_probe(window):\n return []\n" + ), + "orchestration/jobs/daily_orders_job.toml": ( + 'name = "daily_orders_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + } files: dict[str, str] = TYPE_PROOF_FILES | { "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', - "orchestration/orders.py": declaration, + **orchestration, "sinks/orders.py": 'print("Loading orders sink", flush=True)\n', } write_files(project_dir=project, files=files) @@ -134,20 +141,27 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ assert int(compiler_pid.read_text()) == first_interpreter assert "CAST(14 AS INTEGER)" in recovered["nodes"]["model:stg_orders"]["raw_sql"] - _ = (project / "orchestration/orders.py").write_text("invalid Python declaration !\n") + _ = (project / calendar).write_text("invalid Python declaration !\n") wait_for_log(process, log, "Location reload failed; retaining last good snapshot.") _ = wait_for_location_health( api, test_case.target, {"kind": "compile_failed", "generation": 4}, process, log ) assert current_snapshot(api, test_case.target)["record"] == recovered["record"] - _ = (project / "orchestration/orders.py").write_text( - declaration.replace("Build orders.", "Refresh order totals.") + write_files( + project_dir=project, + files=orchestration + | { + "orchestration/jobs/daily_orders_job.toml": orchestration[ + "orchestration/jobs/daily_orders_job.toml" + ].replace("Build orders.", "Refresh order totals.") + }, ) third: dict[str, Any] = wait_for_generation(api, test_case.target, 4, process, log) replacement_interpreter: int = int(compiler_pid.read_text()) assert replacement_interpreter != first_interpreter assert ( - third["documents"]["jobs"]["daily_orders"]["description"] == "Refresh order totals." + third["documents"]["jobs"]["daily_orders_job"]["description"] + == "Refresh order totals." ) _ = subprocess.run( [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py index 4682b6bb2..7a5ebfc24 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py @@ -59,20 +59,28 @@ def test_given_a_partial_group_when_retrying_from_failure_then_successful_nodes_ "MODEL (description 'Independent inventory.', materialized table);\n" "SELECT 29 AS amount\n" ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, job_group, schedule\n" - "staging = job('staging', description='Stage orders.', select='orders broken_orders')\n" - "marts = job('marts', description='Build totals.', select='totals')\n" - "inventory = job('stock', description='Build inventory.', select='inventory')\n" - "annual_orders = job_group('annual_orders', description='Retry boundaries.', " - "jobs=[inventory, marts, staging], schedule=schedule(cron='0 0 * * *', " - "timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "orchestration/jobs/staging_job.toml": ( + 'name = "staging_job"\n' + 'description = "Stage orders."\n' + 'select = "orders broken_orders"\n' + ), + "orchestration/jobs/marts_job.toml": ( + 'name = "marts_job"\ndescription = "Build totals."\nselect = "totals"\n' + ), + "orchestration/jobs/stock_job.toml": ( + 'name = "stock_job"\ndescription = "Build inventory."\nselect = "inventory"\n' + ), + "orchestration/job_groups/annual_orders_job_group.toml": ( + 'name = "annual_orders_job_group"\n' + 'description = "Retry boundaries."\n' + 'jobs = ["stock_job", "marts_job", "staging_job"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: source: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job_group" ) assert source["status"] == "partial", worker.log.read_text() with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: @@ -112,8 +120,8 @@ def test_given_a_partial_group_when_retrying_from_failure_then_successful_nodes_ (run_id,), ).fetchall() assert [(name, json.loads(selection)) for name, selection in children] == [ - ("marts", ["model:totals"]), - ("staging", ["model:broken_orders"]), + ("marts_job", ["model:totals"]), + ("staging_job", ["model:broken_orders"]), ] assert control.execute( "SELECT count(*) FROM sqb_security_audit WHERE action='run.rerun_requested'" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py index 18be7d027..f5af3b526 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py @@ -18,6 +18,7 @@ write_source_freshness_records, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import duckshop_orchestration from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( WorkerExecutionTestCase, ) @@ -41,9 +42,7 @@ [(73,)], expected_origin="sensor", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" - ).read_text(), + **duckshop_orchestration("source_orders"), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), @@ -57,9 +56,7 @@ [(73,)], expected_origin="sensor", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" - ).read_text(), + **duckshop_orchestration("stale_orders"), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/managed_orders.yml" ).read_text(), @@ -79,9 +76,7 @@ [(73,)], expected_origin="sensor", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/changed_source_orders.py" - ).read_text(), + **duckshop_orchestration("changed_source_orders"), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), @@ -190,7 +185,7 @@ def test_given_external_source_versions_when_a_location_resyncs_then_only_change worker.api, worker.process, worker.log, run_id=runs[-1][0] ) assert result["status"] == test_case.expected_status - assert result["job_key"] == "orders_refresh" + assert result["job_key"] == "orders_refresh_job" assert result["node_counts_json"] == {"success": 1} with duckdb.connect( str(worker.project / "warehouse.duckdb"), read_only=True diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py index a379eb200..b581d4661 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py @@ -52,7 +52,7 @@ def test_given_relation_dropped_during_sampling_when_read_fails_then_catalog_can adapter, connection = open_legacy_warehouse(project, "sqlite", "path") _ = adapter.execute(connection=connection, sql=fixture["create_sql"]) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() (project / "models").mkdir() (project / "models/orders.sql").write_text(fixture["model"]) hook: Path = tmp_path / "sampling-hook" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py index 429858e78..2885a202c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py @@ -96,7 +96,12 @@ def test_given_case_distinct_postgres_relations_when_reading_catalog_then_column '[connections.local]\nconninfo = "${ENV:SQLBUILD_CATALOG_TEST_URL}"\n' '[targets.development]\nschema = "main"\n' ), - "orchestration/orders.py": 'raise AssertionError("unexpected declaration import")\n', + "orchestration/calendars/unexpected_calendar.py": ( + "from sqlbuild.orchestration import calendar\n" + 'raise AssertionError("unexpected declaration import")\n' + "@calendar(name='unexpected_calendar', description='Never imported.', horizon='1d')\n" + "def unexpected(window):\n return []\n" + ), }, ) with PostgresContainer(POSTGRES_IMAGE, driver=None) as postgres: @@ -192,7 +197,7 @@ def test_given_served_location_when_catalog_changes_then_facts_and_current_proje ) _ = adapter.execute(connection=connection, sql=fixture["create_sql"]) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() write_files(project_dir=project, files={"models/orders.sql": fixture["model"]}) expected: dict[str, Any] = fixture["record"] | {"relation_type": test_case.expected_table_type} control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 94c53c35f..a20a7e800 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -397,7 +397,7 @@ def test_given_a_served_location_when_state_event_history_is_rebuilt_and_retaine ) test_case.append_history(adapter, connection, fixture) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() write_files(project_dir=project, files={"models/orders.sql": fixture["model"]}) with snapshot_api(tmp_path / "control") as api: log: Path = tmp_path / "location.log" @@ -942,7 +942,7 @@ def test_given_a_served_location_when_warehouse_history_changes_then_periodic_re "wide fixture exceeds one API request before any synchronization" ) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() write_files( project_dir=project, files={"models/orders.sql": fixture["model"]}, @@ -1052,7 +1052,7 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre ) append_legacy_history(adapter, connection, fixture["initial"], fixture["insert_sql"]) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() write_files( project_dir=project, files={"models/orders.sql": "MODEL (description 'Orders.'); SELECT 17 AS amount"}, @@ -1352,7 +1352,7 @@ def test_given_change_tokens_when_a_served_location_resyncs_then_only_changed_or project, test_case.adapter_name, test_case.connection_key ) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() fixture: dict[str, Any] = json.loads( Path( "benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json" @@ -1522,7 +1522,7 @@ def test_given_a_served_location_when_a_sync_has_many_pages_then_pages_run_back_ project, test_case.adapter_name, test_case.connection_key ) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() fixture: dict[str, Any] = json.loads( Path( "benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index b1c6be952..7c1b282dd 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -27,6 +27,7 @@ from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( METADATA, TYPE_PROOF_FILES, + duckshop_orchestration, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( CheckExecutionTestCase, @@ -63,7 +64,7 @@ "SELECT amount FROM orders", "partial", [(11,)], - expected_child_statuses={"staging": ("partial",), "marts": ("partial",)}, + expected_child_statuses={"staging_job": ("partial",), "marts_job": ("partial",)}, ), GroupExecutionTestCase( "successful jobs honor compiled boundaries", @@ -71,7 +72,7 @@ "SELECT amount FROM totals", "success", [(17,)], - expected_child_statuses={"staging": ("success",), "marts": ("success",)}, + expected_child_statuses={"staging_job": ("success",), "marts_job": ("success",)}, expected_totals=("success", None), ), GroupExecutionTestCase( @@ -81,18 +82,20 @@ "success", [(17,)], expected_origin="sensor", - expected_child_statuses={"staging": ("success",), "marts": ("success",)}, + expected_child_statuses={"staging_job": ("success",), "marts_job": ("success",)}, expected_totals=("success", None), files={ - "orchestration/orders.py": ( - "from sqlbuild.orchestration import PlatformEventKind, job, job_group, platform_event, sensor\n" - "staging = job('staging', description='Stage orders.', select='stg_orders broken_orders')\n" - "marts = job('marts', description='Build independent marts.', select='orders totals')\n" - "annual_orders = job_group('annual_orders', description='Stage then build marts.', jobs=[marts, staging])\n" - "health_orders = sensor('health_orders', description='Refresh on platform heartbeat.', " - "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run=annual_orders, " + "orchestration/job_groups/annual_orders_job_group.toml": ( + 'name = "annual_orders_job_group"\n' + 'description = "Stage then build marts."\n' + 'jobs = ["marts_job", "staging_job"]\n' + ), + "orchestration/sensors/health_orders_sensor.py": ( + "from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor\n" + "health_orders = sensor('health_orders_sensor', description='Refresh on platform heartbeat.', " + "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run='annual_orders_job_group', " "minimum_interval_seconds=60)\n" - ) + ), }, ), GroupExecutionTestCase( @@ -101,7 +104,10 @@ "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", "failed", [(0,)], - expected_child_statuses={"staging": ("partial", "cancelled"), "marts": ("cancelled",)}, + expected_child_statuses={ + "staging_job": ("partial", "cancelled"), + "marts_job": ("cancelled",), + }, expected_totals=("cancelled", None), expected_marts_started=False, files={ @@ -138,22 +144,28 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail "MODEL (description 'Dependent totals.', materialized table);\n" 'SELECT amount FROM __ref("broken_orders")\n' ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, job_group, schedule\n" - 'staging = job("staging", description="Stage orders.", ' - 'select="stg_orders broken_orders")\n' - 'marts = job("marts", description="Build independent marts.", select="orders totals")\n' - 'annual_orders = job_group("annual_orders", description="Stage then build marts.", ' - "jobs=[marts, staging], " - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/staging_job.toml": ( + 'name = "staging_job"\n' + 'description = "Stage orders."\n' + 'select = "stg_orders broken_orders"\n' + ), + "orchestration/jobs/marts_job.toml": ( + 'name = "marts_job"\n' + 'description = "Build independent marts."\n' + 'select = "orders totals"\n' + ), + "orchestration/job_groups/annual_orders_job_group.toml": ( + 'name = "annual_orders_job_group"\n' + 'description = "Stage then build marts."\n' + 'jobs = ["marts_job", "staging_job"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), } | test_case.files, ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: parent: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job_group" ) assert parent["status"] == test_case.expected_status, worker.log.read_text() request: Request = Request( @@ -162,14 +174,14 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail ) with urlopen(request, timeout=5) as response: runs: dict[str, Any] = {run["job_key"]: run for run in json.load(response)} - assert set(runs) == {"annual_orders", *test_case.expected_child_statuses} + assert set(runs) == {"annual_orders_job_group", *test_case.expected_child_statuses} assert all( runs[name]["status"] in statuses for name, statuses in test_case.expected_child_statuses.items() ) - assert (runs["marts"]["started_at"] is not None) == test_case.expected_marts_started - assert datetime.fromisoformat(runs["staging"]["ended_at"]) <= datetime.fromisoformat( - runs["marts"]["started_at"] or runs["staging"]["ended_at"] + assert (runs["marts_job"]["started_at"] is not None) == test_case.expected_marts_started + assert datetime.fromisoformat(runs["staging_job"]["ended_at"]) <= datetime.fromisoformat( + runs["marts_job"]["started_at"] or runs["staging_job"]["ended_at"] ) with sqlite3.connect(worker.control_store) as control: assert control.execute( @@ -179,18 +191,18 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail control.execute( "SELECT parent_run_id, root_run_id FROM sqb_runs WHERE kind='job' ORDER BY job_key" ).fetchall() - == [(runs["annual_orders"]["run_id"],) * 2] * 2 + == [(runs["annual_orders_job_group"]["run_id"],) * 2] * 2 ) assert ( control.execute( "SELECT status, skipped_because_of FROM sqb_run_nodes WHERE run_id=? AND node_key='model:totals'", - (runs["marts"]["run_id"],), + (runs["marts_job"]["run_id"],), ).fetchone() == test_case.expected_totals ) assert control.execute( "SELECT status FROM sqb_run_nodes WHERE run_id=? AND node_key='model:broken_orders'", - (runs["staging"]["run_id"],), + (runs["staging_job"]["run_id"],), ).fetchone() == ({"success": "success"}.get(test_case.expected_status, "failed"),) assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) assert control.execute( @@ -225,13 +237,11 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail "tests/unit/test_stg_orders.sql": Path( "benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test.sql" ).read_text(), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders raw_orders stg_orders order_totals", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders raw_orders stg_orders order_totals"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, expected_run_nodes=[ @@ -269,13 +279,11 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail "tests/unit/test_stg_orders.sql": Path( "benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql" ).read_text(), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders raw_orders stg_orders order_totals", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders raw_orders stg_orders order_totals"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, expected_run_nodes=[ @@ -313,13 +321,11 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail "tests/unit/test_stg_orders.sql": Path( "benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_assert.sql" ).read_text(), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders raw_orders stg_orders order_totals", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders raw_orders stg_orders order_totals"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, expected_run_nodes=[ @@ -362,13 +368,11 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail "tests/unit/test_udf__order_band.sql": Path( "benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql" ).read_text(), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders raw_orders udf__order_band stg_orders order_totals", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders raw_orders udf__order_band stg_orders order_totals"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, expected_run_nodes=[ @@ -415,13 +419,11 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail ) .read_text() .replace("(5, 'large')", "(5, 'small')"), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders raw_orders udf__order_band stg_orders order_totals", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders raw_orders udf__order_band stg_orders order_totals"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, expected_run_nodes=[ @@ -440,7 +442,7 @@ def test_given_a_failing_unit_test_when_a_worker_builds_then_its_model_fails_and ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: run: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert run["status"] == test_case.expected_status, worker.log.read_text() with sqlite3.connect(worker.control_store) as control: @@ -482,13 +484,11 @@ def test_given_a_failing_unit_test_when_a_worker_builds_then_its_model_fails_and "MODEL (description 'Returns.', materialized table);\n" 'SELECT return_id FROM __source("raw_returns")\n' ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders stale_orders late_orders returns", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders stale_orders late_orders returns"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, ) @@ -500,7 +500,7 @@ def test_given_a_stale_source_when_a_worker_builds_then_blocked_nodes_finish_ski ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: run: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert run["status"] == test_case.expected_status, worker.log.read_text() with sqlite3.connect(worker.control_store) as control: @@ -540,13 +540,11 @@ def test_given_a_stale_source_when_a_worker_builds_then_blocked_nodes_finish_ski "MODEL (description 'Coded orders.', materialized table);\n" 'SELECT code FROM __seed("order_codes")\n' ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders order_codes coded_orders", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders order_codes coded_orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, ) @@ -558,7 +556,7 @@ def test_given_a_failing_seed_audit_when_a_worker_builds_then_its_seed_fails_wit ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: run: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert run["status"] == test_case.expected_status, worker.log.read_text() with sqlite3.connect(worker.control_store) as control: @@ -598,18 +596,31 @@ def test_given_a_failing_seed_audit_when_a_worker_builds_then_its_seed_fails_wit "models/inventory.sql": ( "MODEL (description 'Inventory.', materialized table);\nSELECT 23 AS amount\n" ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, job_group, schedule\n" - "orders = job('orders_job', description='Build orders.', select='orders', " - "concurrency_groups=('inventory_limit',))\n" - "inventory = job('inventory_job', description='Build inventory.', select='inventory', " - "concurrency_groups=('orders_limit',))\n" - "annual_orders = job_group('annual_orders', description='Orders group.', jobs=[orders], " - "concurrency_groups=('orders_limit',), " - "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" - "annual_inventory = job_group('annual_inventory', description='Inventory group.', " - "jobs=[inventory], concurrency_groups=('inventory_limit',), " - "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "orchestration/jobs/orders_job.toml": ( + 'name = "orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'concurrency_groups = ["inventory_limit"]\n' + ), + "orchestration/jobs/inventory_job.toml": ( + 'name = "inventory_job"\n' + 'description = "Build inventory."\n' + 'select = "inventory"\n' + 'concurrency_groups = ["orders_limit"]\n' + ), + "orchestration/job_groups/annual_orders_job_group.toml": ( + 'name = "annual_orders_job_group"\n' + 'description = "Orders group."\n' + 'jobs = ["orders_job"]\n' + 'concurrency_groups = ["orders_limit"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + ), + "orchestration/job_groups/annual_inventory_job_group.toml": ( + 'name = "annual_inventory_job_group"\n' + 'description = "Inventory group."\n' + 'jobs = ["inventory_job"]\n' + 'concurrency_groups = ["inventory_limit"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, store_setup=( @@ -627,10 +638,10 @@ def test_given_opposing_group_limits_when_scheduled_then_each_group_holds_every_ ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: orders: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job_group" ) inventory: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_inventory" + worker.api, worker.process, worker.log, job_key="annual_inventory_job_group" ) assert orders["status"] == inventory["status"] == "success", worker.log.read_text() with sqlite3.connect(worker.control_store) as control: @@ -638,10 +649,10 @@ def test_given_opposing_group_limits_when_scheduled_then_each_group_holds_every_ "SELECT r.job_key, c.group_key, max(c.occupied) FROM test_slot_claims c " "JOIN sqb_runs r ON r.run_id=c.run_id GROUP BY r.job_key, c.group_key ORDER BY r.job_key" ).fetchall() == [ - ("annual_inventory", "inventory_limit", 1), - ("annual_inventory", "orders_limit", 1), - ("annual_orders", "inventory_limit", 1), - ("annual_orders", "orders_limit", 1), + ("annual_inventory_job_group", "inventory_limit", 1), + ("annual_inventory_job_group", "orders_limit", 1), + ("annual_orders_job_group", "inventory_limit", 1), + ("annual_orders_job_group", "orders_limit", 1), ] assert control.execute( "SELECT count(*) FROM sqb_runs WHERE status='success'" @@ -804,13 +815,15 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ [(47,)], expected_origin="sensor", files={ - "orchestration/orders.py": ( - "from sqlbuild.orchestration import PlatformEventKind, job, platform_event, sensor\n" - "annual_orders = job('annual_orders', select='orders', description='Build orders.')\n" - "health_orders = sensor('health_orders', description='Refresh on platform heartbeat.', " - "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run=annual_orders, " + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + "orchestration/sensors/health_orders_sensor.py": ( + "from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor\n" + "health_orders = sensor('health_orders_sensor', description='Refresh on platform heartbeat.', " + "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run='annual_orders_job', " "minimum_interval_seconds=60)\n" - ) + ), }, ), WorkerExecutionTestCase( @@ -825,12 +838,16 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ '[connections.local]\ndatabase = "warehouse.duckdb"\n' '[targets.development]\nschema = "main"\n' ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - "annual_orders = job('annual_orders', select='orders', description='Build orders.', " - "settings={'vars': {'batch': 23}, 'reload': True, 'run_tests': False}, " - "schedule=schedule(cron='0 0 * * *', timezone='UTC', " - "starts_at='2000-01-01T00:00:00Z'))\n" + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + "\n" + "[settings]\n" + "vars = { batch = 23 }\n" + "reload = true\n" + "run_tests = false\n" ), }, ), @@ -845,9 +862,7 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ "sources/raw_orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders.py" - ).read_text(), + **duckshop_orchestration("upstream_source_orders"), }, warehouse_setup=lambda path: ( duckdb.connect(str(path)) @@ -867,9 +882,7 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ "sources/raw_orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders.py" - ).read_text(), + **duckshop_orchestration("upstream_source_orders"), }, warehouse_setup=lambda path: ( duckdb.connect(str(path)) @@ -954,12 +967,7 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re "SELECT amount FROM orders", "success", [(9,)], - files=INTERMEDIATE_LOADER_FILES - | { - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py" - ).read_text() - }, + files=INTERMEDIATE_LOADER_FILES | {**duckshop_orchestration("upstream_loaded_orders")}, expected_node_count=3, expected_write_set=frozenset( {"model:orders", "source:raw_shipments", "source:stage_shipments"} @@ -1188,7 +1196,9 @@ def test_given_an_attached_check_when_a_worker_executes_then_its_own_outcome_is_ [(59,)], files={ "python/checks/orders.py": "failed", - "orchestration/orders.py": "on_failure", + "orchestration/jobs/annual_orders_job.toml": "on_failure_job", + "orchestration/jobs/refresh_inventory_job.toml": "on_failure_destination", + "orchestration/sensors/failed_orders_sensor.py": "on_failure_sensor", "models/inventory.sql": "inventory", }, ), @@ -1206,11 +1216,11 @@ def test_given_an_attached_check_failure_when_a_sensor_observes_it_then_its_dest ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: source: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert source["status"] == "failed", worker.log.read_text() destination: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="refresh_inventory" + worker.api, worker.process, worker.log, job_key="refresh_inventory_job" ) assert destination["status"] == test_case.expected_status, worker.log.read_text() assert destination["selection_json"] == ["model:inventory"] @@ -1231,7 +1241,9 @@ def test_given_an_attached_check_failure_when_a_sensor_observes_it_then_its_dest [(17,)], files={ "python/checks/orders.py": "cancelled", - "orchestration/orders.py": "on_failure", + "orchestration/jobs/annual_orders_job.toml": "on_failure_job", + "orchestration/jobs/refresh_inventory_job.toml": "on_failure_destination", + "orchestration/sensors/failed_orders_sensor.py": "on_failure_sensor", "models/inventory.sql": "inventory", }, ), @@ -1248,9 +1260,9 @@ def test_given_an_attached_check_query_when_cancelled_then_it_cannot_trigger_fai test_case, files={path: fixture[key] for path, key in test_case.files.items()} ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: - cancel_running_statement(worker, job_key="annual_orders") + cancel_running_statement(worker, job_key="annual_orders_job") run: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert run["status"] == test_case.expected_status, worker.log.read_text() with sqlite3.connect(worker.control_store) as control: @@ -1291,16 +1303,27 @@ def test_given_an_attached_check_query_when_cancelled_then_it_cannot_trigger_fai [(59,)], expected_origin="sensor", files={ - "orchestration/orders.py": ( - "from sqlbuild.orchestration import RunEventKind, job, job_group, run_event, schedule, sensor\n" - "source = job('source_orders', select='orders', description='Source orders.')\n" - "source_group = job_group('source_group', description='Source group.', jobs=[source], " - "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" - "annual_orders = job('annual_orders', select='orders', description='Refresh orders.', settings={'reload': True})\n" - "on_group = sensor('on_group', description='Refresh after group completion.', " - "when=run_event(job=source_group, kinds=[RunEventKind.COMPLETED, RunEventKind.SUCCESS], executed_successfully=('orders',)), " - "run=annual_orders, minimum_interval_seconds=1)\n" - ) + "orchestration/jobs/source_orders_job.toml": ( + 'name = "source_orders_job"\ndescription = "Source orders."\nselect = "orders"\n' + ), + "orchestration/job_groups/source_job_group.toml": ( + 'name = "source_job_group"\n' + 'description = "Source group."\n' + 'jobs = ["source_orders_job"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + ), + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Refresh orders."\n' + 'select = "orders"\n' + "\n[settings]\nreload = true\n" + ), + "orchestration/sensors/on_group_sensor.py": ( + "from sqlbuild.orchestration import RunEventKind, run_event, sensor\n" + "on_group = sensor('on_group_sensor', description='Refresh after group completion.', " + "when=run_event(job='source_job_group', kinds=[RunEventKind.COMPLETED, RunEventKind.SUCCESS], executed_successfully=('orders',)), " + "run='annual_orders_job', minimum_interval_seconds=1)\n" + ), }, ) ], @@ -1311,7 +1334,7 @@ def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destin ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: destination: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert destination["status"] == test_case.expected_status, worker.log.read_text() result: dict[str, Any] = worker_result(worker, test_case) @@ -1320,7 +1343,7 @@ def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destin assert control.execute("SELECT run_key,run_id FROM sqb_run_requests").fetchall() == [ ( control.execute( - "SELECT run_id FROM sqb_runs WHERE job_key='source_group'" + "SELECT run_id FROM sqb_runs WHERE job_key='source_job_group'" ).fetchone()[0], destination["run_id"], ) @@ -1328,9 +1351,9 @@ def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destin assert control.execute( "SELECT job_key,status FROM sqb_runs ORDER BY job_key" ).fetchall() == [ - ("annual_orders", "success"), - ("source_group", "success"), - ("source_orders", "success"), + ("annual_orders_job", "success"), + ("source_job_group", "success"), + ("source_orders_job", "success"), ] @@ -1346,15 +1369,26 @@ def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destin expected_origin="sensor", files={ "models/stg_orders.sql": "MODEL (description 'Staging orders.');\nSELECT 71 AS amount\n", - "orchestration/orders.py": ( - "from sqlbuild.orchestration import NodeEventKind, job, job_group, node_event, schedule, sensor\n" - "source = job('source_orders', select='orders', description='Source orders.')\n" - "source_group = job_group('source_group', description='Source group.', jobs=[source], " - "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" - "annual_orders = job('annual_orders', select='stg_orders', description='Refresh staging.', settings={'reload': True})\n" - "on_node = sensor('on_node', description='React to actual orders rebuilds.', " + "orchestration/jobs/source_orders_job.toml": ( + 'name = "source_orders_job"\ndescription = "Source orders."\nselect = "orders"\n' + ), + "orchestration/job_groups/source_job_group.toml": ( + 'name = "source_job_group"\n' + 'description = "Source group."\n' + 'jobs = ["source_orders_job"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + ), + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Refresh staging."\n' + 'select = "stg_orders"\n' + "\n[settings]\nreload = true\n" + ), + "orchestration/sensors/on_node_sensor.py": ( + "from sqlbuild.orchestration import NodeEventKind, node_event, sensor\n" + "on_node = sensor('on_node_sensor', description='React to actual orders rebuilds.', " "when=node_event(select='orders', kinds=[NodeEventKind.SUCCESS]), " - "run=annual_orders, minimum_interval_seconds=1)\n" + "run='annual_orders_job', minimum_interval_seconds=1)\n" ), }, ) @@ -1366,23 +1400,23 @@ def test_given_a_real_group_child_rebuild_when_a_node_sensor_matches_then_only_t ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: destination: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert destination["status"] == test_case.expected_status, worker.log.read_text() with sqlite3.connect(worker.control_store) as control: assert control.execute( "SELECT job_key,status FROM sqb_runs ORDER BY job_key" ).fetchall() == [ - ("annual_orders", "success"), - ("source_group", "success"), - ("source_orders", "success"), + ("annual_orders_job", "success"), + ("source_job_group", "success"), + ("source_orders_job", "success"), ] assert control.execute( "SELECT q.run_key,q.run_id FROM sqb_run_requests q" ).fetchall() == [ ( control.execute( - "SELECT run_id FROM sqb_runs WHERE job_key='source_orders'" + "SELECT run_id FROM sqb_runs WHERE job_key='source_orders_job'" ).fetchone()[0] + "/model:orders", destination["run_id"], @@ -1429,13 +1463,19 @@ def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_s "MODEL (description 'Downstream totals.', materialized table);\n" 'SELECT amount FROM __ref("orders")\n' ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, job_group, schedule\n" - "orders = job('long_orders', description='Long query.', select='orders')\n" - "totals = job('waiting_totals', description='Waiting query.', select='totals')\n" - "annual_orders = job_group('annual_orders', description='Cancellable group.', " - "jobs=[totals, orders], schedule=schedule(cron='0 0 * * *', timezone='UTC', " - "starts_at='2000-01-01T00:00:00Z'))\n" + "orchestration/jobs/long_orders_job.toml": ( + 'name = "long_orders_job"\ndescription = "Long query."\nselect = "orders"\n' + ), + "orchestration/jobs/waiting_totals_job.toml": ( + 'name = "waiting_totals_job"\n' + 'description = "Waiting query."\n' + 'select = "totals"\n' + ), + "orchestration/job_groups/annual_orders_job_group.toml": ( + 'name = "annual_orders_job_group"\n' + 'description = "Cancellable group."\n' + 'jobs = ["waiting_totals_job", "long_orders_job"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, ), @@ -1451,21 +1491,21 @@ def test_given_an_executing_group_when_parent_cancelled_then_all_children_stop_b worker.process, worker.log, pending=frozenset({"queued"}), - job_key="annual_orders", + job_key="annual_orders_job_group", ) assert parent["status"] == "running", worker.log.read_text() - cancel_running_statement(worker, job_key="long_orders", run_id=parent["run_id"]) + cancel_running_statement(worker, job_key="long_orders_job", run_id=parent["run_id"]) completed: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job_group" ) assert completed["status"] == test_case.expected_status, worker.log.read_text() with sqlite3.connect(worker.control_store) as control: assert control.execute( "SELECT job_key, status FROM sqb_runs ORDER BY job_key" ).fetchall() == [ - ("annual_orders", "cancelled"), - ("long_orders", "cancelled"), - ("waiting_totals", "cancelled"), + ("annual_orders_job_group", "cancelled"), + ("long_orders_job", "cancelled"), + ("waiting_totals_job", "cancelled"), ] assert control.execute( "SELECT count(*) FROM sqb_runs WHERE parent_run_id=? AND ended_at>?", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 06e61dc87..81ec3d42b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -4,7 +4,6 @@ import json import re -from dataclasses import replace from pathlib import Path from typing import Any @@ -22,17 +21,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration import ( - RunEventKind, - compile_snapshot, - job, - job_group, - on_failure, - run_event, - schedule, - sensor, -) -from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup +from sqlbuild.orchestration import compile_snapshot from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( PythonCheckJobRejectionTestCase, PythonCheckJobTestCase, @@ -55,8 +44,8 @@ FUNCTION_PROJECT_FILES, METADATA, PARITY_FILES, - PYTHON_PIPELINE_FILES, TYPE_PROOF_FILES, + duckshop_orchestration, published_texts, python_pipeline_snapshot, ) @@ -64,12 +53,29 @@ ORCHESTRATION_NAME_CONFLICT: str = ( "; model, source, seed, function, loader, task, asset, check, provider, hook, unit test, " - "scenario, job, job group, sensor, and calendar schedule names must be globally unique; " - "rename one of them (a job, job group or sensor takes its name from its first argument, and " - "a calendar schedule from the function it decorates)" + "scenario, job, job group, sensor, and calendar names must be globally unique; rename one of " + "them (a job or job group takes its name from `name` in its TOML file, and a calendar or " + "sensor from its `name` argument)" ) _DUCKSHOP: Path = Path("benchmarks/duckshop/project") + + +_DAILY_BUILD: dict[str, str] = { + "name": '"daily_build_job"', + "description": '"Orders."', + "select": '"orders"', + "schedule": '{ cron = "0 2 * * *", timezone = "UTC" }', +} + + +def _edited(fixture: str, old: str, new: str) -> dict[str, str]: + """A DuckShop orchestration fixture with one edit to the file that holds `old`.""" + files: dict[str, str] = duckshop_orchestration(fixture) + (path,) = [path for path, text in files.items() if old in text] + return files | {path: files[path].replace(old, new)} + + _PROVIDER_PROJECT: dict[str, str] = { "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', "models/orders.sql": "MODEL (description 'Orders.');\n\nSELECT 1 AS order_id\n", @@ -179,9 +185,7 @@ "DuckShop polling callback with nested code is relocatable", TYPE_PROOF_FILES | { - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/custom_orders.py" - ).read_text(), + **duckshop_orchestration("custom_orders"), }, frozenset({"model"}), ), @@ -836,22 +840,23 @@ def test_given_a_dependency_lock_when_it_changes_then_environment_identity_chang [ SnapshotScheduleTestCase( description="daylight saving zone", - schedule=schedule(cron="15 2 * * *", timezone="Europe/London"), + schedule='{ cron = "15 2 * * *", timezone = "Europe/London" }', + expected_timezone="Europe/London", expected_selection=("model:orders",), ), SnapshotScheduleTestCase( description="fixed offset zone", - schedule=schedule(cron="15 2 * * *", timezone="UTC"), + schedule='{ cron = "15 2 * * *", timezone = "UTC" }', + expected_timezone="UTC", expected_selection=("model:orders",), ), SnapshotScheduleTestCase( description="paused with a zoned start", - schedule=schedule( - cron="15 2 * * *", - timezone="Europe/London", - starts_at="2026-07-03T01:17:23.456789+05:30", - default_status="paused", + schedule=( + '{ cron = "15 2 * * *", timezone = "Europe/London", ' + 'starts_at = "2026-07-03T01:17:23.456789+05:30", default_status = "paused" }' ), + expected_timezone="Europe/London", expected_selection=("model:orders",), expected_controls={ "starts_at": "2026-07-02T19:47:23.456789Z", @@ -871,32 +876,46 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) - nightly: Job = job( - "nightly_orders", - description="Refresh order reporting.", - select="+orders", - exclude="stg_orders", - settings={"run_tests": False, "vars": {"batch": 3}, "warehouse": "reporting"}, - concurrency=4, - concurrency_groups=("order_reporting",), - schedule=test_case.schedule, - targets=("development",), - owner="Orders team", - overlap="skip", - on_failure=on_failure(retries=2, retry_delay_seconds=600), - ) - excluded: Job = replace(nightly, name="other_target", targets=("production",)) - first: dict[str, Any] = json.loads( - compile_snapshot( - project=project, - metadata_json=json.dumps(METADATA), + + def snapshot(schedule: str) -> dict[str, Any]: + write_files( project_dir=tmp_path, - declarations=[nightly, excluded], - discovered_inputs=discovered, - ).to_json() - ) + files={ + "orchestration/jobs/nightly_orders_job.toml": ( + 'name = "nightly_orders_job"\n' + 'description = "Refresh order reporting."\n' + 'select = "+orders"\n' + 'exclude = "stg_orders"\n' + 'targets = ["development"]\n' + 'owner = "Orders team"\n' + 'overlap = "skip"\n' + 'concurrency_groups = ["order_reporting"]\n' + f"schedule = {schedule}\n" + "\n[on_failure]\nretries = 2\nretry_delay_seconds = 600\n" + '\n[settings]\nrun_tests = false\nvars = { batch = 3 }\nwarehouse = "reporting"\n' + "concurrency = 4\n" + ), + "orchestration/jobs/other_target_job.toml": ( + 'name = "other_target_job"\n' + 'description = "Refresh order reporting."\n' + 'select = "+orders"\n' + 'targets = ["production"]\n' + f"schedule = {schedule}\n" + ), + }, + ) + return json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + ).to_json() + ) + + first: dict[str, Any] = snapshot(test_case.schedule) assert first["documents"]["jobs"] == { - "nightly_orders": { + "nightly_orders_job": { "description": "Refresh order reporting.", "selection": list(test_case.expected_selection), "settings": { @@ -912,29 +931,21 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select "owner": "Orders team", "overlap": "skip", "on_failure": {"retries": 2, "retry_delay_seconds": 600}, + "declared_at": {"path": "orchestration/jobs/nightly_orders_job.toml", "line": 1}, } } - trigger: dict[str, Any] = first["documents"]["triggers"]["nightly_orders"] - assert trigger["job_key"] == "nightly_orders" + trigger: dict[str, Any] = first["documents"]["triggers"]["nightly_orders_job"] + assert trigger["job_key"] == "nightly_orders_job" assert trigger["targets"] == ["development"] assert trigger["schedule"] == { "kind": "cron", "expression": "15 2 * * *", - "timezone": test_case.schedule.timezone, + "timezone": test_case.expected_timezone, } assert {key: trigger[key] for key in test_case.expected_controls} == test_case.expected_controls - edited: Job = replace(nightly, schedule=schedule(cron="30 3 * * *", timezone="UTC")) - second: dict[str, Any] = json.loads( - compile_snapshot( - project=project, - metadata_json=json.dumps(METADATA), - project_dir=tmp_path, - declarations=[edited], - discovered_inputs=discovered, - ).to_json() - ) + second: dict[str, Any] = snapshot('{ cron = "30 3 * * *", timezone = "UTC" }') assert second["documents"]["triggers"].keys() == first["documents"]["triggers"].keys() - assert second["documents"]["triggers"]["nightly_orders"]["schedule"]["expression"] == ( + assert second["documents"]["triggers"]["nightly_orders_job"]["schedule"]["expression"] == ( "30 3 * * *" ) assert second["nodes"] == first["nodes"] @@ -945,102 +956,115 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select [ SnapshotDeclarationRejectionTestCase( description="blank description", - job_description=" ", - expected_message="needs a description", + fields={"description": '" "'}, + expected_message="2:15: description must not be blank", ), SnapshotDeclarationRejectionTestCase( - description="blank selection", select="", expected_message="needs a selection" + description="blank selection", + fields={"select": '""'}, + expected_message="3:10: select must not be blank", ), SnapshotDeclarationRejectionTestCase( - description="missing model", select="missing", expected_message="missing" + description="missing model", + fields={"select": '"missing"'}, + expected_message="missing", ), SnapshotDeclarationRejectionTestCase( - description="invalid cron", cron="not a cron", expected_message="cron" + description="invalid cron", + fields={"schedule": '{ cron = "not a cron", timezone = "UTC" }'}, + expected_message="cron", ), SnapshotDeclarationRejectionTestCase( - description="invalid timezone", timezone="Unknown/Zone", expected_message="time zone" + description="invalid timezone", + fields={"schedule": '{ cron = "0 2 * * *", timezone = "Unknown/Zone" }'}, + expected_message="time zone", ), SnapshotDeclarationRejectionTestCase( description="start needs an explicit offset", - schedule=schedule(cron="0 2 * * *", timezone="UTC", starts_at="2026-07-03T01:17:23"), + fields={ + "schedule": '{ cron = "0 2 * * *", timezone = "UTC", starts_at = "2026-07-03T01:17:23" }' + }, expected_message="starts_at", ), SnapshotDeclarationRejectionTestCase( - description="duplicate names", copies=2, expected_message="duplicate job daily_build" + description="duplicate names", + copies=2, + expected_message="name 'daily_build_job' is declared as both job", ), SnapshotDeclarationRejectionTestCase( description="unknown setting", - settings={"concurreny": 4}, + tables="[settings]\nconcurreny = 4\n", expected_message="unknown build setting concurreny", ), SnapshotDeclarationRejectionTestCase( description="selection cannot escape its pinned run", - settings={"select": "customers"}, + tables='[settings]\nselect = "customers"\n', expected_message="owned by the run request", ), SnapshotDeclarationRejectionTestCase( description="zero concurrency", - concurrency=0, + tables="[settings]\nconcurrency = 0\n", expected_message="must be positive", ), SnapshotDeclarationRejectionTestCase( description="boolean is not concurrency", - concurrency=True, + tables="[settings]\nconcurrency = true\n", expected_message="concurrency must be an integer", ), SnapshotDeclarationRejectionTestCase( description="integer is not a boolean flag", - settings={"full_refresh": 1}, + tables="[settings]\nfull_refresh = 1\n", expected_message="must be boolean", ), SnapshotDeclarationRejectionTestCase( description="integer strings are not typed settings", - settings={"concurrency": "3"}, + tables='[settings]\nconcurrency = "3"\n', expected_message="must be an integer", ), SnapshotDeclarationRejectionTestCase( description="variables need an object", - settings={"vars": ["orders"]}, + tables='[settings]\nvars = ["orders"]\n', expected_message="must be a JSON object", ), SnapshotDeclarationRejectionTestCase( description="mutually exclusive load settings", - settings={"load_sources": False, "reload": True}, + tables="[settings]\nload_sources = false\nreload = true\n", expected_message="mutually exclusive", ), SnapshotDeclarationRejectionTestCase( description="conflicting cursor families", - settings={"start_cursor": "2", "end_cursor_int": "4"}, + tables='[settings]\nstart_cursor = "2"\nend_cursor_int = "4"\n', expected_message="cannot be combined", ), SnapshotDeclarationRejectionTestCase( - description="blank owner", owner=" ", expected_message="owner must not be blank" + description="blank owner", + fields={"owner": '" "'}, + expected_message="owner must not be blank", ), SnapshotDeclarationRejectionTestCase( description="unknown overlap", - overlap="coalesce", - expected_message="overlap must be queue or skip, not coalesce", + fields={"overlap": '"coalesce"'}, + expected_message="unknown variant `coalesce`, expected `queue` or `skip`", ), SnapshotDeclarationRejectionTestCase( description="no retries", - on_failure=on_failure(retries=0), + tables="[on_failure]\nretries = 0\n", expected_message="retries must be a whole number from 1 to 10", ), SnapshotDeclarationRejectionTestCase( description="too many retries", - on_failure=on_failure(retries=11), + tables="[on_failure]\nretries = 11\n", expected_message="retries must be a whole number from 1 to 10", ), SnapshotDeclarationRejectionTestCase( description="retry delay over a day", - on_failure=on_failure(retries=1, retry_delay_seconds=86_401), + tables="[on_failure]\nretries = 1\nretry_delay_seconds = 86401\n", expected_message="retry_delay_seconds must be a whole number from 0 to 86400", ), SnapshotDeclarationRejectionTestCase( - description="duplicate concurrency declarations", - concurrency=3, - settings={"concurrency": 4}, - expected_message="declared twice", + description="concurrency belongs under settings", + fields={"concurrency": "3"}, + expected_message="unknown job key 'concurrency'", ), ], ids=lambda case: case.description, @@ -1048,33 +1072,24 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select def test_given_invalid_declarations_when_compiling_then_native_validation_rejects_them( tmp_path: Path, test_case: SnapshotDeclarationRejectionTestCase ) -> None: - write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discovered, - adapter=DuckDbAdapter(), - resolved_connection={"database": "warehouse", "schema": "main"}, + text: str = "".join( + f"{key} = {value}\n" for key, value in (_DAILY_BUILD | test_case.fields).items() ) - declarations: list[Job] = [ - job( - "daily_build", - description=test_case.job_description, - select=test_case.select, - settings=test_case.settings, - concurrency=test_case.concurrency, - schedule=test_case.schedule - or schedule(cron=test_case.cron, timezone=test_case.timezone), - owner=test_case.owner, - overlap=test_case.overlap, - on_failure=test_case.on_failure, - ) - ] * test_case.copies + copies: dict[str, str] = { + f"orchestration/jobs/{'copy/' * copy}daily_build_job.toml": text + "\n" + test_case.tables + for copy in range(test_case.copies) + } + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | copies) with pytest.raises(ValueError, match=test_case.expected_message): + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) compile_snapshot( - project=project, + project=build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ), metadata_json=json.dumps(METADATA), project_dir=tmp_path, - declarations=declarations, discovered_inputs=discovered, ) @@ -1083,53 +1098,38 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "test_case", [ SnapshotDiscoveryTestCase( - description="relative imports aliases and factory containers", + description="group-only references retain lineage waits", files={ - "orchestration/shared.py": ( - "from sqlbuild.orchestration import job\n" - "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + "orchestration/jobs/staging_job.toml": ( + 'name = "staging_job"\ndescription = "Stage orders."\nselect = "stg_orders"\n' ), - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job\n" - "from .shared import staging\n" - "alias = staging\n" - "def factory():\n" - " return [job('daily', description='Daily orders.', select='orders')]\n" - "jobs = [factory(), staging]\n" - "jobs.append(jobs)\n" + "orchestration/jobs/inventory_job.toml": ( + 'name = "inventory_job"\n' + 'description = "Repeat staging."\n' + 'select = "stg_orders"\n' ), - }, - expected_jobs=("daily", "staging"), - ), - SnapshotDiscoveryTestCase( - description="group-only references retain lineage waits", - files={ - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job, job_group\n" - "def factory():\n" - " staging = job('staging', description='Stage orders.', select='stg_orders')\n" - " marts = job('marts', description='Build orders.', select='orders')\n" - " inventory = job('inventory', description='Repeat staging.', select='stg_orders')\n" - " return job_group('daily', description='Ordered jobs.', " - "jobs=[marts, inventory, staging])\n" - "daily = factory()\n" + "orchestration/jobs/marts_job.toml": ( + 'name = "marts_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + "orchestration/job_groups/daily_job_group.toml": ( + 'name = "daily_job_group"\n' + 'description = "Ordered jobs."\n' + 'jobs = ["marts_job", "inventory_job", "staging_job"]\n' ), }, - expected_jobs=("inventory", "marts", "staging"), + expected_jobs=("inventory_job", "marts_job", "staging_job"), expected_groups={ - "daily": { - "inventory": [], - "marts": ["inventory", "staging"], - "staging": [], + "daily_job_group": { + "inventory_job": [], + "marts_job": ["inventory_job", "staging_job"], + "staging_job": [], } }, ), SnapshotDiscoveryTestCase( description="DuckShop layer jobs that share a function wait only by lineage", files={ - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py" - ).read_text(), + **duckshop_orchestration("shared_function_layers"), "functions/sql/order_cents.sql": ( "FUNCTION (description 'Amount in cents.'," " arguments (amount INTEGER), returns INTEGER);\nSELECT amount * 100\n" @@ -1148,142 +1148,154 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject 'SELECT order_id, __udf("order_cents")(cents) AS cents\nFROM __ref("stg_orders")\n' ), }, - expected_jobs=("marts_layer", "staging_layer"), + expected_jobs=("marts_layer_job", "staging_layer_job"), expected_groups={ - "layered_orders": {"marts_layer": ["staging_layer"], "staging_layer": []} + "layered_orders_job_group": { + "marts_layer_job": ["staging_layer_job"], + "staging_layer_job": [], + } }, ), - SnapshotDiscoveryTestCase( - description="explicit empty declarations do not execute project files", - files={"orchestration/jobs.py": "raise RuntimeError('must not import')\n"}, - expected_jobs=(), - declarations=(), - ), SnapshotDiscoveryTestCase( description="DuckShop edge-case declaration containers", files={ - "orchestration/edge_cases/definitions.py": Path( - "benchmarks/duckshop/project/orchestration/edge_cases/definitions.py" - ).read_text(), + **{ + path.relative_to(_DUCKSHOP).as_posix(): path.read_text() + for path in (_DUCKSHOP / "orchestration").rglob("*") + if path.is_file() + }, "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( "MODEL (", "MODEL (tags ['edge_case'],", 1 ), }, expected_jobs=( - "edge_snapshot_inventory", - "edge_snapshot_orders", - "edge_snapshot_products", + "edge_snapshot_inventory_job", + "edge_snapshot_orders_job", + "edge_snapshot_products_job", ), expected_groups={ - "edge_snapshot_group": {"edge_snapshot_orders": []}, - "edge_inventory_group": {"edge_snapshot_inventory": []}, - "edge_products_group": {"edge_snapshot_products": []}, + "edge_snapshot_job_group": {"edge_snapshot_orders_job": []}, + "edge_inventory_job_group": {"edge_snapshot_inventory_job": []}, + "edge_products_job_group": {"edge_snapshot_products_job": []}, }, expected_sensors={ - "edge_platform_orders": { + "edge_platform_orders_sensor": { "description": "Catch up retained platform observations with a stopped group sensor.", - "job_key": "edge_snapshot_group", + "job_key": "edge_snapshot_job_group", "when": { "kind": "platform_event", "kinds": ["tick_overdue", "location_broken", "launch_failed"], }, "minimum_interval_seconds": 7, "default_status": "paused", - "declared_at": {"path": "orchestration/edge_cases/definitions.py", "line": 61}, + "declared_at": { + "path": "orchestration/sensors/edge_cases/edge_platform_orders_sensor.py", + "line": 5, + }, }, - "edge_job_outcomes": { + "edge_job_outcomes_sensor": { "description": "Keep warning and partial outcomes distinct from success.", - "job_key": "edge_snapshot_inventory", + "job_key": "edge_snapshot_inventory_job", "when": { "kind": "run_event", - "job_key": "edge_snapshot_orders", + "job_key": "edge_snapshot_orders_job", "kinds": ["warn", "partial"], }, "minimum_interval_seconds": 30, "default_status": "paused", - "declared_at": {"path": "orchestration/edge_cases/definitions.py", "line": 77}, + "declared_at": { + "path": "orchestration/sensors/edge_cases/edge_job_outcomes_sensor.py", + "line": 5, + }, }, - "edge_group_outcomes": { + "edge_group_outcomes_sensor": { "description": "Observe group completion including cancellation and presumed failure.", - "job_key": "edge_snapshot_inventory", + "job_key": "edge_snapshot_inventory_job", "when": { "kind": "run_event", - "job_key": "edge_snapshot_group", + "job_key": "edge_snapshot_job_group", "kinds": ["completed"], "executed_successfully": ["model:orders"], }, "minimum_interval_seconds": 30, "default_status": "paused", - "declared_at": {"path": "orchestration/edge_cases/definitions.py", "line": 84}, + "declared_at": { + "path": "orchestration/sensors/edge_cases/edge_group_outcomes_sensor.py", + "line": 5, + }, }, }, - expected_schedules=("edge_snapshot_orders",), + expected_schedules=("edge_snapshot_orders_job",), expected_trigger_declared_at={ - "edge_snapshot_orders": { - "path": "orchestration/edge_cases/definitions.py", - "line": 14, + "edge_snapshot_orders_job": { + "path": "orchestration/jobs/edge_cases/edge_snapshot_orders_job.toml", + "line": 2, }, }, expected_declared_at={ - "edge_snapshot_orders": { - "path": "orchestration/edge_cases/definitions.py", - "line": 14, + "edge_snapshot_orders_job": { + "path": "orchestration/jobs/edge_cases/edge_snapshot_orders_job.toml", + "line": 2, }, - "edge_snapshot_group": { - "path": "orchestration/edge_cases/definitions.py", - "line": 30, + "edge_snapshot_job_group": { + "path": "orchestration/job_groups/edge_cases/edge_snapshot_job_group.toml", + "line": 1, }, }, ), SnapshotDiscoveryTestCase( - description="sensor-only roots collect source groups and destination jobs natively", - files={}, - declarations=( - sensor( - "on_orders", - description="Build inventory after group completion.", - when=run_event( - job=job_group( - "source_group", - description="Source orders.", - jobs=[job("source_orders", description="Orders.", select="orders")], - ), - kinds=[RunEventKind.COMPLETED], - executed_successfully=("orders", "orders*"), - ), - run=job("inventory", description="Inventory.", select="stg_orders"), + description="a run_event on a group freezes its members' execution selectors", + files={ + "orchestration/jobs/source_orders_job.toml": ( + 'name = "source_orders_job"\ndescription = "Orders."\nselect = "orders"\n' ), - ), - expected_jobs=("inventory", "source_orders"), - expected_declared_at={"inventory": None, "source_group": None}, - expected_groups={"source_group": {"source_orders": []}}, + "orchestration/jobs/inventory_job.toml": ( + 'name = "inventory_job"\ndescription = "Inventory."\nselect = "stg_orders"\n' + ), + "orchestration/job_groups/source_job_group.toml": ( + 'name = "source_job_group"\ndescription = "Source orders."\n' + 'jobs = ["source_orders_job"]\n' + ), + "orchestration/sensors/on_orders_sensor.py": ( + "from sqlbuild.orchestration import RunEventKind, run_event, sensor\n" + "on_orders = sensor(\n" + " 'on_orders_sensor',\n" + " description='Build inventory after group completion.',\n" + " when=run_event(\n" + " job='source_job_group',\n" + " kinds=[RunEventKind.COMPLETED],\n" + " executed_successfully=('orders', 'orders*'),\n" + " ),\n" + " run='inventory_job',\n" + ")\n" + ), + }, + expected_jobs=("inventory_job", "source_orders_job"), + expected_groups={"source_job_group": {"source_orders_job": []}}, expected_sensors={ - "on_orders": { + "on_orders_sensor": { "description": "Build inventory after group completion.", - "job_key": "inventory", + "job_key": "inventory_job", "when": { "kind": "run_event", - "job_key": "source_group", + "job_key": "source_job_group", "kinds": ["completed"], "executed_successfully": ["model:orders"], }, "minimum_interval_seconds": 30, "default_status": "running", + "declared_at": {"path": "orchestration/sensors/on_orders_sensor.py", "line": 2}, } }, ), SnapshotDiscoveryTestCase( description="any-run node selectors freeze canonical keys without a source job reference", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" - ).read_text() - }, - expected_jobs=("inventory", "source_orders"), + files={**duckshop_orchestration("node_orders")}, + expected_jobs=("inventory_job", "source_orders_job"), expected_sensors={ - "on_orders": { + "on_orders_sensor": { "description": "Build inventory for every genuine orders rebuild.", - "job_key": "inventory", + "job_key": "inventory_job", "when": { "kind": "node_event", "selection": ["model:orders"], @@ -1291,24 +1303,24 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "running", - "declared_at": {"path": "orchestration/orders.py", "line": 7}, + "declared_at": {"path": "orchestration/sensors/on_orders_sensor.py", "line": 5}, } }, ), SnapshotDiscoveryTestCase( description="failure and recovery selectors preserve their individual transition kinds", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + **_edited( + "node_orders", + "NodeEventKind.SUCCESS", + "NodeEventKind.FAILED, NodeEventKind.RECOVERED", ) - .read_text() - .replace("NodeEventKind.SUCCESS", "NodeEventKind.FAILED, NodeEventKind.RECOVERED") }, - expected_jobs=("inventory", "source_orders"), + expected_jobs=("inventory_job", "source_orders_job"), expected_sensors={ - "on_orders": { + "on_orders_sensor": { "description": "Build inventory for every genuine orders rebuild.", - "job_key": "inventory", + "job_key": "inventory_job", "when": { "kind": "node_event", "selection": ["model:orders"], @@ -1316,47 +1328,46 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "running", - "declared_at": {"path": "orchestration/orders.py", "line": 7}, + "declared_at": {"path": "orchestration/sensors/on_orders_sensor.py", "line": 5}, } }, ), SnapshotDiscoveryTestCase( description="source_new_data selectors freeze only source nodes without event kinds", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" - ).read_text(), + **duckshop_orchestration("source_orders"), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, - expected_jobs=("orders_refresh",), + expected_jobs=("orders_refresh_job",), expected_sensors={ - "on_source_orders": { + "on_source_orders_sensor": { "description": "Refresh orders for the first source version and each changed version.", - "job_key": "orders_refresh", + "job_key": "orders_refresh_job", "when": {"kind": "source_new_data", "selection": ["source:raw_orders"]}, "minimum_interval_seconds": 1, "default_status": "running", - "declared_at": {"path": "orchestration/orders.py", "line": 10}, + "declared_at": { + "path": "orchestration/sensors/on_source_orders_sensor.py", + "line": 5, + }, } }, ), SnapshotDiscoveryTestCase( description="source_stale freezes source nodes and a positive timestamp-age threshold", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" - ).read_text(), + **duckshop_orchestration("stale_orders"), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, - expected_jobs=("orders_refresh",), + expected_jobs=("orders_refresh_job",), expected_sensors={ - "on_stale_orders": { + "on_stale_orders_sensor": { "description": "Refresh orders once for each timestamp version older than one minute.", - "job_key": "orders_refresh", + "job_key": "orders_refresh_job", "when": { "kind": "source_stale", "selection": ["source:raw_orders"], @@ -1364,51 +1375,49 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 1, "default_status": "running", - "declared_at": {"path": "orchestration/orders.py", "line": 7}, + "declared_at": { + "path": "orchestration/sensors/on_stale_orders_sensor.py", + "line": 5, + }, } }, ), SnapshotDiscoveryTestCase( description="source-job execution selectors compile to frozen canonical keys", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" - ).read_text() - }, - expected_jobs=("inventory", "source_orders"), + files={**duckshop_orchestration("run_orders")}, + expected_jobs=("inventory_job", "source_orders_job"), expected_sensors={ - "on_orders": { + "on_orders_sensor": { "description": "Build inventory only after orders actually succeeds.", - "job_key": "inventory", + "job_key": "inventory_job", "when": { "kind": "run_event", - "job_key": "source_orders", + "job_key": "source_orders_job", "kinds": ["completed"], "executed_successfully": ["model:orders"], }, "minimum_interval_seconds": 30, "default_status": "running", - "declared_at": {"path": "orchestration/orders.py", "line": 7}, + "declared_at": {"path": "orchestration/sensors/on_orders_sensor.py", "line": 5}, } }, ), SnapshotDiscoveryTestCase( description="sensor-only roots preserve referenced jobs and filter targets", - files={ - "orchestration/health.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" - ).read_text() - }, - expected_jobs=("health_orders",), + files={**duckshop_orchestration("platform_orders")}, + expected_jobs=("health_orders_job",), expected_sensors={ - "overdue_orders": { + "overdue_orders_sensor": { "description": "Catch up overdue order reporting.", - "job_key": "health_orders", + "job_key": "health_orders_job", "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, "minimum_interval_seconds": 7, "default_status": "paused", "targets": ["development"], - "declared_at": {"path": "orchestration/health.py", "line": 9}, + "declared_at": { + "path": "orchestration/sensors/overdue_orders_sensor.py", + "line": 8, + }, } }, ), @@ -1435,7 +1444,6 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects project=project, metadata_json=json.dumps(METADATA), project_dir=tmp_path, - declarations=test_case.declarations, discovered_inputs=discovered, ).to_json() ) @@ -1470,21 +1478,19 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects [ SnapshotDiscoveryRejectionTestCase( description="group members are jobs, not nested groups", - files={ - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/nested_groups.py" - ).read_text(), - }, - expected_error=TypeError, - expected_message="group members must be Job objects", + files=duckshop_orchestration("nested_groups"), + expected_error=ValueError, + expected_message="daily_orders_job_group", ), SnapshotDiscoveryRejectionTestCase( description="a polling sensor needs an actual callback", files={ - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, sensor\n" - "orders = job('refresh_orders', description='Build orders.', select='orders')\n" - "poll = sensor('poll', description='Poll orders.', run=orders)\n" + "orchestration/jobs/refresh_orders_job.toml": ( + 'name = "refresh_orders_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + "orchestration/sensors/poll_sensor.py": ( + "from sqlbuild.orchestration import sensor\n" + "poll = sensor('poll_sensor', description='Poll orders.', run='refresh_orders_job')\n" ), }, expected_error=ValueError, @@ -1493,10 +1499,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="callbacks with event conditions remain explicitly unsupported", files={ - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, sensor, platform_event, PlatformEventKind\n" - "orders = job('refresh_orders', description='Build orders.', select='orders')\n" - "@sensor('poll', description='Poll orders.', run=orders, " + "orchestration/jobs/refresh_orders_job.toml": ( + 'name = "refresh_orders_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + "orchestration/sensors/poll_sensor.py": ( + "from sqlbuild.orchestration import sensor, platform_event, PlatformEventKind\n" + "@sensor('poll_sensor', description='Poll orders.', run='refresh_orders_job', " "when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]))\n" "def poll(ctx):\n return []\n" ), @@ -1507,113 +1515,90 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="a job description past max_description_lines cannot publish", files={ - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job\n" - "daily = job('daily', description='Build orders. ' * 100, select='orders')\n" + "orchestration/jobs/daily_job.toml": ( + 'name = "daily_job"\n' + 'description = """Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders."""\n' + 'select = "orders"\n' ), }, expected_error=ValueError, expected_message=re.escape( - "job 'daily' description exceeds 10 lines; shorten it to 10 lines" + "job 'daily_job' description exceeds 10 lines; shorten it to 10 lines" ), ), SnapshotDiscoveryRejectionTestCase( - description="a group declaring after is refused at its module", + description="a group cannot declare an order between its jobs", files={ - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job, job_group\n" - "staging = job('staging', description='Stage orders.', select='stg_orders')\n" - "marts = job('marts', description='Build orders.', select='orders')\n" - "daily = job_group('daily', description='Ordered jobs.', " - "jobs=[marts, staging], after={staging: [marts]})\n" + "orchestration/jobs/staging_job.toml": ( + 'name = "staging_job"\ndescription = "Stage orders."\nselect = "stg_orders"\n' + ), + "orchestration/jobs/marts_job.toml": ( + 'name = "marts_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + "orchestration/job_groups/daily_job_group.toml": ( + 'name = "daily_job_group"\n' + 'description = "Ordered jobs."\n' + 'jobs = ["marts_job", "staging_job"]\n' + 'after = { staging_job = ["marts_job"] }\n' ), }, expected_error=ValueError, - expected_message=re.escape( - "Failed to import orchestration file orchestration/jobs.py: JobGroup.__init__() " - "got an unexpected keyword argument 'after'" - ), + expected_message="after", ), SnapshotDiscoveryRejectionTestCase( - description="distinct objects cannot share a job name", + description="two job files cannot share a name", files={ - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job\n" - "jobs = [job('daily', description='Orders.', select='orders') for _ in range(2)]\n" + "orchestration/jobs/daily_job.toml": ( + 'name = "daily_job"\ndescription = "Orders."\nselect = "orders"\n' + ), + "orchestration/jobs/copy/daily_job.toml": ( + 'name = "daily_job"\ndescription = "Orders."\nselect = "orders"\n' ), }, expected_error=DiscoveryConflictError, expected_message=re.escape( - "Project resource name 'daily' is declared as both job in orchestration/jobs.py " - "and job in orchestration/jobs.py" + ORCHESTRATION_NAME_CONFLICT + "Project resource name 'daily_job' is declared as both job in " + "orchestration/jobs/copy/daily_job.toml and job in orchestration/jobs/daily_job.toml" + + ORCHESTRATION_NAME_CONFLICT ), ), SnapshotDiscoveryRejectionTestCase( description="DuckShop jobs cannot reuse a model name", - files={ - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/name_collision.py" - ).read_text() - }, + files=duckshop_orchestration("name_collision") + | {"models/orders_job.sql": "MODEL (description 'Orders.');\nSELECT 1 AS order_id\n"}, expected_error=DiscoveryConflictError, expected_message=re.escape( - "Project resource name 'orders' is declared as both model in models/orders.sql " - "and job in orchestration/jobs.py" + ORCHESTRATION_NAME_CONFLICT - ), - ), - SnapshotDiscoveryRejectionTestCase( - description="a sensor cannot reuse its job's name", - files={ - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job, sensor\n" - "daily = job('daily', description='Orders.', select='orders')\n" - "poll = sensor('daily', description='Poll orders.', run=daily)\n" - ), - }, - expected_error=DiscoveryConflictError, - expected_message=re.escape( - "Project resource name 'daily' is declared as both sensor in orchestration/jobs.py " - "and job in orchestration/jobs.py" + ORCHESTRATION_NAME_CONFLICT + "Project resource name 'orders_job' is declared as both model in " + "models/orders_job.sql and job in orchestration/jobs/orders_job.toml" + + ORCHESTRATION_NAME_CONFLICT ), ), SnapshotDiscoveryRejectionTestCase( description="broken imports fail on every retry", - files={ - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/failed_import.py" - ).read_text() - }, + files=duckshop_orchestration("failed_import"), expected_error=PythonNodeDiscoveryError, - expected_message="Failed to import orchestration file orchestration/jobs.py", + expected_message=( + "Failed to import orchestration file orchestration/sensors/broken_orders_sensor.py" + ), ), SnapshotDiscoveryRejectionTestCase( description="DuckShop settings cannot replace the selection", - files={ - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py" - ).read_text() - }, + files={**duckshop_orchestration("invalid_settings")}, expected_error=ValueError, expected_message="owned by the run request", ), SnapshotDiscoveryRejectionTestCase( description="DuckShop mutually exclusive settings fail on every retry", - files={ - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py" - ).read_text() - }, + files={**duckshop_orchestration("conflicting_settings")}, expected_error=ValueError, expected_message="mutually exclusive", ), SnapshotDiscoveryRejectionTestCase( description="a zero sensor interval cannot publish", files={ - "orchestration/health.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + **_edited( + "platform_orders", "minimum_interval_seconds=7", "minimum_interval_seconds=0" ) - .read_text() - .replace("minimum_interval_seconds=7", "minimum_interval_seconds=0") }, expected_error=ValueError, expected_message="nonzero u32", @@ -1621,11 +1606,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="empty platform conditions cannot publish", files={ - "orchestration/health.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" - ) - .read_text() - .replace("kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=[]") + **_edited("platform_orders", "kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=[]") }, expected_error=ValueError, expected_message="nonempty event kinds", @@ -1633,35 +1614,25 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="platform kinds are a closed vocabulary", files={ - "orchestration/health.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + **_edited( + "platform_orders", + "kinds=[PlatformEventKind.TICK_OVERDUE]", + "kinds=['unknown_event']", ) - .read_text() - .replace("kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=['unknown_event']") }, expected_error=ValueError, expected_message="unknown variant", ), SnapshotDiscoveryRejectionTestCase( description="empty node selectors cannot broaden matching to all nodes", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" - ) - .read_text() - .replace('select="orders orders*"', 'select=" "') - }, + files={**_edited("node_orders", 'select="orders orders*"', 'select=" "')}, expected_error=ValueError, expected_message="node_event needs nonempty selectors", ), SnapshotDiscoveryRejectionTestCase( description="unknown node event kinds cannot publish", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" - ) - .read_text() - .replace("kinds=[NodeEventKind.SUCCESS]", "kinds=['unknown_event']") + **_edited("node_orders", "kinds=[NodeEventKind.SUCCESS]", "kinds=['unknown_event']") }, expected_error=ValueError, expected_message="unknown variant", @@ -1669,11 +1640,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="source_new_data rejects model selections", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" - ) - .read_text() - .replace('select="source:raw_orders"', 'select="orders"'), + **_edited("source_orders", 'select="source:raw_orders"', 'select="orders"'), }, expected_error=ValueError, expected_message="source_new_data must select only source nodes", @@ -1681,11 +1648,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="source_new_data rejects blank selections", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" - ) - .read_text() - .replace('select="source:raw_orders"', 'select=" "'), + **_edited("source_orders", 'select="source:raw_orders"', 'select=" "'), }, expected_error=ValueError, expected_message="source_new_data needs nonempty selectors", @@ -1693,140 +1656,84 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="source_stale rejects a zero threshold", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" - ) - .read_text() - .replace("after_seconds=60", "after_seconds=0"), + **_edited("stale_orders", "after_seconds=60", "after_seconds=0"), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, expected_error=ValueError, - expected_message="sensor on_stale_orders", + expected_message="sensor on_stale_orders_sensor", ), SnapshotDiscoveryRejectionTestCase( description="source_stale rejects a boolean threshold", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" - ) - .read_text() - .replace("after_seconds=60", "after_seconds=True"), + **_edited("stale_orders", "after_seconds=60", "after_seconds=True"), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, expected_error=ValueError, - expected_message="sensor on_stale_orders", + expected_message="sensor on_stale_orders_sensor", ), SnapshotDiscoveryRejectionTestCase( description="source_stale rejects an overflowing threshold", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" - ) - .read_text() - .replace("after_seconds=60", "after_seconds=2**32"), + **_edited("stale_orders", "after_seconds=60", "after_seconds=2**32"), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, expected_error=ValueError, - expected_message="sensor on_stale_orders", + expected_message="sensor on_stale_orders_sensor", ), SnapshotDiscoveryRejectionTestCase( description="source_stale rejects model selections", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" - ) - .read_text() - .replace('select="source:raw_orders"', 'select="orders"'), + **_edited("stale_orders", 'select="source:raw_orders"', 'select="orders"'), }, expected_error=ValueError, expected_message="source_stale must select only source nodes", ), SnapshotDiscoveryRejectionTestCase( description="explicit empty execution selectors cannot broaden matching", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" - ) - .read_text() - .replace('("orders",)', "()") - }, + files={**_edited("run_orders", '("orders",)', "()")}, expected_error=ValueError, expected_message="executed_successfully needs nonempty selectors", ), SnapshotDiscoveryRejectionTestCase( description="blank execution selectors cannot broaden matching", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" - ) - .read_text() - .replace('("orders",)', '(" ",)') - }, + files={**_edited("run_orders", '("orders",)', '(" ",)')}, expected_error=ValueError, expected_message="executed_successfully needs nonempty selectors", ), SnapshotDiscoveryRejectionTestCase( description="unresolved execution selectors fail closed", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" - ) - .read_text() - .replace('("orders",)', '("unknown_orders",)') - }, + files={**_edited("run_orders", '("orders",)', '("unknown_orders",)')}, expected_error=ValueError, expected_message="unknown selector name 'unknown_orders'", ), SnapshotDiscoveryRejectionTestCase( description="unmatched execution patterns cannot become an omitted filter", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" - ) - .read_text() - .replace('("orders",)', '("unknown_*",)') - }, + files={**_edited("run_orders", '("orders",)', '("unknown_*",)')}, expected_error=ValueError, expected_message="executed_successfully: unknown selector name 'unknown_\\*'", ), SnapshotDiscoveryRejectionTestCase( description="invalid execution selector syntax fails closed", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" - ) - .read_text() - .replace('("orders",)', '("tag:",)') - }, + files={**_edited("run_orders", '("orders",)', '("tag:",)')}, expected_error=ValueError, expected_message="has empty value after ':'", ), SnapshotDiscoveryRejectionTestCase( description="execution selectors outside the source job fail closed", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" - ) - .read_text() - .replace('("orders",)', '("stg_orders",)') - }, + files={**_edited("run_orders", '("orders",)', '("stg_orders",)')}, expected_error=ValueError, expected_message="executed_successfully must select nonempty source job/group nodes", ), SnapshotDiscoveryRejectionTestCase( description="sensor code defaults use running or paused", files={ - "orchestration/health.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" - ) - .read_text() - .replace('default_status="paused"', 'default_status="stopped"') + **_edited("platform_orders", 'default_status="paused"', 'default_status="stopped"') }, expected_error=ValueError, expected_message="unknown variant", @@ -1838,16 +1745,15 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho tmp_path: Path, test_case: SnapshotDiscoveryRejectionTestCase ) -> None: write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | test_case.files) - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discovered, - adapter=DuckDbAdapter(), - resolved_connection={"database": "warehouse", "schema": "main"}, - ) for _ in range(2): with pytest.raises(test_case.expected_error, match=test_case.expected_message): + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) compile_snapshot( - project=project, + project=build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ), metadata_json=json.dumps(METADATA), project_dir=tmp_path, discovered_inputs=discovered, @@ -1903,22 +1809,8 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_cli( tmp_path: Path, test_case: PythonJobSelectionTestCase ) -> None: - write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES) - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discovered, - adapter=DuckDbAdapter(), - resolved_connection={"database": "warehouse", "schema": "main"}, - ) - snapshot: dict[str, Any] = json.loads( - compile_snapshot( - project=project, - metadata_json=json.dumps(METADATA), - project_dir=tmp_path, - declarations=[job("orders_job", description="Orders.", select=test_case.select)], - discovered_inputs=discovered, - ).to_json() + python_pipeline_snapshot(tmp_path, {"orders_job": test_case.select}).to_json() ) assert snapshot["documents"]["jobs"]["orders_job"]["selection"] == list( @@ -1936,7 +1828,7 @@ def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_c ("carrier_sync_job", "python/fulfillment/carrier_sync"), ), expected_groups={ - "orders_nightly": { + "orders_nightly_job_group": { "carrier_sync_job": ["orders_export_job"], "orders_export_job": [], } @@ -1946,7 +1838,7 @@ def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_c description="a Python node's SQL dependency orders it after the model's job", jobs=(("orders_export_job", "asset:partner_orders_file"), ("models_job", "+orders")), expected_groups={ - "orders_nightly": {"models_job": [], "orders_export_job": ["models_job"]} + "orders_nightly_job_group": {"models_job": [], "orders_export_job": ["models_job"]} }, ), ], @@ -1955,13 +1847,11 @@ def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_c def test_given_python_dependencies_across_jobs_when_publishing_a_group_then_jobs_wait_for_them( tmp_path: Path, test_case: PythonJobGroupTestCase ) -> None: - jobs: list[Job] = [ - job(name, description="Orders.", select=select) for name, select in test_case.jobs - ] - group: JobGroup = job_group("orders_nightly", description="Nightly orders.", jobs=jobs) - + jobs: dict[str, str] = dict(test_case.jobs) snapshot: dict[str, Any] = json.loads( - python_pipeline_snapshot(tmp_path, [*jobs, group]).to_json() + python_pipeline_snapshot( + tmp_path, jobs, {"orders_nightly_job_group": tuple(jobs)} + ).to_json() ) assert { @@ -1979,7 +1869,7 @@ def test_given_python_dependencies_across_jobs_when_publishing_a_group_then_jobs ("export_job", "asset:partner_orders_file"), ), expected_message=( - "group dependencies contain a cycle in job group orders_nightly: " + "group dependencies contain a cycle in job group orders_nightly_job_group: " "export_job waits for staging_job (model:orders reads model:stg_orders); " "staging_job waits for export_job " "(task:request_carrier_collection reads asset:partner_orders_file)" @@ -1991,13 +1881,9 @@ def test_given_python_dependencies_across_jobs_when_publishing_a_group_then_jobs def test_given_a_python_dependency_cycle_across_jobs_when_publishing_a_group_then_it_is_rejected( tmp_path: Path, test_case: PythonJobGroupCycleTestCase ) -> None: - jobs: list[Job] = [ - job(name, description="Orders.", select=select) for name, select in test_case.jobs - ] - group: JobGroup = job_group("orders_nightly", description="Nightly orders.", jobs=jobs) - + jobs: dict[str, str] = dict(test_case.jobs) with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): - python_pipeline_snapshot(tmp_path, [*jobs, group]) + python_pipeline_snapshot(tmp_path, jobs, {"orders_nightly_job_group": tuple(jobs)}) @pytest.mark.parametrize( @@ -2014,11 +1900,11 @@ def test_given_a_python_dependency_cycle_across_jobs_when_publishing_a_group_the def test_given_a_job_matching_a_check_and_its_node_when_publishing_then_the_check_runs_with_it( tmp_path: Path, test_case: PythonCheckJobTestCase ) -> None: - nightly: Job = job("nightly_orders", description="Orders.", select=test_case.select) - - snapshot: dict[str, Any] = json.loads(python_pipeline_snapshot(tmp_path, [nightly]).to_json()) + snapshot: dict[str, Any] = json.loads( + python_pipeline_snapshot(tmp_path, {"nightly_orders_job": test_case.select}).to_json() + ) - assert snapshot["documents"]["jobs"]["nightly_orders"]["selection"] == ( + assert snapshot["documents"]["jobs"]["nightly_orders_job"]["selection"] == ( test_case.expected_selection ) @@ -2030,7 +1916,7 @@ def test_given_a_job_matching_a_check_and_its_node_when_publishing_then_the_chec description="a job matching a check without the model it checks names both", select="tag:order_checks", expected_message=( - "job nightly_orders: Python check 'orders_present' checks model:orders, which " + "job nightly_orders_job: Python check 'orders_present' checks model:orders, which " "the selection does not include; select it too, or narrow the selector so it no " "longer matches the check." ), @@ -2041,7 +1927,5 @@ def test_given_a_job_matching_a_check_and_its_node_when_publishing_then_the_chec def test_given_a_job_matching_a_check_without_its_node_when_publishing_then_it_is_rejected( tmp_path: Path, test_case: PythonCheckJobRejectionTestCase ) -> None: - nightly: Job = job("nightly_orders", description="Orders.", select=test_case.select) - with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): - python_pipeline_snapshot(tmp_path, [nightly]) + python_pipeline_snapshot(tmp_path, {"nightly_orders_job": test_case.select}) diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index c8956b692..d323e30c6 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -143,18 +143,18 @@ belongs to a scheduled job without the changed model. That job's next run refres model is held back: ``` - staging_daily (schedule) finance_daily (schedule) + staging_daily_job (schedule) finance_daily_job (schedule) stg_orders* ─▶ int_orders ✓ ──────────▶ fct_revenue ⏸ ─▶ rpt_pnl ⏸ │ └──▶ orders_export ✓ (in no scheduled job) ``` -- A job counts when a schedule, calendar schedule or sensor starts it, directly or through its +- A job counts when a schedule, calendar or sensor starts it, directly or through its group, for the target you build. A job nothing starts counts as no job. - A model held back by one job runs if another job that holds the change also contains it. - A model that reads a held-back model is held back too. Unchanged parents never hold a model back. - A job never holds back its own models while it runs: a scheduled run, or a build that selects the - job, such as `sqb build finance_daily --changed`. + job, such as `sqb build finance_daily_job --changed`. - With no scheduled jobs, everything downstream runs. `--reach all` builds everything downstream of a change. `--reach` without `--changed` is refused. diff --git a/website/src/content/docs/docs/cli/rename.mdx b/website/src/content/docs/docs/cli/rename.mdx index a1bfdc847..24b828d4e 100644 --- a/website/src/content/docs/docs/cli/rename.mdx +++ b/website/src/content/docs/docs/cli/rename.mdx @@ -82,6 +82,7 @@ Renaming or moving a model rewrites: - `__ref__` and `__expected__` fixture CTEs in tests and scenarios; - `cursor_inputs` keys, `relationships (to ...)` audits, and `__ref("")` inside quoted SQL in `MODEL` headers and reusable `SCHEMA` declarations; - `relationships` audits (`to:` and `field:`) and `__ref("")` in source and seed YAML. +- the model's name and folder in job and job group `select` and `exclude` selectors; see [Rename models in jobs](/docs/concepts/orchestration#rename-models-in-jobs). The model's own unit tests, the test files whose every `TEST` block targets only that model, follow it: diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 0cb5dc46d..47fc368ce 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -354,18 +354,18 @@ Details: Declare who owns a job, whether scheduled requests wait for a running copy, and how often a failed run retries: -```python -from sqlbuild.orchestration import job, on_failure, schedule - -nightly_orders = job( - "nightly_orders", - description="Orders and their daily totals", - select="+order_totals", - owner="Orders team", - overlap="skip", - on_failure=on_failure(retries=2, retry_delay_seconds=600), - schedule=schedule(cron="0 2 * * *", timezone="UTC"), -) +```toml +# orchestration/jobs/nightly_orders_job.toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" +owner = "Orders team" +overlap = "skip" +schedule = { cron = "0 2 * * *", timezone = "UTC" } + +[on_failure] +retries = 2 +retry_delay_seconds = 600 ``` | Setting | Values | Default | Effect | @@ -393,8 +393,8 @@ Details: - A group retries at group level, with the group's own `on_failure`: each retry is one rerun of the group's failed and skipped nodes. A member job's `on_failure` applies only when that job runs on its own. -- An invalid value fails publication with the job's name, such as - `job nightly_orders overlap must be queue or skip, not coalesce`. +- An invalid value is a `D017` error at its line, such as + ``orchestration/jobs/nightly_orders_job.toml:6:11: unknown variant `coalesce`, expected `queue` or `skip` ``. ## Limit worker containers @@ -418,15 +418,15 @@ pids = 512 max_memory = "4g" ``` -```python -from sqlbuild.orchestration import job, resources - -nightly_orders = job( - "nightly_orders", - description="Orders and their daily totals", - select="+order_totals", - resources=resources(memory="3g", cpus=2), -) +```toml +# orchestration/jobs/nightly_orders_job.toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" + +[resources] +memory = "3g" +cpus = 2 ``` | Setting | `run` table | `resources` | Value | @@ -446,7 +446,7 @@ nightly_orders = job( `no-new-privileges`. - A job in a group uses its own `resources`. Local launchers ignore `resources`. - An invalid value fails publication with the job's name, such as - `job nightly_orders resources: cpus must be a positive number`. + `job nightly_orders_job resources: cpus must be a positive number`. ## Run ad-hoc commands diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index 88f265321..16f00240f 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -74,7 +74,7 @@ error[P010]: model 'orders_summary' has no description A description may take at most `[format] max_description_lines` lines (default `10`) once wrapped at the line width, the same limit `sqb format --check` applies to headers. A longer Python description, from a decorator or a docstring, is a `P027` compile error at its declaration. A job, -job group, sensor or calendar schedule with a longer description cannot publish. +job group, sensor or calendar with a longer description cannot publish. SQL tests, audits, macros, enums, constants, materializations and event or output sinks do not need a description. Descriptions do not change fingerprints, so adding or editing one does not rebuild a diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx new file mode 100644 index 000000000..37da5681a --- /dev/null +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -0,0 +1,213 @@ +--- +title: "Orchestration" +description: "Declare jobs, job groups and schedules in TOML, and calendars and sensors in Python, for a SQLBuild server to run." +--- + +A job names the nodes a server builds together, and a schedule says when. Write each job as a TOML +file under `orchestration/jobs/`; `sqb compile` checks it and a code location publishes it. + +```toml +# orchestration/jobs/nightly_orders_job.toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" +schedule = { cron = "0 2 * * *", timezone = "UTC" } +``` + +```bash +sqb build nightly_orders_job # build what the job builds, locally +``` + +## Project layout + +| Folder | Holds | Name suffix | +| --- | --- | --- | +| `orchestration/jobs/` | One job per `.toml` file | `_job` | +| `orchestration/job_groups/` | One job group per `.toml` file | `_job_group` | +| `orchestration/calendars/` | One calendar per `.py` module | `_calendar` | +| `orchestration/sensors/` | One sensor per `.py` module | `_sensor` | + +- Subfolders are allowed, such as `orchestration/jobs/finance/revenue_job.toml`. +- A name must end with its folder's suffix. Names are unique across the whole project, with models, + sources and every other resource. +- A Python module whose file name starts with `_` is a private helper and declares nothing. +- Any other file under `orchestration/`, or a folder not in the table, is a `D017` error. + +## Declare a job + +A job builds its `select` minus its `exclude`, with the same [selectors](/docs/concepts/selectors) +as `sqb build`. + +```toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" +exclude = "tag:deprecated" +targets = ["production"] +owner = "Orders team" +overlap = "skip" +concurrency_groups = ["warehouse_heavy"] +schedule = { cron = "0 2 * * *", timezone = "Europe/London" } + +[on_failure] +retries = 2 +retry_delay_seconds = 600 + +[settings] +concurrency = 4 +vars = { batch_size = 500 } + +[resources] +memory = "3g" +cpus = 2 +``` + +| Key | Required | Value | +| --- | --- | --- | +| `name` | Yes | The job's name, ending in `_job` | +| `description` | Yes | Shown on the job's page | +| `select` | Yes | The nodes the job builds | +| `exclude` | No | Nodes removed from `select` | +| `targets` | No | The targets the job exists in; all targets when omitted | +| `owner` | No | Shown on the job's page and in the job list | +| `overlap` | No | `queue` (default) or `skip`; see [Skip overlapping runs](/docs/cli/server#skip-overlapping-runs-and-retry-failures) | +| `concurrency_groups` | No | Server concurrency limits the job's runs share | +| `schedule` | No | When the server starts the job; see below | +| `[on_failure]` | No | `retries` from 1 to 10, `retry_delay_seconds` from 0 to 86400 | +| `[settings]` | No | Build options for each run, such as `concurrency`, `vars`, `full_refresh` or `changed` | +| `[resources]` | No | Docker worker limits; see [Limit worker containers](/docs/cli/server#limit-worker-containers) | + +A job's `[settings]` cannot set `select` or `exclude`: the job's own selection is the run's +selection. + +## Group jobs + +A job group runs its jobs as one run. A job waits for another job in the group when it reads a node +that job builds. + +```toml +# orchestration/job_groups/daily_orders_job_group.toml +name = "daily_orders_job_group" +description = "Stage orders, then build the marts" +jobs = ["staging_orders_job", "orders_marts_job"] +schedule = { cron = "0 3 * * *", timezone = "UTC" } +``` + +A group takes `name`, `description` and `jobs`, and the same `targets`, `owner`, `overlap`, +`concurrency_groups`, `schedule` and `[on_failure]` as a job. Its members are jobs, never other +groups. + +## Schedule a job + +`schedule` takes a cron expression or the name of a calendar, and always a timezone. + +| Key | Value | +| --- | --- | +| `cron` | A five-field cron expression | +| `calendar` | The name of a calendar in `orchestration/calendars/` | +| `timezone` | An IANA timezone, such as `"UTC"` or `"America/New_York"` | +| `starts_at` | An RFC 3339 time with an offset; no ticks before it | +| `default_status` | `running` (default) or `paused` when the schedule is first published | + +```toml +schedule = { cron = "15 2 * * *", timezone = "Europe/London", starts_at = "2026-07-03T00:00:00Z" } +``` + +## Schedule from a calendar + +A calendar returns the upcoming times a job runs, such as fulfillment cut-offs that come from +another system. The code location evaluates it when it publishes and again before its times run +out. + +```python +# orchestration/calendars/fulfillment_calendar.py +from sqlbuild.orchestration import calendar + + +@calendar(name="fulfillment_calendar", description="Upcoming fulfillment times", horizon="1d") +def fulfillment_calendar(window, fulfillment): + return fulfillment.upcoming(window) +``` + +```toml +schedule = { calendar = "fulfillment_calendar", timezone = "America/New_York" } +``` + +The function receives a `window` with timezone-aware `start` and `end`, in the job's timezone, and +returns timezone-aware datetimes inside it. Other parameters name +[providers](/docs/concepts/python-nodes/providers). `horizon` sets how far ahead the window reaches. + +## Start a job from a sensor + +A sensor starts a job or job group when something happens, named by `run`. + +```python +# orchestration/sensors/on_new_orders_sensor.py +from sqlbuild.orchestration import sensor, source_new_data + +on_new_orders = sensor( + "on_new_orders_sensor", + description="Refresh orders for each new source version", + when=source_new_data(select="source:raw_orders"), + run="orders_refresh_job", + minimum_interval_seconds=60, +) +``` + +| Condition | Starts a run for | +| --- | --- | +| `platform_event(kinds=[...])` | Server events, such as an overdue tick | +| `run_event(job="...", kinds=[...])` | Outcomes of another job or group, optionally only when `executed_successfully` nodes ran and succeeded | +| `node_event(select="...", kinds=[...])` | Each success, failure or recovery of the selected nodes in any run | +| `source_new_data(select="...")` | Each new version of the selected sources | +| `source_stale(select="...", after_seconds=...)` | Each source version older than `after_seconds` | + +A sensor without `when` decorates a function that polls and returns `RunRequest(run_key=...)` +values or a `SkipReason`. A sensor also takes `minimum_interval_seconds` (default 30), +`default_status` and `targets`. + +## Rename models in jobs + +`sqb rename` and `sqb mv` update job and job group selectors with the models they rename or move: + +```bash +sqb rename stg_orders stg_orders_clean +``` + +```text +Rename model stg_orders -> stg_orders_clean +├── models/marts/fact_orders.sql +│ └── 10:13 reference __ref("stg_orders") -> __ref("stg_orders_clean") +├── models/staging/stg_orders_clean.sql (moved from models/staging/stg_orders.sql) +│ └── 1:8 migration + migrate_from stg_orders +└── orchestration/jobs/orders_refresh_job.toml + └── 3:10 selector "+stg_orders+ fact_orders" -> "+stg_orders_clean+ fact_orders" +``` + +- Exact names are rewritten, including inside graph operators such as `+stg_orders+`, `,` and `~`. + Tags, globs and `job:` selectors stay as written. +- A folder selector, such as `models/staging` or `path:models/staging/`, moves when every model + under it moves to the same folder. When only some leave, the command lists a note at the + selector to check by hand. +- A selector in a multi-line TOML string is reported for a manual edit and blocks the refactor. + +## Errors + +Every problem with an orchestration file is a `D017` error at its file, line and column, reported by +`sqb compile` and every other command that reads the project. + +| Error | Fix | +| --- | --- | +| ``orchestration/jobs/nightly_orders_job.toml:4:11: unknown variant `coalesce`, expected `queue` or `skip` `` | Use one of the listed values | +| `files cannot sit directly in orchestration/` | Move jobs and groups to TOML files in `orchestration/jobs/` or `orchestration/job_groups/` | +| `` `job` was removed from sqlbuild.orchestration`` | Declare the job in TOML and name it in the sensor's `run` | +| `declares a second sensor; each module declares exactly one` | Split the module | + +Details: + +- `sqb build` and the other commands read job and group files without running project Python, so + a job selector never imports a calendar or sensor module. +- A job or group declared with `targets` is selectable only when the command's target is one of + them, as on the server. +- A job's resources are checked when a location publishes, such as + `job nightly_orders_job resources: cpus must be a positive number`. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index c7ec219c1..e6f3e46ef 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -713,7 +713,7 @@ Code that extends SQLBuild itself keeps its own root: | `hooks/python/` | Python model lifecycle hooks | | `functions/python/` | Python UDFs | | `macros/python/`, `enums/`, `constants/` and scoped `_macros/python/`, `_enums/`, `_constants/` | Python macros and declarations | -| `orchestration/` | Jobs, job groups, sensors and calendar schedules. SQLBuild imports these modules only when a selector names a job or an unknown name, and when a location publishes. See [Job selectors](/docs/concepts/selectors#job) | +| `orchestration/` | Jobs and job groups as TOML in `jobs/` and `job_groups/`; calendars and sensors as Python in `calendars/` and `sensors/`. See [Orchestration](/docs/concepts/orchestration) | Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create an unofficial project structure. diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 9d215b702..1c575fb56 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -146,7 +146,7 @@ sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` -Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendar schedules. +Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendars. ## Lifecycle: run, build, check diff --git a/website/src/content/docs/docs/concepts/selectors.mdx b/website/src/content/docs/docs/concepts/selectors.mdx index 7f598d7a3..2d7e7e2a1 100644 --- a/website/src/content/docs/docs/concepts/selectors.mdx +++ b/website/src/content/docs/docs/concepts/selectors.mdx @@ -69,13 +69,13 @@ sqb build --select source:raw__orders ### Job -Select the SQL and Python nodes a job or job group declared under `orchestration/` builds. Job names are globally +Select the SQL and Python nodes a [job or job group](/docs/concepts/orchestration) builds. Job names are globally unique, so a bare name works; `job:` is the explicit form and accepts glob patterns: ```bash -sqb build nightly_orders -sqb build --select job:nightly_orders --exclude order_totals -sqb build --select "job:nightly_orders,tag:finance" +sqb build nightly_orders_job +sqb build --select job:nightly_orders_job --exclude order_totals +sqb build --select "job:nightly_orders_job,tag:finance" ``` A job selects its own `select` minus its own `exclude`. A job group selects the union of its jobs. @@ -84,8 +84,7 @@ them, as on the server. Job selectors compose with `+`, `,` and `--exclude` like any other selector. A job's own selection cannot name another job, and path-between selectors cannot start or end at a job. -SQLBuild imports the modules under `orchestration/` only when a selector names a job or a name no -model, source, seed or function has. +SQLBuild reads job and job group files without running project Python. ### Unit test diff --git a/website/src/sidebar.json b/website/src/sidebar.json index cee743e40..30e1b55f5 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -94,7 +94,8 @@ "label": "Review a branch", "link": "/docs/cli/review/" }, - "docs/concepts/column-lineage" + "docs/concepts/column-lineage", + "docs/concepts/orchestration" ] }, { From 6b38c83768de651f3450f60c3a0194b58c8ea30d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:55:02 +0000 Subject: [PATCH 26/34] fix: check TOML job values at compile and keep sensors, calendars and selectors honest - Compile checks schedule cron, time zone and starts_at, build settings, worker resources and selector syntax, as D017 at the value's line and column, with the checks publication applies. - Only calls on the sqlbuild.orchestration module count as calendar, sensor or run_event declarations; a provider or stdlib method named calendar no longer reads as a second declaration. - rename and mv no longer rewrite a folder selector into a folder that keeps or receives other models; they report it instead. - Publication refuses a calendar or sensor that compile declared but import did not define, or the reverse, instead of dropping it. - Restore the check that model selectors never import orchestration Python, and match the nested-group rejection exactly. --- .../calendars/fulfillment_calendar.py | 3 +- .../src/orchestration_files/_helpers/raw.rs | 10 +- .../orchestration_files/_helpers/reading.rs | 6 +- .../orchestration_files/main/read_job_file.rs | 19 ++- .../src/orchestration_files/models.rs | 12 +- .../orchestration_modules/_helpers/calls.rs | 57 ++++++++- .../main/orchestration_module_declarations.rs | 9 +- .../test_orchestration_module_declarations.rs | 25 ++++ .../compiled_project/snapshot_declarations.rs | 49 +++++++- .../src/bindings/_helpers/discovery/mod.rs | 1 + .../_helpers/discovery/orchestration_files.rs | 5 +- .../discovery/orchestration_values.rs | 110 ++++++++++++++++++ .../native_host/orchestration_triggers.rs | 2 +- .../_helpers/planning/job_selectors.rs | 13 ++- .../refactoring/tests/test_job_selectors.rs | 18 +++ .../compile/test_python_project_layout.py | 48 ++++++++ .../cli/commands/main/test_job_selectors.py | 14 +++ .../compiled_project/publication/helpers.py | 16 --- .../publication/test_calendar.py | 2 +- .../test_orchestration_snapshot.py | 24 +++- 20 files changed, 389 insertions(+), 54 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/calendars/fulfillment_calendar.py b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/calendars/fulfillment_calendar.py index ddb78ea4d..bd368ad3c 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/calendars/fulfillment_calendar.py +++ b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/calendars/fulfillment_calendar.py @@ -9,4 +9,5 @@ horizon="8s", ) def fulfillment_calendar(window, fulfillment): - return fulfillment.upcoming(window) + # A provider method named `calendar` is not a second calendar declaration. + return fulfillment.calendar(window) diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs index d4473e2f7..b21378fb2 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs @@ -1,5 +1,7 @@ //! The file shapes serde reads before their values are checked. +use std::collections::BTreeMap; + use serde::Deserialize; use toml::Spanned; @@ -8,10 +10,10 @@ use crate::orchestration_files::models::{InitialStatus, Overlap}; #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct RawSchedule { - pub(crate) cron: Option, + pub(crate) cron: Option>, pub(crate) calendar: Option>, - pub(crate) timezone: String, - pub(crate) starts_at: Option, + pub(crate) timezone: Spanned, + pub(crate) starts_at: Option>, #[serde(default)] pub(crate) default_status: InitialStatus, } @@ -46,7 +48,7 @@ pub(crate) struct RawJob { pub(crate) exclude: Option>, pub(crate) resources: Option>, #[serde(default)] - pub(crate) settings: toml::Table, + pub(crate) settings: BTreeMap>, } /// The group-only fields; the shared ones are read as `RawRun` from the same text. diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs index 6e366565f..296b09aad 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs @@ -100,7 +100,7 @@ fn schedule(text: &str, raw: Spanned) -> Result ScheduleSource::Cron(cron), + (Some(cron), None) => ScheduleSource::Cron(positioned(text, cron)), (None, Some(calendar)) => { ScheduleSource::Calendar(nonblank(text, calendar, "schedule calendar")?) } @@ -114,8 +114,8 @@ fn schedule(text: &str, raw: Spanned) -> Result Result { @@ -23,10 +25,16 @@ pub fn read_job_file(text: &str) -> Result { position: position(text, start), } }); - let settings = serde_json::to_value(&raw.settings) - .ok() - .and_then(|value| serde_json::from_value(value).ok()) - .unwrap_or_default(); + let mut settings: BTreeMap = BTreeMap::new(); + let mut setting_positions: BTreeMap = BTreeMap::new(); + for (key, value) in raw.settings { + let value = positioned(text, value); + settings.insert( + key.clone(), + serde_json::to_value(value.value).unwrap_or_default(), + ); + setting_positions.insert(key, value.position); + } Ok(JobFile { run: run_file(text, run)?, select: nonblank(text, raw.select, "select")?, @@ -36,5 +44,6 @@ pub fn read_job_file(text: &str) -> Result { .map(|exclude| positioned(text, exclude)), resources, settings, + setting_positions, }) } diff --git a/crates/sqlbuild-discovery/src/orchestration_files/models.rs b/crates/sqlbuild-discovery/src/orchestration_files/models.rs index 95f4b3990..4e40a8914 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/models.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/models.rs @@ -57,7 +57,7 @@ pub struct FileError { /// What starts a job or group on its own: a cron expression or a named calendar. #[derive(Clone, Debug, PartialEq, Eq)] pub enum ScheduleSource { - Cron(String), + Cron(Positioned), Calendar(Positioned), } @@ -65,9 +65,9 @@ pub enum ScheduleSource { #[derive(Clone, Debug, PartialEq, Eq)] pub struct ScheduleFile { pub source: ScheduleSource, - pub timezone: String, - /// RFC 3339 as written; publication parses it. - pub starts_at: Option, + pub timezone: Positioned, + /// RFC 3339 as written; compile checks it and publication parses it. + pub starts_at: Option>, pub default_status: InitialStatus, } @@ -90,10 +90,12 @@ pub struct JobFile { pub run: RunFile, pub select: Positioned, pub exclude: Option>, - /// Worker container limits as written; publication checks their values. + /// Worker container limits as written; compile checks their values. pub resources: Option>>, /// `sqb build` flags by name, as JSON values. pub settings: BTreeMap, + /// Where each setting's value is written. + pub setting_positions: BTreeMap, } /// One `orchestration/job_groups/**.toml` file. diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs index 4c1ec660d..9122be97c 100644 --- a/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs +++ b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs @@ -1,6 +1,6 @@ //! Orchestration authoring calls and their literal arguments, read from the ruff tree. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use ruff_python_ast::{Expr, ExprCall, Stmt}; use ruff_text_size::{Ranged, TextSize}; @@ -51,14 +51,63 @@ pub(crate) fn imported_names(body: &[Stmt]) -> Vec<(String, String, TextSize)> { imported } -/// Which authoring call `call` is: an imported `calendar`, `sensor` or `run_event`, or one reached as an attribute. +/// The dotted names a top-level `import sqlbuild.orchestration [as x]` or `from sqlbuild import orchestration [as x]` binds the module to. +pub(crate) fn module_names(body: &[Stmt]) -> BTreeSet { + let (package, module) = ORCHESTRATION_MODULE + .rsplit_once('.') + .unwrap_or_default(); + let mut names: BTreeSet = BTreeSet::new(); + for statement in body { + match statement { + Stmt::Import(import) => names.extend( + import + .names + .iter() + .filter(|alias| alias.name.as_str() == ORCHESTRATION_MODULE) + .map(|alias| alias.asname.as_ref().unwrap_or(&alias.name).to_string()), + ), + Stmt::ImportFrom(import) + if import.level == 0 + && import + .module + .as_ref() + .is_some_and(|name| name.as_str() == package) => + { + names.extend( + import + .names + .iter() + .filter(|alias| alias.name.as_str() == module) + .map(|alias| alias.asname.as_ref().unwrap_or(&alias.name).to_string()), + ); + } + _ => {} + } + } + names +} + +/// `expression` as dotted text when it is a name or a chain of attributes on one. +fn dotted(expression: &Expr) -> Option { + match expression { + Expr::Name(name) => Some(name.id.to_string()), + Expr::Attribute(attribute) => { + Some(format!("{}.{}", dotted(&attribute.value)?, attribute.attr)) + } + _ => None, + } +} + +/// Which authoring call `call` is: an imported `calendar`, `sensor` or `run_event`, or one reached as an attribute of the orchestration module. pub(crate) fn authoring_call( call: &ExprCall, - imported: &BTreeMap<&str, &str>, + (imported, modules): (&BTreeMap<&str, &str>, &BTreeSet), ) -> Option<&'static str> { let name: &str = match call.func.as_ref() { Expr::Name(name) => imported.get(name.id.as_str())?, - Expr::Attribute(attribute) => attribute.attr.as_str(), + Expr::Attribute(attribute) if modules.contains(&dotted(&attribute.value)?) => { + attribute.attr.as_str() + } _ => return None, }; AUTHORING_CALLS.into_iter().find(|call| *call == name) diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs b/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs index 8e0f6298f..ebb78252a 100644 --- a/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs +++ b/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs @@ -1,6 +1,6 @@ //! The calendars and sensors a module declares, and the job names they use, without importing it. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use ruff_python_ast::Expr; use ruff_text_size::{Ranged, TextSize}; @@ -9,7 +9,7 @@ use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; use crate::cpython_tree::models::{CpythonNode, ParsedSource}; use crate::orchestration_modules::_helpers::calls::{ - authoring_call, imported_names, literal, literal_strings, position, + authoring_call, imported_names, literal, literal_strings, module_names, position, }; use crate::orchestration_modules::constants::{ CALENDAR_CALL, JOB_KEYWORD, NAME_KEYWORD, REMOVED_NAMES, RUN_EVENT_CALL, RUN_KEYWORD, @@ -31,6 +31,7 @@ pub fn orchestration_module_declarations( .iter() .map(|(local, name, _)| (local.as_str(), name.as_str())) .collect(); + let modules: BTreeSet = module_names(&parsed.module.body); let removed_imports = imports .iter() .filter(|(_, name, _)| REMOVED_NAMES.contains(&name.as_str())) @@ -41,7 +42,7 @@ pub fn orchestration_module_declarations( let CpythonNode::Expr(Expr::Call(call)) = node else { continue; }; - let kind = match authoring_call(call, &imported) { + let kind = match authoring_call(call, (&imported, &modules)) { Some(CALENDAR_CALL) => PythonDeclarationKind::Calendar, Some(SENSOR_CALL) => PythonDeclarationKind::Sensor, _ => continue, @@ -75,7 +76,7 @@ pub fn orchestration_module_declarations( let CpythonNode::Expr(Expr::Call(inner)) = inner else { continue; }; - if authoring_call(inner, &imported) != Some(RUN_EVENT_CALL) { + if authoring_call(inner, (&imported, &modules)) != Some(RUN_EVENT_CALL) { continue; } if let Some(job) = inner.arguments.find_keyword(JOB_KEYWORD) { diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs index 75681c782..811828cb5 100644 --- a/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs +++ b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs @@ -95,3 +95,28 @@ fn given_syntax_error_when_scanning_then_returns_none() { None ); } + +#[test] +fn given_other_objects_calendar_and_sensor_methods_when_scanning_then_only_orchestration_calls_declare() { + let module = scan( + "import calendar\nfrom sqlbuild.orchestration import calendar as declare\n\n@declare(name=\"days_calendar\", description=\"d\", horizon=\"7d\")\ndef days(window, exchange):\n calendar.calendar(2026)\n return exchange.calendar(window.start)\n", + ); + assert_eq!(module.declarations.len(), 1); + assert_eq!(module.declarations[0].name, Some(text("days_calendar", 4, 15))); +} + +#[test] +fn given_orchestration_module_attributes_when_scanning_then_each_import_form_declares() { + let module = scan( + "import sqlbuild.orchestration\nfrom sqlbuild import orchestration as authoring\n\na = sqlbuild.orchestration.sensor(\"a_sensor\", description=\"d\", run=\"a_job\")\nb = authoring.sensor(\"b_sensor\", description=\"d\", run=\"b_job\")\n", + ); + let names: Vec> = module + .declarations + .iter() + .map(|declaration| declaration.name.as_ref().and_then(|name| name.value.clone())) + .collect(); + assert_eq!( + names, + vec![Some("a_sensor".to_owned()), Some("b_sensor".to_owned())] + ); +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index cd3dbb253..817708e28 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -9,7 +9,7 @@ use serde_json::Value; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_discovery::orchestration_files::models::{ - FilePosition, InitialStatus, OrchestrationInventory, RunFile, ScheduleSource, + FilePosition, InitialStatus, NamedModule, OrchestrationInventory, RunFile, ScheduleSource, }; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; @@ -77,7 +77,7 @@ fn run_schedule( let starts_at = schedule .starts_at .as_ref() - .map(|starts_at| serde_json::from_value(Value::String(starts_at.clone()))) + .map(|starts_at| serde_json::from_value(Value::String(starts_at.value.clone()))) .transpose() .map_err(|error| PyValueError::new_err(format!("{path}: schedule starts_at: {error}")))?; let trigger = TriggerDefinition { @@ -85,7 +85,7 @@ fn run_schedule( job_key: name.to_owned(), schedule: Schedule::Cron { expression: String::new(), - timezone: schedule.timezone.clone(), + timezone: schedule.timezone.value.clone(), }, starts_at, default_status: match schedule.default_status { @@ -101,8 +101,8 @@ fn run_schedule( name.to_owned(), TriggerDefinition { schedule: Schedule::Cron { - expression: expression.clone(), - timezone: schedule.timezone.clone(), + expression: expression.value.clone(), + timezone: schedule.timezone.value.clone(), }, ..trigger }, @@ -110,13 +110,39 @@ fn run_schedule( } ScheduleSource::Calendar(calendar) => calendars.push(CalendarRequest { calendar: calendar.value.clone(), - timezone: schedule.timezone.clone(), + timezone: schedule.timezone.value.clone(), trigger, }), } Ok(()) } +/// Refuse a calendar or sensor that compile declared but import did not define, or the reverse. +fn matching_imports( + (kind, declared): (&str, &[NamedModule]), + imported: BTreeSet<&str>, +) -> PyResult<()> { + if let Some(module) = declared + .iter() + .find(|module| !imported.contains(module.name.as_str())) + { + let FilePosition { line, column } = module.position; + return Err(PyValueError::new_err(format!( + "{}:{line}:{column}: {kind} '{}' is declared here, but importing the module does not \ + define it; assign it to a module-level name or list", + module.path, module.name + ))); + } + let names: BTreeSet<&str> = declared.iter().map(|module| module.name.as_str()).collect(); + if let Some(name) = imported.difference(&names).next() { + return Err(PyValueError::new_err(format!( + "{kind} '{name}' is defined when its module is imported, but no module under \ + orchestration/ declares it with a literal name; call {kind}(...) in the module itself" + ))); + } + Ok(()) +} + pub(crate) fn compile_declarations( py: Python<'_>, mut input: SnapshotInput, @@ -152,6 +178,17 @@ pub(crate) fn compile_declarations( } } } + matching_imports( + ("calendar", &inventory.calendars), + authored_calendars.keys().map(String::as_str).collect(), + )?; + matching_imports( + ("sensor", &inventory.sensors), + authored_sensors + .iter() + .map(|sensor| sensor.name.as_str()) + .collect(), + )?; let format: FormatSettings = resolve_format_settings(project_dir).map_err(PyValueError::new_err)?; let mut described: Vec<(&str, String, String)> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 60b884fc5..4fe23096f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -18,6 +18,7 @@ pub(crate) mod locations; pub(crate) mod model_file_record; pub(crate) mod model_records; pub(crate) mod orchestration_files; +pub(crate) mod orchestration_values; pub(crate) mod project_config; pub(crate) mod project_facts; pub(crate) mod project_files; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs index d4bd4cf51..7cc3388f8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs @@ -9,6 +9,7 @@ use sqlbuild_discovery::orchestration_files::models::{ }; use sqlbuild_discovery::tree::models::ProjectTree; +use crate::bindings::_helpers::discovery::orchestration_values::check_orchestration_values; use crate::bindings::_helpers::discovery::project_files::path_object; use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; @@ -21,7 +22,9 @@ pub(crate) fn orchestration_declarations( let tree = ProjectTree::new(&root); let version = py.version_info(); let inventory = py.detach(|| orchestration_inventory(&tree, (version.major, version.minor))); - inventory.map_err(|failure| orchestration_error(&tree, project_dir, failure)) + let inventory = inventory.map_err(|failure| orchestration_error(&tree, project_dir, failure))?; + check_orchestration_values(py, &inventory).map_err(|failure| failure_error(py, failure))?; + Ok(inventory) } fn orchestration_error( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs new file mode 100644 index 000000000..f870bfde1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs @@ -0,0 +1,110 @@ +//! Check the values of TOML jobs and groups at compile, with the checks publication applies. + +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use chrono_tz::Tz; +use pyo3::prelude::Python; +use serde_json::Value; +use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; +use sqlbuild_discovery::models::{DiscoveryFailure, FailureKind}; +use sqlbuild_discovery::orchestration_files::models::{ + FilePosition, OrchestrationInventory, Positioned, RunFile, ScheduleSource, +}; +use sqlbuild_orchestration::scheduler::main::parse_cron::parse_cron; +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::bindings::_helpers::native_host::orchestration_declarations::run_resources; + +/// A `D017` at `position` of `path`, worded as discovery words its own. +type ValueError = (FilePosition, String); + +/// The first invalid schedule, setting, resource limit or selector, as `D017` at its file, line and column. +pub(crate) fn check_orchestration_values( + py: Python<'_>, + inventory: &OrchestrationInventory, +) -> Result<(), DiscoveryFailure> { + let runs = inventory + .jobs + .iter() + .map(|(path, job)| (path, &job.run)) + .chain(inventory.groups.iter().map(|(path, group)| (path, &group.run))); + for (path, run) in runs { + schedule_values(run).map_err(|error| failure(path, error))?; + } + for (path, job) in &inventory.jobs { + for selector in std::iter::once(&job.select).chain(&job.exclude) { + selector_syntax(selector).map_err(|error| failure(path, error))?; + } + let mut settings: BTreeMap = BTreeMap::new(); + for (key, value) in &job.settings { + settings.insert(key.clone(), value.clone()); + job_settings_arguments(&BTreeMap::from([(key.clone(), value.clone())])) + .map_err(|message| failure(path, (job.setting_positions[key], message)))?; + } + if let Some(first) = job.setting_positions.values().min() { + job_settings_arguments(&settings) + .map_err(|message| failure(path, (*first, message)))?; + } + if let Some(resources) = &job.resources { + run_resources(&job.run.name.value, &resources.value).map_err(|error| { + failure(path, (resources.position, error.value(py).to_string())) + })?; + } + } + Ok(()) +} + +fn failure(path: &str, (FilePosition { line, column }, message): ValueError) -> DiscoveryFailure { + DiscoveryFailure { + kind: FailureKind::Orchestration, + message: format!("{path}:{line}:{column}: {message}"), + help: None, + } +} + +fn schedule_values(run: &RunFile) -> Result<(), ValueError> { + let Some(schedule) = &run.schedule else { + return Ok(()); + }; + let timezone: &Positioned = &schedule.timezone; + timezone.value.parse::().map_err(|error| { + ( + timezone.position, + format!("schedule timezone is not an IANA time zone: {error}"), + ) + })?; + if let ScheduleSource::Cron(cron) = &schedule.source { + parse_cron(&cron.value, &timezone.value).map_err(|error| { + let reason: String = match error { + ControlError::Invalid(reason) => reason, + other => other.to_string(), + }; + (cron.position, format!("schedule cron: {reason}")) + })?; + } + if let Some(starts_at) = &schedule.starts_at { + serde_json::from_value::>(Value::String(starts_at.value.clone())).map_err( + |error| { + ( + starts_at.position, + format!("schedule starts_at must be an RFC 3339 timestamp: {error}"), + ) + }, + )?; + } + Ok(()) +} + +fn selector_syntax(selector: &Positioned) -> Result<(), ValueError> { + let parts = selector + .value + .split_whitespace() + .flat_map(|token| token.split(',')) + .filter(|part| !part.is_empty()); + for part in parts { + parse_selector(part).map_err(|error| (selector.position, error.message))?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 89c275ebe..a76334202 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -1,6 +1,6 @@ //! Authored trigger controls and bounded calendars evaluated only in the location interpreter. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use chrono::{DateTime, TimeDelta, Timelike, Utc}; diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs index d153692b8..d0975bd4b 100644 --- a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs @@ -152,12 +152,23 @@ fn changes(models: &[ModelFacts], renamed: &[(&ModelFacts, &str, &str)]) -> Chan to.strip_suffix(rest).map(str::to_owned) }; let targets: Vec> = below.iter().map(|model| moved_to(model)).collect(); + // A destination that keeps or receives other models would widen the selector to them. + let shared = |to: &String| { + models.iter().any(|model| { + let lands: String = destinations + .get(model.path.as_str()) + .cloned() + .unwrap_or_else(|| parent(&model.path)); + within(&lands, to) && !within(&parent(&model.path), &folder) + }) + }; let whole = targets .first() .cloned() .flatten() .filter(|to| *to != folder && !within(to, &folder)) - .filter(|to| targets.iter().all(|target| target.as_ref() == Some(to))); + .filter(|to| targets.iter().all(|target| target.as_ref() == Some(to))) + .filter(|to| !shared(to)); match whole { Some(to) => { folders.insert(folder, to); diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_job_selectors.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_job_selectors.rs index ce6104ae4..0c8a5f6c1 100644 --- a/crates/sqlbuild-refactor/src/refactoring/tests/test_job_selectors.rs +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_job_selectors.rs @@ -83,6 +83,24 @@ fn given_job_selectors_when_models_are_renamed_or_moved_then_exact_parts_follow( "2:10 this job selects folder 'models/staging', and orders moved out of it; check the job still selects what it should", ], }, + JobSelectorTestCase { + description: "a folder moving into one that holds other models stays and is reported", + models: &[ + ( + "orders", + "models/staging/orders.sql", + "orders", + "models/marts/orders.sql", + ), + STAYS, + ], + job: "name = \"x_job\"\nselect = \"models/staging\"\nexclude = \"models/staging\"\n", + expected_job: "name = \"x_job\"\nselect = \"models/staging\"\nexclude = \"models/staging\"\n", + expected_manual: &[ + "2:10 this job selects folder 'models/staging', and orders moved out of it; check the job still selects what it should", + "3:11 this job selects folder 'models/staging', and orders moved out of it; check the job still selects what it should", + ], + }, JobSelectorTestCase { description: "a multi-line selector is reported instead of rewritten", models: &[( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py index ed0930ad4..f3e84056e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py @@ -68,6 +68,54 @@ "https://sqlbuild.com/docs/concepts/orchestration/", ), ), + *( + PythonProjectLayoutCompileTestCase( + description=description, + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "models/orders.sql": "MODEL (description 'Orders.');\nSELECT 1 AS order_id\n", + "orchestration/jobs/orders_job.toml": ( + 'name = "orders_job"\ndescription = "Orders."\n' + job_values + ), + }, + expected_exit_code=1, + expected_stderr_fragments=("error[D017]", location, message), + ) + for description, job_values, location, message in ( + ( + "an invalid cron is reported at its value", + 'select = "orders"\nschedule = { cron = "not a cron", timezone = "UTC" }\n', + "orchestration/jobs/orders_job.toml:4:21:", + "schedule cron: invalid cron expression", + ), + ( + "an unknown time zone is reported at its value", + 'select = "orders"\n' + 'schedule = { cron = "0 2 * * *", timezone = "Mars/Olympus" }\n', + "orchestration/jobs/orders_job.toml:4:45:", + "schedule timezone is not an IANA time zone", + ), + ( + "a starts_at that is not RFC 3339 is reported at its value", + 'select = "orders"\n' + 'schedule = { cron = "0 2 * * *", timezone = "UTC", starts_at = "yesterday" }\n', + "orchestration/jobs/orders_job.toml:4:64:", + "schedule starts_at must be an RFC 3339 timestamp", + ), + ( + "an invalid build setting is reported at its value", + 'select = "orders"\n\n[settings]\nconcurrency = "lots"\n', + "orchestration/jobs/orders_job.toml:6:15:", + "concurrency", + ), + ( + "an unknown selector type is reported at the selector", + 'select = "orders tagg:finance"\n', + "orchestration/jobs/orders_job.toml:3:10:", + "unknown selector type 'tagg' in 'tagg:finance'", + ), + ) + ), PythonProjectLayoutCompileTestCase( description="removed tasks Python root", repo_files={ diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py index bf12cbf56..9a8aaf725 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py @@ -41,6 +41,13 @@ "orchestration/job_groups/daily_job_group.toml": ( 'name = "daily_job_group"\ndescription = "Everything."\njobs = ["nightly_job", "crm_job"]\n' ), + "orchestration/sensors/overdue_sensor.py": ( + "from pathlib import Path\n" + "from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor\n" + "Path(__file__).parents[2].joinpath('sensor_imported').write_text('yes')\n" + "overdue = sensor('overdue_sensor', description='Catch up.', " + "when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), run='nightly_job')\n" + ), } @@ -90,6 +97,12 @@ expected_exit_code=0, expected_tables=frozenset({"customers"}), ), + JobSelectorBuildTestCase( + description="model selectors build without job declarations", + build_args=("--select", "+orders"), + expected_exit_code=0, + expected_tables=frozenset({"orders", "stg_orders"}), + ), JobSelectorBuildTestCase( description="an unknown job suggests a known one", build_args=("--select", "job:nightyl"), @@ -117,6 +130,7 @@ def test_given_declared_jobs_when_building_by_job_then_only_their_nodes_build( assert exit_code == test_case.expected_exit_code, captured assert test_case.expected_fragment in captured + assert not (tmp_path / "sensor_imported").exists() with duckdb.connect(str(tmp_path / "demo.duckdb")) as warehouse: rows: list[tuple[str]] = warehouse.sql( "SELECT table_name FROM duckdb_tables() " diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 725a5677d..4d0a0e168 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1639,22 +1639,6 @@ def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: } -INTERMEDIATE_LOADER_FILES: dict[str, str] = { - "sources/shipments.yml": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml" - ).read_text(), - "python/loaders/shipments.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py" - ).read_text(), - "models/orders.sql": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql" - ).read_text(), - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" - ).read_text(), -} - - def without_loads(control: sqlite3.Connection, node_key: str) -> None: """Drop a published node's `loads` in place; its content address and the worker's check stay.""" for node_hash, body in control.execute( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index 799952b08..3818bbc16 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -169,7 +169,7 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr " def record(self, value):\n" " with Path(os.environ['SQLBUILD_CALENDAR_EVALUATIONS']).open('a') as out:\n" " out.write(json.dumps(value | {'pid': os.getpid()}) + '\\n')\n" - " def upcoming(self, window):\n" + " def calendar(self, window):\n" " self.record({'phase': 'evaluate', 'start': window.start.isoformat(), 'end': window.end.isoformat(), 'zone': window.start.tzinfo.key})\n" " data = json.loads(Path(os.environ['SQLBUILD_FULFILLMENT_DATA']).read_text())\n" " if data.get('fail'):\n raise ValueError('synthetic calendar unavailable')\n" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 54ed11bca..ba96265ec 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -44,7 +44,6 @@ FUNCTION_PROJECT_FILES, METADATA, PARITY_FILES, - PYTHON_PIPELINE_FILES, TYPE_PROOF_FILES, duckshop_orchestration, published_texts, @@ -1481,7 +1480,28 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects description="group members are jobs, not nested groups", files=duckshop_orchestration("nested_groups"), expected_error=ValueError, - expected_message="daily_orders_job_group", + expected_message=( + "names 'daily_orders_job_group', which is not a job in orchestration/jobs/" + ), + ), + SnapshotDiscoveryRejectionTestCase( + description="a sensor import does not define is refused, not dropped", + files={ + "orchestration/jobs/refresh_orders_job.toml": ( + 'name = "refresh_orders_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + "orchestration/sensors/overdue_sensor.py": ( + "from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor\n" + "SENSORS = {'overdue': sensor('overdue_sensor', description='Catch up.', " + "when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), " + "run='refresh_orders_job')}\n" + ), + }, + expected_error=ValueError, + expected_message=( + "sensor 'overdue_sensor' is declared here, but importing the module does not " + "define it" + ), ), SnapshotDiscoveryRejectionTestCase( description="a polling sensor needs an actual callback", From ffb8a24ef152908fb547246c76c4eb5fd60d201f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:18:27 +0000 Subject: [PATCH 27/34] fix: reject empty selector parts at compile and report calendar time zones at their line --- .../bindings/_helpers/discovery/orchestration_values.rs | 3 +-- .../sqlbuild/references/docs/concepts/orchestration.md | 8 ++++++-- .../commands/main/compile/test_python_project_layout.py | 6 ++++++ .../compiled_project/publication/test_calendar.py | 4 +++- website/src/content/docs/docs/concepts/orchestration.mdx | 8 ++++++-- 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs index f870bfde1..ad73ccbb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs @@ -101,8 +101,7 @@ fn selector_syntax(selector: &Positioned) -> Result<(), ValueError> { let parts = selector .value .split_whitespace() - .flat_map(|token| token.split(',')) - .filter(|part| !part.is_empty()); + .flat_map(|token| token.split(',')); for part in parts { parse_selector(part).map_err(|error| (selector.position, error.message))?; } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index 85cf73fd4..d37f7e9a8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -215,6 +215,9 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `files cannot sit directly in orchestration/` | Move jobs and groups to TOML files in `orchestration/jobs/` or `orchestration/job_groups/` | | `` `job` was removed from sqlbuild.orchestration`` | Declare the job in TOML and name it in the sensor's `run` | | `declares a second sensor; each module declares exactly one` | Split the module | +| `schedule cron: invalid cron expression: Pattern must have 5 fields when seconds are disallowed.` | Write a five-field cron, such as `0 2 * * *` | +| `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | +| `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | Details: @@ -222,5 +225,6 @@ Details: a job selector never imports a calendar or sensor module. - A job or group declared with `targets` is selectable only when the command's target is one of them, as on the server. -- A job's resources are checked when a location publishes, such as - `job nightly_orders_job resources: cpus must be a positive number`. +- Selectors are checked for syntax at compile. A selector that names a node the project does not + have fails when a location publishes, such as `job nightly_orders_job: unknown selector name + 'order_total'`. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py index f3e84056e..f47d268e9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py @@ -114,6 +114,12 @@ "orchestration/jobs/orders_job.toml:3:10:", "unknown selector type 'tagg' in 'tagg:finance'", ), + ( + "an empty intersection part is reported at the selector", + 'select = "tag:finance,"\n', + "orchestration/jobs/orders_job.toml:3:10:", + "empty selector", + ), ) ), PythonProjectLayoutCompileTestCase( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index 3818bbc16..3282ad96a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -59,7 +59,9 @@ "positive fixed duration", horizon="999999999999999999999999999d", ), - CalendarRejectionTestCase("unknown zone", "[]", "IANA timezone", timezone="Unknown/Orders"), + CalendarRejectionTestCase( + "unknown zone", "[]", "is not an IANA time zone", timezone="Unknown/Orders" + ), ], ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index 37da5681a..3aee7cb79 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -202,6 +202,9 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `files cannot sit directly in orchestration/` | Move jobs and groups to TOML files in `orchestration/jobs/` or `orchestration/job_groups/` | | `` `job` was removed from sqlbuild.orchestration`` | Declare the job in TOML and name it in the sensor's `run` | | `declares a second sensor; each module declares exactly one` | Split the module | +| `schedule cron: invalid cron expression: Pattern must have 5 fields when seconds are disallowed.` | Write a five-field cron, such as `0 2 * * *` | +| `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | +| `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | Details: @@ -209,5 +212,6 @@ Details: a job selector never imports a calendar or sensor module. - A job or group declared with `targets` is selectable only when the command's target is one of them, as on the server. -- A job's resources are checked when a location publishes, such as - `job nightly_orders_job resources: cpus must be a positive number`. +- Selectors are checked for syntax at compile. A selector that names a node the project does not + have fails when a location publishes, such as `job nightly_orders_job: unknown selector name + 'order_total'`. From 8a2e5747fcb0343ad63ce9dc4eef1f72e149e588 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:29:07 +0000 Subject: [PATCH 28/34] refactor: share the relative-path read error between discovery stages and orchestration --- .../_helpers/discovery/orchestration_files.rs | 15 +++++-------- .../src/bindings/_helpers/discovery/raised.rs | 21 +++++++++++++------ 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs index 7cc3388f8..9c71f624b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs @@ -10,8 +10,9 @@ use sqlbuild_discovery::orchestration_files::models::{ use sqlbuild_discovery::tree::models::ProjectTree; use crate::bindings::_helpers::discovery::orchestration_values::check_orchestration_values; -use crate::bindings::_helpers::discovery::project_files::path_object; -use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; +use crate::bindings::_helpers::discovery::raised::{ + failure_error, relative_read_error, stage_error, +}; /// Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). pub(crate) fn orchestration_declarations( @@ -32,18 +33,12 @@ fn orchestration_error( project_dir: &Bound<'_, PyAny>, failure: OrchestrationFailure, ) -> PyErr { - let py: Python<'_> = project_dir.py(); match failure { - OrchestrationFailure::Declaration(failure) => failure_error(py, failure), + OrchestrationFailure::Declaration(failure) => failure_error(project_dir.py(), failure), OrchestrationFailure::Unreadable { relative_path, failure, - } => match path_object(py, tree, relative_path) - .and_then(|relative| project_dir.div(relative.bind(py))) - { - Ok(file_path) => read_error(failure, &file_path), - Err(error) => error, - }, + } => relative_read_error((tree, project_dir), relative_path, failure), OrchestrationFailure::Stage(failure) => stage_error(tree, project_dir, failure), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs index 2920bebf3..cd32e4ceb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs @@ -113,16 +113,25 @@ pub(crate) fn stage_error( StageFailure::Unlistable { relative_path, error, - } => match path_object(project_dir.py(), tree, relative_path) - .and_then(|relative| project_dir.div(relative.bind(project_dir.py()))) - { - Ok(directory) => read_error(error, &directory), - Err(error) => error, - }, + } => relative_read_error((tree, project_dir), relative_path, error), StageFailure::Internal(reason) => PyRuntimeError::new_err(reason), } } +/// `read_error` for `relative_path` below `project_dir`, or the error building that path raised. +pub(crate) fn relative_read_error( + (tree, project_dir): (&ProjectTree, &Bound<'_, PyAny>), + relative_path: String, + failure: ReadFailure, +) -> PyErr { + match path_object(project_dir.py(), tree, relative_path) + .and_then(|relative| project_dir.div(relative.bind(project_dir.py()))) + { + Ok(path) => read_error(failure, &path), + Err(error) => error, + } +} + /// The error `read_error` builds for an operating-system read failure, for the platform oracle. #[pyfunction] #[pyo3(signature = (errno, winerror, message, *, file_path))] From 4d24002ec03a04485292c8a27a56f880b9d33b5e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:36:39 +0000 Subject: [PATCH 29/34] style: format the orchestration value checks --- .../src/orchestration_modules/_helpers/calls.rs | 4 +--- .../test_orchestration_module_declarations.rs | 15 ++++++++++++--- .../_helpers/discovery/orchestration_files.rs | 3 ++- .../_helpers/discovery/orchestration_values.rs | 7 ++++++- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs index 9122be97c..d4fd1b078 100644 --- a/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs +++ b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs @@ -53,9 +53,7 @@ pub(crate) fn imported_names(body: &[Stmt]) -> Vec<(String, String, TextSize)> { /// The dotted names a top-level `import sqlbuild.orchestration [as x]` or `from sqlbuild import orchestration [as x]` binds the module to. pub(crate) fn module_names(body: &[Stmt]) -> BTreeSet { - let (package, module) = ORCHESTRATION_MODULE - .rsplit_once('.') - .unwrap_or_default(); + let (package, module) = ORCHESTRATION_MODULE.rsplit_once('.').unwrap_or_default(); let mut names: BTreeSet = BTreeSet::new(); for statement in body { match statement { diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs index 811828cb5..4f79a0de1 100644 --- a/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs +++ b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs @@ -97,12 +97,16 @@ fn given_syntax_error_when_scanning_then_returns_none() { } #[test] -fn given_other_objects_calendar_and_sensor_methods_when_scanning_then_only_orchestration_calls_declare() { +fn given_other_objects_calendar_and_sensor_methods_when_scanning_then_only_orchestration_calls_declare() + { let module = scan( "import calendar\nfrom sqlbuild.orchestration import calendar as declare\n\n@declare(name=\"days_calendar\", description=\"d\", horizon=\"7d\")\ndef days(window, exchange):\n calendar.calendar(2026)\n return exchange.calendar(window.start)\n", ); assert_eq!(module.declarations.len(), 1); - assert_eq!(module.declarations[0].name, Some(text("days_calendar", 4, 15))); + assert_eq!( + module.declarations[0].name, + Some(text("days_calendar", 4, 15)) + ); } #[test] @@ -113,7 +117,12 @@ fn given_orchestration_module_attributes_when_scanning_then_each_import_form_dec let names: Vec> = module .declarations .iter() - .map(|declaration| declaration.name.as_ref().and_then(|name| name.value.clone())) + .map(|declaration| { + declaration + .name + .as_ref() + .and_then(|name| name.value.clone()) + }) .collect(); assert_eq!( names, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs index 9c71f624b..91ec16535 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs @@ -23,7 +23,8 @@ pub(crate) fn orchestration_declarations( let tree = ProjectTree::new(&root); let version = py.version_info(); let inventory = py.detach(|| orchestration_inventory(&tree, (version.major, version.minor))); - let inventory = inventory.map_err(|failure| orchestration_error(&tree, project_dir, failure))?; + let inventory = + inventory.map_err(|failure| orchestration_error(&tree, project_dir, failure))?; check_orchestration_values(py, &inventory).map_err(|failure| failure_error(py, failure))?; Ok(inventory) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs index ad73ccbb9..f4f0288eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs @@ -29,7 +29,12 @@ pub(crate) fn check_orchestration_values( .jobs .iter() .map(|(path, job)| (path, &job.run)) - .chain(inventory.groups.iter().map(|(path, group)| (path, &group.run))); + .chain( + inventory + .groups + .iter() + .map(|(path, group)| (path, &group.run)), + ); for (path, run) in runs { schedule_values(run).map_err(|error| failure(path, error))?; } From 2aa8e1a5b94f1255fdad4ba94c53eda5d938904e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:18:18 +0000 Subject: [PATCH 30/34] refactor: bring the orchestration readers and their tests in line with the project rules Move job and group file reading into the discovery helpers, split the inventory into named phases, keep Python object access at the native host edge, return schedules instead of filling caller collections, and restructure the new Rust and Python tests as branch-free case tables. --- .../_helpers/inventory_files.rs | 193 +++++++++ .../src/orchestration_files/_helpers/mod.rs | 3 + .../_helpers/python_modules.rs | 3 +- .../{main => _helpers}/read_group_file.rs | 5 +- .../{main => _helpers}/read_job_file.rs | 43 +- .../orchestration_files/_helpers/reading.rs | 42 +- .../_helpers/references.rs | 3 +- .../src/orchestration_files/constants.rs | 7 + .../src/orchestration_files/errors.rs | 11 + .../src/orchestration_files/main/mod.rs | 2 - .../main/orchestration_inventory.rs | 186 +-------- .../src/orchestration_files/mod.rs | 1 + .../src/orchestration_files/models.rs | 16 +- .../src/orchestration_files/tests.rs | 2 + .../src/orchestration_files/tests/helpers.rs | 107 +++++ .../tests/test_orchestration_inventory.rs | 391 +++++++++--------- .../tests/test_read_files.rs | 302 +++++++++----- .../orchestration_files/tests/test_types.rs | 43 +- .../main/orchestration_module_declarations.rs | 4 +- .../src/orchestration_modules/tests.rs | 4 + .../orchestration_modules/tests/helpers.rs | 40 ++ .../test_orchestration_module_declarations.rs | 238 +++++------ .../orchestration_modules/tests/test_types.rs | 7 + .../_helpers/compiled_project/job_catalog.rs | 1 - .../compiled_project/snapshot_declarations.rs | 94 ++--- .../native_host/orchestration_declarations.rs | 41 +- .../_helpers/planning/job_selectors.rs | 62 +-- src/sqlbuild/orchestration.py | 6 +- .../models/authoring.py | 10 +- .../cli/commands/main/compile/helpers.py | 11 + .../compile/test_python_project_layout.py | 121 +++--- .../cli/commands/main/refactor/helpers.py | 5 + .../main/refactor/test_job_selectors.py | 20 +- .../compiler/compiled_project/helpers.py | 12 +- .../publication/_test_types.py | 12 + .../compiled_project/publication/helpers.py | 14 +- .../publication/test_backfill.py | 1 + .../publication/test_calendar.py | 1 + .../publication/test_changed.py | 4 +- .../publication/test_custom_sensor.py | 1 + .../publication/test_docker_worker.py | 2 + .../publication/test_job_subsets.py | 1 + .../publication/test_local_recovery.py | 1 + .../publication/test_publication.py | 6 +- .../publication/test_rerun.py | 1 + .../publication/test_source_sensor.py | 3 + .../publication/test_worker.py | 6 + .../test_orchestration_snapshot.py | 144 +++++-- 48 files changed, 1365 insertions(+), 868 deletions(-) create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs rename crates/sqlbuild-discovery/src/orchestration_files/{main => _helpers}/read_group_file.rs (87%) rename crates/sqlbuild-discovery/src/orchestration_files/{main => _helpers}/read_job_file.rs (56%) create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/errors.rs create mode 100644 crates/sqlbuild-discovery/src/orchestration_files/tests/helpers.rs create mode 100644 crates/sqlbuild-pysource/src/orchestration_modules/tests/helpers.rs create mode 100644 crates/sqlbuild-pysource/src/orchestration_modules/tests/test_types.rs diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs new file mode 100644 index 000000000..85ba6a6b4 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs @@ -0,0 +1,193 @@ +//! The phases of the orchestration inventory: the folder layout, then each kind's files. + +use sqlbuild_pysource::orchestration_modules::models::PythonDeclarationKind; + +use crate::_helpers::reading::read_authored_text; +use crate::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::orchestration_files::_helpers::python_modules::module_declaration; +use crate::orchestration_files::_helpers::read_group_file::read_group_file; +use crate::orchestration_files::_helpers::read_job_file::read_job_file; +use crate::orchestration_files::constants::{ + BYTECODE_DIRECTORY, CALENDARS_DIRECTORY, GROUPS_DIRECTORY, HIDDEN_PREFIX, JOBS_DIRECTORY, + ROLE_DIRECTORIES, SENSORS_DIRECTORY, TOML_FILE_SUFFIX, TOML_JOBS_HELP, +}; +use crate::orchestration_files::errors::FileError; +use crate::orchestration_files::models::{ + FilePosition, GroupFile, JobFile, NamedModule, OrchestrationFailure, PythonModules, +}; +use crate::tree::main::children::children; +use crate::tree::main::rglob::rglob; +use crate::tree::models::{ProjectTree, TreeEntry}; + +/// `D017` at a file and position, with optional help. +pub(crate) fn failure(path: &str, error: FileError) -> OrchestrationFailure { + let FilePosition { line, column } = error.position; + OrchestrationFailure::Declaration(DiscoveryFailure { + kind: FailureKind::Orchestration, + message: format!("{path}:{line}:{column}: {}", error.message), + help: error.help, + }) +} + +fn layout(path: &str, message: String, help: Option) -> OrchestrationFailure { + failure( + path, + FileError { + message, + position: FilePosition { line: 1, column: 1 }, + help, + }, + ) +} + +fn ignored(entry: &TreeEntry) -> bool { + entry.name.starts_with(HIDDEN_PREFIX) || entry.name == BYTECODE_DIRECTORY +} + +fn read(tree: &ProjectTree, path: &str) -> Result { + if tree.is_undecodable(path) { + return Err(layout( + path, + "path is not valid UTF-8; rename it so SQLBuild can read it".to_owned(), + None, + )); + } + read_authored_text(&tree.absolute(path)).map_err(|failure| OrchestrationFailure::Unreadable { + relative_path: path.to_owned(), + failure, + }) +} + +/// Refuse a file directly in `orchestration/` or a folder no declaration kind owns. +pub(crate) fn check_top_level(tree: &ProjectTree) -> Result<(), OrchestrationFailure> { + for (path, entry) in children(tree, ORCHESTRATION_DIRECTORY)? { + if ignored(&entry) { + continue; + } + if !entry.is_dir { + return Err(layout( + &path, + "files cannot sit directly in orchestration/".to_owned(), + Some(if entry.name.ends_with(PYTHON_FILE_SUFFIX) { + TOML_JOBS_HELP.to_owned() + } else { + format!( + "move it into one of orchestration/{}/", + ROLE_DIRECTORIES.join("/, orchestration/") + ) + }), + )); + } + if !ROLE_DIRECTORIES.contains(&entry.name.as_str()) { + let holds_python: bool = !rglob(tree, &path, |found| { + found.name.ends_with(PYTHON_FILE_SUFFIX) + })? + .is_empty(); + return Err(layout( + &path, + format!("unknown orchestration folder '{}'", entry.name), + Some(if holds_python { + TOML_JOBS_HELP.to_owned() + } else { + format!("use one of: {}", ROLE_DIRECTORIES.join(", ")) + }), + )); + } + } + Ok(()) +} + +/// The files below one role folder, refusing any without `suffix`. +fn role_files( + tree: &ProjectTree, + role: &str, + suffix: &str, +) -> Result, OrchestrationFailure> { + let root: String = format!("{ORCHESTRATION_DIRECTORY}/{role}"); + let files: Vec = rglob(tree, &root, |entry| !entry.is_dir)?; + let mut kept: Vec = Vec::new(); + for path in files { + let hidden: bool = path + .split('/') + .any(|part| part.starts_with(HIDDEN_PREFIX) || part == BYTECODE_DIRECTORY); + if hidden { + continue; + } + if !path.ends_with(suffix) { + let python_job: bool = path.ends_with(PYTHON_FILE_SUFFIX) + && matches!(role, JOBS_DIRECTORY | GROUPS_DIRECTORY); + return Err(layout( + &path, + format!("orchestration/{role}/ holds only {suffix} files"), + Some(match role { + _ if python_job => TOML_JOBS_HELP.to_owned(), + JOBS_DIRECTORY | GROUPS_DIRECTORY => { + "jobs and job groups are TOML; calendars and sensors are Python in orchestration/calendars/ and orchestration/sensors/".to_owned() + } + _ => "jobs and job groups are TOML in orchestration/jobs/ and orchestration/job_groups/".to_owned(), + }), + )); + } + kept.push(path); + } + Ok(kept) +} + +/// Every job file with its path. +pub(crate) fn read_jobs( + tree: &ProjectTree, +) -> Result, OrchestrationFailure> { + let mut jobs: Vec<(String, JobFile)> = Vec::new(); + for path in role_files(tree, JOBS_DIRECTORY, TOML_FILE_SUFFIX)? { + let job: JobFile = + read_job_file(&read(tree, &path)?).map_err(|error| failure(&path, error))?; + jobs.push((path, job)); + } + Ok(jobs) +} + +/// Every job group file with its path. +pub(crate) fn read_groups( + tree: &ProjectTree, +) -> Result, OrchestrationFailure> { + let mut groups: Vec<(String, GroupFile)> = Vec::new(); + for path in role_files(tree, GROUPS_DIRECTORY, TOML_FILE_SUFFIX)? { + let group: GroupFile = + read_group_file(&read(tree, &path)?).map_err(|error| failure(&path, error))?; + groups.push((path, group)); + } + Ok(groups) +} + +/// Every calendar and sensor module, read without importing it. +pub(crate) fn read_python_modules( + tree: &ProjectTree, + python_version: (u8, u8), +) -> Result { + let mut found = PythonModules::default(); + for (role, kind) in [ + (CALENDARS_DIRECTORY, PythonDeclarationKind::Calendar), + (SENSORS_DIRECTORY, PythonDeclarationKind::Sensor), + ] { + for path in role_files(tree, role, PYTHON_FILE_SUFFIX)? { + let stem: &str = path + .rsplit('/') + .next() + .and_then(|name| name.strip_suffix(PYTHON_FILE_SUFFIX)) + .unwrap_or_default(); + let declared: Option = + module_declaration(&read(tree, &path)?, stem, kind, python_version) + .map_err(|error| failure(&path, error))?; + if let Some(mut module) = declared { + module.path = path.clone(); + match kind { + PythonDeclarationKind::Calendar => found.calendars.push(module), + PythonDeclarationKind::Sensor => found.sensors.push(module), + } + } + found.modules.push(path); + } + } + Ok(found) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/mod.rs index 68584816f..bfe96c904 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/mod.rs @@ -1,4 +1,7 @@ +pub(crate) mod inventory_files; pub(crate) mod python_modules; pub(crate) mod raw; +pub(crate) mod read_group_file; +pub(crate) mod read_job_file; pub(crate) mod reading; pub(crate) mod references; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs index 4fc66ffad..f7c3b11ab 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs @@ -8,7 +8,8 @@ use sqlbuild_pysource::orchestration_modules::models::{ use crate::orchestration_files::constants::{ CALENDAR_KIND, PRIVATE_PREFIX, SENSOR_KIND, TOML_JOBS_HELP, }; -use crate::orchestration_files::models::{FileError, FilePosition, NamedModule, Positioned}; +use crate::orchestration_files::errors::FileError; +use crate::orchestration_files::models::{FilePosition, NamedModule, Positioned}; fn file_position(position: SourcePosition) -> FilePosition { FilePosition { diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/read_group_file.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/read_group_file.rs similarity index 87% rename from crates/sqlbuild-discovery/src/orchestration_files/main/read_group_file.rs rename to crates/sqlbuild-discovery/src/orchestration_files/_helpers/read_group_file.rs index 503b8f2e0..2b0a6e40c 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/main/read_group_file.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/read_group_file.rs @@ -7,10 +7,11 @@ use crate::orchestration_files::_helpers::reading::{ check_keys, error_at, nonblank, run_file, toml_error, }; use crate::orchestration_files::constants::GROUP_KEYS; -use crate::orchestration_files::models::{FileError, GroupFile, Positioned}; +use crate::orchestration_files::errors::FileError; +use crate::orchestration_files::models::{GroupFile, Positioned}; /// The job group `text` declares, or the first problem with it and where it is. -pub fn read_group_file(text: &str) -> Result { +pub(crate) fn read_group_file(text: &str) -> Result { check_keys(text, &GROUP_KEYS, "job group")?; let run: RawRun = toml::from_str(text).map_err(|error| toml_error(text, &error))?; let raw: RawGroup = toml::from_str(text).map_err(|error| toml_error(text, &error))?; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/read_job_file.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/read_job_file.rs similarity index 56% rename from crates/sqlbuild-discovery/src/orchestration_files/main/read_job_file.rs rename to crates/sqlbuild-discovery/src/orchestration_files/_helpers/read_job_file.rs index 1436c6fe8..a288ee813 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/main/read_job_file.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/read_job_file.rs @@ -2,29 +2,25 @@ use std::collections::BTreeMap; +use toml::Spanned; + use crate::orchestration_files::_helpers::raw::{RawJob, RawRun}; use crate::orchestration_files::_helpers::reading::{ - check_keys, nonblank, position, positioned, run_file, toml_error, + check_keys, error_at, nonblank, position, positioned, run_file, toml_error, }; use crate::orchestration_files::constants::JOB_KEYS; -use crate::orchestration_files::models::{FileError, FilePosition, JobFile, Positioned}; +use crate::orchestration_files::errors::FileError; +use crate::orchestration_files::models::{FilePosition, JobFile, Positioned}; /// The job `text` declares, or the first problem with it and where it is. -pub fn read_job_file(text: &str) -> Result { +pub(crate) fn read_job_file(text: &str) -> Result { check_keys(text, &JOB_KEYS, "job")?; let run: RawRun = toml::from_str(text).map_err(|error| toml_error(text, &error))?; let raw: RawJob = toml::from_str(text).map_err(|error| toml_error(text, &error))?; - let resources = raw.resources.map(|resources| { - let start: usize = resources.span().start; - let table = serde_json::to_value(resources.into_inner()) - .ok() - .and_then(|value| value.as_object().cloned()) - .unwrap_or_default(); - Positioned { - value: table, - position: position(text, start), - } - }); + let resources: Option>> = raw + .resources + .map(|resources| json_table(text, resources)) + .transpose()?; let mut settings: BTreeMap = BTreeMap::new(); let mut setting_positions: BTreeMap = BTreeMap::new(); for (key, value) in raw.settings { @@ -47,3 +43,22 @@ pub fn read_job_file(text: &str) -> Result { setting_positions, }) } + +/// A TOML table as JSON, positioned where it is written. +fn json_table( + text: &str, + table: Spanned, +) -> Result>, FileError> { + let start: usize = table.span().start; + match serde_json::to_value(table.into_inner()) { + Ok(serde_json::Value::Object(value)) => Ok(Positioned { + value, + position: position(text, start), + }), + Ok(_) | Err(_) => Err(error_at( + text, + start, + "resources must be a table of plain values".to_owned(), + )), + } +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs index 296b09aad..d44c1aec1 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs @@ -10,8 +10,9 @@ use crate::orchestration_files::_helpers::raw::{RawOnFailure, RawRun, RawSchedul use crate::orchestration_files::constants::{ CONCURRENCY_KEY, RETRIES_RANGE, RETRY_DELAY_RANGE, SUGGESTION_CUTOFF, }; +use crate::orchestration_files::errors::FileError; use crate::orchestration_files::models::{ - FileError, FilePosition, OnFailure, Positioned, RunFile, ScheduleFile, ScheduleSource, + FilePosition, OnFailure, Positioned, RunFile, ScheduleFile, ScheduleSource, }; /// The 1-based line and character column of byte `offset` in `text`. @@ -130,27 +131,20 @@ fn on_failure(text: &str, raw: Spanned) -> Result) -> Result Result { - let owner: Option = raw + let owner: Option> = raw .owner - .map(|owner| nonblank(text, owner, "owner").map(|owner| owner.value)) + .map(|owner| nonblank(text, owner, "owner")) .transpose()?; let mut groups: BTreeSet<&str> = BTreeSet::new(); if let Some(repeated) = raw @@ -179,7 +173,7 @@ pub(crate) fn run_file(text: &str, raw: RawRun) -> Result { name: nonblank(text, raw.name, "name")?, description: nonblank(text, raw.description, "description")?.value, targets: raw.targets, - owner, + owner: owner.map(|owner| owner.value), overlap: raw.overlap, concurrency_groups: raw.concurrency_groups, schedule: raw diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs index acd076023..c8df4e054 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs @@ -5,8 +5,9 @@ use std::collections::BTreeSet; use crate::orchestration_files::constants::{ CALENDAR_KIND, CALENDARS_DIRECTORY, GROUP_KIND, JOB_KIND, JOBS_DIRECTORY, SENSOR_KIND, }; +use crate::orchestration_files::errors::FileError; use crate::orchestration_files::models::{ - FileError, FilePosition, OrchestrationInventory, Positioned, RunFile, ScheduleSource, + FilePosition, OrchestrationInventory, Positioned, RunFile, ScheduleSource, }; /// The first problem with a name or reference, and the file it is in. diff --git a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs index 078def0d3..c722549fb 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs @@ -54,3 +54,10 @@ pub(crate) const BYTECODE_DIRECTORY: &str = "__pycache__"; /// The prefix of hidden files and folders, and of private support modules. pub(crate) const HIDDEN_PREFIX: char = '.'; pub(crate) const PRIVATE_PREFIX: char = '_'; +/// Every folder `orchestration/` may hold. +pub const ROLE_DIRECTORIES: [&str; 4] = [ + JOBS_DIRECTORY, + GROUPS_DIRECTORY, + CALENDARS_DIRECTORY, + SENSORS_DIRECTORY, +]; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/errors.rs b/crates/sqlbuild-discovery/src/orchestration_files/errors.rs new file mode 100644 index 000000000..231cad2bf --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/errors.rs @@ -0,0 +1,11 @@ +//! Why a declaration file cannot be read. + +use crate::orchestration_files::models::FilePosition; + +/// Why a declaration file cannot be read, at the position it points to. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FileError { + pub message: String, + pub position: FilePosition, + pub help: Option, +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs index 619ee67d4..086c012c6 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs @@ -1,3 +1 @@ pub mod orchestration_inventory; -pub mod read_group_file; -pub mod read_job_file; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs index 280112973..d1837fa2b 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs @@ -1,180 +1,38 @@ //! Discover, read and check every orchestration declaration without importing project Python. -use sqlbuild_pysource::orchestration_modules::models::PythonDeclarationKind; - -use crate::_helpers::reading::read_authored_text; -use crate::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; -use crate::models::{DiscoveryFailure, FailureKind}; -use crate::orchestration_files::_helpers::python_modules::module_declaration; -use crate::orchestration_files::_helpers::references::check_references; -use crate::orchestration_files::constants::{ - BYTECODE_DIRECTORY, CALENDARS_DIRECTORY, GROUPS_DIRECTORY, HIDDEN_PREFIX, JOBS_DIRECTORY, - SENSORS_DIRECTORY, TOML_FILE_SUFFIX, TOML_JOBS_HELP, +use crate::constants::ORCHESTRATION_DIRECTORY; +use crate::orchestration_files::_helpers::inventory_files::{ + check_top_level, failure, read_groups, read_jobs, read_python_modules, }; -use crate::orchestration_files::main::read_group_file::read_group_file; -use crate::orchestration_files::main::read_job_file::read_job_file; +use crate::orchestration_files::_helpers::references::check_references; use crate::orchestration_files::models::{ - FileError, FilePosition, OrchestrationFailure, OrchestrationInventory, + GroupFile, JobFile, OrchestrationFailure, OrchestrationInventory, PythonModules, }; -use crate::tree::main::children::children; -use crate::tree::main::rglob::rglob; -use crate::tree::models::{ProjectTree, TreeEntry}; - -const ROLE_DIRECTORIES: [&str; 4] = [ - JOBS_DIRECTORY, - GROUPS_DIRECTORY, - CALENDARS_DIRECTORY, - SENSORS_DIRECTORY, -]; - -/// `D017` at a file and position, with optional help. -fn failure(path: &str, error: FileError) -> OrchestrationFailure { - let FilePosition { line, column } = error.position; - OrchestrationFailure::Declaration(DiscoveryFailure { - kind: FailureKind::Orchestration, - message: format!("{path}:{line}:{column}: {}", error.message), - help: error.help, - }) -} - -fn layout(path: &str, message: String, help: Option) -> OrchestrationFailure { - failure( - path, - FileError { - message, - position: FilePosition { line: 1, column: 1 }, - help, - }, - ) -} - -fn ignored(entry: &TreeEntry) -> bool { - entry.name.starts_with(HIDDEN_PREFIX) || entry.name == BYTECODE_DIRECTORY -} - -fn read(tree: &ProjectTree, path: &str) -> Result { - if tree.is_undecodable(path) { - return Err(layout( - path, - "path is not valid UTF-8; rename it so SQLBuild can read it".to_owned(), - None, - )); - } - read_authored_text(&tree.absolute(path)).map_err(|failure| OrchestrationFailure::Unreadable { - relative_path: path.to_owned(), - failure, - }) -} - -/// The files below one role folder, refusing any without `suffix`. -fn role_files( - tree: &ProjectTree, - role: &str, - suffix: &str, -) -> Result, OrchestrationFailure> { - let root: String = format!("{ORCHESTRATION_DIRECTORY}/{role}"); - let files: Vec = rglob(tree, &root, |entry| !entry.is_dir)?; - let mut kept: Vec = Vec::new(); - for path in files { - let hidden: bool = path - .split('/') - .any(|part| part.starts_with(HIDDEN_PREFIX) || part == BYTECODE_DIRECTORY); - if hidden { - continue; - } - if !path.ends_with(suffix) { - let python_job: bool = path.ends_with(PYTHON_FILE_SUFFIX) - && matches!(role, JOBS_DIRECTORY | GROUPS_DIRECTORY); - return Err(layout( - &path, - format!("orchestration/{role}/ holds only {suffix} files"), - Some(match role { - _ if python_job => TOML_JOBS_HELP.to_owned(), - JOBS_DIRECTORY | GROUPS_DIRECTORY => { - "jobs and job groups are TOML; calendars and sensors are Python in orchestration/calendars/ and orchestration/sensors/".to_owned() - } - _ => "jobs and job groups are TOML in orchestration/jobs/ and orchestration/job_groups/".to_owned(), - }), - )); - } - kept.push(path); - } - Ok(kept) -} +use crate::tree::models::ProjectTree; /// Every job, group, calendar and sensor below `orchestration/`, or the first problem with them. pub fn orchestration_inventory( tree: &ProjectTree, python_version: (u8, u8), ) -> Result { - let mut inventory = OrchestrationInventory::default(); if !tree.is_dir(ORCHESTRATION_DIRECTORY) { - return Ok(inventory); - } - for (path, entry) in children(tree, ORCHESTRATION_DIRECTORY)? { - if ignored(&entry) { - continue; - } - if !entry.is_dir { - return Err(layout( - &path, - "files cannot sit directly in orchestration/".to_owned(), - Some(if entry.name.ends_with(PYTHON_FILE_SUFFIX) { - TOML_JOBS_HELP.to_owned() - } else { - format!( - "move it into one of orchestration/{}/", - ROLE_DIRECTORIES.join("/, orchestration/") - ) - }), - )); - } - if !ROLE_DIRECTORIES.contains(&entry.name.as_str()) { - let holds_python: bool = !rglob(tree, &path, |found| { - found.name.ends_with(PYTHON_FILE_SUFFIX) - })? - .is_empty(); - return Err(layout( - &path, - format!("unknown orchestration folder '{}'", entry.name), - Some(if holds_python { - TOML_JOBS_HELP.to_owned() - } else { - format!("use one of: {}", ROLE_DIRECTORIES.join(", ")) - }), - )); - } - } - for path in role_files(tree, JOBS_DIRECTORY, TOML_FILE_SUFFIX)? { - let job = read_job_file(&read(tree, &path)?).map_err(|error| failure(&path, error))?; - inventory.jobs.push((path, job)); - } - for path in role_files(tree, GROUPS_DIRECTORY, TOML_FILE_SUFFIX)? { - let group = read_group_file(&read(tree, &path)?).map_err(|error| failure(&path, error))?; - inventory.groups.push((path, group)); - } - for (role, kind) in [ - (CALENDARS_DIRECTORY, PythonDeclarationKind::Calendar), - (SENSORS_DIRECTORY, PythonDeclarationKind::Sensor), - ] { - for path in role_files(tree, role, PYTHON_FILE_SUFFIX)? { - let stem: &str = path - .rsplit('/') - .next() - .and_then(|name| name.strip_suffix(PYTHON_FILE_SUFFIX)) - .unwrap_or_default(); - let declared = module_declaration(&read(tree, &path)?, stem, kind, python_version) - .map_err(|error| failure(&path, error))?; - if let Some(mut module) = declared { - module.path = path.clone(); - match kind { - PythonDeclarationKind::Calendar => inventory.calendars.push(module), - PythonDeclarationKind::Sensor => inventory.sensors.push(module), - } - } - inventory.modules.push(path); - } + return Ok(OrchestrationInventory::default()); } + check_top_level(tree)?; + let jobs: Vec<(String, JobFile)> = read_jobs(tree)?; + let groups: Vec<(String, GroupFile)> = read_groups(tree)?; + let PythonModules { + calendars, + sensors, + modules, + } = read_python_modules(tree, python_version)?; + let inventory: OrchestrationInventory = OrchestrationInventory { + jobs, + groups, + calendars, + sensors, + modules, + }; check_references(&inventory).map_err(|(path, error)| failure(&path, error))?; Ok(inventory) } diff --git a/crates/sqlbuild-discovery/src/orchestration_files/mod.rs b/crates/sqlbuild-discovery/src/orchestration_files/mod.rs index 9ed28e529..3c98387f0 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/mod.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod _helpers; pub mod constants; +pub mod errors; pub mod main; pub mod models; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/models.rs b/crates/sqlbuild-discovery/src/orchestration_files/models.rs index 4e40a8914..26fbd25f5 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/models.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/models.rs @@ -46,14 +46,6 @@ pub struct Positioned { pub position: FilePosition, } -/// Why a declaration file cannot be read, at the position it points to. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct FileError { - pub message: String, - pub position: FilePosition, - pub help: Option, -} - /// What starts a job or group on its own: a cron expression or a named calendar. #[derive(Clone, Debug, PartialEq, Eq)] pub enum ScheduleSource { @@ -117,6 +109,14 @@ pub struct NamedModule { pub targets: Option>, } +/// The calendars, sensors and every module below their folders. +#[derive(Debug, Default)] +pub(crate) struct PythonModules { + pub(crate) calendars: Vec, + pub(crate) sensors: Vec, + pub(crate) modules: Vec, +} + /// Every orchestration declaration in a project, checked for layout, names and references. #[derive(Clone, Debug, Default, PartialEq)] pub struct OrchestrationInventory { diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests.rs index 5fc6ae351..5995ff0d9 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_orchestration_inventory.rs"] mod orchestration_inventory; #[path = "tests/test_read_files.rs"] diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/helpers.rs new file mode 100644 index 000000000..5095f0de3 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/helpers.rs @@ -0,0 +1,107 @@ +use std::path::Path; + +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::orchestration_files::_helpers::read_group_file::read_group_file; +use crate::orchestration_files::main::orchestration_inventory::orchestration_inventory; +use crate::orchestration_files::models::{ + FilePosition, OrchestrationFailure, OrchestrationInventory, Positioned, +}; +use crate::tree::models::ProjectTree; + +pub(super) const EXPORT_JOB: &str = "name = \"orders_export_job\"\ndescription = \"d\"\nselect = \"orders\"\nschedule = { calendar = \"trading_days_calendar\", timezone = \"UTC\" }\n"; +pub(super) const NIGHTLY_GROUP: &str = + "name = \"orders_nightly_job_group\"\ndescription = \"d\"\njobs = [\"orders_export_job\"]\n"; +pub(super) const CALENDAR: &str = "from sqlbuild.orchestration import calendar\n\n@calendar(name=\"trading_days_calendar\", description=\"d\", horizon=\"7d\")\ndef trading_days(window):\n return []\n"; +pub(super) const SENSOR: &str = "from sqlbuild.orchestration import run_event, sensor\n\nstale = sensor(name=\"stock_stale_sensor\", description=\"d\", when=run_event(job=\"orders_export_job\", kinds=[]), run=\"orders_nightly_job_group\")\n"; + +/// A 1-based line and column. +pub(super) fn at(line: usize, column: usize) -> FilePosition { + FilePosition { line, column } +} + +/// A string value written at `line` and `column`. +pub(super) fn written(value: &str, line: usize, column: usize) -> Positioned { + Positioned { + value: value.to_owned(), + position: at(line, column), + } +} + +/// A group's members with their positions, or its error's message and position. +pub(super) fn group_members( + text: &str, +) -> Result, (String, FilePosition)> { + read_group_file(text) + .map(|group| group.jobs.into_iter().map(member).collect()) + .map_err(|error| (error.message, error.position)) +} + +/// A group member's name and position. +fn member(job: Positioned) -> (String, FilePosition) { + (job.value, job.position) +} + +/// A temporary project holding `files`. +pub(super) fn project(files: &[(&str, &str)]) -> tempfile::TempDir { + let directory = tempfile::tempdir().expect("temporary project"); + for (path, text) in files { + let path = directory.path().join(path); + std::fs::create_dir_all(path.parent().expect("parent")).expect("folders"); + std::fs::write(path, text).expect("file"); + } + directory +} + +/// The inventory of the project at `root`. +pub(super) fn inventory(root: &Path) -> Result { + orchestration_inventory(&ProjectTree::new(root), (3, 12)) +} + +/// Why the inventory of a project holding `files` fails. +pub(super) fn inventory_failure(files: &[(&str, &str)]) -> OrchestrationFailure { + let directory = project(files); + inventory(directory.path()).expect_err("invalid project") +} + +/// A `D017` declaration failure with `message` and `help`. +pub(super) fn declaration_failure(message: &str, help: Option<&str>) -> OrchestrationFailure { + OrchestrationFailure::Declaration(DiscoveryFailure { + kind: FailureKind::Orchestration, + message: message.to_owned(), + help: help.map(str::to_owned), + }) +} + +/// A declared name with its kind and file. +pub(super) fn named(name: &str, kind: &'static str, path: &str) -> (String, &'static str, String) { + (name.to_owned(), kind, path.to_owned()) +} + +/// A project declaring one of each kind, plus files the inventory skips. +pub(super) fn valid() -> Vec<(&'static str, &'static str)> { + vec![ + ("orchestration/jobs/orders/export.toml", EXPORT_JOB), + ( + "orchestration/job_groups/orders/nightly.toml", + NIGHTLY_GROUP, + ), + ("orchestration/calendars/trading_days.py", CALENDAR), + ("orchestration/calendars/_holidays.py", "HOLIDAYS = []\n"), + ("orchestration/sensors/inventory/stock.py", SENSOR), + ( + "orchestration/sensors/__pycache__/stock.cpython-312.pyc", + "x", + ), + ] +} + +/// The valid project with the file at `path` replaced by, or added as, `text`. +pub(super) fn with_file( + path: &'static str, + text: &'static str, +) -> Vec<(&'static str, &'static str)> { + let mut files: Vec<(&'static str, &'static str)> = valid(); + files.retain(|(existing, _)| *existing != path); + files.push((path, text)); + files +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs index 2c8cd3ab1..0e63db921 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs @@ -1,219 +1,200 @@ -use std::path::Path; +use crate::orchestration_files::tests::helpers::{ + EXPORT_JOB, SENSOR, declaration_failure, inventory, inventory_failure, named, project, valid, + with_file, +}; +use crate::orchestration_files::tests::test_types::{InventoryErrorTestCase, InventoryTestCase}; -use crate::orchestration_files::main::orchestration_inventory::orchestration_inventory; -use crate::orchestration_files::models::{OrchestrationFailure, OrchestrationInventory}; -use crate::orchestration_files::tests::test_types::InventoryErrorCase; -use crate::tree::models::ProjectTree; - -const EXPORT_JOB: &str = "name = \"orders_export_job\"\ndescription = \"d\"\nselect = \"orders\"\nschedule = { calendar = \"trading_days_calendar\", timezone = \"UTC\" }\n"; -const NIGHTLY_GROUP: &str = - "name = \"orders_nightly_job_group\"\ndescription = \"d\"\njobs = [\"orders_export_job\"]\n"; -const CALENDAR: &str = "from sqlbuild.orchestration import calendar\n\n@calendar(name=\"trading_days_calendar\", description=\"d\", horizon=\"7d\")\ndef trading_days(window):\n return []\n"; -const SENSOR: &str = "from sqlbuild.orchestration import run_event, sensor\n\nstale = sensor(name=\"stock_stale_sensor\", description=\"d\", when=run_event(job=\"orders_export_job\", kinds=[]), run=\"orders_nightly_job_group\")\n"; - -fn project(files: &[(&str, &str)]) -> tempfile::TempDir { - let directory = tempfile::tempdir().expect("temporary project"); - for (path, text) in files { - let path = directory.path().join(path); - std::fs::create_dir_all(path.parent().expect("parent")).expect("folders"); - std::fs::write(path, text).expect("file"); - } - directory -} - -fn inventory(root: &Path) -> Result { - orchestration_inventory(&ProjectTree::new(root), (3, 12)) -} - -fn message(files: &[(&str, &str)]) -> (String, Option) { - let directory = project(files); - match inventory(directory.path()).expect_err("invalid project") { - OrchestrationFailure::Declaration(failure) => (failure.message, failure.help), - other => panic!("unexpected failure {other:?}"), +#[test] +fn given_projects_when_inventorying_then_reads_each_kind_and_names_them() { + let test_cases = [ + InventoryTestCase { + description: "one declaration of each kind, skipping private modules and bytecode", + files: valid(), + expected_names: vec![ + named( + "orders_export_job", + "job", + "orchestration/jobs/orders/export.toml", + ), + named( + "orders_nightly_job_group", + "job group", + "orchestration/job_groups/orders/nightly.toml", + ), + named( + "trading_days_calendar", + "calendar", + "orchestration/calendars/trading_days.py", + ), + named( + "stock_stale_sensor", + "sensor", + "orchestration/sensors/inventory/stock.py", + ), + ], + expected_modules: vec![ + "orchestration/calendars/_holidays.py", + "orchestration/calendars/trading_days.py", + "orchestration/sensors/inventory/stock.py", + ], + expected_empty: false, + }, + InventoryTestCase { + description: "no orchestration folder", + files: vec![("models/orders.sql", "select 1")], + expected_names: Vec::new(), + expected_modules: Vec::new(), + expected_empty: true, + }, + ]; + for test_case in test_cases { + let directory = project(&test_case.files); + let found = inventory(directory.path()).expect(test_case.description); + let names: Vec<(String, &str, String)> = found + .names() + .into_iter() + .map(|fact| (fact.name, fact.kind, fact.path)) + .collect(); + assert_eq!(names, test_case.expected_names, "{}", test_case.description); + assert_eq!( + found.modules, test_case.expected_modules, + "{}", + test_case.description + ); + assert_eq!( + found.is_empty(), + test_case.expected_empty, + "{}", + test_case.description + ); } } -fn valid() -> Vec<(&'static str, &'static str)> { - vec![ - ("orchestration/jobs/orders/export.toml", EXPORT_JOB), - ( - "orchestration/job_groups/orders/nightly.toml", - NIGHTLY_GROUP, - ), - ("orchestration/calendars/trading_days.py", CALENDAR), - ("orchestration/calendars/_holidays.py", "HOLIDAYS = []\n"), - ("orchestration/sensors/inventory/stock.py", SENSOR), - ( - "orchestration/sensors/__pycache__/stock.cpython-312.pyc", - "x", - ), - ] -} - #[test] -fn given_valid_layout_when_inventorying_then_reads_each_kind_and_names_them() { - let directory = project(&valid()); - let found = inventory(directory.path()).expect("valid project"); - let names: Vec<(String, &str, String)> = found - .names() - .into_iter() - .map(|fact| (fact.name, fact.kind, fact.path)) - .collect(); - assert_eq!( - names, - [ - ( - "orders_export_job".into(), - "job", - "orchestration/jobs/orders/export.toml".into() +fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_fix() { + let test_cases = [ + InventoryErrorTestCase { + description: "loose file", + files: with_file("orchestration/jobs.py", "x = 1\n"), + expected_message: "orchestration/jobs.py:1:1: files cannot sit directly in orchestration/", + expected_help: Some( + "jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/", ), - ( - "orders_nightly_job_group".into(), - "job group", - "orchestration/job_groups/orders/nightly.toml".into() + }, + InventoryErrorTestCase { + description: "loose non-Python file", + files: with_file("orchestration/nightly.toml", EXPORT_JOB), + expected_message: "orchestration/nightly.toml:1:1: files cannot sit directly in orchestration/", + expected_help: Some( + "move it into one of orchestration/jobs/, orchestration/job_groups/, orchestration/calendars/, orchestration/sensors/", ), - ( - "trading_days_calendar".into(), - "calendar", - "orchestration/calendars/trading_days.py".into() + }, + InventoryErrorTestCase { + description: "unknown folder of Python modules", + files: with_file("orchestration/pipelines/daily/orders.py", "x = 1\n"), + expected_message: "orchestration/pipelines:1:1: unknown orchestration folder 'pipelines'", + expected_help: Some( + "jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/", ), - ( - "stock_stale_sensor".into(), - "sensor", - "orchestration/sensors/inventory/stock.py".into() + }, + InventoryErrorTestCase { + description: "unknown folder", + files: with_file("orchestration/schedules/nightly.toml", EXPORT_JOB), + expected_message: "orchestration/schedules:1:1: unknown orchestration folder 'schedules'", + expected_help: Some("use one of: jobs, job_groups, calendars, sensors"), + }, + InventoryErrorTestCase { + description: "python job", + files: with_file("orchestration/jobs/orders/legacy.py", "x = 1\n"), + expected_message: "orchestration/jobs/orders/legacy.py:1:1: orchestration/jobs/ holds only .toml files", + expected_help: Some( + "jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/", ), - ] - ); - assert_eq!( - found.modules, - [ - "orchestration/calendars/_holidays.py", - "orchestration/calendars/trading_days.py", - "orchestration/sensors/inventory/stock.py", - ] - ); -} - -#[test] -fn given_no_orchestration_folder_when_inventorying_then_is_empty() { - let directory = project(&[("models/orders.sql", "select 1")]); - assert!( - inventory(directory.path()) - .expect("no declarations") - .is_empty() - ); -} - -#[test] -fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_fix() { - let mut cases: Vec = Vec::new(); - let with = |path: &'static str, text: &'static str| { - let mut files = valid(); - files.retain(|(existing, _)| *existing != path); - files.push((path, text)); - files - }; - cases.push(InventoryErrorCase { - description: "loose file", - files: with("orchestration/jobs.py", "x = 1\n"), - expected_message: "orchestration/jobs.py:1:1: files cannot sit directly in orchestration/", - expected_help: Some("jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/"), - }); - cases.push(InventoryErrorCase { - description: "loose non-Python file", - files: with("orchestration/nightly.toml", EXPORT_JOB), - expected_message: "orchestration/nightly.toml:1:1: files cannot sit directly in orchestration/", - expected_help: Some("move it into one of orchestration/jobs/, orchestration/job_groups/, orchestration/calendars/, orchestration/sensors/"), - }); - cases.push(InventoryErrorCase { - description: "unknown folder of Python modules", - files: with("orchestration/pipelines/daily/orders.py", "x = 1\n"), - expected_message: "orchestration/pipelines:1:1: unknown orchestration folder 'pipelines'", - expected_help: Some("jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/"), - }); - cases.push(InventoryErrorCase { - description: "unknown folder", - files: with("orchestration/schedules/nightly.toml", EXPORT_JOB), - expected_message: "orchestration/schedules:1:1: unknown orchestration folder 'schedules'", - expected_help: Some("use one of: jobs, job_groups, calendars, sensors"), - }); - cases.push(InventoryErrorCase { - description: "python job", - files: with("orchestration/jobs/orders/legacy.py", "x = 1\n"), - expected_message: "orchestration/jobs/orders/legacy.py:1:1: orchestration/jobs/ holds only .toml files", - expected_help: Some("jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/"), - }); - cases.push(InventoryErrorCase { - description: "non-TOML job file", - files: with("orchestration/jobs/orders/export.yaml", "name: orders_export_job\n"), - expected_message: "orchestration/jobs/orders/export.yaml:1:1: orchestration/jobs/ holds only .toml files", - expected_help: Some("jobs and job groups are TOML; calendars and sensors are Python in orchestration/calendars/ and orchestration/sensors/"), - }); - cases.push(InventoryErrorCase { - description: "missing suffix", - files: with( - "orchestration/jobs/orders/export.toml", - "name = \"orders_export\"\ndescription = \"d\"\nselect = \"orders\"\n", - ), - expected_message: "orchestration/jobs/orders/export.toml:1:8: job 'orders_export' must be named with the suffix '_job'", - expected_help: Some("rename it to 'orders_export_job'"), - }); - cases.push(InventoryErrorCase { - description: "unknown group member", - files: with( - "orchestration/job_groups/orders/nightly.toml", - "name = \"orders_nightly_job_group\"\ndescription = \"d\"\njobs = [\"orders_export_job\", \"carrier_sync_job\"]\n", - ), - expected_message: "orchestration/job_groups/orders/nightly.toml:3:30: job group 'orders_nightly_job_group' names 'carrier_sync_job', which is not a job in orchestration/jobs/", - expected_help: None, - }); - cases.push(InventoryErrorCase { - description: "unknown calendar", - files: with( - "orchestration/calendars/trading_days.py", - "from sqlbuild.orchestration import calendar\n\n@calendar(name=\"working_days_calendar\", description=\"d\", horizon=\"7d\")\ndef f(window):\n return []\n", - ), - expected_message: "orchestration/jobs/orders/export.toml:4:25: schedule names 'trading_days_calendar', which is not a calendar in orchestration/calendars/", - expected_help: None, - }); - cases.push(InventoryErrorCase { - description: "non-literal sensor job", - files: with( - "orchestration/sensors/inventory/stock.py", - "from sqlbuild.orchestration import sensor\nJOB = \"orders_export_job\"\nstale = sensor(name=\"stock_stale_sensor\", description=\"d\", run=JOB)\n", - ), - expected_message: "orchestration/sensors/inventory/stock.py:3:64: sensor run job must be a string literal", - expected_help: Some("write the sensor run job as a quoted string, such as \"orders_job\", so the compiler can check it without running the module"), - }); - cases.push(InventoryErrorCase { - description: "sensor in calendars", - files: with("orchestration/calendars/stock.py", SENSOR), - expected_message: "orchestration/calendars/stock.py:3:9: declares a sensor, which belongs in orchestration/sensors/", - expected_help: None, - }); - cases.push(InventoryErrorCase { - description: "public module with nothing", - files: with("orchestration/sensors/helpers.py", "def helper():\n return 1\n"), - expected_message: "orchestration/sensors/helpers.py:1:1: declares no sensor", - expected_help: Some("declare one sensor, or start the file name with `_` to make it a private support module"), - }); - cases.push(InventoryErrorCase { - description: "removed Python job API", - files: with( - "orchestration/sensors/inventory/stock.py", - "from sqlbuild.orchestration import job, sensor\nstale = sensor(name=\"stock_stale_sensor\", description=\"d\", run=\"orders_export_job\")\n", - ), - expected_message: "orchestration/sensors/inventory/stock.py:1:36: `job` was removed from sqlbuild.orchestration; jobs, job groups and schedules are declared in TOML", - expected_help: Some("jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/"), - }); - for case in cases { - let (actual, actual_help) = message(&case.files); - assert_eq!(actual, case.expected_message, "{}", case.description); + }, + InventoryErrorTestCase { + description: "non-TOML job file", + files: with_file( + "orchestration/jobs/orders/export.yaml", + "name: orders_export_job\n", + ), + expected_message: "orchestration/jobs/orders/export.yaml:1:1: orchestration/jobs/ holds only .toml files", + expected_help: Some( + "jobs and job groups are TOML; calendars and sensors are Python in orchestration/calendars/ and orchestration/sensors/", + ), + }, + InventoryErrorTestCase { + description: "missing suffix", + files: with_file( + "orchestration/jobs/orders/export.toml", + "name = \"orders_export\"\ndescription = \"d\"\nselect = \"orders\"\n", + ), + expected_message: "orchestration/jobs/orders/export.toml:1:8: job 'orders_export' must be named with the suffix '_job'", + expected_help: Some("rename it to 'orders_export_job'"), + }, + InventoryErrorTestCase { + description: "unknown group member", + files: with_file( + "orchestration/job_groups/orders/nightly.toml", + "name = \"orders_nightly_job_group\"\ndescription = \"d\"\njobs = [\"orders_export_job\", \"carrier_sync_job\"]\n", + ), + expected_message: "orchestration/job_groups/orders/nightly.toml:3:30: job group 'orders_nightly_job_group' names 'carrier_sync_job', which is not a job in orchestration/jobs/", + expected_help: None, + }, + InventoryErrorTestCase { + description: "unknown calendar", + files: with_file( + "orchestration/calendars/trading_days.py", + "from sqlbuild.orchestration import calendar\n\n@calendar(name=\"working_days_calendar\", description=\"d\", horizon=\"7d\")\ndef f(window):\n return []\n", + ), + expected_message: "orchestration/jobs/orders/export.toml:4:25: schedule names 'trading_days_calendar', which is not a calendar in orchestration/calendars/", + expected_help: None, + }, + InventoryErrorTestCase { + description: "non-literal sensor job", + files: with_file( + "orchestration/sensors/inventory/stock.py", + "from sqlbuild.orchestration import sensor\nJOB = \"orders_export_job\"\nstale = sensor(name=\"stock_stale_sensor\", description=\"d\", run=JOB)\n", + ), + expected_message: "orchestration/sensors/inventory/stock.py:3:64: sensor run job must be a string literal", + expected_help: Some( + "write the sensor run job as a quoted string, such as \"orders_job\", so the compiler can check it without running the module", + ), + }, + InventoryErrorTestCase { + description: "sensor in calendars", + files: with_file("orchestration/calendars/stock.py", SENSOR), + expected_message: "orchestration/calendars/stock.py:3:9: declares a sensor, which belongs in orchestration/sensors/", + expected_help: None, + }, + InventoryErrorTestCase { + description: "public module with nothing", + files: with_file( + "orchestration/sensors/helpers.py", + "def helper():\n return 1\n", + ), + expected_message: "orchestration/sensors/helpers.py:1:1: declares no sensor", + expected_help: Some( + "declare one sensor, or start the file name with `_` to make it a private support module", + ), + }, + InventoryErrorTestCase { + description: "removed Python job API", + files: with_file( + "orchestration/sensors/inventory/stock.py", + "from sqlbuild.orchestration import job, sensor\nstale = sensor(name=\"stock_stale_sensor\", description=\"d\", run=\"orders_export_job\")\n", + ), + expected_message: "orchestration/sensors/inventory/stock.py:1:36: `job` was removed from sqlbuild.orchestration; jobs, job groups and schedules are declared in TOML", + expected_help: Some( + "jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/", + ), + }, + ]; + for test_case in test_cases { assert_eq!( - actual_help.as_deref(), - case.expected_help, + inventory_failure(&test_case.files), + declaration_failure(test_case.expected_message, test_case.expected_help), "{}", - case.description + test_case.description ); } } diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs index 12e5bf25c..f56e400c8 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs @@ -1,12 +1,13 @@ -use crate::orchestration_files::main::read_group_file::read_group_file; -use crate::orchestration_files::main::read_job_file::read_job_file; +use std::collections::BTreeMap; + +use crate::orchestration_files::_helpers::read_job_file::read_job_file; use crate::orchestration_files::models::{ - FilePosition, InitialStatus, OnFailure, Overlap, ScheduleSource, + InitialStatus, OnFailure, Overlap, ScheduleFile, ScheduleSource, +}; +use crate::orchestration_files::tests::helpers::{at, group_members, written}; +use crate::orchestration_files::tests::test_types::{ + FileErrorTestCase, GroupReadTestCase, JobReadTestCase, }; - -fn at(line: usize, column: usize) -> FilePosition { - FilePosition { line, column } -} const FULL_JOB: &str = r#"name = "orders_export_job" description = "Write the daily partner file." @@ -31,119 +32,194 @@ concurrency = 4 "#; #[test] -fn given_full_job_when_reading_then_types_every_field_and_keeps_positions() { - let job = read_job_file(FULL_JOB).expect("valid job"); - assert_eq!(job.run.name.value, "orders_export_job"); - assert_eq!(job.run.name.position, at(1, 8)); - assert_eq!(job.select.position, at(3, 10)); - assert_eq!( - job.exclude.map(|exclude| exclude.value).as_deref(), - Some("tag:slow") - ); - assert_eq!(job.run.overlap, Overlap::Skip); - assert_eq!( - job.run.on_failure, - Some(OnFailure { - retries: 2, - retry_delay_seconds: 0 - }) - ); - let schedule = job.run.schedule.expect("schedule"); - assert_eq!(schedule.default_status, InitialStatus::Paused); - let ScheduleSource::Calendar(calendar) = schedule.source else { - panic!("calendar schedule expected"); - }; - assert_eq!(calendar.value, "trading_days_calendar"); - assert_eq!(calendar.position, at(9, 25)); - assert_eq!( - job.resources - .map(|resources| resources.value["cpus"].clone()), - Some(serde_json::json!(1.5)) - ); - assert_eq!(job.settings["concurrency"], serde_json::json!(4)); - assert_eq!(job.settings["vars"], serde_json::json!({"batch": 23})); +fn given_job_files_when_reading_then_types_every_field_and_keeps_positions() { + let test_cases = [ + JobReadTestCase { + description: "a job using every key", + text: FULL_JOB, + expected_name: written("orders_export_job", 1, 8), + expected_select: written("python/orders/partner_export", 3, 10), + expected_exclude: Some("tag:slow"), + expected_overlap: Overlap::Skip, + expected_on_failure: Some(OnFailure { + retries: 2, + retry_delay_seconds: 0, + }), + expected_schedule: Some(ScheduleFile { + source: ScheduleSource::Calendar(written("trading_days_calendar", 9, 25)), + timezone: written("Europe/London", 9, 61), + starts_at: None, + default_status: InitialStatus::Paused, + }), + expected_cpus: Some(serde_json::json!(1.5)), + expected_settings: BTreeMap::from([ + ("concurrency".to_owned(), serde_json::json!(4)), + ("vars".to_owned(), serde_json::json!({"batch": 23})), + ]), + }, + JobReadTestCase { + description: "a job with only the required keys takes the defaults", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nexclude = \" \"\n", + expected_name: written("a_job", 1, 8), + expected_select: written("orders", 3, 10), + expected_exclude: None, + expected_overlap: Overlap::Queue, + expected_on_failure: None, + expected_schedule: None, + expected_cpus: None, + expected_settings: BTreeMap::new(), + }, + ]; + for test_case in test_cases { + let job = read_job_file(test_case.text).expect(test_case.description); + assert_eq!( + job.run.name, test_case.expected_name, + "{}", + test_case.description + ); + assert_eq!( + job.select, test_case.expected_select, + "{}", + test_case.description + ); + assert_eq!( + job.exclude.map(|exclude| exclude.value).as_deref(), + test_case.expected_exclude, + "{}", + test_case.description + ); + assert_eq!( + job.run.overlap, test_case.expected_overlap, + "{}", + test_case.description + ); + assert_eq!( + job.run.on_failure, test_case.expected_on_failure, + "{}", + test_case.description + ); + assert_eq!( + job.run.schedule, test_case.expected_schedule, + "{}", + test_case.description + ); + assert_eq!( + job.resources + .and_then(|resources| resources.value.get("cpus").cloned()), + test_case.expected_cpus, + "{}", + test_case.description + ); + assert_eq!( + job.settings, test_case.expected_settings, + "{}", + test_case.description + ); + } } #[test] fn given_invalid_job_files_when_reading_then_reports_message_position_and_help() { - let cases: [(&str, &str, &str, FilePosition, Option<&str>); 6] = [ - ( - "misspelt key", - "name = \"a_job\"\ndescription = \"d\"\nselct = \"orders\"\n", - "unknown job key 'selct'", - at(3, 1), - Some("did you mean 'select'?"), - ), - ( - "top-level concurrency", - "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nconcurrency = 2\n", - "unknown job key 'concurrency'", - at(4, 1), - Some("move it under [settings] as `concurrency = ...`"), - ), - ( - "blank select", - "name = \"a_job\"\ndescription = \"d\"\nselect = \" \"\n", - "select must not be blank", - at(3, 10), - None, - ), - ( - "both cron and calendar", - "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nschedule = { cron = \"0 2 * * *\", calendar = \"c_calendar\", timezone = \"UTC\" }\n", - "schedule needs exactly one of `cron` or `calendar`", - at(4, 12), - None, - ), - ( - "retries out of range", - "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\non_failure = { retries = 0 }\n", - "on_failure retries must be a whole number from 1 to 10", - at(4, 14), - None, - ), - ( - "missing description", - "name = \"a_job\"\nselect = \"orders\"\n", - "missing field `description`", - at(1, 1), - None, - ), + let test_cases = [ + FileErrorTestCase { + description: "misspelt key", + text: "name = \"a_job\"\ndescription = \"d\"\nselct = \"orders\"\n", + expected_message: "unknown job key 'selct'", + expected_position: at(3, 1), + expected_help: Some("did you mean 'select'?"), + }, + FileErrorTestCase { + description: "top-level concurrency", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nconcurrency = 2\n", + expected_message: "unknown job key 'concurrency'", + expected_position: at(4, 1), + expected_help: Some("move it under [settings] as `concurrency = ...`"), + }, + FileErrorTestCase { + description: "blank select", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \" \"\n", + expected_message: "select must not be blank", + expected_position: at(3, 10), + expected_help: None, + }, + FileErrorTestCase { + description: "both cron and calendar", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nschedule = { cron = \"0 2 * * *\", calendar = \"c_calendar\", timezone = \"UTC\" }\n", + expected_message: "schedule needs exactly one of `cron` or `calendar`", + expected_position: at(4, 12), + expected_help: None, + }, + FileErrorTestCase { + description: "retries out of range", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\non_failure = { retries = 0 }\n", + expected_message: "on_failure retries must be a whole number from 1 to 10", + expected_position: at(4, 14), + expected_help: None, + }, + FileErrorTestCase { + description: "retry delay beyond a day", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\non_failure = { retries = 1, retry_delay_seconds = 86401 }\n", + expected_message: "on_failure retry_delay_seconds must be a whole number from 0 to 86400", + expected_position: at(4, 14), + expected_help: None, + }, + FileErrorTestCase { + description: "missing description", + text: "name = \"a_job\"\nselect = \"orders\"\n", + expected_message: "missing field `description`", + expected_position: at(1, 1), + expected_help: None, + }, ]; - for (description, text, message, position, help) in cases { - let error = read_job_file(text).expect_err(description); - assert_eq!(error.message, message, "{description}"); - assert_eq!(error.position, position, "{description}"); - assert_eq!(error.help.as_deref(), help, "{description}"); + for test_case in test_cases { + let error = read_job_file(test_case.text).expect_err(test_case.description); + assert_eq!( + error.message, test_case.expected_message, + "{}", + test_case.description + ); + assert_eq!( + error.position, test_case.expected_position, + "{}", + test_case.description + ); + assert_eq!( + error.help.as_deref(), + test_case.expected_help, + "{}", + test_case.description + ); } } #[test] -fn given_group_when_reading_then_keeps_member_positions_and_refuses_repeats() { - let group = read_group_file( - "name = \"nightly_job_group\"\ndescription = \"d\"\njobs = [\"export_job\", \"carrier_job\"]\n", - ) - .expect("valid group"); - let members: Vec<(&str, FilePosition)> = group - .jobs - .iter() - .map(|job| (job.value.as_str(), job.position)) - .collect(); - assert_eq!( - members, - [("export_job", at(3, 9)), ("carrier_job", at(3, 23))] - ); - - let repeated = read_group_file( - "name = \"nightly_job_group\"\ndescription = \"d\"\njobs = [\"export_job\", \"export_job\"]\n", - ) - .expect_err("repeated member"); - assert_eq!(repeated.message, "job 'export_job' is listed twice"); - assert_eq!(repeated.position, at(3, 23)); - - let selecting = read_group_file( - "name = \"nightly_job_group\"\ndescription = \"d\"\njobs = [\"export_job\"]\nselect = \"orders\"\n", - ) - .expect_err("groups do not select"); - assert_eq!(selecting.message, "unknown job group key 'select'"); +fn given_group_files_when_reading_then_keeps_member_positions_and_refuses_repeats() { + let test_cases = [ + GroupReadTestCase { + description: "members keep their positions", + text: "name = \"nightly_job_group\"\ndescription = \"d\"\njobs = [\"export_job\", \"carrier_job\"]\n", + expected_members: Ok(vec![ + ("export_job".to_owned(), at(3, 9)), + ("carrier_job".to_owned(), at(3, 23)), + ]), + }, + GroupReadTestCase { + description: "a repeated member is refused at the repeat", + text: "name = \"nightly_job_group\"\ndescription = \"d\"\njobs = [\"export_job\", \"export_job\"]\n", + expected_members: Err(("job 'export_job' is listed twice".to_owned(), at(3, 23))), + }, + GroupReadTestCase { + description: "groups do not select", + text: "name = \"nightly_job_group\"\ndescription = \"d\"\njobs = [\"export_job\"]\nselect = \"orders\"\n", + expected_members: Err(("unknown job group key 'select'".to_owned(), at(4, 1))), + }, + ]; + for test_case in test_cases { + assert_eq!( + group_members(test_case.text), + test_case.expected_members, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs index 338e12426..8cd4f4868 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs @@ -1,4 +1,45 @@ -pub(super) struct InventoryErrorCase { +use std::collections::BTreeMap; + +use crate::orchestration_files::models::{ + FilePosition, OnFailure, Overlap, Positioned, ScheduleFile, +}; + +pub(super) struct JobReadTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_name: Positioned, + pub(super) expected_select: Positioned, + pub(super) expected_exclude: Option<&'static str>, + pub(super) expected_overlap: Overlap, + pub(super) expected_on_failure: Option, + pub(super) expected_schedule: Option, + pub(super) expected_cpus: Option, + pub(super) expected_settings: BTreeMap, +} + +pub(super) struct FileErrorTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_message: &'static str, + pub(super) expected_position: FilePosition, + pub(super) expected_help: Option<&'static str>, +} + +pub(super) struct GroupReadTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_members: Result, (String, FilePosition)>, +} + +pub(super) struct InventoryTestCase { + pub(super) description: &'static str, + pub(super) files: Vec<(&'static str, &'static str)>, + pub(super) expected_names: Vec<(String, &'static str, String)>, + pub(super) expected_modules: Vec<&'static str>, + pub(super) expected_empty: bool, +} + +pub(super) struct InventoryErrorTestCase { pub(super) description: &'static str, pub(super) files: Vec<(&'static str, &'static str)>, pub(super) expected_message: &'static str, diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs b/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs index ebb78252a..c05f52ade 100644 --- a/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs +++ b/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs @@ -16,7 +16,7 @@ use crate::orchestration_modules::constants::{ SENSOR_CALL, TARGETS_KEYWORD, }; use crate::orchestration_modules::models::{ - LiteralArgument, ModuleDeclarations, PythonDeclaration, PythonDeclarationKind, + LiteralArgument, ModuleDeclarations, PythonDeclaration, PythonDeclarationKind, SourcePosition, }; /// Every `calendar(...)` and `sensor(...)` call with its name and job references, in source order; `None` when unparsable. @@ -32,7 +32,7 @@ pub fn orchestration_module_declarations( .map(|(local, name, _)| (local.as_str(), name.as_str())) .collect(); let modules: BTreeSet = module_names(&parsed.module.body); - let removed_imports = imports + let removed_imports: Vec<(String, SourcePosition)> = imports .iter() .filter(|(_, name, _)| REMOVED_NAMES.contains(&name.as_str())) .map(|(_, name, start)| (name.clone(), position(&parsed.lines, *start))) diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/tests.rs b/crates/sqlbuild-pysource/src/orchestration_modules/tests.rs index 959e94a42..985ec8cc9 100644 --- a/crates/sqlbuild-pysource/src/orchestration_modules/tests.rs +++ b/crates/sqlbuild-pysource/src/orchestration_modules/tests.rs @@ -1,2 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_orchestration_module_declarations.rs"] mod orchestration_module_declarations; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/tests/helpers.rs b/crates/sqlbuild-pysource/src/orchestration_modules/tests/helpers.rs new file mode 100644 index 000000000..ac4918ab9 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/tests/helpers.rs @@ -0,0 +1,40 @@ +use crate::orchestration_modules::models::{ + LiteralArgument, PythonDeclaration, PythonDeclarationKind, SourcePosition, +}; + +/// A 1-based line and column. +pub(super) fn at(line: usize, column: usize) -> SourcePosition { + SourcePosition { line, column } +} + +/// A string literal argument written at `line` and `column`. +pub(super) fn text(value: &str, line: usize, column: usize) -> LiteralArgument { + LiteralArgument { + value: Some(value.to_owned()), + position: at(line, column), + } +} + +/// An argument that is not a string literal, written at `line` and `column`. +pub(super) fn non_literal(line: usize, column: usize) -> LiteralArgument { + LiteralArgument { + value: None, + position: at(line, column), + } +} + +/// A declaration whose call starts at `position` and passes no `targets`. +pub(super) fn declaration( + kind: PythonDeclarationKind, + name: LiteralArgument, + position: SourcePosition, + job_references: Vec<(&'static str, LiteralArgument)>, +) -> PythonDeclaration { + PythonDeclaration { + kind, + name: Some(name), + position, + job_references, + targets: Some(Vec::new()), + } +} diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs index 4f79a0de1..ab2164368 100644 --- a/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs +++ b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs @@ -1,131 +1,115 @@ use crate::orchestration_modules::main::orchestration_module_declarations::orchestration_module_declarations; -use crate::orchestration_modules::models::{ - LiteralArgument, ModuleDeclarations, PythonDeclarationKind, SourcePosition, -}; - -fn at(line: usize, column: usize) -> SourcePosition { - SourcePosition { line, column } -} - -fn text(value: &str, line: usize, column: usize) -> LiteralArgument { - LiteralArgument { - value: Some(value.to_owned()), - position: at(line, column), - } -} - -fn scan(contents: &str) -> ModuleDeclarations { - orchestration_module_declarations(contents, (3, 12)).expect("module parses") -} +use crate::orchestration_modules::models::{ModuleDeclarations, PythonDeclarationKind}; +use crate::orchestration_modules::tests::helpers::{at, declaration, non_literal, text}; +use crate::orchestration_modules::tests::test_types::ScanTestCase; #[test] -fn given_decorated_calendar_when_scanning_then_reads_its_literal_name() { - let module = scan( - "from sqlbuild.orchestration import calendar\n\n@calendar(name=\"trading_days_calendar\", description=\"d\", horizon=\"7d\")\ndef trading_days(window):\n return []\n", - ); - assert_eq!(module.declarations.len(), 1); - let calendar = &module.declarations[0]; - assert_eq!(calendar.kind, PythonDeclarationKind::Calendar); - assert_eq!(calendar.name, Some(text("trading_days_calendar", 3, 16))); - assert_eq!(calendar.position, at(3, 2)); - assert!(calendar.job_references.is_empty()); -} - -#[test] -fn given_sensor_when_scanning_then_reads_run_and_watched_job_in_order() { - let module = scan( - "from sqlbuild.orchestration import RunEventKind, run_event as watched, sensor\n\nstock = sensor(\n \"stock_sensor\",\n description=\"d\",\n when=watched(job=\"orders_job\", kinds=[RunEventKind.SUCCESS]),\n run=\"inventory_job\",\n)\n", - ); - let sensor = &module.declarations[0]; - assert_eq!(sensor.kind, PythonDeclarationKind::Sensor); - assert_eq!(sensor.name, Some(text("stock_sensor", 4, 5))); - assert_eq!( - sensor.job_references, - vec![ - ("run", text("inventory_job", 7, 9)), - ("run_event", text("orders_job", 6, 22)), - ] - ); -} - -#[test] -fn given_non_literal_references_when_scanning_then_keeps_their_positions_without_values() { - let module = scan( - "import sqlbuild.orchestration as orchestration\n\nNAME = \"x_sensor\"\nJOB = \"x_job\"\nx = orchestration.sensor(name=NAME, description=\"d\", run=JOB)\n", - ); - let sensor = &module.declarations[0]; - assert_eq!( - sensor.name, - Some(LiteralArgument { - value: None, - position: at(5, 31), - }) - ); - assert_eq!(sensor.job_references[0].1.value, None); - assert_eq!(sensor.job_references[0].1.position, at(5, 58)); -} - -#[test] -fn given_unrelated_calls_when_scanning_then_ignores_them() { - let module = scan( - "import calendar\nfrom other import sensor\n\ndays = calendar.monthrange(2026, 2)\nsensor(\"not_ours\")\n", - ); - assert!(module.declarations.is_empty()); - assert!(module.removed_imports.is_empty()); -} - -#[test] -fn given_removed_authoring_imports_when_scanning_then_reports_each() { - let module = scan( - "from sqlbuild.orchestration import (\n job,\n job_group as group,\n sensor,\n)\n", - ); - assert_eq!( - module.removed_imports, - vec![ - ("job".to_owned(), at(2, 5)), - ("job_group".to_owned(), at(3, 5)) - ] - ); -} - -#[test] -fn given_syntax_error_when_scanning_then_returns_none() { - assert_eq!( - orchestration_module_declarations("def f(:\n", (3, 12)), - None - ); -} - -#[test] -fn given_other_objects_calendar_and_sensor_methods_when_scanning_then_only_orchestration_calls_declare() - { - let module = scan( - "import calendar\nfrom sqlbuild.orchestration import calendar as declare\n\n@declare(name=\"days_calendar\", description=\"d\", horizon=\"7d\")\ndef days(window, exchange):\n calendar.calendar(2026)\n return exchange.calendar(window.start)\n", - ); - assert_eq!(module.declarations.len(), 1); - assert_eq!( - module.declarations[0].name, - Some(text("days_calendar", 4, 15)) - ); -} - -#[test] -fn given_orchestration_module_attributes_when_scanning_then_each_import_form_declares() { - let module = scan( - "import sqlbuild.orchestration\nfrom sqlbuild import orchestration as authoring\n\na = sqlbuild.orchestration.sensor(\"a_sensor\", description=\"d\", run=\"a_job\")\nb = authoring.sensor(\"b_sensor\", description=\"d\", run=\"b_job\")\n", - ); - let names: Vec> = module - .declarations - .iter() - .map(|declaration| { - declaration - .name - .as_ref() - .and_then(|name| name.value.clone()) - }) - .collect(); - assert_eq!( - names, - vec![Some("a_sensor".to_owned()), Some("b_sensor".to_owned())] - ); +fn given_orchestration_modules_when_scanning_then_reads_declarations_and_removed_imports() { + let test_cases = [ + ScanTestCase { + description: "a decorated calendar has its literal name and decorator position", + contents: "from sqlbuild.orchestration import calendar\n\n@calendar(name=\"trading_days_calendar\", description=\"d\", horizon=\"7d\")\ndef trading_days(window):\n return []\n", + expected_module: Some(ModuleDeclarations { + declarations: vec![declaration( + PythonDeclarationKind::Calendar, + text("trading_days_calendar", 3, 16), + at(3, 2), + Vec::new(), + )], + removed_imports: Vec::new(), + }), + }, + ScanTestCase { + description: "a sensor reads its run job, then each watched job", + contents: "from sqlbuild.orchestration import RunEventKind, run_event as watched, sensor\n\nstock = sensor(\n \"stock_sensor\",\n description=\"d\",\n when=watched(job=\"orders_job\", kinds=[RunEventKind.SUCCESS]),\n run=\"inventory_job\",\n)\n", + expected_module: Some(ModuleDeclarations { + declarations: vec![declaration( + PythonDeclarationKind::Sensor, + text("stock_sensor", 4, 5), + at(3, 9), + vec![ + ("run", text("inventory_job", 7, 9)), + ("run_event", text("orders_job", 6, 22)), + ], + )], + removed_imports: Vec::new(), + }), + }, + ScanTestCase { + description: "non-literal names and jobs keep their positions without values", + contents: "import sqlbuild.orchestration as orchestration\n\nNAME = \"x_sensor\"\nJOB = \"x_job\"\nx = orchestration.sensor(name=NAME, description=\"d\", run=JOB)\n", + expected_module: Some(ModuleDeclarations { + declarations: vec![declaration( + PythonDeclarationKind::Sensor, + non_literal(5, 31), + at(5, 5), + vec![("run", non_literal(5, 58))], + )], + removed_imports: Vec::new(), + }), + }, + ScanTestCase { + description: "calls of other calendar and sensor objects declare nothing", + contents: "import calendar\nfrom other import sensor\n\ndays = calendar.monthrange(2026, 2)\nsensor(\"not_ours\")\n", + expected_module: Some(ModuleDeclarations::default()), + }, + ScanTestCase { + description: "each removed authoring import is reported at its name", + contents: "from sqlbuild.orchestration import (\n job,\n job_group as group,\n sensor,\n)\n", + expected_module: Some(ModuleDeclarations { + declarations: Vec::new(), + removed_imports: vec![ + ("job".to_owned(), at(2, 5)), + ("job_group".to_owned(), at(3, 5)), + ], + }), + }, + ScanTestCase { + description: "a syntax error reads as no module", + contents: "def f(:\n", + expected_module: None, + }, + ScanTestCase { + description: "calendar methods of other objects do not declare beside an aliased import", + contents: "import calendar\nfrom sqlbuild.orchestration import calendar as declare\n\n@declare(name=\"days_calendar\", description=\"d\", horizon=\"7d\")\ndef days(window, exchange):\n calendar.calendar(2026)\n return exchange.calendar(window.start)\n", + expected_module: Some(ModuleDeclarations { + declarations: vec![declaration( + PythonDeclarationKind::Calendar, + text("days_calendar", 4, 15), + at(4, 2), + Vec::new(), + )], + removed_imports: Vec::new(), + }), + }, + ScanTestCase { + description: "attributes of the orchestration module declare under each import form", + contents: "import sqlbuild.orchestration\nfrom sqlbuild import orchestration as authoring\n\na = sqlbuild.orchestration.sensor(\"a_sensor\", description=\"d\", run=\"a_job\")\nb = authoring.sensor(\"b_sensor\", description=\"d\", run=\"b_job\")\n", + expected_module: Some(ModuleDeclarations { + declarations: vec![ + declaration( + PythonDeclarationKind::Sensor, + text("a_sensor", 4, 35), + at(4, 5), + vec![("run", text("a_job", 4, 68))], + ), + declaration( + PythonDeclarationKind::Sensor, + text("b_sensor", 5, 22), + at(5, 5), + vec![("run", text("b_job", 5, 55))], + ), + ], + removed_imports: Vec::new(), + }), + }, + ]; + for test_case in test_cases { + assert_eq!( + orchestration_module_declarations(test_case.contents, (3, 12)), + test_case.expected_module, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_types.rs b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_types.rs new file mode 100644 index 000000000..a0838a465 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_types.rs @@ -0,0 +1,7 @@ +use crate::orchestration_modules::models::ModuleDeclarations; + +pub(super) struct ScanTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) expected_module: Option, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs index c6ef9d997..dc6026816 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs @@ -202,7 +202,6 @@ fn inventory_catalog(inventory: &OrchestrationInventory, target: Option<&str>) - ); } for sensor in &inventory.sensors { - // A sensor whose targets are not literals counts for every target. if sensor.targets.as_deref().is_none_or(declared_for) { triggered.extend( sensor diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 817708e28..93935aecf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; @@ -21,9 +21,8 @@ use sqlbuild_orchestration::declarations::models::{ use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; -use crate::bindings::_helpers::native_host::constants::{AUTHORING_MODULE, SENSOR_CLASS}; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - AuthoredSensor, module_declarations, run_controls, run_resources, + ImportedDeclarations, imported_declarations, module_declarations, run_controls, run_resources, }; use crate::bindings::_helpers::native_host::orchestration_triggers::{ CalendarRequest, compile_calendars, compile_sensors, @@ -64,14 +63,12 @@ fn declared_for(targets: &[String], target: &str) -> bool { targets.is_empty() || targets.iter().any(|name| name == target) } -/// A run's cron trigger, or the calendar it asks for, keyed by the run's name. +/// A run's cron trigger, or the calendar it asks for; neither when it has no schedule. fn run_schedule( (path, run): (&str, &RunFile), - triggers: &mut BTreeMap, - calendars: &mut Vec, -) -> PyResult<()> { +) -> PyResult<(Option, Option)> { let Some(schedule) = &run.schedule else { - return Ok(()); + return Ok((None, None)); }; let name: &str = &run.name.value; let starts_at = schedule @@ -95,26 +92,26 @@ fn run_schedule( targets: run.targets.iter().cloned().collect(), declared_at: file_site(path, run.name.position), }; - match &schedule.source { - ScheduleSource::Cron(expression) => { - triggers.insert( - name.to_owned(), - TriggerDefinition { - schedule: Schedule::Cron { - expression: expression.value.clone(), - timezone: schedule.timezone.value.clone(), - }, - ..trigger + Ok(match &schedule.source { + ScheduleSource::Cron(expression) => ( + Some(TriggerDefinition { + schedule: Schedule::Cron { + expression: expression.value.clone(), + timezone: schedule.timezone.value.clone(), }, - ); - } - ScheduleSource::Calendar(calendar) => calendars.push(CalendarRequest { - calendar: calendar.value.clone(), - timezone: schedule.timezone.value.clone(), - trigger, - }), - } - Ok(()) + ..trigger + }), + None, + ), + ScheduleSource::Calendar(calendar) => ( + None, + Some(CalendarRequest { + calendar: calendar.value.clone(), + timezone: schedule.timezone.value.clone(), + trigger, + }), + ), + }) } /// Refuse a calendar or sensor that compile declared but import did not define, or the reverse. @@ -160,24 +157,11 @@ pub(crate) fn compile_declarations( "authored declarations cannot overwrite snapshot job or trigger documents", )); } - let sensor_class = py.import(AUTHORING_MODULE)?.getattr(SENSOR_CLASS)?; - let mut authored_calendars: BTreeMap> = BTreeMap::new(); - let mut authored_sensors: Vec = Vec::new(); - for declaration in declarations { - if declaration.is_instance(&sensor_class)? { - authored_sensors.push(declaration.extract()?); - } else { - let name: String = declaration.getattr("name")?.extract()?; - if authored_calendars - .insert(name.clone(), declaration) - .is_some() - { - return Err(PyValueError::new_err(format!( - "calendar '{name}' is declared twice" - ))); - } - } - } + let ImportedDeclarations { + calendars: authored_calendars, + calendar_descriptions, + sensors: authored_sensors, + } = imported_declarations(py, declarations)?; matching_imports( ("calendar", &inventory.calendars), authored_calendars.keys().map(String::as_str).collect(), @@ -205,13 +189,11 @@ pub(crate) fn compile_declarations( .iter() .map(|sensor| ("sensor", sensor.name.clone(), sensor.description.clone())), ); - for (name, calendar) in &authored_calendars { - described.push(( - "calendar", - name.clone(), - calendar.getattr("description")?.extract()?, - )); - } + described.extend( + calendar_descriptions + .into_iter() + .map(|(name, description)| ("calendar", name, description)), + ); for (kind, name, description) in described { let limit: i64 = format.max_description_lines; let lines: usize = description_line_count(&description, format.line_width); @@ -256,7 +238,9 @@ pub(crate) fn compile_declarations( .as_ref() .map(|resources| run_resources(name, &resources.value)) .transpose()?; - run_schedule((path, run), &mut triggers, &mut calendar_requests)?; + let (trigger, calendar) = run_schedule((path, run))?; + triggers.extend(trigger.map(|trigger| (name.to_owned(), trigger))); + calendar_requests.extend(calendar); let controls = run_controls(run); jobs.insert( name.to_owned(), @@ -295,7 +279,9 @@ pub(crate) fn compile_declarations( .iter() .flat_map(|member| jobs[member].selection.iter().cloned()) .collect(); - run_schedule((path, run), &mut triggers, &mut calendar_requests)?; + let (trigger, calendar) = run_schedule((path, run))?; + triggers.extend(trigger.map(|trigger| (name.to_owned(), trigger))); + calendar_requests.extend(calendar); let controls = run_controls(run); groups.insert( name.to_owned(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index f91523bf0..9b4d86606 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -1,6 +1,6 @@ //! The public authoring records enter native compilation here, never scheduler execution. -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::path::{Component, Path}; use pyo3::exceptions::PyValueError; @@ -150,6 +150,45 @@ pub(crate) fn run_resources( Ok(resources) } +/// The calendars, by name with their descriptions, and the sensors that importing the modules defines. +pub(crate) struct ImportedDeclarations<'py> { + pub(crate) calendars: BTreeMap>, + pub(crate) calendar_descriptions: BTreeMap, + pub(crate) sensors: Vec, +} + +/// Sort imported declarations into calendars and sensors, refusing a calendar name defined twice. +pub(crate) fn imported_declarations<'py>( + py: Python<'py>, + declarations: Vec>, +) -> PyResult> { + let sensor_class = py.import(AUTHORING_MODULE)?.getattr(SENSOR_CLASS)?; + let mut imported = ImportedDeclarations { + calendars: BTreeMap::new(), + calendar_descriptions: BTreeMap::new(), + sensors: Vec::new(), + }; + for declaration in declarations { + if declaration.is_instance(&sensor_class)? { + imported.sensors.push(declaration.extract()?); + continue; + } + let name: String = declaration.getattr("name")?.extract()?; + let description: String = declaration.getattr("description")?.extract()?; + if imported + .calendars + .insert(name.clone(), declaration) + .is_some() + { + return Err(PyValueError::new_err(format!( + "calendar '{name}' is declared twice" + ))); + } + imported.calendar_descriptions.insert(name, description); + } + Ok(imported) +} + /// The calendars and sensors one module defines; only module execution crosses into Python. pub(crate) fn module_declarations<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs index d0975bd4b..73a4806d0 100644 --- a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs @@ -44,10 +44,7 @@ struct Changes { partial: BTreeMap>, } -/// Rewrite exact model names and wholly moved folders in every job's `select` and `exclude`. -/// -/// `renamed` holds each target model and its new name and path. Tags, globs, `job:` and -/// other kinds are left alone; a folder only some of its models leave becomes a note. +/// Rewrite renamed models and wholly moved folders in job selectors; a partly moved folder becomes a note. pub(crate) fn job_selector_parts( project_dir: &Path, models: &[ModelFacts], @@ -70,7 +67,6 @@ pub(crate) fn job_selector_parts( continue; } }; - // A job file that does not parse fails compile with its own error. let Ok(job) = toml::from_str::(&contents) else { continue; }; @@ -131,8 +127,8 @@ fn changes(models: &[ModelFacts], renamed: &[(&ModelFacts, &str, &str)]) -> Chan .iter() .map(|(model, _, destination)| (model.path.as_str(), parent(destination))) .collect(); - let mut folders = BTreeMap::new(); - let mut partial = BTreeMap::new(); + let mut folders: BTreeMap = BTreeMap::new(); + let mut partial: BTreeMap> = BTreeMap::new(); let moved_from = renamed .iter() .map(|(model, _, destination)| (parent(&model.path), parent(destination))) @@ -152,23 +148,12 @@ fn changes(models: &[ModelFacts], renamed: &[(&ModelFacts, &str, &str)]) -> Chan to.strip_suffix(rest).map(str::to_owned) }; let targets: Vec> = below.iter().map(|model| moved_to(model)).collect(); - // A destination that keeps or receives other models would widen the selector to them. - let shared = |to: &String| { - models.iter().any(|model| { - let lands: String = destinations - .get(model.path.as_str()) - .cloned() - .unwrap_or_else(|| parent(&model.path)); - within(&lands, to) && !within(&parent(&model.path), &folder) - }) - }; - let whole = targets - .first() - .cloned() - .flatten() - .filter(|to| *to != folder && !within(to, &folder)) - .filter(|to| targets.iter().all(|target| target.as_ref() == Some(to))) - .filter(|to| !shared(to)); + let whole: Option = targets.first().cloned().flatten().filter(|to| { + *to != folder + && !within(to, &folder) + && all_move_to(&targets, to) + && !receives_other_models(models, &destinations, &folder, to) + }); match whole { Some(to) => { folders.insert(folder, to); @@ -196,6 +181,27 @@ fn changes(models: &[ModelFacts], renamed: &[(&ModelFacts, &str, &str)]) -> Chan } } +/// Whether every model below a folder moves to `to`. +fn all_move_to(targets: &[Option], to: &str) -> bool { + targets.iter().all(|target| target.as_deref() == Some(to)) +} + +/// Whether a model from outside `folder` keeps or gets a place in `to`, so selecting `to` would widen to it. +fn receives_other_models( + models: &[ModelFacts], + destinations: &BTreeMap<&str, String>, + folder: &str, + to: &str, +) -> bool { + models.iter().any(|model| { + let lands: String = destinations + .get(model.path.as_str()) + .cloned() + .unwrap_or_else(|| parent(&model.path)); + within(&lands, to) && !within(&parent(&model.path), folder) + }) +} + /// `folder` and each folder above it, up to the model root. fn ancestors(folder: &str) -> Vec { let parts: Vec<&str> = folder.split('/').collect(); @@ -213,7 +219,13 @@ fn selected_folders(selector: &str) -> impl Iterator + '_ { selector .split_whitespace() .flat_map(|token| token.split(',')) - .filter_map(|part| folder_of(part.trim_matches('+')).map(|(_, folder)| folder)) + .filter_map(selected_folder) +} + +/// The normalized folder one selector part names, ignoring its graph operators. +fn selected_folder(part: &str) -> Option { + let (_, folder) = folder_of(part.trim_matches('+'))?; + Some(folder) } /// The `path:` prefix (if any) and normalized folder of a folder selector part. diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index af83e60b2..73abd5f7f 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -1,8 +1,4 @@ -"""Public authoring and snapshot compilation API for orchestration declarations. - -Jobs, job groups and their schedules are TOML files under `orchestration/jobs/` and -`orchestration/job_groups/`; calendars and sensors are declared here. -""" +"""Calendar and sensor authoring and snapshot compilation; jobs and job groups are TOML files.""" from sqlbuild.orchestration_declarations.main.compile_snapshot import compile_snapshot from sqlbuild.orchestration_declarations.models.authoring import Calendar as calendar diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index 79fa6a044..a51b33d16 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -21,10 +21,7 @@ class CalendarWindow: @dataclass(frozen=True, kw_only=True, eq=False) class Calendar: - """Decorate a location callback returning upcoming timezone-aware datetimes. - - Jobs and job groups name the calendar in their TOML `schedule`, which also sets the timezone. - """ + """Decorate a location callback returning upcoming timezone-aware datetimes for schedules.""" name: str description: str @@ -49,10 +46,7 @@ class PlatformEventCondition: @dataclass(frozen=True, kw_only=True) class RunEventCondition: - """Match scoped terminal outcomes of the named job or job group. - - `executed_successfully` optionally requires nodes that genuinely executed and succeeded. - """ + """Match a job's or group's terminal outcomes; listed nodes must have run and succeeded.""" job: str kinds: Sequence[RunEventKind] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index d3cfbc3b5..5fba89969 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5588,3 +5588,14 @@ def truncate_sql_test_cache(project_dir: Path) -> None: for cache in sorted((project_dir / "target/cache").rglob("sql-test-artifacts.json")): text: str = cache.read_text() _ = cache.write_text(text[: len(text) // 2]) + + +def orders_job_project(job_values: str) -> dict[str, str]: + """Return a one-model project whose `orders_job` TOML file continues with `job_values`.""" + return { + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "models/orders.sql": "MODEL (description 'Orders.');\nSELECT 1 AS order_id\n", + "orchestration/jobs/orders_job.toml": ( + 'name = "orders_job"\ndescription = "Orders."\n' + job_values + ), + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py index f47d268e9..d36210219 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py @@ -13,6 +13,7 @@ PythonProjectLayoutCompileTestCase, UnrelatedPythonPackageBuildTestCase, ) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import orders_job_project from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -68,59 +69,73 @@ "https://sqlbuild.com/docs/concepts/orchestration/", ), ), - *( - PythonProjectLayoutCompileTestCase( - description=description, - repo_files={ - "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', - "models/orders.sql": "MODEL (description 'Orders.');\nSELECT 1 AS order_id\n", - "orchestration/jobs/orders_job.toml": ( - 'name = "orders_job"\ndescription = "Orders."\n' + job_values - ), - }, - expected_exit_code=1, - expected_stderr_fragments=("error[D017]", location, message), - ) - for description, job_values, location, message in ( - ( - "an invalid cron is reported at its value", - 'select = "orders"\nschedule = { cron = "not a cron", timezone = "UTC" }\n', - "orchestration/jobs/orders_job.toml:4:21:", - "schedule cron: invalid cron expression", - ), - ( - "an unknown time zone is reported at its value", - 'select = "orders"\n' - 'schedule = { cron = "0 2 * * *", timezone = "Mars/Olympus" }\n', - "orchestration/jobs/orders_job.toml:4:45:", - "schedule timezone is not an IANA time zone", - ), - ( - "a starts_at that is not RFC 3339 is reported at its value", - 'select = "orders"\n' - 'schedule = { cron = "0 2 * * *", timezone = "UTC", starts_at = "yesterday" }\n', - "orchestration/jobs/orders_job.toml:4:64:", - "schedule starts_at must be an RFC 3339 timestamp", - ), - ( - "an invalid build setting is reported at its value", - 'select = "orders"\n\n[settings]\nconcurrency = "lots"\n', - "orchestration/jobs/orders_job.toml:6:15:", - "concurrency", - ), - ( - "an unknown selector type is reported at the selector", - 'select = "orders tagg:finance"\n', - "orchestration/jobs/orders_job.toml:3:10:", - "unknown selector type 'tagg' in 'tagg:finance'", - ), - ( - "an empty intersection part is reported at the selector", - 'select = "tag:finance,"\n', - "orchestration/jobs/orders_job.toml:3:10:", - "empty selector", - ), - ) + PythonProjectLayoutCompileTestCase( + description="an invalid cron is reported at its value", + repo_files=orders_job_project( + 'select = "orders"\nschedule = { cron = "not a cron", timezone = "UTC" }\n' + ), + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs/orders_job.toml:4:21:", + "schedule cron: invalid cron expression", + ), + ), + PythonProjectLayoutCompileTestCase( + description="an unknown time zone is reported at its value", + repo_files=orders_job_project( + 'select = "orders"\nschedule = { cron = "0 2 * * *", timezone = "Mars/Olympus" }\n' + ), + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs/orders_job.toml:4:45:", + "schedule timezone is not an IANA time zone", + ), + ), + PythonProjectLayoutCompileTestCase( + description="a starts_at that is not RFC 3339 is reported at its value", + repo_files=orders_job_project( + 'select = "orders"\nschedule = { cron = "0 2 * * *", timezone = "UTC", starts_at = "yesterday" }\n' + ), + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs/orders_job.toml:4:64:", + "schedule starts_at must be an RFC 3339 timestamp", + ), + ), + PythonProjectLayoutCompileTestCase( + description="an invalid build setting is reported at its value", + repo_files=orders_job_project( + 'select = "orders"\n\n[settings]\nconcurrency = "lots"\n' + ), + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs/orders_job.toml:6:15:", + "concurrency", + ), + ), + PythonProjectLayoutCompileTestCase( + description="an unknown selector type is reported at the selector", + repo_files=orders_job_project('select = "orders tagg:finance"\n'), + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs/orders_job.toml:3:10:", + "unknown selector type 'tagg' in 'tagg:finance'", + ), + ), + PythonProjectLayoutCompileTestCase( + description="an empty intersection part is reported at the selector", + repo_files=orders_job_project('select = "tag:finance,"\n'), + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs/orders_job.toml:3:10:", + "empty selector", + ), ), PythonProjectLayoutCompileTestCase( description="removed tasks Python root", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py index 0a429d21d..da2c79c8b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py @@ -698,3 +698,8 @@ def layout_view(name: str, source: str = '__ref("stg_orders")') -> str: for folder, name in (("fin_a", "a"), ("fin_b", "b"), ("core", "c")) }, } + + +def refresh_job(select: str) -> str: + """Return a job file whose `select` is the TOML value `select`.""" + return f'name = "orders_refresh_job"\ndescription = "Refresh orders."\nselect = {select}\n' diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py index de4e65ace..9202e5299 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py @@ -10,7 +10,11 @@ from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( JobSelectorRefactorE2ETestCase, ) -from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import sqb, write_orders_project +from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( + refresh_job, + sqb, + write_orders_project, +) _JOB: str = "orchestration/jobs/orders/refresh.toml" _UNRELATED: str = "orchestration/jobs/orders/totals.toml" @@ -22,12 +26,6 @@ ) -def _job(select: str, exclude: str = "") -> str: - return f'name = "orders_refresh_job"\ndescription = "Refresh orders."\nselect = {select}\n' + ( - f"exclude = {exclude}\n" if exclude else "" - ) - - @pytest.mark.parametrize( "test_case", [ @@ -44,15 +42,15 @@ def _job(select: str, exclude: str = "") -> str: JobSelectorRefactorE2ETestCase( description="moving every model out of a folder moves its folder selectors", command=("mv", "stg_orders", "models/core/"), - job=_job('"models/staging +path:models/staging/ stg_orders"'), - expected_job=_job('"models/core +path:models/core/ stg_orders"'), + job=refresh_job('"models/staging +path:models/staging/ stg_orders"'), + expected_job=refresh_job('"models/core +path:models/core/ stg_orders"'), expected_output=("selector", _JOB), ), JobSelectorRefactorE2ETestCase( description="moving some models out of a folder leaves it and asks for a check", command=("mv", "fact_orders", "models/finance/"), - job=_job('"+models/marts"'), - expected_job=_job('"+models/marts"'), + job=refresh_job('"+models/marts"'), + expected_job=refresh_job('"+models/marts"'), expected_output=( "Notes", f"{_JOB}:3:10", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 897269da8..8c8165381 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -431,13 +431,19 @@ def expected_summary(name: str) -> dict[str, object]: def duckshop_orchestration(fixture: str) -> dict[str, str]: """A DuckShop orchestration fixture tree as project files below `orchestration/`.""" root: Path = Path("benchmarks/duckshop/edge_cases/orchestration") / fixture + paths: list[Path] = sorted([*root.rglob("*.py"), *root.rglob("*.toml")]) return { - f"orchestration/{path.relative_to(root).as_posix()}": path.read_text() - for path in sorted(root.rglob("*")) - if path.is_file() + f"orchestration/{path.relative_to(root).as_posix()}": path.read_text() for path in paths } +def edited_duckshop_orchestration(fixture: str, path: str, old: str, new: str) -> dict[str, str]: + """A DuckShop orchestration fixture with `old` replaced by `new` in the file at `path`.""" + files: dict[str, str] = duckshop_orchestration(fixture) + assert old in files[path], f"{old!r} is not in {path}" + return files | {path: files[path].replace(old, new)} + + def copy_project(*, fixtures: tuple[str, ...], files: dict[str, str], project_dir: Path) -> Path: """Copy each fixture into `project_dir`, then write `files` over them.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index de0a58434..16f5045ad 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -117,6 +117,18 @@ class WorkerExecutionTestCase: expected_node_counts: dict[str, int] | None = None expected_origin: Literal["schedule", "sensor"] = "schedule" files: dict[str, str] = field(default_factory=dict) + # The scheduled job written before `files`; cases that declare their own jobs set it to {}. + default_job_files: dict[str, str] = field( + default_factory=lambda: { + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", ' + 'starts_at = "2000-01-01T00:00:00Z" }\n' + ), + } + ) files_after_publication: dict[str, str] = field(default_factory=dict) store_setup: str = "" store_edit: Callable[[sqlite3.Connection], None] = lambda _: None diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 4d0a0e168..4007281a0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -816,19 +816,7 @@ def scheduled_worker( " assert 'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL' not in os.environ\n" ), } - | ( - {} - if any(path.startswith("orchestration/jobs/") for path in test_case.files) - else { - "orchestration/jobs/annual_orders_job.toml": ( - 'name = "annual_orders_job"\n' - 'description = "Build orders."\n' - 'select = "orders"\n' - 'schedule = { cron = "0 0 * * *", timezone = "UTC", ' - 'starts_at = "2000-01-01T00:00:00Z" }\n' - ), - } - ) + | test_case.default_job_files | test_case.files, ) test_case.warehouse_setup(project / "warehouse.duckdb") diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py index 51988adfa..b8c6978b0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py @@ -88,6 +88,7 @@ def test_given_explicit_backfill_bounds_when_running_and_rerunning_a_group_then_ 'jobs = ["windowed_orders_job"]\n' ), }, + default_job_files={}, ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index 3282ad96a..829ce52e7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -313,6 +313,7 @@ def test_given_a_published_calendar_when_a_real_worker_compiles_then_it_does_not " marker.write_text('evaluated')\n return []\n" ), }, + default_job_files={}, ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: assert evaluated.read_text() == "evaluated" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py index 68695dd67..10f655d91 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py @@ -80,7 +80,9 @@ def test_given_a_changed_job_when_inputs_change_between_runs_then_only_their_consumers_run( test_case: ChangedJobTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - test_case = replace(test_case, files=changed_job_files(test_case.project_toml)) + test_case = replace( + test_case, files=changed_job_files(test_case.project_toml), default_job_files={} + ) observed: list[tuple[str, frozenset[str], dict[str, int]]] = [] with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: warehouse: Path = worker.project / test_case.warehouse_file diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py index 2b651b832..6b543dd1c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py @@ -70,6 +70,7 @@ " return [RunRequest(run_key='orders-1'), RunRequest(run_key='orders-1')]\n" ), }, + default_job_files={}, ) ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py index 91e07f168..f0e16d790 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -67,6 +67,7 @@ [(41,)], launcher="docker", files={**duckshop_orchestration("container_resources_orders")}, + default_job_files={}, launcher_settings={ "run": {"memory": "768m", "cpus": 2, "pids": 256, "max_memory": "1g"} }, @@ -80,6 +81,7 @@ [(43,)], launcher="docker", files={**duckshop_orchestration("container_resources_orders")}, + default_job_files={}, expected_host_config={"Memory": 768 << 20, "NanoCpus": 500_000_000}, ), WorkerExecutionTestCase( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py index 7af8561cc..995ca03d7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py @@ -38,6 +38,7 @@ 'select = "orders customers"\n' ), }, + default_job_files={}, ) ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py index 584d7c405..0feaabe88 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py @@ -116,6 +116,7 @@ def test_given_a_running_local_worker_when_scheduler_dies_then_restart_adopts_an files={ **duckshop_orchestration("local_recovery_orders"), }, + default_job_files={}, store_setup=Path( "benchmarks/duckshop/edge_cases/orchestration/local_recovery_publication_fault.sql" ).read_text(), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index 5de785eb7..a4e413d3e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -67,8 +67,10 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical TYPE_PROOF_FILES | { path.relative_to(duckshop).as_posix(): path.read_text() - for path in (duckshop / "orchestration").rglob("*") - if path.is_file() + for path in [ + *(duckshop / "orchestration").rglob("*.py"), + *(duckshop / "orchestration").rglob("*.toml"), + ] } | { "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py index 7a5ebfc24..9c5caf7b7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py @@ -77,6 +77,7 @@ def test_given_a_partial_group_when_retrying_from_failure_then_successful_nodes_ 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, + default_job_files={}, ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: source: dict[str, Any] = wait_for_run( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py index f5af3b526..54e7ee6a1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py @@ -47,6 +47,7 @@ "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, + default_job_files={}, ), WorkerExecutionTestCase( "stale external timestamp versions", @@ -67,6 +68,7 @@ "benchmarks/duckshop/edge_cases/orchestration/source_stale_journey.json" ).read_text(), }, + default_job_files={}, ), WorkerExecutionTestCase( "a changed: True job still rebuilds on new source data", @@ -81,6 +83,7 @@ "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, + default_job_files={}, warehouse_setup=create_raw_orders, ), ], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 7c1b282dd..4b469b733 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -162,6 +162,7 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail ), } | test_case.files, + default_job_files={}, ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: parent: dict[str, Any] = wait_for_run( @@ -623,6 +624,7 @@ def test_given_a_failing_seed_audit_when_a_worker_builds_then_its_seed_fails_wit 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, + default_job_files={}, store_setup=( "CREATE TABLE test_slot_claims (run_id TEXT, group_key TEXT, occupied INTEGER);\n" "CREATE TRIGGER test_observe_slots AFTER INSERT ON sqb_concurrency_slots BEGIN " @@ -864,6 +866,7 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ ).read_text(), **duckshop_orchestration("upstream_source_orders"), }, + default_job_files={}, warehouse_setup=lambda path: ( duckdb.connect(str(path)) .execute("CREATE TABLE raw_orders AS SELECT CAST(23 AS BIGINT) AS order_id") @@ -884,6 +887,7 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ ).read_text(), **duckshop_orchestration("upstream_source_orders"), }, + default_job_files={}, warehouse_setup=lambda path: ( duckdb.connect(str(path)) .execute("CREATE TABLE raw_orders AS SELECT CAST(23 AS BIGINT) AS order_id") @@ -968,6 +972,7 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re "success", [(9,)], files=INTERMEDIATE_LOADER_FILES | {**duckshop_orchestration("upstream_loaded_orders")}, + default_job_files={}, expected_node_count=3, expected_write_set=frozenset( {"model:orders", "source:raw_shipments", "source:stage_shipments"} @@ -1478,6 +1483,7 @@ def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_s 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, + default_job_files={}, ), ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index ba96265ec..aefebb42f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -46,6 +46,7 @@ PARITY_FILES, TYPE_PROOF_FILES, duckshop_orchestration, + edited_duckshop_orchestration, published_texts, python_pipeline_snapshot, ) @@ -69,13 +70,6 @@ } -def _edited(fixture: str, old: str, new: str) -> dict[str, str]: - """A DuckShop orchestration fixture with one edit to the file that holds `old`.""" - files: dict[str, str] = duckshop_orchestration(fixture) - (path,) = [path for path, text in files.items() if old in text] - return files | {path: files[path].replace(old, new)} - - _PROVIDER_PROJECT: dict[str, str] = { "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', "models/orders.sql": "MODEL (description 'Orders.');\n\nSELECT 1 AS order_id\n", @@ -1310,8 +1304,9 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject SnapshotDiscoveryTestCase( description="failure and recovery selectors preserve their individual transition kinds", files={ - **_edited( + **edited_duckshop_orchestration( "node_orders", + "orchestration/sensors/on_orders_sensor.py", "NodeEventKind.SUCCESS", "NodeEventKind.FAILED, NodeEventKind.RECOVERED", ) @@ -1617,8 +1612,11 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="a zero sensor interval cannot publish", files={ - **_edited( - "platform_orders", "minimum_interval_seconds=7", "minimum_interval_seconds=0" + **edited_duckshop_orchestration( + "platform_orders", + "orchestration/sensors/overdue_orders_sensor.py", + "minimum_interval_seconds=7", + "minimum_interval_seconds=0", ) }, expected_error=ValueError, @@ -1627,7 +1625,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="empty platform conditions cannot publish", files={ - **_edited("platform_orders", "kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=[]") + **edited_duckshop_orchestration( + "platform_orders", + "orchestration/sensors/overdue_orders_sensor.py", + "kinds=[PlatformEventKind.TICK_OVERDUE]", + "kinds=[]", + ) }, expected_error=ValueError, expected_message="nonempty event kinds", @@ -1635,8 +1638,9 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="platform kinds are a closed vocabulary", files={ - **_edited( + **edited_duckshop_orchestration( "platform_orders", + "orchestration/sensors/overdue_orders_sensor.py", "kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=['unknown_event']", ) @@ -1646,14 +1650,26 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects ), SnapshotDiscoveryRejectionTestCase( description="empty node selectors cannot broaden matching to all nodes", - files={**_edited("node_orders", 'select="orders orders*"', 'select=" "')}, + files={ + **edited_duckshop_orchestration( + "node_orders", + "orchestration/sensors/on_orders_sensor.py", + 'select="orders orders*"', + 'select=" "', + ) + }, expected_error=ValueError, expected_message="node_event needs nonempty selectors", ), SnapshotDiscoveryRejectionTestCase( description="unknown node event kinds cannot publish", files={ - **_edited("node_orders", "kinds=[NodeEventKind.SUCCESS]", "kinds=['unknown_event']") + **edited_duckshop_orchestration( + "node_orders", + "orchestration/sensors/on_orders_sensor.py", + "kinds=[NodeEventKind.SUCCESS]", + "kinds=['unknown_event']", + ) }, expected_error=ValueError, expected_message="unknown variant", @@ -1661,7 +1677,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="source_new_data rejects model selections", files={ - **_edited("source_orders", 'select="source:raw_orders"', 'select="orders"'), + **edited_duckshop_orchestration( + "source_orders", + "orchestration/sensors/on_source_orders_sensor.py", + 'select="source:raw_orders"', + 'select="orders"', + ), }, expected_error=ValueError, expected_message="source_new_data must select only source nodes", @@ -1669,7 +1690,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="source_new_data rejects blank selections", files={ - **_edited("source_orders", 'select="source:raw_orders"', 'select=" "'), + **edited_duckshop_orchestration( + "source_orders", + "orchestration/sensors/on_source_orders_sensor.py", + 'select="source:raw_orders"', + 'select=" "', + ), }, expected_error=ValueError, expected_message="source_new_data needs nonempty selectors", @@ -1677,7 +1703,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="source_stale rejects a zero threshold", files={ - **_edited("stale_orders", "after_seconds=60", "after_seconds=0"), + **edited_duckshop_orchestration( + "stale_orders", + "orchestration/sensors/on_stale_orders_sensor.py", + "after_seconds=60", + "after_seconds=0", + ), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), @@ -1688,7 +1719,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="source_stale rejects a boolean threshold", files={ - **_edited("stale_orders", "after_seconds=60", "after_seconds=True"), + **edited_duckshop_orchestration( + "stale_orders", + "orchestration/sensors/on_stale_orders_sensor.py", + "after_seconds=60", + "after_seconds=True", + ), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), @@ -1699,7 +1735,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="source_stale rejects an overflowing threshold", files={ - **_edited("stale_orders", "after_seconds=60", "after_seconds=2**32"), + **edited_duckshop_orchestration( + "stale_orders", + "orchestration/sensors/on_stale_orders_sensor.py", + "after_seconds=60", + "after_seconds=2**32", + ), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), @@ -1710,51 +1751,100 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="source_stale rejects model selections", files={ - **_edited("stale_orders", 'select="source:raw_orders"', 'select="orders"'), + **edited_duckshop_orchestration( + "stale_orders", + "orchestration/sensors/on_stale_orders_sensor.py", + 'select="source:raw_orders"', + 'select="orders"', + ), }, expected_error=ValueError, expected_message="source_stale must select only source nodes", ), SnapshotDiscoveryRejectionTestCase( description="explicit empty execution selectors cannot broaden matching", - files={**_edited("run_orders", '("orders",)', "()")}, + files={ + **edited_duckshop_orchestration( + "run_orders", "orchestration/sensors/on_orders_sensor.py", '("orders",)', "()" + ) + }, expected_error=ValueError, expected_message="executed_successfully needs nonempty selectors", ), SnapshotDiscoveryRejectionTestCase( description="blank execution selectors cannot broaden matching", - files={**_edited("run_orders", '("orders",)', '(" ",)')}, + files={ + **edited_duckshop_orchestration( + "run_orders", + "orchestration/sensors/on_orders_sensor.py", + '("orders",)', + '(" ",)', + ) + }, expected_error=ValueError, expected_message="executed_successfully needs nonempty selectors", ), SnapshotDiscoveryRejectionTestCase( description="unresolved execution selectors fail closed", - files={**_edited("run_orders", '("orders",)', '("unknown_orders",)')}, + files={ + **edited_duckshop_orchestration( + "run_orders", + "orchestration/sensors/on_orders_sensor.py", + '("orders",)', + '("unknown_orders",)', + ) + }, expected_error=ValueError, expected_message="unknown selector name 'unknown_orders'", ), SnapshotDiscoveryRejectionTestCase( description="unmatched execution patterns cannot become an omitted filter", - files={**_edited("run_orders", '("orders",)', '("unknown_*",)')}, + files={ + **edited_duckshop_orchestration( + "run_orders", + "orchestration/sensors/on_orders_sensor.py", + '("orders",)', + '("unknown_*",)', + ) + }, expected_error=ValueError, expected_message="executed_successfully: unknown selector name 'unknown_\\*'", ), SnapshotDiscoveryRejectionTestCase( description="invalid execution selector syntax fails closed", - files={**_edited("run_orders", '("orders",)', '("tag:",)')}, + files={ + **edited_duckshop_orchestration( + "run_orders", + "orchestration/sensors/on_orders_sensor.py", + '("orders",)', + '("tag:",)', + ) + }, expected_error=ValueError, expected_message="has empty value after ':'", ), SnapshotDiscoveryRejectionTestCase( description="execution selectors outside the source job fail closed", - files={**_edited("run_orders", '("orders",)', '("stg_orders",)')}, + files={ + **edited_duckshop_orchestration( + "run_orders", + "orchestration/sensors/on_orders_sensor.py", + '("orders",)', + '("stg_orders",)', + ) + }, expected_error=ValueError, expected_message="executed_successfully must select nonempty source job/group nodes", ), SnapshotDiscoveryRejectionTestCase( description="sensor code defaults use running or paused", files={ - **_edited("platform_orders", 'default_status="paused"', 'default_status="stopped"') + **edited_duckshop_orchestration( + "platform_orders", + "orchestration/sensors/overdue_orders_sensor.py", + 'default_status="paused"', + 'default_status="stopped"', + ) }, expected_error=ValueError, expected_message="unknown variant", From c92c4069c52f8dd60bbb47a5dbe761b3550ae817 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:25:41 +0000 Subject: [PATCH 31/34] refactor: read job and job group files through one TOML reader --- .../_helpers/inventory_files.rs | 36 ++++++------------- .../main/orchestration_inventory.rs | 10 ++++-- 2 files changed, 18 insertions(+), 28 deletions(-) diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs index 85ba6a6b4..57c463b88 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs @@ -6,15 +6,13 @@ use crate::_helpers::reading::read_authored_text; use crate::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; use crate::models::{DiscoveryFailure, FailureKind}; use crate::orchestration_files::_helpers::python_modules::module_declaration; -use crate::orchestration_files::_helpers::read_group_file::read_group_file; -use crate::orchestration_files::_helpers::read_job_file::read_job_file; use crate::orchestration_files::constants::{ BYTECODE_DIRECTORY, CALENDARS_DIRECTORY, GROUPS_DIRECTORY, HIDDEN_PREFIX, JOBS_DIRECTORY, ROLE_DIRECTORIES, SENSORS_DIRECTORY, TOML_FILE_SUFFIX, TOML_JOBS_HELP, }; use crate::orchestration_files::errors::FileError; use crate::orchestration_files::models::{ - FilePosition, GroupFile, JobFile, NamedModule, OrchestrationFailure, PythonModules, + FilePosition, NamedModule, OrchestrationFailure, PythonModules, }; use crate::tree::main::children::children; use crate::tree::main::rglob::rglob; @@ -134,30 +132,18 @@ fn role_files( Ok(kept) } -/// Every job file with its path. -pub(crate) fn read_jobs( +/// Every TOML file below one role folder, read by `reader`, with its path. +pub(crate) fn read_toml_files( tree: &ProjectTree, -) -> Result, OrchestrationFailure> { - let mut jobs: Vec<(String, JobFile)> = Vec::new(); - for path in role_files(tree, JOBS_DIRECTORY, TOML_FILE_SUFFIX)? { - let job: JobFile = - read_job_file(&read(tree, &path)?).map_err(|error| failure(&path, error))?; - jobs.push((path, job)); - } - Ok(jobs) -} - -/// Every job group file with its path. -pub(crate) fn read_groups( - tree: &ProjectTree, -) -> Result, OrchestrationFailure> { - let mut groups: Vec<(String, GroupFile)> = Vec::new(); - for path in role_files(tree, GROUPS_DIRECTORY, TOML_FILE_SUFFIX)? { - let group: GroupFile = - read_group_file(&read(tree, &path)?).map_err(|error| failure(&path, error))?; - groups.push((path, group)); + role: &str, + reader: fn(&str) -> Result, +) -> Result, OrchestrationFailure> { + let mut files: Vec<(String, T)> = Vec::new(); + for path in role_files(tree, role, TOML_FILE_SUFFIX)? { + let file: T = reader(&read(tree, &path)?).map_err(|error| failure(&path, error))?; + files.push((path, file)); } - Ok(groups) + Ok(files) } /// Every calendar and sensor module, read without importing it. diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs index d1837fa2b..8759e9e69 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs @@ -2,9 +2,12 @@ use crate::constants::ORCHESTRATION_DIRECTORY; use crate::orchestration_files::_helpers::inventory_files::{ - check_top_level, failure, read_groups, read_jobs, read_python_modules, + check_top_level, failure, read_python_modules, read_toml_files, }; +use crate::orchestration_files::_helpers::read_group_file::read_group_file; +use crate::orchestration_files::_helpers::read_job_file::read_job_file; use crate::orchestration_files::_helpers::references::check_references; +use crate::orchestration_files::constants::{GROUPS_DIRECTORY, JOBS_DIRECTORY}; use crate::orchestration_files::models::{ GroupFile, JobFile, OrchestrationFailure, OrchestrationInventory, PythonModules, }; @@ -19,8 +22,9 @@ pub fn orchestration_inventory( return Ok(OrchestrationInventory::default()); } check_top_level(tree)?; - let jobs: Vec<(String, JobFile)> = read_jobs(tree)?; - let groups: Vec<(String, GroupFile)> = read_groups(tree)?; + let jobs: Vec<(String, JobFile)> = read_toml_files(tree, JOBS_DIRECTORY, read_job_file)?; + let groups: Vec<(String, GroupFile)> = + read_toml_files(tree, GROUPS_DIRECTORY, read_group_file)?; let PythonModules { calendars, sensors, From 557de017062f433d44aff75a644cf050badd8771 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:22:02 +0000 Subject: [PATCH 32/34] test: cover the orchestration declaration error in the failure corpus The failure corpus must reach every discovery error code. D017, which rejects Python job modules and invalid TOML orchestration declarations, had no case, so the corpus coverage E2E failed in the merge queue. --- .../_helpers/corpus/discovery_failure_cases.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py index b420c64c0..a72a2b462 100644 --- a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py @@ -389,6 +389,17 @@ def _python_cases() -> tuple[FailureCase, ...]: for module in ("first", "second") }, ), + failure_case( + name="orchestration-python-job-module", + expected_code="D017", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "nightly = job('nightly', description='Orders.', select='stg_orders')\n" + ) + }, + expected_message="files cannot sit directly in orchestration/", + ), failure_case( name="legacy-event-exporters-directory", expected_code="D015", From f629221020fd297b4c0de30773803fe1e7f9a575 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 18:58:04 +0000 Subject: [PATCH 33/34] test: record the compile output of a Python job module under orchestration/ --- .../orchestration-python-job-module.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/goldens/compiler/failure/orchestration-python-job-module.json diff --git a/tests/goldens/compiler/failure/orchestration-python-job-module.json b/tests/goldens/compiler/failure/orchestration-python-job-module.json new file mode 100644 index 000000000..b811a2c18 --- /dev/null +++ b/tests/goldens/compiler/failure/orchestration-python-job-module.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "D017", + "message": "orchestration/jobs.py:1:1: files cannot sit directly in orchestration/", + "help": "jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/" + } + ] + } + ], + "compiled": {}, + "manifest": null +} From 05fbb891d308871fa47962461e5849d5c75fdc2d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 19:13:42 +0000 Subject: [PATCH 34/34] test: wait for the TOML job key in the ad-hoc load and variables test --- .../compiler/compiled_project/publication/test_adhoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index 8c63d9234..f761f6232 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -475,7 +475,7 @@ def test_given_ad_hoc_loads_and_variables_when_launched_then_only_permitted_peop server_arguments=("--password-origin", "https://control.example.com"), ) as worker: scheduled: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert scheduled["status"] == "success", worker.log.read_text() admin: dict[str, Any] = provision_person(