diff --git a/Cargo.lock b/Cargo.lock index 3f8fbd50c..772d66f4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4288,12 +4288,16 @@ version = "0.0.0" dependencies = [ "junction", "rayon", + "serde", + "serde_json", "sqlbuild-cache", "sqlbuild-config", "sqlbuild-core", "sqlbuild-model-config", + "sqlbuild-pysource", "sqlbuild-sqltext", "tempfile", + "toml", ] [[package]] @@ -4453,6 +4457,7 @@ dependencies = [ "sqlbuild-pysource", "sqlbuild-sqltext", "tempfile", + "toml", ] [[package]] diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs.py b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs.py deleted file mode 100644 index 24b96260d..000000000 --- a/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Two scheduled jobs split the orders lineage and share int_orders; a third job has no trigger.""" - -from sqlbuild.orchestration import job, schedule - -daily = schedule(cron="0 2 * * *", timezone="UTC") -staging = job( - "staging_daily", description="Stage orders.", select="stg_orders int_orders", schedule=daily -) -finance = job( - "finance_daily", - description="Build revenue reports.", - select="int_orders fct_revenue rpt_pnl", - schedule=daily, -) -exports = job("adhoc_exports", description="Export orders on request.", select="orders_export") diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/adhoc_exports_job.toml b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/adhoc_exports_job.toml new file mode 100644 index 000000000..e6c4a3963 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/adhoc_exports_job.toml @@ -0,0 +1,3 @@ +name = "adhoc_exports_job" +description = "Export orders on request." +select = "orders_export" diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/finance_daily_job.toml b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/finance_daily_job.toml new file mode 100644 index 000000000..5075f6d97 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/finance_daily_job.toml @@ -0,0 +1,4 @@ +name = "finance_daily_job" +description = "Build revenue reports." +select = "int_orders fct_revenue rpt_pnl" +schedule = { cron = "0 2 * * *", timezone = "UTC" } diff --git a/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/staging_daily_job.toml b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/staging_daily_job.toml new file mode 100644 index 000000000..a53b42295 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_reach/orchestration/jobs/staging_daily_job.toml @@ -0,0 +1,5 @@ +# Two scheduled jobs split the orders lineage and share int_orders; a third job has no trigger. +name = "staging_daily_job" +description = "Stage orders." +select = "stg_orders int_orders" +schedule = { cron = "0 2 * * *", timezone = "UTC" } diff --git a/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs.py b/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs.py deleted file mode 100644 index be6161714..000000000 --- a/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Two scheduled jobs each own one staging model; customer_orders reads both, in crm_daily.""" - -from sqlbuild.orchestration import job, schedule - -daily = schedule(cron="0 2 * * *", timezone="UTC") -crm = job( - "crm_daily", - description="Refresh customers.", - select="stg_customers customer_orders", - schedule=daily, -) -sales = job( - "sales_daily", - description="Refresh sales.", - select="stg_orders customer_sales", - schedule=daily, -) diff --git a/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/crm_daily_job.toml b/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/crm_daily_job.toml new file mode 100644 index 000000000..534d1a0ab --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/crm_daily_job.toml @@ -0,0 +1,5 @@ +# Two scheduled jobs each own one staging model; customer_orders reads both, in crm_daily_job. +name = "crm_daily_job" +description = "Refresh customers." +select = "stg_customers customer_orders" +schedule = { cron = "0 2 * * *", timezone = "UTC" } diff --git a/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/sales_daily_job.toml b/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/sales_daily_job.toml new file mode 100644 index 000000000..66eb43b1a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/changed_builds_two_up/orchestration/jobs/sales_daily_job.toml @@ -0,0 +1,4 @@ +name = "sales_daily_job" +description = "Refresh sales." +select = "stg_orders customer_sales" +schedule = { cron = "0 2 * * *", timezone = "UTC" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json index f111fc629..76c5dad56 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json +++ b/benchmarks/duckshop/edge_cases/orchestration/attached_checks.json @@ -3,6 +3,8 @@ "warned": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.warn(message='Review order totals.')\n", "failed": "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Order totals are valid.')\ndef orders_valid(ctx):\n return ctx.fail(message='Order totals are invalid.')\n", "cancelled": "import signal\nfrom sqlbuild.checks import check\nfrom sqlbuild.refs import model\n@check(depends_on=model('orders'), description='Heartbeat cancellation interrupts the driver without SIGTERM.')\ndef orders_valid(ctx):\n signal.signal(signal.SIGTERM, signal.SIG_IGN)\n ctx.query('WITH RECURSIVE pending(i) AS ((SELECT i FROM range(10000000000) AS seed(i) LIMIT 1) UNION ALL SELECT i FROM pending) SELECT sum(i) FROM pending')\n return ctx.pass_()\n", - "on_failure": "from sqlbuild.orchestration import NodeEventKind, job, node_event, schedule, sensor\nannual_orders = job('annual_orders', select='orders', description='Build orders.', schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\nrefresh_inventory = job('refresh_inventory', select='inventory', description='Refresh inventory.')\nfailed_orders = sensor('failed_orders', description='React to an actual orders check failure.', when=node_event(select='check:orders_valid', kinds=[NodeEventKind.FAILED]), run=refresh_inventory, minimum_interval_seconds=1)\n", + "on_failure_job": "name = \"annual_orders_job\"\ndescription = \"Build orders.\"\nselect = \"orders\"\nschedule = { cron = \"0 0 * * *\", timezone = \"UTC\", starts_at = \"2000-01-01T00:00:00Z\" }\n", + "on_failure_destination": "name = \"refresh_inventory_job\"\ndescription = \"Refresh inventory.\"\nselect = \"inventory\"\n", + "on_failure_sensor": "from sqlbuild.orchestration import NodeEventKind, node_event, sensor\n\nfailed_orders = sensor(\n 'failed_orders_sensor',\n description='React to an actual orders check failure.',\n when=node_event(select='check:orders_valid', kinds=[NodeEventKind.FAILED]),\n run='refresh_inventory_job',\n minimum_interval_seconds=1,\n)\n", "inventory": "MODEL (description 'Inventory after check failure.', materialized table);\nSELECT 59 AS amount;\n" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py deleted file mode 100644 index ce5a9b672..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Provider-backed fulfillment times evaluated in the code location, never by the scheduler.""" - -from sqlbuild.orchestration import calendar_schedule, job - -orders = job("order_reporting", description="Build order reporting.", select="orders") - - -@calendar_schedule( - job=orders, - description="Refresh orders at upcoming fulfillment times.", - horizon="8s", - timezone="America/New_York", -) -def fulfillment_calendar(window, fulfillment): - return fulfillment.upcoming(window) - - -# Declared for another target without decorating a function: development publication skips it. -production_calendar = calendar_schedule( - job=orders, - description="Production fulfillment times, not yet attached to a provider.", - horizon="8s", - timezone="UTC", - targets=("production",), -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/calendars/fulfillment_calendar.py b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/calendars/fulfillment_calendar.py new file mode 100644 index 000000000..bd368ad3c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/calendars/fulfillment_calendar.py @@ -0,0 +1,13 @@ +"""Upcoming fulfillment times from the provider, evaluated in the code location.""" + +from sqlbuild.orchestration import calendar + + +@calendar( + name="fulfillment_calendar", + description="Refresh orders at upcoming fulfillment times.", + horizon="8s", +) +def fulfillment_calendar(window, fulfillment): + # A provider method named `calendar` is not a second calendar declaration. + return fulfillment.calendar(window) diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/jobs/order_reporting_job.toml b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/jobs/order_reporting_job.toml new file mode 100644 index 000000000..80fe02628 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders/jobs/order_reporting_job.toml @@ -0,0 +1,5 @@ +# Provider-backed fulfillment times evaluated in the code location, never by the scheduler. +name = "order_reporting_job" +description = "Build order reporting." +select = "orders" +schedule = { calendar = "fulfillment_calendar", timezone = "America/New_York" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders.py b/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders.py deleted file mode 100644 index 377130a8b..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders.py +++ /dev/null @@ -1,17 +0,0 @@ -"""A sensor on new source data starts a job that builds only what changed, source data included.""" - -from sqlbuild.orchestration import job, sensor, source_new_data - -orders_refresh = job( - "orders_refresh", - description="Refresh orders when the source has new data.", - select="orders", - settings={"changed": True}, -) -on_source_orders = sensor( - "on_source_orders", - description="Refresh orders for each changed source version.", - when=source_new_data(select="source:raw_orders"), - run=orders_refresh, - minimum_interval_seconds=1, -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/jobs/orders_refresh_job.toml b/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/jobs/orders_refresh_job.toml new file mode 100644 index 000000000..188309855 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/jobs/orders_refresh_job.toml @@ -0,0 +1,7 @@ +# A sensor on new source data starts a job that builds only what changed, source data included. +name = "orders_refresh_job" +description = "Refresh orders when the source has new data." +select = "orders" + +[settings] +changed = true diff --git a/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/sensors/on_source_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/sensors/on_source_orders_sensor.py new file mode 100644 index 000000000..a8abd7568 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/changed_source_orders/sensors/on_source_orders_sensor.py @@ -0,0 +1,11 @@ +"""Start the changed-only refresh for each new source version.""" + +from sqlbuild.orchestration import sensor, source_new_data + +on_source_orders = sensor( + "on_source_orders_sensor", + description="Refresh orders for each changed source version.", + when=source_new_data(select="source:raw_orders"), + run="orders_refresh_job", + minimum_interval_seconds=1, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py deleted file mode 100644 index e864cf753..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Negative fixture: mutually exclusive build flags must fail before publication.""" - -from sqlbuild.orchestration import job - -orders = job( - "refresh_orders", - description="Refresh orders.", - select="orders", - settings={"load_sources": False, "reload": True}, -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings/jobs/refresh_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings/jobs/refresh_orders_job.toml new file mode 100644 index 000000000..82c136e70 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/conflicting_settings/jobs/refresh_orders_job.toml @@ -0,0 +1,8 @@ +# Negative fixture: mutually exclusive build flags must fail before publication. +name = "refresh_orders_job" +description = "Refresh orders." +select = "orders" + +[settings] +load_sources = false +reload = true diff --git a/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py b/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py deleted file mode 100644 index b419a1f81..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py +++ /dev/null @@ -1,11 +0,0 @@ -"""A job asks for more memory than its worker may have and fewer CPUs than the launcher default.""" - -from sqlbuild.orchestration import job, resources, schedule - -annual_orders = job( - "annual_orders", - select="orders", - description="Build orders.", - resources=resources(memory="2g", cpus=0.5), - schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..0bae046ba --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/container_resources_orders/jobs/annual_orders_job.toml @@ -0,0 +1,9 @@ +# A job asks for more memory than its worker may have and fewer CPUs than the launcher default. +name = "annual_orders_job" +description = "Build orders." +select = "orders" +schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" } + +[resources] +memory = "2g" +cpus = 0.5 diff --git a/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py b/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py deleted file mode 100644 index f448f2b85..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/custom_orders.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Polling callbacks return durable run keys and keep their cursor in the native transaction.""" - -from sqlbuild.orchestration import job, sensor - -orders = job("annual_orders", description="Build orders after source inspection.", select="orders") - - -@sensor( - "poll_orders", description="Poll fulfillment readiness.", run=orders, minimum_interval_seconds=1 -) -def poll_orders(ctx, fulfillment): - def poll(): - return fulfillment.poll(ctx) - - return poll() diff --git a/benchmarks/duckshop/edge_cases/orchestration/custom_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/custom_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..2f4d35d6c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/custom_orders/jobs/annual_orders_job.toml @@ -0,0 +1,3 @@ +name = "annual_orders_job" +description = "Build orders after source inspection." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/custom_orders/sensors/poll_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/custom_orders/sensors/poll_orders_sensor.py new file mode 100644 index 000000000..ad888268d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/custom_orders/sensors/poll_orders_sensor.py @@ -0,0 +1,16 @@ +"""Polling callbacks return durable run keys and keep their cursor in the native transaction.""" + +from sqlbuild.orchestration import sensor + + +@sensor( + "poll_orders_sensor", + description="Poll fulfillment readiness.", + run="annual_orders_job", + minimum_interval_seconds=1, +) +def poll_orders(ctx, fulfillment): + def poll(): + return fulfillment.poll(ctx) + + return poll() diff --git a/benchmarks/duckshop/edge_cases/orchestration/development_orders.py b/benchmarks/duckshop/edge_cases/orchestration/development_orders.py deleted file mode 100644 index 7306a2147..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/development_orders.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Development defaults pause code-enabled triggers without changing their definitions.""" - -from sqlbuild.orchestration import job, schedule - -annual = schedule(cron="0 0 1 1 *", timezone="UTC", starts_at="2099-01-01T00:00:00Z") -annual_orders = job( - "annual_orders", description="Build annual orders.", select="+orders", schedule=annual -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/development_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/development_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..c5d320cbe --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/development_orders/jobs/annual_orders_job.toml @@ -0,0 +1,5 @@ +# Development defaults pause code-enabled triggers without changing their definitions. +name = "annual_orders_job" +description = "Build annual orders." +select = "+orders" +schedule = { cron = "0 0 1 1 *", timezone = "UTC", starts_at = "2099-01-01T00:00:00Z" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/failed_import.py b/benchmarks/duckshop/edge_cases/orchestration/failed_import.py deleted file mode 100644 index 6483f0b2a..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/failed_import.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Negative fixture: a failed declaration import must fail every publication retry.""" - -raise RuntimeError("broken declaration") diff --git a/benchmarks/duckshop/edge_cases/orchestration/failed_import/jobs/orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/failed_import/jobs/orders_job.toml new file mode 100644 index 000000000..e2acd270d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/failed_import/jobs/orders_job.toml @@ -0,0 +1,3 @@ +name = "orders_job" +description = "Build orders." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/failed_import/sensors/broken_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/failed_import/sensors/broken_orders_sensor.py new file mode 100644 index 000000000..454fea133 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/failed_import/sensors/broken_orders_sensor.py @@ -0,0 +1,12 @@ +"""Negative fixture: a failed declaration import must fail every publication retry.""" + +from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor + +raise RuntimeError("broken declaration") + +broken_orders = sensor( + "broken_orders_sensor", + description="Never declared: the module fails first.", + when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), + run="orders_job", +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py deleted file mode 100644 index 621d03807..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Negative fixture: job settings must not override the frozen selection.""" - -from sqlbuild.orchestration import job - -orders = job( - "refresh_orders", - description="Refresh orders.", - select="orders", - settings={"select": "customers"}, -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/invalid_settings/jobs/refresh_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings/jobs/refresh_orders_job.toml new file mode 100644 index 000000000..c7dd89f19 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/invalid_settings/jobs/refresh_orders_job.toml @@ -0,0 +1,7 @@ +# Negative fixture: job settings must not override the frozen selection. +name = "refresh_orders_job" +description = "Refresh orders." +select = "orders" + +[settings] +select = "customers" diff --git a/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py deleted file mode 100644 index 2941037d7..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py +++ /dev/null @@ -1,14 +0,0 @@ -"""A tag-selected job whose model loads a loader chain and a hook-read source. - -The build loads raw_orders and the raw_customers loader it depends on, and raw_returns, which only -the model's pre-hook reads. The job's write set claims those three and not raw_inventory. -""" - -from sqlbuild.orchestration import job, schedule - -annual_orders = job( - "annual_orders", - description="Build the marts layer and the sources it loads.", - select="tag:marts", - schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/loaded_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..66b4cb5e0 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders/jobs/annual_orders_job.toml @@ -0,0 +1,4 @@ +name = "annual_orders_job" +description = "Build the marts layer and the sources it loads." +select = "tag:marts" +schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py deleted file mode 100644 index 6724f992a..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Keep a populated concurrency slot until recovered terminal publication commits.""" - -from sqlbuild.orchestration import job, schedule - -annual_orders = job( - "annual_orders", - description="Build orders under a shared concurrency limit.", - select="orders", - concurrency_groups=("orders_limit",), - schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..7f3e260da --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders/jobs/annual_orders_job.toml @@ -0,0 +1,6 @@ +# Keep a populated concurrency slot until recovered terminal publication commits. +name = "annual_orders_job" +description = "Build orders under a shared concurrency limit." +select = "orders" +concurrency_groups = ["orders_limit"] +schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/name_collision.py b/benchmarks/duckshop/edge_cases/orchestration/name_collision.py deleted file mode 100644 index fb0e024e9..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/name_collision.py +++ /dev/null @@ -1,5 +0,0 @@ -"""A job may not reuse a model's name: jobs share the project's one resource namespace.""" - -from sqlbuild.orchestration import job - -orders = job("orders", description="Build orders.", select="orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/name_collision/jobs/orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/name_collision/jobs/orders_job.toml new file mode 100644 index 000000000..48d401f69 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/name_collision/jobs/orders_job.toml @@ -0,0 +1,4 @@ +# A job may not reuse another resource's name: jobs share the project's one resource namespace. +name = "orders_job" +description = "Build orders." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py b/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py deleted file mode 100644 index 35727b209..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/nested_groups.py +++ /dev/null @@ -1,7 +0,0 @@ -"""A group can contain jobs, but cannot contain another group.""" - -from sqlbuild.orchestration import job, job_group - -orders = job("orders_job", description="Build orders.", select="orders") -daily = job_group("daily_orders", description="Build daily orders.", jobs=[orders]) -weekly = job_group("weekly_orders", description="Unsupported nested membership.", jobs=[daily]) diff --git a/benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/daily_orders_job_group.toml b/benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/daily_orders_job_group.toml new file mode 100644 index 000000000..3b33d97cb --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/daily_orders_job_group.toml @@ -0,0 +1,3 @@ +name = "daily_orders_job_group" +description = "Build daily orders." +jobs = ["orders_job"] diff --git a/benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/weekly_orders_job_group.toml b/benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/weekly_orders_job_group.toml new file mode 100644 index 000000000..693cdd117 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/nested_groups/job_groups/weekly_orders_job_group.toml @@ -0,0 +1,4 @@ +# A group can contain jobs, but cannot contain another group. +name = "weekly_orders_job_group" +description = "Unsupported nested membership." +jobs = ["daily_orders_job_group"] diff --git a/benchmarks/duckshop/edge_cases/orchestration/nested_groups/jobs/orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/nested_groups/jobs/orders_job.toml new file mode 100644 index 000000000..e2acd270d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/nested_groups/jobs/orders_job.toml @@ -0,0 +1,3 @@ +name = "orders_job" +description = "Build orders." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_orders.py b/benchmarks/duckshop/edge_cases/orchestration/node_orders.py deleted file mode 100644 index ac8f6cb8f..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/node_orders.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Trigger on each real orders rebuild, not on freshness-only success.""" - -from sqlbuild.orchestration import NodeEventKind, job, node_event, sensor - -source_orders = job("source_orders", description="Source orders.", select="orders") -inventory = job("inventory", description="Destination inventory.", select="stg_orders") -on_orders = sensor( - "on_orders", - description="Build inventory for every genuine orders rebuild.", - when=node_event(select="orders orders*", kinds=[NodeEventKind.SUCCESS]), - run=inventory, -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/inventory_job.toml b/benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/inventory_job.toml new file mode 100644 index 000000000..6d273e69a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/inventory_job.toml @@ -0,0 +1,3 @@ +name = "inventory_job" +description = "Destination inventory." +select = "stg_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/source_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/source_orders_job.toml new file mode 100644 index 000000000..aa4b407a4 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_orders/jobs/source_orders_job.toml @@ -0,0 +1,3 @@ +name = "source_orders_job" +description = "Source orders." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/node_orders/sensors/on_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/node_orders/sensors/on_orders_sensor.py new file mode 100644 index 000000000..40d4d909f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/node_orders/sensors/on_orders_sensor.py @@ -0,0 +1,10 @@ +"""Trigger on each real orders rebuild, not on freshness-only success.""" + +from sqlbuild.orchestration import NodeEventKind, node_event, sensor + +on_orders = sensor( + "on_orders_sensor", + description="Build inventory for every genuine orders rebuild.", + when=node_event(select="orders orders*", kinds=[NodeEventKind.SUCCESS]), + run="inventory_job", +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py b/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py deleted file mode 100644 index 575d5ccaa..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/platform_orders.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Sensor-only roots retain referenced jobs while target filters and event enums stay native.""" - -from sqlbuild.orchestration import PlatformEventKind, job, platform_event, sensor - - -def definitions(): - refresh = job("health_orders", description="Refresh orders.", select="orders") - return [ - sensor( - "overdue_orders", - description="Catch up overdue order reporting.", - when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), - run=refresh, - minimum_interval_seconds=7, - default_status="paused", - targets=("development",), - ), - sensor( - "other_target_orders", - description="Refresh another target.", - when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), - run=refresh, - targets=("production",), - ), - ] - - -triggers = definitions() -alias = triggers[0] -triggers.append(triggers) diff --git a/benchmarks/duckshop/edge_cases/orchestration/platform_orders/jobs/health_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/platform_orders/jobs/health_orders_job.toml new file mode 100644 index 000000000..a74fd6136 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/platform_orders/jobs/health_orders_job.toml @@ -0,0 +1,4 @@ +# Sensor-only roots retain referenced jobs while target filters and event enums stay native. +name = "health_orders_job" +description = "Refresh orders." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/other_target_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/other_target_orders_sensor.py new file mode 100644 index 000000000..b3301c551 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/other_target_orders_sensor.py @@ -0,0 +1,11 @@ +"""A sensor declared for another target is left out of this target's snapshot.""" + +from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor + +other_target_orders = sensor( + "other_target_orders_sensor", + description="Refresh another target.", + when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), + run="health_orders_job", + targets=("production",), +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/overdue_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/overdue_orders_sensor.py new file mode 100644 index 000000000..6f2d75933 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/platform_orders/sensors/overdue_orders_sensor.py @@ -0,0 +1,22 @@ +"""A sensor returned from a factory, aliased and held in a cyclic container, is found once.""" + +from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor + + +def definitions(): + return [ + sensor( + "overdue_orders_sensor", + description="Catch up overdue order reporting.", + when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), + run="health_orders_job", + minimum_interval_seconds=7, + default_status="paused", + targets=("development",), + ) + ] + + +triggers = definitions() +alias = triggers[0] +triggers.append(triggers) diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py deleted file mode 100644 index d153e434f..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Jobs whose selections are Python nodes, which `--select job:` must select as published.""" - -from sqlbuild.orchestration import job - -customer_counts = job( - "customer_counts", - description="Count customers.", - select="task:count_customers asset:customers_extract", -) -order_counts = job("order_counts", description="Count orders.", select="+task:count_orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/customer_counts_job.toml b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/customer_counts_job.toml new file mode 100644 index 000000000..e552003d6 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/customer_counts_job.toml @@ -0,0 +1,4 @@ +# Jobs whose selections are Python nodes, which `--select job:` must select as published. +name = "customer_counts_job" +description = "Count customers." +select = "task:count_customers asset:customers_extract" diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/order_counts_job.toml b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/order_counts_job.toml new file mode 100644 index 000000000..3a0104dc8 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs/jobs/order_counts_job.toml @@ -0,0 +1,3 @@ +name = "order_counts_job" +description = "Count orders." +select = "+task:count_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/run_orders.py b/benchmarks/duckshop/edge_cases/orchestration/run_orders.py deleted file mode 100644 index bf6d37618..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/run_orders.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Freeze source-job selectors without executing user code in the scheduler.""" - -from sqlbuild.orchestration import RunEventKind, job, run_event, sensor - -source_orders = job("source_orders", description="Source orders.", select="orders") -inventory = job("inventory", description="Destination inventory.", select="stg_orders") -on_orders = sensor( - "on_orders", - description="Build inventory only after orders actually succeeds.", - when=run_event( - job=source_orders, - kinds=[RunEventKind.COMPLETED], - executed_successfully=("orders",), - ), - run=inventory, -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/inventory_job.toml b/benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/inventory_job.toml new file mode 100644 index 000000000..6d273e69a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/inventory_job.toml @@ -0,0 +1,3 @@ +name = "inventory_job" +description = "Destination inventory." +select = "stg_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/source_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/source_orders_job.toml new file mode 100644 index 000000000..aa4b407a4 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/run_orders/jobs/source_orders_job.toml @@ -0,0 +1,3 @@ +name = "source_orders_job" +description = "Source orders." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/run_orders/sensors/on_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/run_orders/sensors/on_orders_sensor.py new file mode 100644 index 000000000..d8fbf6746 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/run_orders/sensors/on_orders_sensor.py @@ -0,0 +1,14 @@ +"""Freeze source-job selectors without executing user code in the scheduler.""" + +from sqlbuild.orchestration import RunEventKind, run_event, sensor + +on_orders = sensor( + "on_orders_sensor", + description="Build inventory only after orders actually succeeds.", + when=run_event( + job="source_orders_job", + kinds=[RunEventKind.COMPLETED], + executed_successfully=("orders",), + ), + run="inventory_job", +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json b/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json index ab9c620d9..396777d00 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json +++ b/benchmarks/duckshop/edge_cases/orchestration/scheduler_crash_recovery.json @@ -1,9 +1,9 @@ { "description": "Three minutely cron jobs start ten hours back with the default latest catch-up. Two installed schedulers catch up while being SIGKILLed at seeded random moments; every lapse of a killed leader's lease is simulated. After a final uninterrupted scheduler, each trigger has ticked its latest due minute exactly once and nothing off schedule, each tick requests or is covered by exactly one run, and every requested run has one request event.", "jobs": [ - "refresh_orders", - "refresh_customers", - "refresh_inventory" + "refresh_orders_job", + "refresh_customers_job", + "refresh_inventory_job" ], "backlog_minutes": 600, "rounds": 16, diff --git a/benchmarks/duckshop/edge_cases/orchestration/selector_rewrite/jobs/orders_refresh_job.toml b/benchmarks/duckshop/edge_cases/orchestration/selector_rewrite/jobs/orders_refresh_job.toml new file mode 100644 index 000000000..6ae2bd8d9 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/selector_rewrite/jobs/orders_refresh_job.toml @@ -0,0 +1,6 @@ +# Renaming stg_orders rewrites every exact name in these selectors, inside graph operators, unions, +# intersections and path endpoints, keeps each string's quote style, and leaves the glob alone. +name = "orders_refresh_job" +description = "Refresh orders." +select = "+stg_orders+ fact_orders,+stg_orders stg_orders~customer_totals stg_ord*" +exclude = 'stg_orders+,customer_totals' diff --git a/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py deleted file mode 100644 index 77a937120..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Layer jobs selected by tag share the function their models call, and still order by lineage. - -A tag selection brings in the functions its models call, so both jobs build the shared function. A -job that builds a node itself does not wait for that node's other builders, so the group runs the -marts layer after the staging layer instead of reporting a cycle. -""" - -from sqlbuild.orchestration import job, job_group - -staging = job("staging_layer", description="Build the staging layer.", select="tag:staging") -marts = job("marts_layer", description="Build the marts layer.", select="tag:marts") -layers = job_group( - "layered_orders", description="Build staging, then marts.", jobs=[marts, staging] -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/job_groups/layered_orders_job_group.toml b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/job_groups/layered_orders_job_group.toml new file mode 100644 index 000000000..fe34d0d60 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/job_groups/layered_orders_job_group.toml @@ -0,0 +1,4 @@ +# Layer jobs that share a function wait for each other only by lineage. +name = "layered_orders_job_group" +description = "Build staging, then marts." +jobs = ["marts_layer_job", "staging_layer_job"] diff --git a/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/marts_layer_job.toml b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/marts_layer_job.toml new file mode 100644 index 000000000..dccd96a30 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/marts_layer_job.toml @@ -0,0 +1,3 @@ +name = "marts_layer_job" +description = "Build the marts layer." +select = "tag:marts" diff --git a/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/staging_layer_job.toml b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/staging_layer_job.toml new file mode 100644 index 000000000..f26d0ab3e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers/jobs/staging_layer_job.toml @@ -0,0 +1,3 @@ +name = "staging_layer_job" +description = "Build the staging layer." +select = "tag:staging" diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_orders/jobs/orders_refresh_job.toml b/benchmarks/duckshop/edge_cases/orchestration/source_orders/jobs/orders_refresh_job.toml new file mode 100644 index 000000000..7eabe333f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/source_orders/jobs/orders_refresh_job.toml @@ -0,0 +1,3 @@ +name = "orders_refresh_job" +description = "Refresh orders for a changed source version." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/source_orders.py b/benchmarks/duckshop/edge_cases/orchestration/source_orders/sensors/on_source_orders_sensor.py similarity index 54% rename from benchmarks/duckshop/edge_cases/orchestration/source_orders.py rename to benchmarks/duckshop/edge_cases/orchestration/source_orders/sensors/on_source_orders_sensor.py index c700b7d23..7ee39f8dd 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/source_orders.py +++ b/benchmarks/duckshop/edge_cases/orchestration/source_orders/sensors/on_source_orders_sensor.py @@ -1,16 +1,11 @@ """Consume changed warehouse source versions without treating a sync poll as new data.""" -from sqlbuild.orchestration import job, sensor, source_new_data +from sqlbuild.orchestration import sensor, source_new_data -orders_refresh = job( - "orders_refresh", - description="Refresh orders for a changed source version.", - select="orders", -) on_source_orders = sensor( - "on_source_orders", + "on_source_orders_sensor", description="Refresh orders for the first source version and each changed version.", when=source_new_data(select="source:raw_orders"), - run=orders_refresh, + run="orders_refresh_job", minimum_interval_seconds=1, ) diff --git a/benchmarks/duckshop/edge_cases/orchestration/stale_orders.py b/benchmarks/duckshop/edge_cases/orchestration/stale_orders.py deleted file mode 100644 index bf17c3428..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/stale_orders.py +++ /dev/null @@ -1,13 +0,0 @@ -from sqlbuild.orchestration import job, sensor, source_stale - -orders_refresh = job( - "orders_refresh", description="Refresh orders from stale timestamp data.", select="orders" -) - -on_stale_orders = sensor( - "on_stale_orders", - description="Refresh orders once for each timestamp version older than one minute.", - when=source_stale(select="source:raw_orders", after_seconds=60), - run=orders_refresh, - minimum_interval_seconds=1, -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/stale_orders/jobs/orders_refresh_job.toml b/benchmarks/duckshop/edge_cases/orchestration/stale_orders/jobs/orders_refresh_job.toml new file mode 100644 index 000000000..a574364e9 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/stale_orders/jobs/orders_refresh_job.toml @@ -0,0 +1,3 @@ +name = "orders_refresh_job" +description = "Refresh orders from stale timestamp data." +select = "orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/stale_orders/sensors/on_stale_orders_sensor.py b/benchmarks/duckshop/edge_cases/orchestration/stale_orders/sensors/on_stale_orders_sensor.py new file mode 100644 index 000000000..01ad50030 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/stale_orders/sensors/on_stale_orders_sensor.py @@ -0,0 +1,11 @@ +"""Start one refresh for each timestamp source version older than a minute.""" + +from sqlbuild.orchestration import sensor, source_stale + +on_stale_orders = sensor( + "on_stale_orders_sensor", + description="Refresh orders once for each timestamp version older than one minute.", + when=source_stale(select="source:raw_orders", after_seconds=60), + run="orders_refresh_job", + minimum_interval_seconds=1, +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py deleted file mode 100644 index 0723fa403..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py +++ /dev/null @@ -1,15 +0,0 @@ -"""A job that reaches a managed source through `+`, so its run selects the source's loader chain. - -`+orders` selects the model, its managed source raw_shipments, the raw_shipments loader and the -intermediate stage_shipments loader it depends on. The run records each loader as the source it -writes, which is the node its load reports under. -""" - -from sqlbuild.orchestration import job, schedule - -annual_orders = job( - "annual_orders", - description="Build orders and everything upstream of it, loaders included.", - select="+orders", - schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..81a343141 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders/jobs/annual_orders_job.toml @@ -0,0 +1,4 @@ +name = "annual_orders_job" +description = "Build orders and everything upstream of it, loaders included." +select = "+orders" +schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" } diff --git a/benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders.py b/benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders.py deleted file mode 100644 index 7fc6d7150..000000000 --- a/benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Schedule a job whose selection reaches an upstream source that a build never executes.""" - -from sqlbuild.orchestration import job, schedule - -annual_orders = job( - "annual_orders", - description="Build orders together with the source it reads.", - select="+orders", - schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), -) diff --git a/benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders/jobs/annual_orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders/jobs/annual_orders_job.toml new file mode 100644 index 000000000..c7e5aafa3 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders/jobs/annual_orders_job.toml @@ -0,0 +1,5 @@ +# Schedule a job whose selection reaches an upstream source that a build never executes. +name = "annual_orders_job" +description = "Build orders together with the source it reads." +select = "+orders" +schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" } diff --git a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py b/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py deleted file mode 100644 index 8c5069c1d..000000000 --- a/benchmarks/duckshop/project/orchestration/edge_cases/definitions.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Exercise alias deduplication and cyclic factory containers during native discovery.""" - -from sqlbuild.orchestration import ( - PlatformEventKind, - RunEventKind, - job, - job_group, - platform_event, - run_event, - schedule, - sensor, -) - -edge_orders = job( - "edge_snapshot_orders", - description="Refresh only the synthetic edge-case models.", - select="tag:edge_case", - settings={"run_tests": False, "vars": {"edge_delivery_batch": "1"}, "reload": True}, - concurrency=2, - schedule=schedule( - cron="15 2 * * *", - timezone="UTC", - starts_at="2026-07-03T01:17:23.456789+05:30", - default_status="paused", - ), -) -edge_alias = edge_orders -edge_factory_results = [[edge_orders]] -edge_factory_results.append(edge_factory_results) -edge_group = job_group( - "edge_snapshot_group", - description="Exercise group discovery through shared, cyclic declaration containers.", - jobs=[edge_orders], -) -edge_factory_results.append(edge_group) - -edge_inventory = job( - "edge_snapshot_inventory", - description="Exercise a child limit independent of its parent.", - select="tag:edge_case", - concurrency_groups=("edge_products_limit",), -) -edge_products = job( - "edge_snapshot_products", - description="Exercise the opposing parent and child acquisition order.", - select="tag:edge_case", - concurrency_groups=("edge_inventory_limit",), -) -edge_inventory_group = job_group( - "edge_inventory_group", - description="Complete inventory without reserving its child's product limit.", - jobs=[edge_inventory], - concurrency_groups=("edge_inventory_limit",), -) -edge_products_group = job_group( - "edge_products_group", - description="Wait for safe admission when both installation limits are one.", - jobs=[edge_products], - concurrency_groups=("edge_products_limit",), -) -edge_health = sensor( - "edge_platform_orders", - description="Catch up retained platform observations with a stopped group sensor.", - when=platform_event( - kinds=[ - PlatformEventKind.TICK_OVERDUE, - PlatformEventKind.LOCATION_BROKEN, - PlatformEventKind.LAUNCH_FAILED, - ] - ), - run=edge_group, - minimum_interval_seconds=7, - default_status="paused", -) -edge_factory_results.append(edge_health) - -edge_job_outcomes = sensor( - "edge_job_outcomes", - description="Keep warning and partial outcomes distinct from success.", - when=run_event(job=edge_orders, kinds=[RunEventKind.WARN, RunEventKind.PARTIAL]), - run=edge_inventory, - default_status="paused", -) -edge_group_outcomes = sensor( - "edge_group_outcomes", - description="Observe group completion including cancellation and presumed failure.", - when=run_event( - job=edge_group, - kinds=[RunEventKind.COMPLETED], - executed_successfully=("tag:edge_case",), - ), - run=edge_inventory, - default_status="paused", -) -edge_factory_results.extend([edge_job_outcomes, edge_group_outcomes]) diff --git a/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_inventory_job_group.toml b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_inventory_job_group.toml new file mode 100644 index 000000000..91d5adf5b --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_inventory_job_group.toml @@ -0,0 +1,4 @@ +name = "edge_inventory_job_group" +description = "Complete inventory without reserving its child's product limit." +jobs = ["edge_snapshot_inventory_job"] +concurrency_groups = ["edge_inventory_limit"] diff --git a/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_products_job_group.toml b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_products_job_group.toml new file mode 100644 index 000000000..8177a6402 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_products_job_group.toml @@ -0,0 +1,4 @@ +name = "edge_products_job_group" +description = "Wait for safe admission when both installation limits are one." +jobs = ["edge_snapshot_products_job"] +concurrency_groups = ["edge_products_limit"] diff --git a/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_snapshot_job_group.toml b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_snapshot_job_group.toml new file mode 100644 index 000000000..cae323412 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/job_groups/edge_cases/edge_snapshot_job_group.toml @@ -0,0 +1,3 @@ +name = "edge_snapshot_job_group" +description = "Run the edge-case refresh as a group that sensors start and observe." +jobs = ["edge_snapshot_orders_job"] diff --git a/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_inventory_job.toml b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_inventory_job.toml new file mode 100644 index 000000000..4b624fe69 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_inventory_job.toml @@ -0,0 +1,4 @@ +name = "edge_snapshot_inventory_job" +description = "Exercise a child limit independent of its parent." +select = "tag:edge_case" +concurrency_groups = ["edge_products_limit"] diff --git a/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_orders_job.toml b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_orders_job.toml new file mode 100644 index 000000000..85b15b929 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_orders_job.toml @@ -0,0 +1,11 @@ +# Refresh only the synthetic edge-case models, with nested settings and a paused offset-start schedule. +name = "edge_snapshot_orders_job" +description = "Refresh only the synthetic edge-case models." +select = "tag:edge_case" +schedule = { cron = "15 2 * * *", timezone = "UTC", starts_at = "2026-07-03T01:17:23.456789+05:30", default_status = "paused" } + +[settings] +run_tests = false +vars = { edge_delivery_batch = "1" } +reload = true +concurrency = 2 diff --git a/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_products_job.toml b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_products_job.toml new file mode 100644 index 000000000..68c81fe2e --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/jobs/edge_cases/edge_snapshot_products_job.toml @@ -0,0 +1,4 @@ +name = "edge_snapshot_products_job" +description = "Exercise the opposing parent and child acquisition order." +select = "tag:edge_case" +concurrency_groups = ["edge_inventory_limit"] diff --git a/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_group_outcomes_sensor.py b/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_group_outcomes_sensor.py new file mode 100644 index 000000000..a1f8a750d --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_group_outcomes_sensor.py @@ -0,0 +1,15 @@ +"""Observe group completion, including cancellation and presumed failure.""" + +from sqlbuild.orchestration import RunEventKind, run_event, sensor + +edge_group_outcomes = sensor( + "edge_group_outcomes_sensor", + description="Observe group completion including cancellation and presumed failure.", + when=run_event( + job="edge_snapshot_job_group", + kinds=[RunEventKind.COMPLETED], + executed_successfully=("tag:edge_case",), + ), + run="edge_snapshot_inventory_job", + default_status="paused", +) diff --git a/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_job_outcomes_sensor.py b/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_job_outcomes_sensor.py new file mode 100644 index 000000000..fc35482e5 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_job_outcomes_sensor.py @@ -0,0 +1,11 @@ +"""Keep warning and partial job outcomes distinct from success.""" + +from sqlbuild.orchestration import RunEventKind, run_event, sensor + +edge_job_outcomes = sensor( + "edge_job_outcomes_sensor", + description="Keep warning and partial outcomes distinct from success.", + when=run_event(job="edge_snapshot_orders_job", kinds=[RunEventKind.WARN, RunEventKind.PARTIAL]), + run="edge_snapshot_inventory_job", + default_status="paused", +) diff --git a/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_platform_orders_sensor.py b/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_platform_orders_sensor.py new file mode 100644 index 000000000..5275a4581 --- /dev/null +++ b/benchmarks/duckshop/project/orchestration/sensors/edge_cases/edge_platform_orders_sensor.py @@ -0,0 +1,21 @@ +"""Exercise alias deduplication and cyclic containers while discovering a sensor module.""" + +from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor + +edge_health = sensor( + "edge_platform_orders_sensor", + description="Catch up retained platform observations with a stopped group sensor.", + when=platform_event( + kinds=[ + PlatformEventKind.TICK_OVERDUE, + PlatformEventKind.LOCATION_BROKEN, + PlatformEventKind.LAUNCH_FAILED, + ] + ), + run="edge_snapshot_job_group", + minimum_interval_seconds=7, + default_status="paused", +) +edge_alias = edge_health +edge_factory_results = [[edge_health], (edge_alias,)] +edge_factory_results.append(edge_factory_results) diff --git a/crates/sqlbuild-discovery/Cargo.toml b/crates/sqlbuild-discovery/Cargo.toml index 0cd082a7f..5ca9425e6 100644 --- a/crates/sqlbuild-discovery/Cargo.toml +++ b/crates/sqlbuild-discovery/Cargo.toml @@ -9,11 +9,15 @@ rust-version.workspace = true [dependencies] rayon.workspace = true +serde.workspace = true +serde_json.workspace = true sqlbuild-cache.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-pysource.workspace = true sqlbuild-sqltext.workspace = true +toml.workspace = true [target.'cfg(windows)'.dependencies] junction.workspace = true diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index 0f9a9b835..8bb9b900a 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -10,6 +10,7 @@ pub mod header_changes; pub mod key_checks; pub mod model_files; pub mod models; +pub mod orchestration_files; pub mod project_validation; pub mod python_paths; pub mod sql_tests; diff --git a/crates/sqlbuild-discovery/src/models.rs b/crates/sqlbuild-discovery/src/models.rs index f00fcbd4d..44627f8c2 100644 --- a/crates/sqlbuild-discovery/src/models.rs +++ b/crates/sqlbuild-discovery/src/models.rs @@ -30,6 +30,8 @@ pub enum FailureKind { Conflict, /// `SeedDiscoveryError` (D008). Seed, + /// `OrchestrationDeclarationError` (D017). + Orchestration, } impl FailureKind { @@ -48,6 +50,7 @@ impl FailureKind { Self::ResourceIdentity => "resource_identity", Self::Conflict => "conflict", Self::Seed => "seed", + Self::Orchestration => "orchestration", } } } diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs new file mode 100644 index 000000000..57c463b88 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/inventory_files.rs @@ -0,0 +1,179 @@ +//! The phases of the orchestration inventory: the folder layout, then each kind's files. + +use sqlbuild_pysource::orchestration_modules::models::PythonDeclarationKind; + +use crate::_helpers::reading::read_authored_text; +use crate::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::orchestration_files::_helpers::python_modules::module_declaration; +use crate::orchestration_files::constants::{ + BYTECODE_DIRECTORY, CALENDARS_DIRECTORY, GROUPS_DIRECTORY, HIDDEN_PREFIX, JOBS_DIRECTORY, + ROLE_DIRECTORIES, SENSORS_DIRECTORY, TOML_FILE_SUFFIX, TOML_JOBS_HELP, +}; +use crate::orchestration_files::errors::FileError; +use crate::orchestration_files::models::{ + FilePosition, NamedModule, OrchestrationFailure, PythonModules, +}; +use crate::tree::main::children::children; +use crate::tree::main::rglob::rglob; +use crate::tree::models::{ProjectTree, TreeEntry}; + +/// `D017` at a file and position, with optional help. +pub(crate) fn failure(path: &str, error: FileError) -> OrchestrationFailure { + let FilePosition { line, column } = error.position; + OrchestrationFailure::Declaration(DiscoveryFailure { + kind: FailureKind::Orchestration, + message: format!("{path}:{line}:{column}: {}", error.message), + help: error.help, + }) +} + +fn layout(path: &str, message: String, help: Option) -> OrchestrationFailure { + failure( + path, + FileError { + message, + position: FilePosition { line: 1, column: 1 }, + help, + }, + ) +} + +fn ignored(entry: &TreeEntry) -> bool { + entry.name.starts_with(HIDDEN_PREFIX) || entry.name == BYTECODE_DIRECTORY +} + +fn read(tree: &ProjectTree, path: &str) -> Result { + if tree.is_undecodable(path) { + return Err(layout( + path, + "path is not valid UTF-8; rename it so SQLBuild can read it".to_owned(), + None, + )); + } + read_authored_text(&tree.absolute(path)).map_err(|failure| OrchestrationFailure::Unreadable { + relative_path: path.to_owned(), + failure, + }) +} + +/// Refuse a file directly in `orchestration/` or a folder no declaration kind owns. +pub(crate) fn check_top_level(tree: &ProjectTree) -> Result<(), OrchestrationFailure> { + for (path, entry) in children(tree, ORCHESTRATION_DIRECTORY)? { + if ignored(&entry) { + continue; + } + if !entry.is_dir { + return Err(layout( + &path, + "files cannot sit directly in orchestration/".to_owned(), + Some(if entry.name.ends_with(PYTHON_FILE_SUFFIX) { + TOML_JOBS_HELP.to_owned() + } else { + format!( + "move it into one of orchestration/{}/", + ROLE_DIRECTORIES.join("/, orchestration/") + ) + }), + )); + } + if !ROLE_DIRECTORIES.contains(&entry.name.as_str()) { + let holds_python: bool = !rglob(tree, &path, |found| { + found.name.ends_with(PYTHON_FILE_SUFFIX) + })? + .is_empty(); + return Err(layout( + &path, + format!("unknown orchestration folder '{}'", entry.name), + Some(if holds_python { + TOML_JOBS_HELP.to_owned() + } else { + format!("use one of: {}", ROLE_DIRECTORIES.join(", ")) + }), + )); + } + } + Ok(()) +} + +/// The files below one role folder, refusing any without `suffix`. +fn role_files( + tree: &ProjectTree, + role: &str, + suffix: &str, +) -> Result, OrchestrationFailure> { + let root: String = format!("{ORCHESTRATION_DIRECTORY}/{role}"); + let files: Vec = rglob(tree, &root, |entry| !entry.is_dir)?; + let mut kept: Vec = Vec::new(); + for path in files { + let hidden: bool = path + .split('/') + .any(|part| part.starts_with(HIDDEN_PREFIX) || part == BYTECODE_DIRECTORY); + if hidden { + continue; + } + if !path.ends_with(suffix) { + let python_job: bool = path.ends_with(PYTHON_FILE_SUFFIX) + && matches!(role, JOBS_DIRECTORY | GROUPS_DIRECTORY); + return Err(layout( + &path, + format!("orchestration/{role}/ holds only {suffix} files"), + Some(match role { + _ if python_job => TOML_JOBS_HELP.to_owned(), + JOBS_DIRECTORY | GROUPS_DIRECTORY => { + "jobs and job groups are TOML; calendars and sensors are Python in orchestration/calendars/ and orchestration/sensors/".to_owned() + } + _ => "jobs and job groups are TOML in orchestration/jobs/ and orchestration/job_groups/".to_owned(), + }), + )); + } + kept.push(path); + } + Ok(kept) +} + +/// Every TOML file below one role folder, read by `reader`, with its path. +pub(crate) fn read_toml_files( + tree: &ProjectTree, + role: &str, + reader: fn(&str) -> Result, +) -> Result, OrchestrationFailure> { + let mut files: Vec<(String, T)> = Vec::new(); + for path in role_files(tree, role, TOML_FILE_SUFFIX)? { + let file: T = reader(&read(tree, &path)?).map_err(|error| failure(&path, error))?; + files.push((path, file)); + } + Ok(files) +} + +/// Every calendar and sensor module, read without importing it. +pub(crate) fn read_python_modules( + tree: &ProjectTree, + python_version: (u8, u8), +) -> Result { + let mut found = PythonModules::default(); + for (role, kind) in [ + (CALENDARS_DIRECTORY, PythonDeclarationKind::Calendar), + (SENSORS_DIRECTORY, PythonDeclarationKind::Sensor), + ] { + for path in role_files(tree, role, PYTHON_FILE_SUFFIX)? { + let stem: &str = path + .rsplit('/') + .next() + .and_then(|name| name.strip_suffix(PYTHON_FILE_SUFFIX)) + .unwrap_or_default(); + let declared: Option = + module_declaration(&read(tree, &path)?, stem, kind, python_version) + .map_err(|error| failure(&path, error))?; + if let Some(mut module) = declared { + module.path = path.clone(); + match kind { + PythonDeclarationKind::Calendar => found.calendars.push(module), + PythonDeclarationKind::Sensor => found.sensors.push(module), + } + } + found.modules.push(path); + } + } + Ok(found) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/mod.rs new file mode 100644 index 000000000..bfe96c904 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod inventory_files; +pub(crate) mod python_modules; +pub(crate) mod raw; +pub(crate) mod read_group_file; +pub(crate) mod read_job_file; +pub(crate) mod reading; +pub(crate) mod references; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs new file mode 100644 index 000000000..f7c3b11ab --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/python_modules.rs @@ -0,0 +1,152 @@ +//! A calendar or sensor module checked against its folder, without running it. + +use sqlbuild_pysource::orchestration_modules::main::orchestration_module_declarations::orchestration_module_declarations; +use sqlbuild_pysource::orchestration_modules::models::{ + LiteralArgument, ModuleDeclarations, PythonDeclaration, PythonDeclarationKind, SourcePosition, +}; + +use crate::orchestration_files::constants::{ + CALENDAR_KIND, PRIVATE_PREFIX, SENSOR_KIND, TOML_JOBS_HELP, +}; +use crate::orchestration_files::errors::FileError; +use crate::orchestration_files::models::{FilePosition, NamedModule, Positioned}; + +fn file_position(position: SourcePosition) -> FilePosition { + FilePosition { + line: position.line, + column: position.column, + } +} + +fn error(position: SourcePosition, message: String, help: Option) -> FileError { + FileError { + message, + position: file_position(position), + help, + } +} + +fn label(kind: PythonDeclarationKind) -> (&'static str, &'static str) { + match kind { + PythonDeclarationKind::Calendar => CALENDAR_KIND, + PythonDeclarationKind::Sensor => SENSOR_KIND, + } +} + +/// A literal string argument, or an error naming `what` and how to fix it. +fn literal(argument: LiteralArgument, what: &str) -> Result, FileError> { + match argument.value { + Some(value) if !value.trim().is_empty() => Ok(Positioned { + value, + position: file_position(argument.position), + }), + Some(_) => Err(error( + argument.position, + format!("{what} must not be blank"), + None, + )), + None => Err(error( + argument.position, + format!("{what} must be a string literal"), + Some(format!( + "write the {what} as a quoted string, such as \"orders_job\", so the compiler can check it without running the module" + )), + )), + } +} + +/// The one declaration a public module in a `kind` folder makes; `None` for a private support module. +pub(crate) fn module_declaration( + text: &str, + stem: &str, + kind: PythonDeclarationKind, + python_version: (u8, u8), +) -> Result, FileError> { + let start = SourcePosition { line: 1, column: 1 }; + let Some(ModuleDeclarations { + declarations, + removed_imports, + }) = orchestration_module_declarations(text, python_version) + else { + return Err(error( + start, + format!( + "is not valid Python {}.{}", + python_version.0, python_version.1 + ), + None, + )); + }; + if let Some((name, position)) = removed_imports.into_iter().next() { + return Err(error( + position, + format!( + "`{name}` was removed from sqlbuild.orchestration; jobs, job groups and schedules are declared in TOML" + ), + Some(TOML_JOBS_HELP.to_owned()), + )); + } + let (noun, suffix) = label(kind); + if let Some(other) = declarations + .iter() + .find(|declaration| declaration.kind != kind) + { + let (other_noun, _) = label(other.kind); + return Err(error( + other.position, + format!("declares a {other_noun}, which belongs in orchestration/{other_noun}s/"), + None, + )); + } + let private: bool = stem.starts_with(PRIVATE_PREFIX); + let mut declarations = declarations.into_iter(); + let first: Option = declarations.next(); + if let Some(second) = declarations.next() { + return Err(error( + second.position, + format!("declares a second {noun}; each module declares exactly one"), + None, + )); + } + let Some(declaration) = first else { + if private { + return Ok(None); + } + return Err(error( + start, + format!("declares no {noun}"), + Some(format!( + "declare one {noun}, or start the file name with `_` to make it a private support module" + )), + )); + }; + if private { + return Err(error( + declaration.position, + format!("private support module declares a {noun}"), + Some("remove the leading `_` from the file name".to_owned()), + )); + } + let name: Positioned = match declaration.name { + Some(name) => literal(name, &format!("{noun} name"))?, + None => { + return Err(error( + declaration.position, + format!("{noun} needs a name"), + Some(format!("pass name=\"{suffix}\"")), + )); + } + }; + let job_references = declaration + .job_references + .into_iter() + .map(|(argument, value)| Ok((argument, literal(value, &format!("sensor {argument} job"))?))) + .collect::, _>>()?; + Ok(Some(NamedModule { + name: name.value, + path: String::new(), + position: name.position, + job_references, + targets: declaration.targets, + })) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs new file mode 100644 index 000000000..b21378fb2 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs @@ -0,0 +1,58 @@ +//! The file shapes serde reads before their values are checked. + +use std::collections::BTreeMap; + +use serde::Deserialize; +use toml::Spanned; + +use crate::orchestration_files::models::{InitialStatus, Overlap}; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawSchedule { + pub(crate) cron: Option>, + pub(crate) calendar: Option>, + pub(crate) timezone: Spanned, + pub(crate) starts_at: Option>, + #[serde(default)] + pub(crate) default_status: InitialStatus, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RawOnFailure { + pub(crate) retries: i64, + #[serde(default)] + pub(crate) retry_delay_seconds: i64, +} + +#[derive(Deserialize)] +pub(crate) struct RawRun { + pub(crate) name: Spanned, + pub(crate) description: Spanned, + #[serde(default)] + pub(crate) targets: Vec, + pub(crate) owner: Option>, + #[serde(default)] + pub(crate) overlap: Overlap, + #[serde(default)] + pub(crate) concurrency_groups: Vec, + pub(crate) schedule: Option>, + pub(crate) on_failure: Option>, +} + +/// The job-only fields; the shared ones are read as `RawRun` from the same text. +#[derive(Deserialize)] +pub(crate) struct RawJob { + pub(crate) select: Spanned, + pub(crate) exclude: Option>, + pub(crate) resources: Option>, + #[serde(default)] + pub(crate) settings: BTreeMap>, +} + +/// The group-only fields; the shared ones are read as `RawRun` from the same text. +#[derive(Deserialize)] +pub(crate) struct RawGroup { + pub(crate) jobs: Vec>, +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/read_group_file.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/read_group_file.rs new file mode 100644 index 000000000..2b0a6e40c --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/read_group_file.rs @@ -0,0 +1,42 @@ +//! Read one job group file. + +use std::collections::BTreeSet; + +use crate::orchestration_files::_helpers::raw::{RawGroup, RawRun}; +use crate::orchestration_files::_helpers::reading::{ + check_keys, error_at, nonblank, run_file, toml_error, +}; +use crate::orchestration_files::constants::GROUP_KEYS; +use crate::orchestration_files::errors::FileError; +use crate::orchestration_files::models::{GroupFile, Positioned}; + +/// The job group `text` declares, or the first problem with it and where it is. +pub(crate) fn read_group_file(text: &str) -> Result { + check_keys(text, &GROUP_KEYS, "job group")?; + let run: RawRun = toml::from_str(text).map_err(|error| toml_error(text, &error))?; + let raw: RawGroup = toml::from_str(text).map_err(|error| toml_error(text, &error))?; + if raw.jobs.is_empty() { + return Err(error_at( + text, + run.name.span().start, + "a job group needs at least one job in `jobs`".to_owned(), + )); + } + let mut seen: BTreeSet = BTreeSet::new(); + let mut jobs: Vec> = Vec::new(); + for job in raw.jobs { + let job: Positioned = nonblank(text, job, "job name")?; + if !seen.insert(job.value.clone()) { + return Err(FileError { + message: format!("job '{}' is listed twice", job.value), + position: job.position, + help: None, + }); + } + jobs.push(job); + } + Ok(GroupFile { + run: run_file(text, run)?, + jobs, + }) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/read_job_file.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/read_job_file.rs new file mode 100644 index 000000000..a288ee813 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/read_job_file.rs @@ -0,0 +1,64 @@ +//! Read one job file. + +use std::collections::BTreeMap; + +use toml::Spanned; + +use crate::orchestration_files::_helpers::raw::{RawJob, RawRun}; +use crate::orchestration_files::_helpers::reading::{ + check_keys, error_at, nonblank, position, positioned, run_file, toml_error, +}; +use crate::orchestration_files::constants::JOB_KEYS; +use crate::orchestration_files::errors::FileError; +use crate::orchestration_files::models::{FilePosition, JobFile, Positioned}; + +/// The job `text` declares, or the first problem with it and where it is. +pub(crate) fn read_job_file(text: &str) -> Result { + check_keys(text, &JOB_KEYS, "job")?; + let run: RawRun = toml::from_str(text).map_err(|error| toml_error(text, &error))?; + let raw: RawJob = toml::from_str(text).map_err(|error| toml_error(text, &error))?; + let resources: Option>> = raw + .resources + .map(|resources| json_table(text, resources)) + .transpose()?; + let mut settings: BTreeMap = BTreeMap::new(); + let mut setting_positions: BTreeMap = BTreeMap::new(); + for (key, value) in raw.settings { + let value = positioned(text, value); + settings.insert( + key.clone(), + serde_json::to_value(value.value).unwrap_or_default(), + ); + setting_positions.insert(key, value.position); + } + Ok(JobFile { + run: run_file(text, run)?, + select: nonblank(text, raw.select, "select")?, + exclude: raw + .exclude + .filter(|exclude| !exclude.get_ref().trim().is_empty()) + .map(|exclude| positioned(text, exclude)), + resources, + settings, + setting_positions, + }) +} + +/// A TOML table as JSON, positioned where it is written. +fn json_table( + text: &str, + table: Spanned, +) -> Result>, FileError> { + let start: usize = table.span().start; + match serde_json::to_value(table.into_inner()) { + Ok(serde_json::Value::Object(value)) => Ok(Positioned { + value, + position: position(text, start), + }), + Ok(_) | Err(_) => Err(error_at( + text, + start, + "resources must be a table of plain values".to_owned(), + )), + } +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs new file mode 100644 index 000000000..d44c1aec1 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs @@ -0,0 +1,188 @@ +//! Positions, unknown keys and value checks shared by job and group files. + +use std::collections::BTreeSet; + +use sqlbuild_core::text::main::close_matches::close_matches; +use toml::Spanned; +use toml::de::DeTable; + +use crate::orchestration_files::_helpers::raw::{RawOnFailure, RawRun, RawSchedule}; +use crate::orchestration_files::constants::{ + CONCURRENCY_KEY, RETRIES_RANGE, RETRY_DELAY_RANGE, SUGGESTION_CUTOFF, +}; +use crate::orchestration_files::errors::FileError; +use crate::orchestration_files::models::{ + FilePosition, OnFailure, Positioned, RunFile, ScheduleFile, ScheduleSource, +}; + +/// The 1-based line and character column of byte `offset` in `text`. +pub(crate) fn position(text: &str, offset: usize) -> FilePosition { + let before: &str = &text[..offset.min(text.len())]; + let line_start: usize = before.rfind('\n').map_or(0, |index| index + 1); + FilePosition { + line: before.matches('\n').count() + 1, + column: before[line_start..].chars().count() + 1, + } +} + +/// An error at byte `offset`. +pub(crate) fn error_at(text: &str, offset: usize, message: String) -> FileError { + FileError { + message, + position: position(text, offset), + help: None, + } +} + +/// A TOML syntax or type error, at the span the parser reports. +pub(crate) fn toml_error(text: &str, error: &toml::de::Error) -> FileError { + error_at( + text, + error.span().map_or(0, |span| span.start), + error.message().trim_end().to_owned(), + ) +} + +/// The value with the position its span starts at. +pub(crate) fn positioned(text: &str, spanned: Spanned) -> Positioned { + let start: usize = spanned.span().start; + Positioned { + value: spanned.into_inner(), + position: position(text, start), + } +} + +/// Refuse a top-level key outside `allowed`, at the key, with the nearest known key as help. +pub(crate) fn check_keys(text: &str, allowed: &[&str], kind: &str) -> Result<(), FileError> { + let table = DeTable::parse(text).map_err(|error| toml_error(text, &error))?; + for key in table.get_ref().keys() { + let name: &str = key.get_ref(); + if allowed.contains(&name) { + continue; + } + let help: String = if name == CONCURRENCY_KEY { + format!("move it under [settings] as `{CONCURRENCY_KEY} = ...`") + } else if let Some(nearest) = close_matches(name, allowed, 1, SUGGESTION_CUTOFF).first() { + format!("did you mean '{nearest}'?") + } else { + let mut sorted: Vec<&str> = allowed.to_vec(); + sorted.sort_unstable(); + format!("supported keys: {}", sorted.join(", ")) + }; + return Err(FileError { + help: Some(help), + ..error_at( + text, + key.span().start, + format!("unknown {kind} key '{name}'"), + ) + }); + } + Ok(()) +} + +/// A string that must hold more than whitespace. +pub(crate) fn nonblank( + text: &str, + value: Spanned, + what: &str, +) -> Result, FileError> { + if value.get_ref().trim().is_empty() { + return Err(error_at( + text, + value.span().start, + format!("{what} must not be blank"), + )); + } + Ok(positioned(text, value)) +} + +fn schedule(text: &str, raw: Spanned) -> Result { + let start: usize = raw.span().start; + let raw: RawSchedule = raw.into_inner(); + let source: ScheduleSource = match (raw.cron, raw.calendar) { + (Some(cron), None) => ScheduleSource::Cron(positioned(text, cron)), + (None, Some(calendar)) => { + ScheduleSource::Calendar(nonblank(text, calendar, "schedule calendar")?) + } + _ => { + return Err(error_at( + text, + start, + "schedule needs exactly one of `cron` or `calendar`".to_owned(), + )); + } + }; + Ok(ScheduleFile { + source, + timezone: positioned(text, raw.timezone), + starts_at: raw.starts_at.map(|starts_at| positioned(text, starts_at)), + default_status: raw.default_status, + }) +} + +fn on_failure(text: &str, raw: Spanned) -> Result { + let start: usize = raw.span().start; + let raw: RawOnFailure = raw.into_inner(); + let invalid = |field: &str, (low, high): (i64, i64)| { + error_at( + text, + start, + format!("on_failure {field} must be a whole number from {low} to {high}"), + ) + }; + let retries_range: (i64, i64) = (i64::from(RETRIES_RANGE.0), i64::from(RETRIES_RANGE.1)); + if !(retries_range.0..=retries_range.1).contains(&raw.retries) { + return Err(invalid("retries", retries_range)); + } + let delay_range: (i64, i64) = ( + i64::from(RETRY_DELAY_RANGE.0), + i64::from(RETRY_DELAY_RANGE.1), + ); + if !(delay_range.0..=delay_range.1).contains(&raw.retry_delay_seconds) { + return Err(invalid("retry_delay_seconds", delay_range)); + } + let retries: u8 = u8::try_from(raw.retries).map_err(|_| invalid("retries", retries_range))?; + let retry_delay_seconds: u32 = u32::try_from(raw.retry_delay_seconds) + .map_err(|_| invalid("retry_delay_seconds", delay_range))?; + Ok(OnFailure { + retries, + retry_delay_seconds, + }) +} + +/// The fields jobs and groups share, checked. +pub(crate) fn run_file(text: &str, raw: RawRun) -> Result { + let owner: Option> = raw + .owner + .map(|owner| nonblank(text, owner, "owner")) + .transpose()?; + let mut groups: BTreeSet<&str> = BTreeSet::new(); + if let Some(repeated) = raw + .concurrency_groups + .iter() + .find(|group| !groups.insert(group.as_str())) + { + return Err(error_at( + text, + 0, + format!("concurrency group '{repeated}' is listed twice"), + )); + } + Ok(RunFile { + name: nonblank(text, raw.name, "name")?, + description: nonblank(text, raw.description, "description")?.value, + targets: raw.targets, + owner: owner.map(|owner| owner.value), + overlap: raw.overlap, + concurrency_groups: raw.concurrency_groups, + schedule: raw + .schedule + .map(|value| schedule(text, value)) + .transpose()?, + on_failure: raw + .on_failure + .map(|value| on_failure(text, value)) + .transpose()?, + }) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs new file mode 100644 index 000000000..c8df4e054 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/references.rs @@ -0,0 +1,154 @@ +//! Name suffixes and the names groups, schedules and sensors refer to. + +use std::collections::BTreeSet; + +use crate::orchestration_files::constants::{ + CALENDAR_KIND, CALENDARS_DIRECTORY, GROUP_KIND, JOB_KIND, JOBS_DIRECTORY, SENSOR_KIND, +}; +use crate::orchestration_files::errors::FileError; +use crate::orchestration_files::models::{ + FilePosition, OrchestrationInventory, Positioned, RunFile, ScheduleSource, +}; + +/// The first problem with a name or reference, and the file it is in. +pub(crate) type ReferenceError = (String, FileError); + +fn at(position: FilePosition, message: String, help: Option) -> FileError { + FileError { + message, + position, + help, + } +} + +fn suffixed( + path: &str, + (noun, suffix): (&str, &str), + name: &str, + position: FilePosition, +) -> Result<(), ReferenceError> { + if name.ends_with(suffix) && name.len() > suffix.len() { + return Ok(()); + } + Err(( + path.to_owned(), + at( + position, + format!("{noun} '{name}' must be named with the suffix '{suffix}'"), + Some(format!("rename it to '{name}{suffix}'")), + ), + )) +} + +fn calendar_exists( + path: &str, + run: &RunFile, + calendars: &BTreeSet<&str>, +) -> Result<(), ReferenceError> { + let Some(ScheduleSource::Calendar(calendar)) = run.schedule.as_ref().map(|s| &s.source) else { + return Ok(()); + }; + if calendars.contains(calendar.value.as_str()) { + return Ok(()); + } + Err(( + path.to_owned(), + at( + calendar.position, + format!( + "schedule names '{}', which is not a calendar in orchestration/{CALENDARS_DIRECTORY}/", + calendar.value + ), + None, + ), + )) +} + +fn member( + path: &str, + reference: &Positioned, + known: &BTreeSet<&str>, + (owner, wanted): (&str, &str), +) -> Result<(), ReferenceError> { + if known.contains(reference.value.as_str()) { + return Ok(()); + } + Err(( + path.to_owned(), + at( + reference.position, + format!("{owner} names '{}', which is not {wanted}", reference.value), + None, + ), + )) +} + +/// Refuse a name without its kind's suffix, then a group member, calendar or sensor job that is not declared. +pub(crate) fn check_references(inventory: &OrchestrationInventory) -> Result<(), ReferenceError> { + for (path, job) in &inventory.jobs { + suffixed(path, JOB_KIND, &job.run.name.value, job.run.name.position)?; + } + for (path, group) in &inventory.groups { + suffixed( + path, + GROUP_KIND, + &group.run.name.value, + group.run.name.position, + )?; + } + for module in &inventory.calendars { + suffixed(&module.path, CALENDAR_KIND, &module.name, module.position)?; + } + for module in &inventory.sensors { + suffixed(&module.path, SENSOR_KIND, &module.name, module.position)?; + } + let jobs: BTreeSet<&str> = inventory + .jobs + .iter() + .map(|(_, job)| job.run.name.value.as_str()) + .collect(); + let runnable: BTreeSet<&str> = jobs + .iter() + .copied() + .chain( + inventory + .groups + .iter() + .map(|(_, group)| group.run.name.value.as_str()), + ) + .collect(); + let calendars: BTreeSet<&str> = inventory + .calendars + .iter() + .map(|module| module.name.as_str()) + .collect(); + for (path, job) in &inventory.jobs { + calendar_exists(path, &job.run, &calendars)?; + } + for (path, group) in &inventory.groups { + calendar_exists(path, &group.run, &calendars)?; + let wanted = format!("a job in orchestration/{JOBS_DIRECTORY}/"); + for reference in &group.jobs { + member( + path, + reference, + &jobs, + (&format!("job group '{}'", group.run.name.value), &wanted), + )?; + } + } + for module in &inventory.sensors { + for (_, reference) in &module.job_references { + member( + &module.path, + reference, + &runnable, + ( + &format!("sensor '{}'", module.name), + "a declared job or job group", + ), + )?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs new file mode 100644 index 000000000..c722549fb --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs @@ -0,0 +1,63 @@ +//! The keys job and job group files accept, and the bounds on their run controls. + +/// Top-level keys of a job file. +pub const JOB_KEYS: [&str; 12] = [ + "name", + "description", + "select", + "exclude", + "targets", + "owner", + "overlap", + "concurrency_groups", + "schedule", + "on_failure", + "resources", + "settings", +]; +/// Top-level keys of a job group file. +pub const GROUP_KEYS: [&str; 9] = [ + "name", + "description", + "jobs", + "targets", + "owner", + "overlap", + "concurrency_groups", + "schedule", + "on_failure", +]; +/// The fewest and most reruns `on_failure.retries` allows. +pub const RETRIES_RANGE: (u8, u8) = (1, 10); +/// The shortest and longest `on_failure.retry_delay_seconds`, a day. +pub const RETRY_DELAY_RANGE: (u32, u32) = (0, 86_400); +/// The job setting that moved under `[settings]`. +pub(crate) const CONCURRENCY_KEY: &str = "concurrency"; +/// How close an unknown key must be to a known one to be suggested. +pub(crate) const SUGGESTION_CUTOFF: f64 = 0.6; +/// The folders below `orchestration/`, one per declaration kind. +pub const JOBS_DIRECTORY: &str = "jobs"; +pub const GROUPS_DIRECTORY: &str = "job_groups"; +pub const CALENDARS_DIRECTORY: &str = "calendars"; +pub const SENSORS_DIRECTORY: &str = "sensors"; +/// The file suffix of job and job group declarations. +pub const TOML_FILE_SUFFIX: &str = ".toml"; +/// Each kind as name conflicts word it, and the suffix its names end with. +pub const JOB_KIND: (&str, &str) = ("job", "_job"); +pub const GROUP_KIND: (&str, &str) = ("job group", "_job_group"); +pub const CALENDAR_KIND: (&str, &str) = ("calendar", "_calendar"); +pub const SENSOR_KIND: (&str, &str) = ("sensor", "_sensor"); +/// Where jobs, job groups and schedules are declared now, for projects that still use Python. +pub const TOML_JOBS_HELP: &str = "jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/"; +/// The folder Python writes bytecode to, which declarations never live in. +pub(crate) const BYTECODE_DIRECTORY: &str = "__pycache__"; +/// The prefix of hidden files and folders, and of private support modules. +pub(crate) const HIDDEN_PREFIX: char = '.'; +pub(crate) const PRIVATE_PREFIX: char = '_'; +/// Every folder `orchestration/` may hold. +pub const ROLE_DIRECTORIES: [&str; 4] = [ + JOBS_DIRECTORY, + GROUPS_DIRECTORY, + CALENDARS_DIRECTORY, + SENSORS_DIRECTORY, +]; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/errors.rs b/crates/sqlbuild-discovery/src/orchestration_files/errors.rs new file mode 100644 index 000000000..231cad2bf --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/errors.rs @@ -0,0 +1,11 @@ +//! Why a declaration file cannot be read. + +use crate::orchestration_files::models::FilePosition; + +/// Why a declaration file cannot be read, at the position it points to. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FileError { + pub message: String, + pub position: FilePosition, + pub help: Option, +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs new file mode 100644 index 000000000..086c012c6 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/mod.rs @@ -0,0 +1 @@ +pub mod orchestration_inventory; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs new file mode 100644 index 000000000..8759e9e69 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/main/orchestration_inventory.rs @@ -0,0 +1,42 @@ +//! Discover, read and check every orchestration declaration without importing project Python. + +use crate::constants::ORCHESTRATION_DIRECTORY; +use crate::orchestration_files::_helpers::inventory_files::{ + check_top_level, failure, read_python_modules, read_toml_files, +}; +use crate::orchestration_files::_helpers::read_group_file::read_group_file; +use crate::orchestration_files::_helpers::read_job_file::read_job_file; +use crate::orchestration_files::_helpers::references::check_references; +use crate::orchestration_files::constants::{GROUPS_DIRECTORY, JOBS_DIRECTORY}; +use crate::orchestration_files::models::{ + GroupFile, JobFile, OrchestrationFailure, OrchestrationInventory, PythonModules, +}; +use crate::tree::models::ProjectTree; + +/// Every job, group, calendar and sensor below `orchestration/`, or the first problem with them. +pub fn orchestration_inventory( + tree: &ProjectTree, + python_version: (u8, u8), +) -> Result { + if !tree.is_dir(ORCHESTRATION_DIRECTORY) { + return Ok(OrchestrationInventory::default()); + } + check_top_level(tree)?; + let jobs: Vec<(String, JobFile)> = read_toml_files(tree, JOBS_DIRECTORY, read_job_file)?; + let groups: Vec<(String, GroupFile)> = + read_toml_files(tree, GROUPS_DIRECTORY, read_group_file)?; + let PythonModules { + calendars, + sensors, + modules, + } = read_python_modules(tree, python_version)?; + let inventory: OrchestrationInventory = OrchestrationInventory { + jobs, + groups, + calendars, + sensors, + modules, + }; + check_references(&inventory).map_err(|(path, error)| failure(&path, error))?; + Ok(inventory) +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/mod.rs b/crates/sqlbuild-discovery/src/orchestration_files/mod.rs new file mode 100644 index 000000000..3c98387f0 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/mod.rs @@ -0,0 +1,10 @@ +//! Orchestration declaration files: job and job group TOML, read with source positions. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/models.rs b/crates/sqlbuild-discovery/src/orchestration_files/models.rs new file mode 100644 index 000000000..26fbd25f5 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/models.rs @@ -0,0 +1,191 @@ +//! Typed job and job group files. + +use std::collections::BTreeMap; + +use serde::Deserialize; + +use crate::models::{DiscoveryFailure, ReadFailure, StageFailure}; +use crate::project_validation::models::OrchestrationNameFact; + +/// Whether a run that starts while one is active waits (`queue`) or is dropped (`skip`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Overlap { + #[default] + Queue, + Skip, +} + +/// Whether a schedule starts running or paused when first published. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum InitialStatus { + #[default] + Running, + Paused, +} + +/// Reruns from failure, `retries` times, `retry_delay_seconds` apart. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct OnFailure { + pub retries: u8, + pub retry_delay_seconds: u32, +} + +/// A 1-based line and column in a declaration file. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +pub struct FilePosition { + pub line: usize, + pub column: usize, +} + +/// A value and where it is written. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Positioned { + pub value: T, + pub position: FilePosition, +} + +/// What starts a job or group on its own: a cron expression or a named calendar. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ScheduleSource { + Cron(Positioned), + Calendar(Positioned), +} + +/// A job's or group's `schedule` table. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScheduleFile { + pub source: ScheduleSource, + pub timezone: Positioned, + /// RFC 3339 as written; compile checks it and publication parses it. + pub starts_at: Option>, + pub default_status: InitialStatus, +} + +/// The fields jobs and groups share. +#[derive(Clone, Debug, PartialEq)] +pub struct RunFile { + pub name: Positioned, + pub description: String, + pub targets: Vec, + pub owner: Option, + pub overlap: Overlap, + pub concurrency_groups: Vec, + pub schedule: Option, + pub on_failure: Option, +} + +/// One `orchestration/jobs/**.toml` file. +#[derive(Clone, Debug, PartialEq)] +pub struct JobFile { + pub run: RunFile, + pub select: Positioned, + pub exclude: Option>, + /// Worker container limits as written; compile checks their values. + pub resources: Option>>, + /// `sqb build` flags by name, as JSON values. + pub settings: BTreeMap, + /// Where each setting's value is written. + pub setting_positions: BTreeMap, +} + +/// One `orchestration/job_groups/**.toml` file. +#[derive(Clone, Debug, PartialEq)] +pub struct GroupFile { + pub run: RunFile, + pub jobs: Vec>, +} + +/// A calendar or sensor module and the declaration it makes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NamedModule { + pub name: String, + pub path: String, + pub position: FilePosition, + /// The jobs and groups a sensor starts (`run`) or watches (`run_event`); empty for a calendar. + pub job_references: Vec<(&'static str, Positioned)>, + /// A sensor's targets: empty for every target, `None` when they are not written as literals. + pub targets: Option>, +} + +/// The calendars, sensors and every module below their folders. +#[derive(Debug, Default)] +pub(crate) struct PythonModules { + pub(crate) calendars: Vec, + pub(crate) sensors: Vec, + pub(crate) modules: Vec, +} + +/// Every orchestration declaration in a project, checked for layout, names and references. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct OrchestrationInventory { + pub jobs: Vec<(String, JobFile)>, + pub groups: Vec<(String, GroupFile)>, + pub calendars: Vec, + pub sensors: Vec, + /// Every Python module under `calendars/` and `sensors/`, which publication imports. + pub modules: Vec, +} + +impl OrchestrationInventory { + /// Each declared name with its kind and file, for the project-wide name check. + #[must_use] + pub fn names(&self) -> Vec { + let fact = |name: &str, kind: (&'static str, &str), path: &str| OrchestrationNameFact { + name: name.to_owned(), + kind: kind.0, + path: path.to_owned(), + }; + use crate::orchestration_files::constants::{ + CALENDAR_KIND, GROUP_KIND, JOB_KIND, SENSOR_KIND, + }; + let mut names: Vec = Vec::new(); + names.extend( + self.jobs + .iter() + .map(|(path, job)| fact(&job.run.name.value, JOB_KIND, path)), + ); + names.extend( + self.groups + .iter() + .map(|(path, group)| fact(&group.run.name.value, GROUP_KIND, path)), + ); + names.extend( + self.calendars + .iter() + .map(|module| fact(&module.name, CALENDAR_KIND, &module.path)), + ); + names.extend( + self.sensors + .iter() + .map(|module| fact(&module.name, SENSOR_KIND, &module.path)), + ); + names + } + + /// Whether the project declares anything to orchestrate. + #[must_use] + pub fn is_empty(&self) -> bool { + self.jobs.is_empty() && self.groups.is_empty() && self.sensors.is_empty() + } +} + +/// Why the orchestration declarations cannot be read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum OrchestrationFailure { + /// A layout, file, name or reference error, as `OrchestrationDeclarationError`. + Declaration(DiscoveryFailure), + /// A file could not be read the way Python's `read_text` reads it. + Unreadable { + relative_path: String, + failure: ReadFailure, + }, + Stage(StageFailure), +} + +impl From for OrchestrationFailure { + fn from(failure: StageFailure) -> Self { + Self::Stage(failure) + } +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests.rs new file mode 100644 index 000000000..5995ff0d9 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_orchestration_inventory.rs"] +mod orchestration_inventory; +#[path = "tests/test_read_files.rs"] +mod read_files; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/helpers.rs new file mode 100644 index 000000000..5095f0de3 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/helpers.rs @@ -0,0 +1,107 @@ +use std::path::Path; + +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::orchestration_files::_helpers::read_group_file::read_group_file; +use crate::orchestration_files::main::orchestration_inventory::orchestration_inventory; +use crate::orchestration_files::models::{ + FilePosition, OrchestrationFailure, OrchestrationInventory, Positioned, +}; +use crate::tree::models::ProjectTree; + +pub(super) const EXPORT_JOB: &str = "name = \"orders_export_job\"\ndescription = \"d\"\nselect = \"orders\"\nschedule = { calendar = \"trading_days_calendar\", timezone = \"UTC\" }\n"; +pub(super) const NIGHTLY_GROUP: &str = + "name = \"orders_nightly_job_group\"\ndescription = \"d\"\njobs = [\"orders_export_job\"]\n"; +pub(super) const CALENDAR: &str = "from sqlbuild.orchestration import calendar\n\n@calendar(name=\"trading_days_calendar\", description=\"d\", horizon=\"7d\")\ndef trading_days(window):\n return []\n"; +pub(super) const SENSOR: &str = "from sqlbuild.orchestration import run_event, sensor\n\nstale = sensor(name=\"stock_stale_sensor\", description=\"d\", when=run_event(job=\"orders_export_job\", kinds=[]), run=\"orders_nightly_job_group\")\n"; + +/// A 1-based line and column. +pub(super) fn at(line: usize, column: usize) -> FilePosition { + FilePosition { line, column } +} + +/// A string value written at `line` and `column`. +pub(super) fn written(value: &str, line: usize, column: usize) -> Positioned { + Positioned { + value: value.to_owned(), + position: at(line, column), + } +} + +/// A group's members with their positions, or its error's message and position. +pub(super) fn group_members( + text: &str, +) -> Result, (String, FilePosition)> { + read_group_file(text) + .map(|group| group.jobs.into_iter().map(member).collect()) + .map_err(|error| (error.message, error.position)) +} + +/// A group member's name and position. +fn member(job: Positioned) -> (String, FilePosition) { + (job.value, job.position) +} + +/// A temporary project holding `files`. +pub(super) fn project(files: &[(&str, &str)]) -> tempfile::TempDir { + let directory = tempfile::tempdir().expect("temporary project"); + for (path, text) in files { + let path = directory.path().join(path); + std::fs::create_dir_all(path.parent().expect("parent")).expect("folders"); + std::fs::write(path, text).expect("file"); + } + directory +} + +/// The inventory of the project at `root`. +pub(super) fn inventory(root: &Path) -> Result { + orchestration_inventory(&ProjectTree::new(root), (3, 12)) +} + +/// Why the inventory of a project holding `files` fails. +pub(super) fn inventory_failure(files: &[(&str, &str)]) -> OrchestrationFailure { + let directory = project(files); + inventory(directory.path()).expect_err("invalid project") +} + +/// A `D017` declaration failure with `message` and `help`. +pub(super) fn declaration_failure(message: &str, help: Option<&str>) -> OrchestrationFailure { + OrchestrationFailure::Declaration(DiscoveryFailure { + kind: FailureKind::Orchestration, + message: message.to_owned(), + help: help.map(str::to_owned), + }) +} + +/// A declared name with its kind and file. +pub(super) fn named(name: &str, kind: &'static str, path: &str) -> (String, &'static str, String) { + (name.to_owned(), kind, path.to_owned()) +} + +/// A project declaring one of each kind, plus files the inventory skips. +pub(super) fn valid() -> Vec<(&'static str, &'static str)> { + vec![ + ("orchestration/jobs/orders/export.toml", EXPORT_JOB), + ( + "orchestration/job_groups/orders/nightly.toml", + NIGHTLY_GROUP, + ), + ("orchestration/calendars/trading_days.py", CALENDAR), + ("orchestration/calendars/_holidays.py", "HOLIDAYS = []\n"), + ("orchestration/sensors/inventory/stock.py", SENSOR), + ( + "orchestration/sensors/__pycache__/stock.cpython-312.pyc", + "x", + ), + ] +} + +/// The valid project with the file at `path` replaced by, or added as, `text`. +pub(super) fn with_file( + path: &'static str, + text: &'static str, +) -> Vec<(&'static str, &'static str)> { + let mut files: Vec<(&'static str, &'static str)> = valid(); + files.retain(|(existing, _)| *existing != path); + files.push((path, text)); + files +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs new file mode 100644 index 000000000..0e63db921 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_orchestration_inventory.rs @@ -0,0 +1,200 @@ +use crate::orchestration_files::tests::helpers::{ + EXPORT_JOB, SENSOR, declaration_failure, inventory, inventory_failure, named, project, valid, + with_file, +}; +use crate::orchestration_files::tests::test_types::{InventoryErrorTestCase, InventoryTestCase}; + +#[test] +fn given_projects_when_inventorying_then_reads_each_kind_and_names_them() { + let test_cases = [ + InventoryTestCase { + description: "one declaration of each kind, skipping private modules and bytecode", + files: valid(), + expected_names: vec![ + named( + "orders_export_job", + "job", + "orchestration/jobs/orders/export.toml", + ), + named( + "orders_nightly_job_group", + "job group", + "orchestration/job_groups/orders/nightly.toml", + ), + named( + "trading_days_calendar", + "calendar", + "orchestration/calendars/trading_days.py", + ), + named( + "stock_stale_sensor", + "sensor", + "orchestration/sensors/inventory/stock.py", + ), + ], + expected_modules: vec![ + "orchestration/calendars/_holidays.py", + "orchestration/calendars/trading_days.py", + "orchestration/sensors/inventory/stock.py", + ], + expected_empty: false, + }, + InventoryTestCase { + description: "no orchestration folder", + files: vec![("models/orders.sql", "select 1")], + expected_names: Vec::new(), + expected_modules: Vec::new(), + expected_empty: true, + }, + ]; + for test_case in test_cases { + let directory = project(&test_case.files); + let found = inventory(directory.path()).expect(test_case.description); + let names: Vec<(String, &str, String)> = found + .names() + .into_iter() + .map(|fact| (fact.name, fact.kind, fact.path)) + .collect(); + assert_eq!(names, test_case.expected_names, "{}", test_case.description); + assert_eq!( + found.modules, test_case.expected_modules, + "{}", + test_case.description + ); + assert_eq!( + found.is_empty(), + test_case.expected_empty, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_layout_and_reference_errors_when_inventorying_then_names_file_line_and_fix() { + let test_cases = [ + InventoryErrorTestCase { + description: "loose file", + files: with_file("orchestration/jobs.py", "x = 1\n"), + expected_message: "orchestration/jobs.py:1:1: files cannot sit directly in orchestration/", + expected_help: Some( + "jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/", + ), + }, + InventoryErrorTestCase { + description: "loose non-Python file", + files: with_file("orchestration/nightly.toml", EXPORT_JOB), + expected_message: "orchestration/nightly.toml:1:1: files cannot sit directly in orchestration/", + expected_help: Some( + "move it into one of orchestration/jobs/, orchestration/job_groups/, orchestration/calendars/, orchestration/sensors/", + ), + }, + InventoryErrorTestCase { + description: "unknown folder of Python modules", + files: with_file("orchestration/pipelines/daily/orders.py", "x = 1\n"), + expected_message: "orchestration/pipelines:1:1: unknown orchestration folder 'pipelines'", + expected_help: Some( + "jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/", + ), + }, + InventoryErrorTestCase { + description: "unknown folder", + files: with_file("orchestration/schedules/nightly.toml", EXPORT_JOB), + expected_message: "orchestration/schedules:1:1: unknown orchestration folder 'schedules'", + expected_help: Some("use one of: jobs, job_groups, calendars, sensors"), + }, + InventoryErrorTestCase { + description: "python job", + files: with_file("orchestration/jobs/orders/legacy.py", "x = 1\n"), + expected_message: "orchestration/jobs/orders/legacy.py:1:1: orchestration/jobs/ holds only .toml files", + expected_help: Some( + "jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/", + ), + }, + InventoryErrorTestCase { + description: "non-TOML job file", + files: with_file( + "orchestration/jobs/orders/export.yaml", + "name: orders_export_job\n", + ), + expected_message: "orchestration/jobs/orders/export.yaml:1:1: orchestration/jobs/ holds only .toml files", + expected_help: Some( + "jobs and job groups are TOML; calendars and sensors are Python in orchestration/calendars/ and orchestration/sensors/", + ), + }, + InventoryErrorTestCase { + description: "missing suffix", + files: with_file( + "orchestration/jobs/orders/export.toml", + "name = \"orders_export\"\ndescription = \"d\"\nselect = \"orders\"\n", + ), + expected_message: "orchestration/jobs/orders/export.toml:1:8: job 'orders_export' must be named with the suffix '_job'", + expected_help: Some("rename it to 'orders_export_job'"), + }, + InventoryErrorTestCase { + description: "unknown group member", + files: with_file( + "orchestration/job_groups/orders/nightly.toml", + "name = \"orders_nightly_job_group\"\ndescription = \"d\"\njobs = [\"orders_export_job\", \"carrier_sync_job\"]\n", + ), + expected_message: "orchestration/job_groups/orders/nightly.toml:3:30: job group 'orders_nightly_job_group' names 'carrier_sync_job', which is not a job in orchestration/jobs/", + expected_help: None, + }, + InventoryErrorTestCase { + description: "unknown calendar", + files: with_file( + "orchestration/calendars/trading_days.py", + "from sqlbuild.orchestration import calendar\n\n@calendar(name=\"working_days_calendar\", description=\"d\", horizon=\"7d\")\ndef f(window):\n return []\n", + ), + expected_message: "orchestration/jobs/orders/export.toml:4:25: schedule names 'trading_days_calendar', which is not a calendar in orchestration/calendars/", + expected_help: None, + }, + InventoryErrorTestCase { + description: "non-literal sensor job", + files: with_file( + "orchestration/sensors/inventory/stock.py", + "from sqlbuild.orchestration import sensor\nJOB = \"orders_export_job\"\nstale = sensor(name=\"stock_stale_sensor\", description=\"d\", run=JOB)\n", + ), + expected_message: "orchestration/sensors/inventory/stock.py:3:64: sensor run job must be a string literal", + expected_help: Some( + "write the sensor run job as a quoted string, such as \"orders_job\", so the compiler can check it without running the module", + ), + }, + InventoryErrorTestCase { + description: "sensor in calendars", + files: with_file("orchestration/calendars/stock.py", SENSOR), + expected_message: "orchestration/calendars/stock.py:3:9: declares a sensor, which belongs in orchestration/sensors/", + expected_help: None, + }, + InventoryErrorTestCase { + description: "public module with nothing", + files: with_file( + "orchestration/sensors/helpers.py", + "def helper():\n return 1\n", + ), + expected_message: "orchestration/sensors/helpers.py:1:1: declares no sensor", + expected_help: Some( + "declare one sensor, or start the file name with `_` to make it a private support module", + ), + }, + InventoryErrorTestCase { + description: "removed Python job API", + files: with_file( + "orchestration/sensors/inventory/stock.py", + "from sqlbuild.orchestration import job, sensor\nstale = sensor(name=\"stock_stale_sensor\", description=\"d\", run=\"orders_export_job\")\n", + ), + expected_message: "orchestration/sensors/inventory/stock.py:1:36: `job` was removed from sqlbuild.orchestration; jobs, job groups and schedules are declared in TOML", + expected_help: Some( + "jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/", + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + inventory_failure(&test_case.files), + declaration_failure(test_case.expected_message, test_case.expected_help), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs new file mode 100644 index 000000000..f56e400c8 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs @@ -0,0 +1,225 @@ +use std::collections::BTreeMap; + +use crate::orchestration_files::_helpers::read_job_file::read_job_file; +use crate::orchestration_files::models::{ + InitialStatus, OnFailure, Overlap, ScheduleFile, ScheduleSource, +}; +use crate::orchestration_files::tests::helpers::{at, group_members, written}; +use crate::orchestration_files::tests::test_types::{ + FileErrorTestCase, GroupReadTestCase, JobReadTestCase, +}; + +const FULL_JOB: &str = r#"name = "orders_export_job" +description = "Write the daily partner file." +select = "python/orders/partner_export" +exclude = "tag:slow" +targets = ["production"] +owner = "Orders team" +overlap = "skip" +concurrency_groups = ["warehouse"] +schedule = { calendar = "trading_days_calendar", timezone = "Europe/London", default_status = "paused" } + +[on_failure] +retries = 2 + +[resources] +memory = "2g" +cpus = 1.5 + +[settings] +vars = { batch = 23 } +concurrency = 4 +"#; + +#[test] +fn given_job_files_when_reading_then_types_every_field_and_keeps_positions() { + let test_cases = [ + JobReadTestCase { + description: "a job using every key", + text: FULL_JOB, + expected_name: written("orders_export_job", 1, 8), + expected_select: written("python/orders/partner_export", 3, 10), + expected_exclude: Some("tag:slow"), + expected_overlap: Overlap::Skip, + expected_on_failure: Some(OnFailure { + retries: 2, + retry_delay_seconds: 0, + }), + expected_schedule: Some(ScheduleFile { + source: ScheduleSource::Calendar(written("trading_days_calendar", 9, 25)), + timezone: written("Europe/London", 9, 61), + starts_at: None, + default_status: InitialStatus::Paused, + }), + expected_cpus: Some(serde_json::json!(1.5)), + expected_settings: BTreeMap::from([ + ("concurrency".to_owned(), serde_json::json!(4)), + ("vars".to_owned(), serde_json::json!({"batch": 23})), + ]), + }, + JobReadTestCase { + description: "a job with only the required keys takes the defaults", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nexclude = \" \"\n", + expected_name: written("a_job", 1, 8), + expected_select: written("orders", 3, 10), + expected_exclude: None, + expected_overlap: Overlap::Queue, + expected_on_failure: None, + expected_schedule: None, + expected_cpus: None, + expected_settings: BTreeMap::new(), + }, + ]; + for test_case in test_cases { + let job = read_job_file(test_case.text).expect(test_case.description); + assert_eq!( + job.run.name, test_case.expected_name, + "{}", + test_case.description + ); + assert_eq!( + job.select, test_case.expected_select, + "{}", + test_case.description + ); + assert_eq!( + job.exclude.map(|exclude| exclude.value).as_deref(), + test_case.expected_exclude, + "{}", + test_case.description + ); + assert_eq!( + job.run.overlap, test_case.expected_overlap, + "{}", + test_case.description + ); + assert_eq!( + job.run.on_failure, test_case.expected_on_failure, + "{}", + test_case.description + ); + assert_eq!( + job.run.schedule, test_case.expected_schedule, + "{}", + test_case.description + ); + assert_eq!( + job.resources + .and_then(|resources| resources.value.get("cpus").cloned()), + test_case.expected_cpus, + "{}", + test_case.description + ); + assert_eq!( + job.settings, test_case.expected_settings, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_invalid_job_files_when_reading_then_reports_message_position_and_help() { + let test_cases = [ + FileErrorTestCase { + description: "misspelt key", + text: "name = \"a_job\"\ndescription = \"d\"\nselct = \"orders\"\n", + expected_message: "unknown job key 'selct'", + expected_position: at(3, 1), + expected_help: Some("did you mean 'select'?"), + }, + FileErrorTestCase { + description: "top-level concurrency", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nconcurrency = 2\n", + expected_message: "unknown job key 'concurrency'", + expected_position: at(4, 1), + expected_help: Some("move it under [settings] as `concurrency = ...`"), + }, + FileErrorTestCase { + description: "blank select", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \" \"\n", + expected_message: "select must not be blank", + expected_position: at(3, 10), + expected_help: None, + }, + FileErrorTestCase { + description: "both cron and calendar", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nschedule = { cron = \"0 2 * * *\", calendar = \"c_calendar\", timezone = \"UTC\" }\n", + expected_message: "schedule needs exactly one of `cron` or `calendar`", + expected_position: at(4, 12), + expected_help: None, + }, + FileErrorTestCase { + description: "retries out of range", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\non_failure = { retries = 0 }\n", + expected_message: "on_failure retries must be a whole number from 1 to 10", + expected_position: at(4, 14), + expected_help: None, + }, + FileErrorTestCase { + description: "retry delay beyond a day", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\non_failure = { retries = 1, retry_delay_seconds = 86401 }\n", + expected_message: "on_failure retry_delay_seconds must be a whole number from 0 to 86400", + expected_position: at(4, 14), + expected_help: None, + }, + FileErrorTestCase { + description: "missing description", + text: "name = \"a_job\"\nselect = \"orders\"\n", + expected_message: "missing field `description`", + expected_position: at(1, 1), + expected_help: None, + }, + ]; + for test_case in test_cases { + let error = read_job_file(test_case.text).expect_err(test_case.description); + assert_eq!( + error.message, test_case.expected_message, + "{}", + test_case.description + ); + assert_eq!( + error.position, test_case.expected_position, + "{}", + test_case.description + ); + assert_eq!( + error.help.as_deref(), + test_case.expected_help, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_group_files_when_reading_then_keeps_member_positions_and_refuses_repeats() { + let test_cases = [ + GroupReadTestCase { + description: "members keep their positions", + text: "name = \"nightly_job_group\"\ndescription = \"d\"\njobs = [\"export_job\", \"carrier_job\"]\n", + expected_members: Ok(vec![ + ("export_job".to_owned(), at(3, 9)), + ("carrier_job".to_owned(), at(3, 23)), + ]), + }, + GroupReadTestCase { + description: "a repeated member is refused at the repeat", + text: "name = \"nightly_job_group\"\ndescription = \"d\"\njobs = [\"export_job\", \"export_job\"]\n", + expected_members: Err(("job 'export_job' is listed twice".to_owned(), at(3, 23))), + }, + GroupReadTestCase { + description: "groups do not select", + text: "name = \"nightly_job_group\"\ndescription = \"d\"\njobs = [\"export_job\"]\nselect = \"orders\"\n", + expected_members: Err(("unknown job group key 'select'".to_owned(), at(4, 1))), + }, + ]; + for test_case in test_cases { + assert_eq!( + group_members(test_case.text), + test_case.expected_members, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs new file mode 100644 index 000000000..8cd4f4868 --- /dev/null +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs @@ -0,0 +1,47 @@ +use std::collections::BTreeMap; + +use crate::orchestration_files::models::{ + FilePosition, OnFailure, Overlap, Positioned, ScheduleFile, +}; + +pub(super) struct JobReadTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_name: Positioned, + pub(super) expected_select: Positioned, + pub(super) expected_exclude: Option<&'static str>, + pub(super) expected_overlap: Overlap, + pub(super) expected_on_failure: Option, + pub(super) expected_schedule: Option, + pub(super) expected_cpus: Option, + pub(super) expected_settings: BTreeMap, +} + +pub(super) struct FileErrorTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_message: &'static str, + pub(super) expected_position: FilePosition, + pub(super) expected_help: Option<&'static str>, +} + +pub(super) struct GroupReadTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_members: Result, (String, FilePosition)>, +} + +pub(super) struct InventoryTestCase { + pub(super) description: &'static str, + pub(super) files: Vec<(&'static str, &'static str)>, + pub(super) expected_names: Vec<(String, &'static str, String)>, + pub(super) expected_modules: Vec<&'static str>, + pub(super) expected_empty: bool, +} + +pub(super) struct InventoryErrorTestCase { + pub(super) description: &'static str, + pub(super) files: Vec<(&'static str, &'static str)>, + pub(super) expected_message: &'static str, + pub(super) expected_help: Option<&'static str>, +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs index 2d10e91f6..6191263dd 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs @@ -142,7 +142,7 @@ fn name_conflict(first: &DeclaredName<'_>, second: &DeclaredName<'_>) -> Validat }; conflict(format!( "Project resource name '{}' is declared as both {} in {} and {} in {}; model, source, seed, function, loader, task, asset, check, provider, \ - hook, unit test, scenario, job, job group, sensor, and calendar schedule names must be \ + hook, unit test, scenario, job, job group, sensor, and calendar names must be \ globally unique; rename one of them{advice}", first.name, first.kind, first.path, second.kind, second.path )) diff --git a/crates/sqlbuild-discovery/src/project_validation/constants.rs b/crates/sqlbuild-discovery/src/project_validation/constants.rs index e2dcb6a95..ff5222415 100644 --- a/crates/sqlbuild-discovery/src/project_validation/constants.rs +++ b/crates/sqlbuild-discovery/src/project_validation/constants.rs @@ -14,5 +14,5 @@ pub(crate) const INITIAL_SEED_READ_BYTES: u64 = 64 * 1024; pub(crate) const RECORDS_FOR_MANY_ROWS: usize = 3; /// How a colliding orchestration declaration is renamed, added to its name conflict. -pub(crate) const ORCHESTRATION_RENAME_ADVICE: &str = " (a job, job group or sensor takes its name \ - from its first argument, and a calendar schedule from the function it decorates)"; +pub(crate) const ORCHESTRATION_RENAME_ADVICE: &str = " (a job or job group takes its name from \ + `name` in its TOML file, and a calendar or sensor from its `name` argument)"; diff --git a/crates/sqlbuild-discovery/src/project_validation/main/validate_orchestration_names.rs b/crates/sqlbuild-discovery/src/project_validation/main/validate_orchestration_names.rs index ac3a340b3..edbb1d5cb 100644 --- a/crates/sqlbuild-discovery/src/project_validation/main/validate_orchestration_names.rs +++ b/crates/sqlbuild-discovery/src/project_validation/main/validate_orchestration_names.rs @@ -3,7 +3,7 @@ use crate::project_validation::_helpers::resource_names::validate_unique_project_resource_names; use crate::project_validation::models::{ProjectFacts, ValidationFailure}; -/// Reject the first name a job, group, sensor or calendar schedule shares with anything else. +/// Reject the first name a job, group, calendar or sensor shares with anything else. pub fn validate_orchestration_names(facts: &ProjectFacts) -> Result<(), ValidationFailure> { validate_unique_project_resource_names(facts) } diff --git a/crates/sqlbuild-discovery/src/project_validation/models.rs b/crates/sqlbuild-discovery/src/project_validation/models.rs index 2b185d25a..813c60d70 100644 --- a/crates/sqlbuild-discovery/src/project_validation/models.rs +++ b/crates/sqlbuild-discovery/src/project_validation/models.rs @@ -178,7 +178,7 @@ pub struct ProjectFacts { pub project_model_paths: Option>, /// Python's `csv.field_size_limit()`, or `None` for its default. pub csv_field_limit: Option, - /// Jobs, job groups, sensors and calendar schedules, known only when a location publishes. + /// Jobs, job groups, calendars and sensors, read from `orchestration/` without importing it. pub orchestration: Vec, } @@ -188,7 +188,7 @@ pub(crate) struct DeclaredName<'a> { pub(crate) name: &'a str, pub(crate) kind: &'a str, pub(crate) path: &'a str, - /// A job, job group, sensor or calendar schedule, whose conflicts say how it is named. + /// A job, job group, calendar or sensor, whose conflicts say how it is named. pub(crate) orchestration: bool, } diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs b/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs index 64c9bc747..d7b42c670 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests/helpers.rs @@ -106,7 +106,7 @@ pub(super) fn collision(message: &str, advice: &str) -> Result<(), String> { FailureKind::Conflict, &format!( "{message}; model, source, seed, function, loader, task, asset, check, provider, \ - hook, unit test, scenario, job, job group, sensor, and calendar schedule names must \ + hook, unit test, scenario, job, job group, sensor, and calendar names must \ be globally unique; rename one of them{advice}" ), ) diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs index eb5505f44..f58a33ad7 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_orchestration_names.rs @@ -5,8 +5,8 @@ use crate::project_validation::models::{ use crate::project_validation::tests::helpers::{collision, declared, model, named}; use crate::project_validation::tests::test_types::ValidationTestCase; -const RENAME_DECLARATION: &str = " (a job, job group or sensor takes its name from its first \ - argument, and a calendar schedule from the function it decorates)"; +const RENAME_DECLARATION: &str = " (a job or job group takes its name from `name` in its TOML \ + file, and a calendar or sensor from its `name` argument)"; #[test] fn given_orchestration_names_when_validating_then_they_share_the_project_namespace() { @@ -19,7 +19,7 @@ fn given_orchestration_names_when_validating_then_they_share_the_project_namespa declared("refresh_orders", "job"), declared("nightly", "job group"), declared("orders_arrived", "sensor"), - declared("fulfillment_calendar", "calendar schedule"), + declared("fulfillment_calendar", "calendar"), ], ..ProjectFacts::default() }, @@ -55,15 +55,15 @@ fn given_orchestration_names_when_validating_then_they_share_the_project_namespa ), }, ValidationTestCase { - description: "a calendar schedule named like a provider", + description: "a calendar named like a provider", facts: ProjectFacts { providers: vec![named("fulfillment", "providers/fulfillment.py")], - orchestration: vec![declared("fulfillment", "calendar schedule")], + orchestration: vec![declared("fulfillment", "calendar")], ..ProjectFacts::default() }, expected_failure: collision( "Project resource name 'fulfillment' is declared as both provider in \ - providers/fulfillment.py and calendar schedule in orchestration/daily.py", + providers/fulfillment.py and calendar in orchestration/daily.py", RENAME_DECLARATION, ), }, diff --git a/crates/sqlbuild-pysource/src/lib.rs b/crates/sqlbuild-pysource/src/lib.rs index 56220f830..0d5cbe413 100644 --- a/crates/sqlbuild-pysource/src/lib.rs +++ b/crates/sqlbuild-pysource/src/lib.rs @@ -8,6 +8,7 @@ pub mod import_rewrites; pub mod literal_sql; pub mod macro_modules; pub mod module_parse; +pub mod orchestration_modules; pub mod rule_evidence; pub mod string_literals; pub mod udf_functions; diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs new file mode 100644 index 000000000..d4fd1b078 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/calls.rs @@ -0,0 +1,139 @@ +//! Orchestration authoring calls and their literal arguments, read from the ruff tree. + +use std::collections::{BTreeMap, BTreeSet}; + +use ruff_python_ast::{Expr, ExprCall, Stmt}; +use ruff_text_size::{Ranged, TextSize}; + +use crate::cpython_tree::main::source_position::source_position; +use crate::cpython_tree::models::SourceLines; +use crate::orchestration_modules::constants::{ + CALENDAR_CALL, ORCHESTRATION_MODULE, RUN_EVENT_CALL, SENSOR_CALL, +}; +use crate::orchestration_modules::models::{LiteralArgument, SourcePosition}; + +/// The calls a module's `calendar`, `sensor` and `run_event` names can make. +const AUTHORING_CALLS: [&str; 3] = [CALENDAR_CALL, SENSOR_CALL, RUN_EVENT_CALL]; + +/// The 1-based line and column of `offset`. +pub(crate) fn position(lines: &SourceLines, offset: TextSize) -> SourcePosition { + let (line, column) = source_position(lines, offset); + SourcePosition { + line, + column: column + 1, + } +} + +/// Each local name a top-level `from sqlbuild.orchestration import ...` binds, with the name it imports and where. +pub(crate) fn imported_names(body: &[Stmt]) -> Vec<(String, String, TextSize)> { + let mut imported: Vec<(String, String, TextSize)> = Vec::new(); + for statement in body { + let Stmt::ImportFrom(import) = statement else { + continue; + }; + if import.level != 0 + || import + .module + .as_ref() + .is_none_or(|module| module.as_str() != ORCHESTRATION_MODULE) + { + continue; + } + for alias in &import.names { + let local = alias.asname.as_ref().unwrap_or(&alias.name); + imported.push(( + local.as_str().to_owned(), + alias.name.as_str().to_owned(), + alias.range().start(), + )); + } + } + imported +} + +/// The dotted names a top-level `import sqlbuild.orchestration [as x]` or `from sqlbuild import orchestration [as x]` binds the module to. +pub(crate) fn module_names(body: &[Stmt]) -> BTreeSet { + let (package, module) = ORCHESTRATION_MODULE.rsplit_once('.').unwrap_or_default(); + let mut names: BTreeSet = BTreeSet::new(); + for statement in body { + match statement { + Stmt::Import(import) => names.extend( + import + .names + .iter() + .filter(|alias| alias.name.as_str() == ORCHESTRATION_MODULE) + .map(|alias| alias.asname.as_ref().unwrap_or(&alias.name).to_string()), + ), + Stmt::ImportFrom(import) + if import.level == 0 + && import + .module + .as_ref() + .is_some_and(|name| name.as_str() == package) => + { + names.extend( + import + .names + .iter() + .filter(|alias| alias.name.as_str() == module) + .map(|alias| alias.asname.as_ref().unwrap_or(&alias.name).to_string()), + ); + } + _ => {} + } + } + names +} + +/// `expression` as dotted text when it is a name or a chain of attributes on one. +fn dotted(expression: &Expr) -> Option { + match expression { + Expr::Name(name) => Some(name.id.to_string()), + Expr::Attribute(attribute) => { + Some(format!("{}.{}", dotted(&attribute.value)?, attribute.attr)) + } + _ => None, + } +} + +/// Which authoring call `call` is: an imported `calendar`, `sensor` or `run_event`, or one reached as an attribute of the orchestration module. +pub(crate) fn authoring_call( + call: &ExprCall, + (imported, modules): (&BTreeMap<&str, &str>, &BTreeSet), +) -> Option<&'static str> { + let name: &str = match call.func.as_ref() { + Expr::Name(name) => imported.get(name.id.as_str())?, + Expr::Attribute(attribute) if modules.contains(&dotted(&attribute.value)?) => { + attribute.attr.as_str() + } + _ => return None, + }; + AUTHORING_CALLS.into_iter().find(|call| *call == name) +} + +/// `expression` as a string literal argument, keeping where it starts whatever it is. +pub(crate) fn literal(expression: &Expr, lines: &SourceLines) -> LiteralArgument { + LiteralArgument { + value: match expression { + Expr::StringLiteral(literal) => Some(literal.value.to_str().to_owned()), + _ => None, + }, + position: position(lines, expression.range().start()), + } +} + +/// `expression` as a literal tuple or list of strings; `None` for anything else. +pub(crate) fn literal_strings(expression: &Expr) -> Option> { + let elements = match expression { + Expr::Tuple(tuple) => &tuple.elts, + Expr::List(list) => &list.elts, + _ => return None, + }; + elements + .iter() + .map(|element| match element { + Expr::StringLiteral(literal) => Some(literal.value.to_str().to_owned()), + _ => None, + }) + .collect() +} diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/mod.rs b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/mod.rs new file mode 100644 index 000000000..aedb5c489 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod calls; diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/constants.rs b/crates/sqlbuild-pysource/src/orchestration_modules/constants.rs new file mode 100644 index 000000000..bf2be39eb --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/constants.rs @@ -0,0 +1,27 @@ +//! The orchestration authoring calls a module is read for. + +/// The decorator or call that declares a calendar. +pub(crate) const CALENDAR_CALL: &str = "calendar"; +/// The decorator or call that declares a sensor. +pub(crate) const SENSOR_CALL: &str = "sensor"; +/// The sensor condition that names the job whose runs it watches. +pub(crate) const RUN_EVENT_CALL: &str = "run_event"; +/// The keyword naming a declaration. +pub(crate) const NAME_KEYWORD: &str = "name"; +/// The sensor keyword naming the job it starts. +pub(crate) const RUN_KEYWORD: &str = "run"; +/// The `run_event` keyword naming the job it watches. +pub(crate) const JOB_KEYWORD: &str = "job"; +/// The sensor keyword naming the targets it publishes for. +pub(crate) const TARGETS_KEYWORD: &str = "targets"; +/// The public module the authoring API is imported from. +pub(crate) const ORCHESTRATION_MODULE: &str = "sqlbuild.orchestration"; +/// Authoring names that TOML declarations replaced. +pub(crate) const REMOVED_NAMES: [&str; 6] = [ + "job", + "job_group", + "schedule", + "calendar_schedule", + "on_failure", + "resources", +]; diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/main/mod.rs b/crates/sqlbuild-pysource/src/orchestration_modules/main/mod.rs new file mode 100644 index 000000000..8630a3052 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/main/mod.rs @@ -0,0 +1 @@ +pub mod orchestration_module_declarations; diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs b/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs new file mode 100644 index 000000000..c05f52ade --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/main/orchestration_module_declarations.rs @@ -0,0 +1,101 @@ +//! The calendars and sensors a module declares, and the job names they use, without importing it. + +use std::collections::{BTreeMap, BTreeSet}; + +use ruff_python_ast::Expr; +use ruff_text_size::{Ranged, TextSize}; + +use crate::cpython_tree::main::parse_cpython_module::parse_cpython_module; +use crate::cpython_tree::main::walk_breadth_first::walk_breadth_first; +use crate::cpython_tree::models::{CpythonNode, ParsedSource}; +use crate::orchestration_modules::_helpers::calls::{ + authoring_call, imported_names, literal, literal_strings, module_names, position, +}; +use crate::orchestration_modules::constants::{ + CALENDAR_CALL, JOB_KEYWORD, NAME_KEYWORD, REMOVED_NAMES, RUN_EVENT_CALL, RUN_KEYWORD, + SENSOR_CALL, TARGETS_KEYWORD, +}; +use crate::orchestration_modules::models::{ + LiteralArgument, ModuleDeclarations, PythonDeclaration, PythonDeclarationKind, SourcePosition, +}; + +/// Every `calendar(...)` and `sensor(...)` call with its name and job references, in source order; `None` when unparsable. +#[must_use] +pub fn orchestration_module_declarations( + contents: &str, + python_version: (u8, u8), +) -> Option { + let parsed: ParsedSource = parse_cpython_module(contents, python_version)?; + let imports: Vec<(String, String, TextSize)> = imported_names(&parsed.module.body); + let imported: BTreeMap<&str, &str> = imports + .iter() + .map(|(local, name, _)| (local.as_str(), name.as_str())) + .collect(); + let modules: BTreeSet = module_names(&parsed.module.body); + let removed_imports: Vec<(String, SourcePosition)> = imports + .iter() + .filter(|(_, name, _)| REMOVED_NAMES.contains(&name.as_str())) + .map(|(_, name, start)| (name.clone(), position(&parsed.lines, *start))) + .collect(); + let mut declarations: Vec = Vec::new(); + for node in walk_breadth_first(parsed.module.body.iter().map(CpythonNode::Stmt).collect()) { + let CpythonNode::Expr(Expr::Call(call)) = node else { + continue; + }; + let kind = match authoring_call(call, (&imported, &modules)) { + Some(CALENDAR_CALL) => PythonDeclarationKind::Calendar, + Some(SENSOR_CALL) => PythonDeclarationKind::Sensor, + _ => continue, + }; + let name: Option = call + .arguments + .find_argument_value(NAME_KEYWORD, 0) + .map(|value| literal(value, &parsed.lines)); + let mut job_references: Vec<(&'static str, LiteralArgument)> = Vec::new(); + let targets: Option> = match call.arguments.find_keyword(TARGETS_KEYWORD) { + Some(targets) => literal_strings(&targets.value), + None => Some(Vec::new()), + }; + if kind == PythonDeclarationKind::Sensor { + if let Some(run) = call.arguments.find_keyword(RUN_KEYWORD) { + job_references.push((RUN_KEYWORD, literal(&run.value, &parsed.lines))); + } + let conditions: Vec> = call + .arguments + .keywords + .iter() + .filter(|keyword| { + keyword + .arg + .as_ref() + .is_some_and(|arg| arg.as_str() != RUN_KEYWORD) + }) + .map(|keyword| CpythonNode::Expr(&keyword.value)) + .collect(); + for inner in walk_breadth_first(conditions) { + let CpythonNode::Expr(Expr::Call(inner)) = inner else { + continue; + }; + if authoring_call(inner, (&imported, &modules)) != Some(RUN_EVENT_CALL) { + continue; + } + if let Some(job) = inner.arguments.find_keyword(JOB_KEYWORD) { + job_references.push((RUN_EVENT_CALL, literal(&job.value, &parsed.lines))); + } + } + } + declarations.push(PythonDeclaration { + kind, + name, + position: position(&parsed.lines, call.range().start()), + job_references, + targets, + }); + } + declarations + .sort_by_key(|declaration| (declaration.position.line, declaration.position.column)); + Some(ModuleDeclarations { + declarations, + removed_imports, + }) +} diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/mod.rs b/crates/sqlbuild-pysource/src/orchestration_modules/mod.rs new file mode 100644 index 000000000..1e64f3d11 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/models.rs b/crates/sqlbuild-pysource/src/orchestration_modules/models.rs new file mode 100644 index 000000000..108760147 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/models.rs @@ -0,0 +1,43 @@ +//! The orchestration declarations a Python module makes, read without running it. + +/// What a module-level `calendar(...)` or `sensor(...)` call declares. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PythonDeclarationKind { + Calendar, + Sensor, +} + +/// A 1-based line and column in the module source. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SourcePosition { + pub line: usize, + pub column: usize, +} + +/// A string argument of a declaration; `None` when it is not a string literal. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LiteralArgument { + pub value: Option, + pub position: SourcePosition, +} + +/// One calendar or sensor and the job names it refers to. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PythonDeclaration { + pub kind: PythonDeclarationKind, + /// The `name` argument, or `None` when the call passes no name. + pub name: Option, + pub position: SourcePosition, + /// `run=` of a sensor, then each `run_event(job=...)`, labelled by the argument. + pub job_references: Vec<(&'static str, LiteralArgument)>, + /// A sensor's `targets`: empty when omitted, `None` when not a literal tuple or list of strings. + pub targets: Option>, +} + +/// The orchestration facts of one module. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModuleDeclarations { + pub declarations: Vec, + /// Each removed authoring name the module imports from `sqlbuild.orchestration`. + pub removed_imports: Vec<(String, SourcePosition)>, +} diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/tests.rs b/crates/sqlbuild-pysource/src/orchestration_modules/tests.rs new file mode 100644 index 000000000..985ec8cc9 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_orchestration_module_declarations.rs"] +mod orchestration_module_declarations; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/tests/helpers.rs b/crates/sqlbuild-pysource/src/orchestration_modules/tests/helpers.rs new file mode 100644 index 000000000..ac4918ab9 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/tests/helpers.rs @@ -0,0 +1,40 @@ +use crate::orchestration_modules::models::{ + LiteralArgument, PythonDeclaration, PythonDeclarationKind, SourcePosition, +}; + +/// A 1-based line and column. +pub(super) fn at(line: usize, column: usize) -> SourcePosition { + SourcePosition { line, column } +} + +/// A string literal argument written at `line` and `column`. +pub(super) fn text(value: &str, line: usize, column: usize) -> LiteralArgument { + LiteralArgument { + value: Some(value.to_owned()), + position: at(line, column), + } +} + +/// An argument that is not a string literal, written at `line` and `column`. +pub(super) fn non_literal(line: usize, column: usize) -> LiteralArgument { + LiteralArgument { + value: None, + position: at(line, column), + } +} + +/// A declaration whose call starts at `position` and passes no `targets`. +pub(super) fn declaration( + kind: PythonDeclarationKind, + name: LiteralArgument, + position: SourcePosition, + job_references: Vec<(&'static str, LiteralArgument)>, +) -> PythonDeclaration { + PythonDeclaration { + kind, + name: Some(name), + position, + job_references, + targets: Some(Vec::new()), + } +} diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs new file mode 100644 index 000000000..ab2164368 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_orchestration_module_declarations.rs @@ -0,0 +1,115 @@ +use crate::orchestration_modules::main::orchestration_module_declarations::orchestration_module_declarations; +use crate::orchestration_modules::models::{ModuleDeclarations, PythonDeclarationKind}; +use crate::orchestration_modules::tests::helpers::{at, declaration, non_literal, text}; +use crate::orchestration_modules::tests::test_types::ScanTestCase; + +#[test] +fn given_orchestration_modules_when_scanning_then_reads_declarations_and_removed_imports() { + let test_cases = [ + ScanTestCase { + description: "a decorated calendar has its literal name and decorator position", + contents: "from sqlbuild.orchestration import calendar\n\n@calendar(name=\"trading_days_calendar\", description=\"d\", horizon=\"7d\")\ndef trading_days(window):\n return []\n", + expected_module: Some(ModuleDeclarations { + declarations: vec![declaration( + PythonDeclarationKind::Calendar, + text("trading_days_calendar", 3, 16), + at(3, 2), + Vec::new(), + )], + removed_imports: Vec::new(), + }), + }, + ScanTestCase { + description: "a sensor reads its run job, then each watched job", + contents: "from sqlbuild.orchestration import RunEventKind, run_event as watched, sensor\n\nstock = sensor(\n \"stock_sensor\",\n description=\"d\",\n when=watched(job=\"orders_job\", kinds=[RunEventKind.SUCCESS]),\n run=\"inventory_job\",\n)\n", + expected_module: Some(ModuleDeclarations { + declarations: vec![declaration( + PythonDeclarationKind::Sensor, + text("stock_sensor", 4, 5), + at(3, 9), + vec![ + ("run", text("inventory_job", 7, 9)), + ("run_event", text("orders_job", 6, 22)), + ], + )], + removed_imports: Vec::new(), + }), + }, + ScanTestCase { + description: "non-literal names and jobs keep their positions without values", + contents: "import sqlbuild.orchestration as orchestration\n\nNAME = \"x_sensor\"\nJOB = \"x_job\"\nx = orchestration.sensor(name=NAME, description=\"d\", run=JOB)\n", + expected_module: Some(ModuleDeclarations { + declarations: vec![declaration( + PythonDeclarationKind::Sensor, + non_literal(5, 31), + at(5, 5), + vec![("run", non_literal(5, 58))], + )], + removed_imports: Vec::new(), + }), + }, + ScanTestCase { + description: "calls of other calendar and sensor objects declare nothing", + contents: "import calendar\nfrom other import sensor\n\ndays = calendar.monthrange(2026, 2)\nsensor(\"not_ours\")\n", + expected_module: Some(ModuleDeclarations::default()), + }, + ScanTestCase { + description: "each removed authoring import is reported at its name", + contents: "from sqlbuild.orchestration import (\n job,\n job_group as group,\n sensor,\n)\n", + expected_module: Some(ModuleDeclarations { + declarations: Vec::new(), + removed_imports: vec![ + ("job".to_owned(), at(2, 5)), + ("job_group".to_owned(), at(3, 5)), + ], + }), + }, + ScanTestCase { + description: "a syntax error reads as no module", + contents: "def f(:\n", + expected_module: None, + }, + ScanTestCase { + description: "calendar methods of other objects do not declare beside an aliased import", + contents: "import calendar\nfrom sqlbuild.orchestration import calendar as declare\n\n@declare(name=\"days_calendar\", description=\"d\", horizon=\"7d\")\ndef days(window, exchange):\n calendar.calendar(2026)\n return exchange.calendar(window.start)\n", + expected_module: Some(ModuleDeclarations { + declarations: vec![declaration( + PythonDeclarationKind::Calendar, + text("days_calendar", 4, 15), + at(4, 2), + Vec::new(), + )], + removed_imports: Vec::new(), + }), + }, + ScanTestCase { + description: "attributes of the orchestration module declare under each import form", + contents: "import sqlbuild.orchestration\nfrom sqlbuild import orchestration as authoring\n\na = sqlbuild.orchestration.sensor(\"a_sensor\", description=\"d\", run=\"a_job\")\nb = authoring.sensor(\"b_sensor\", description=\"d\", run=\"b_job\")\n", + expected_module: Some(ModuleDeclarations { + declarations: vec![ + declaration( + PythonDeclarationKind::Sensor, + text("a_sensor", 4, 35), + at(4, 5), + vec![("run", text("a_job", 4, 68))], + ), + declaration( + PythonDeclarationKind::Sensor, + text("b_sensor", 5, 22), + at(5, 5), + vec![("run", text("b_job", 5, 55))], + ), + ], + removed_imports: Vec::new(), + }), + }, + ]; + for test_case in test_cases { + assert_eq!( + orchestration_module_declarations(test_case.contents, (3, 12)), + test_case.expected_module, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_types.rs b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_types.rs new file mode 100644 index 000000000..a0838a465 --- /dev/null +++ b/crates/sqlbuild-pysource/src/orchestration_modules/tests/test_types.rs @@ -0,0 +1,7 @@ +use crate::orchestration_modules::models::ModuleDeclarations; + +pub(super) struct ScanTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) expected_module: Option, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs index 910140339..a265293a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/constants.rs @@ -11,3 +11,6 @@ pub(crate) const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"]; pub(crate) const SELECTOR_INTERSECTION: char = ','; /// The key prefix of a SQL unit test's result, as the snapshot names it. pub(crate) const SQL_TEST_KEY_PREFIX: &str = "sql_test:"; + +/// The sensor argument naming the job it starts, as the orchestration inventory labels it. +pub(crate) const SENSOR_RUN_ARGUMENT: &str = "run"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs index 79b2b2e30..dc6026816 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs @@ -5,22 +5,21 @@ use std::cell::RefCell; use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; +use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::main::parse_selector::parse_selector; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; use sqlbuild_analysis::graph::models::{JobSelection, ParsedSelector, ProjectGraph}; use sqlbuild_cli::native_commands::main::effective_target::effective_target; +use sqlbuild_discovery::orchestration_files::models::{OrchestrationInventory, RunFile}; use crate::bindings::_helpers::compiled_project::constants::{ - JOB_SELECTOR_KINDS, SELECTOR_INTERSECTION, -}; -use crate::bindings::_helpers::compiled_project::snapshot_declarations::discover_declarations; -use crate::bindings::_helpers::native_host::orchestration_declarations::{ - job_selections, triggered_job_names, + JOB_SELECTOR_KINDS, SELECTOR_INTERSECTION, SENSOR_RUN_ARGUMENT, }; +use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; /// Each job and group's selection, and the names of those a schedule or sensor starts. type Catalog = (BTreeMap, BTreeSet); @@ -154,11 +153,9 @@ fn job_catalog(py: Python<'_>) -> PyResult> { }; let target: Option = effective_target(&project_dir, selected_target).map_err(PyValueError::new_err)?; - let roots: Vec<(Bound<'_, PyAny>, String)> = discover_declarations(py, &project_dir)?; - let catalog: Catalog = ( - job_selections(py, &roots, target.as_deref())?, - triggered_job_names(py, &roots, target.as_deref())?, - ); + let inventory: OrchestrationInventory = + orchestration_declarations(project_dir.clone().into_pyobject(py)?.as_any())?; + let catalog: Catalog = inventory_catalog(&inventory, target.as_deref()); JOB_SCOPE.with(|scope| { if let Some(scope) = scope.borrow_mut().as_mut() { scope.catalog = Some(catalog.clone()); @@ -166,3 +163,54 @@ fn job_catalog(py: Python<'_>) -> PyResult> { }); Ok(Some(catalog)) } + +/// Each job's selector tokens and each group's members declared for `target`, and the names a schedule or sensor starts there. +fn inventory_catalog(inventory: &OrchestrationInventory, target: Option<&str>) -> Catalog { + let declared_for = |targets: &[String]| { + targets.is_empty() || target.is_some_and(|target| targets.iter().any(|name| name == target)) + }; + let tokens = |value: &str| value.split_whitespace().map(str::to_owned).collect(); + let mut catalog: BTreeMap = BTreeMap::new(); + let mut triggered: BTreeSet = BTreeSet::new(); + let mut add = |run: &RunFile, selection: JobSelection| { + if declared_for(&run.targets) { + if run.schedule.is_some() { + triggered.insert(run.name.value.clone()); + } + catalog.insert(run.name.value.clone(), selection); + } + }; + for (_, job) in &inventory.jobs { + add( + &job.run, + JobSelection::Job { + select: tokens(&job.select.value), + exclude: job + .exclude + .as_ref() + .map(|exclude| tokens(&exclude.value)) + .unwrap_or_default(), + }, + ); + } + for (_, group) in &inventory.groups { + add( + &group.run, + JobSelection::Group { + members: group.jobs.iter().map(|job| job.value.clone()).collect(), + }, + ); + } + for sensor in &inventory.sensors { + if sensor.targets.as_deref().is_none_or(declared_for) { + triggered.extend( + sensor + .job_references + .iter() + .filter(|(argument, _)| *argument == SENSOR_RUN_ARGUMENT) + .map(|(_, job)| job.value.clone()), + ); + } + } + (catalog, triggered) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 47304d5c3..93935aecf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -1,72 +1,152 @@ -//! Compile authoring records against retained graph data before publication. +//! Compile TOML jobs and groups and Python calendars and sensors against retained graph data before publication. use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; -use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; -use sqlbuild_discovery::constants::{ORCHESTRATION_DIRECTORY, PYTHON_FILE_SUFFIX}; -use sqlbuild_discovery::tree::main::rglob::rglob; +use sqlbuild_discovery::orchestration_files::models::{ + FilePosition, InitialStatus, NamedModule, OrchestrationInventory, RunFile, ScheduleSource, +}; use sqlbuild_discovery::tree::models::ProjectTree; use sqlbuild_orchestration::declarations::main::group_dependencies::group_dependencies; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - DeclaredAt, Definitions, GroupDefinition, JobDefinition, TriggerDeclaration, TriggerDefinition, + DeclaredAt, Definitions, GroupDefinition, JobDefinition, Schedule, TriggerDeclaration, + TriggerDefinition, TriggerStatus, }; use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, declared_at, job_resources, - module_jobs, run_controls, + ImportedDeclarations, imported_declarations, module_declarations, run_controls, run_resources, }; use crate::bindings::_helpers::native_host::orchestration_triggers::{ - calendar_descriptions, compile_calendars, compile_cron, compile_sensors, + CalendarRequest, compile_calendars, compile_sensors, }; use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; use sqlbuild_rules::lint_text::models::FormatSettings; -/// Dedicated declaration modules, imported by publication and by selectors that name a job. -pub(crate) fn discover_declarations<'py>( +/// The calendars and sensors of the inventory's modules, imported by publication and by sensor evaluation. +pub(crate) fn python_declarations<'py>( py: Python<'py>, project_dir: &Path, -) -> PyResult, String)>> { + inventory: &OrchestrationInventory, +) -> PyResult>> { let tree = ProjectTree::new(project_dir); - let files = py - .detach(|| { - rglob(&tree, ORCHESTRATION_DIRECTORY, |entry| { - !entry.is_dir - && entry.name.ends_with(PYTHON_FILE_SUFFIX) - && entry.name.strip_suffix(PYTHON_FILE_SUFFIX) != Some(PYTHON_INIT_MODULE_STEM) - }) - }) - .map_err(|error| { - PyValueError::new_err(format!("orchestration discovery failed: {error:?}")) - })?; - let mut declarations: Vec<(Bound<'py, PyAny>, String)> = Vec::new(); + let mut declarations: Vec> = Vec::new(); let mut seen: BTreeSet = BTreeSet::new(); - for file in files { - for job in module_jobs(py, &tree.absolute(&file), project_dir)? { - if seen.insert(job.as_ptr() as usize) { - declarations.push((job, file.clone())); + for module in inventory.calendars.iter().chain(&inventory.sensors) { + for declaration in module_declarations(py, &tree.absolute(&module.path), project_dir)? { + if seen.insert(declaration.as_ptr() as usize) { + declarations.push(declaration); } } } Ok(declarations) } +/// Where a TOML job or group is declared: its file and the line of its `name`. +fn file_site(path: &str, position: FilePosition) -> Option { + Some(DeclaredAt { + path: path.to_owned(), + line: u32::try_from(position.line).unwrap_or(u32::MAX), + }) +} + +/// Whether a declaration with `targets` applies to `target`. +fn declared_for(targets: &[String], target: &str) -> bool { + targets.is_empty() || targets.iter().any(|name| name == target) +} + +/// A run's cron trigger, or the calendar it asks for; neither when it has no schedule. +fn run_schedule( + (path, run): (&str, &RunFile), +) -> PyResult<(Option, Option)> { + let Some(schedule) = &run.schedule else { + return Ok((None, None)); + }; + let name: &str = &run.name.value; + let starts_at = schedule + .starts_at + .as_ref() + .map(|starts_at| serde_json::from_value(Value::String(starts_at.value.clone()))) + .transpose() + .map_err(|error| PyValueError::new_err(format!("{path}: schedule starts_at: {error}")))?; + let trigger = TriggerDefinition { + description: run.description.clone(), + job_key: name.to_owned(), + schedule: Schedule::Cron { + expression: String::new(), + timezone: schedule.timezone.value.clone(), + }, + starts_at, + default_status: match schedule.default_status { + InitialStatus::Running => TriggerStatus::Running, + InitialStatus::Paused => TriggerStatus::Paused, + }, + targets: run.targets.iter().cloned().collect(), + declared_at: file_site(path, run.name.position), + }; + Ok(match &schedule.source { + ScheduleSource::Cron(expression) => ( + Some(TriggerDefinition { + schedule: Schedule::Cron { + expression: expression.value.clone(), + timezone: schedule.timezone.value.clone(), + }, + ..trigger + }), + None, + ), + ScheduleSource::Calendar(calendar) => ( + None, + Some(CalendarRequest { + calendar: calendar.value.clone(), + timezone: schedule.timezone.value.clone(), + trigger, + }), + ), + }) +} + +/// Refuse a calendar or sensor that compile declared but import did not define, or the reverse. +fn matching_imports( + (kind, declared): (&str, &[NamedModule]), + imported: BTreeSet<&str>, +) -> PyResult<()> { + if let Some(module) = declared + .iter() + .find(|module| !imported.contains(module.name.as_str())) + { + let FilePosition { line, column } = module.position; + return Err(PyValueError::new_err(format!( + "{}:{line}:{column}: {kind} '{}' is declared here, but importing the module does not \ + define it; assign it to a module-level name or list", + module.path, module.name + ))); + } + let names: BTreeSet<&str> = declared.iter().map(|module| module.name.as_str()).collect(); + if let Some(name) = imported.difference(&names).next() { + return Err(PyValueError::new_err(format!( + "{kind} '{name}' is defined when its module is imported, but no module under \ + orchestration/ declares it with a literal name; call {kind}(...) in the module itself" + ))); + } + Ok(()) +} + pub(crate) fn compile_declarations( py: Python<'_>, mut input: SnapshotInput, (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), - declarations: Vec, + (inventory, declarations): (&OrchestrationInventory, Vec>), ) -> PyResult { - if declarations.is_empty() { + if inventory.is_empty() && declarations.is_empty() { return Ok(input); } if ["jobs", "groups", "triggers"] @@ -77,41 +157,42 @@ pub(crate) fn compile_declarations( "authored declarations cannot overwrite snapshot job or trigger documents", )); } - let mut jobs: BTreeMap = BTreeMap::new(); - let mut authored_groups: Vec = Vec::new(); - let mut authored_jobs: Vec = Vec::new(); - let mut authored_calendars: Vec> = Vec::new(); - let mut authored_sensors: Vec = Vec::new(); - for declaration in declarations { - match declaration { - AuthoredDeclaration::Job(job) => authored_jobs.push(job), - AuthoredDeclaration::Group(group) => authored_groups.push(group), - AuthoredDeclaration::Calendar(calendar) => authored_calendars.push(calendar), - AuthoredDeclaration::Sensor(sensor) => authored_sensors.push(sensor), - } - } + let ImportedDeclarations { + calendars: authored_calendars, + calendar_descriptions, + sensors: authored_sensors, + } = imported_declarations(py, declarations)?; + matching_imports( + ("calendar", &inventory.calendars), + authored_calendars.keys().map(String::as_str).collect(), + )?; + matching_imports( + ("sensor", &inventory.sensors), + authored_sensors + .iter() + .map(|sensor| sensor.name.as_str()) + .collect(), + )?; let format: FormatSettings = resolve_format_settings(project_dir).map_err(PyValueError::new_err)?; let mut described: Vec<(&str, String, String)> = Vec::new(); - described.extend( - authored_jobs - .iter() - .map(|job| ("job", job.name.clone(), job.description.clone())), - ); - described.extend( - authored_groups - .iter() - .map(|group| ("job group", group.name.clone(), group.description.clone())), - ); + described.extend(inventory.jobs.iter().map(|(_, job)| { + let run = &job.run; + ("job", run.name.value.clone(), run.description.clone()) + })); + described.extend(inventory.groups.iter().map(|(_, group)| { + let run = &group.run; + ("job group", run.name.value.clone(), run.description.clone()) + })); described.extend( authored_sensors .iter() .map(|sensor| ("sensor", sensor.name.clone(), sensor.description.clone())), ); described.extend( - calendar_descriptions(py, &authored_calendars)? + calendar_descriptions .into_iter() - .map(|(name, description)| ("calendar schedule", name, description)), + .map(|(name, description)| ("calendar", name, description)), ); for (kind, name, description) in described { let limit: i64 = format.max_description_lines; @@ -123,70 +204,55 @@ pub(crate) fn compile_declarations( ))); } } + let tokens = |value: &str| { + value + .split_whitespace() + .map(str::to_owned) + .collect::>() + }; + let mut jobs: BTreeMap = BTreeMap::new(); let mut triggers: BTreeMap = BTreeMap::new(); - for job in authored_jobs { - if !job.targets.is_empty() && !job.targets.contains(&input.target) { + let mut calendar_requests: Vec = Vec::new(); + for (path, job) in &inventory.jobs { + let run = &job.run; + if !declared_for(&run.targets, &input.target) { continue; } - let mut settings: BTreeMap = serde_json::from_value(job.settings) - .map_err(|error| PyValueError::new_err(error.to_string()))?; - if !job.concurrency.is_null() { - if settings.contains_key("concurrency") { - return Err(PyValueError::new_err("job concurrency is declared twice")); - } - settings.insert("concurrency".into(), job.concurrency); - } - job_settings_arguments(&settings).map_err(PyValueError::new_err)?; - let tokens = |value: &str| { - value - .split_whitespace() - .map(str::to_owned) - .collect::>() - }; - if job.select.trim().is_empty() { - return Err(PyValueError::new_err(format!( - "job {} needs a selection", - job.name - ))); - } + let name: &str = &run.name.value; + job_settings_arguments(&job.settings) + .map_err(|error| PyValueError::new_err(format!("job {name} settings: {error}")))?; + let exclude: Option = job + .exclude + .as_ref() + .map(|exclude| exclude.value.clone()) + .filter(|exclude| !exclude.trim().is_empty()); let selection: BTreeSet = selected - .run_keys(graph, &tokens(&job.select), &tokens(&job.exclude)) - .map_err(|error| PyValueError::new_err(format!("job {}: {error}", job.name)))?; - let declared_at: Option = declared_at(project_dir, job.declared_at)?; - let controls = run_controls( - &job.name, - ( - job.owner, - &job.overlap, - job.on_failure.as_ref().map(|policy| policy.bind(py)), - ), - )?; - let resources = job_resources( - &job.name, - job.resources.as_ref().map(|resources| resources.bind(py)), - )?; - let targets: BTreeSet = job.targets.into_iter().collect(); - if let Some(schedule) = job.schedule { - triggers.insert( - job.name.clone(), - TriggerDefinition { - targets: targets.clone(), - declared_at: declared_at.clone(), - ..compile_cron(schedule.bind(py), &job.name, &job.description)? - }, - ); - } + .run_keys( + graph, + &tokens(&job.select.value), + &tokens(exclude.as_deref().unwrap_or_default()), + ) + .map_err(|error| PyValueError::new_err(format!("job {name}: {error}")))?; + let resources = job + .resources + .as_ref() + .map(|resources| run_resources(name, &resources.value)) + .transpose()?; + let (trigger, calendar) = run_schedule((path, run))?; + triggers.extend(trigger.map(|trigger| (name.to_owned(), trigger))); + calendar_requests.extend(calendar); + let controls = run_controls(run); jobs.insert( - job.name, + name.to_owned(), JobDefinition { - description: job.description, + description: run.description.clone(), selection, - settings, - concurrency_groups: job.concurrency_groups.into_iter().collect(), - select: Some(job.select), - exclude: Some(job.exclude).filter(|exclude| !exclude.trim().is_empty()), - targets, - declared_at, + settings: job.settings.clone(), + concurrency_groups: run.concurrency_groups.iter().cloned().collect(), + select: Some(job.select.value.clone()), + exclude, + targets: run.targets.iter().cloned().collect(), + declared_at: file_site(path, run.name.position), owner: controls.owner, overlap: controls.overlap, on_failure: controls.on_failure, @@ -195,52 +261,40 @@ pub(crate) fn compile_declarations( ); } let mut groups: BTreeMap = BTreeMap::new(); - for group in authored_groups { - if !group.targets.is_empty() && !group.targets.contains(&input.target) { + for (path, group) in &inventory.groups { + let run = &group.run; + if !declared_for(&run.targets, &input.target) { continue; } - let members: BTreeSet = group.jobs.iter().map(|job| job.name.clone()).collect(); + let name: &str = &run.name.value; + let members: BTreeSet = group.jobs.iter().map(|job| job.value.clone()).collect(); if members.len() != group.jobs.len() { - return Err(PyValueError::new_err("group members must be distinct")); + return Err(PyValueError::new_err(format!( + "job group {name} lists a job twice" + ))); } - let dependencies = group_dependencies((&group.name, &input), &jobs, &members) + let dependencies = group_dependencies((name, &input), &jobs, &members) .map_err(|error| PyValueError::new_err(error.to_string()))?; let selection: BTreeSet = members .iter() - .flat_map(|name| jobs[name].selection.iter().cloned()) + .flat_map(|member| jobs[member].selection.iter().cloned()) .collect(); - let declared_at: Option = declared_at(project_dir, group.declared_at)?; - let controls = run_controls( - &group.name, - ( - group.owner, - &group.overlap, - group.on_failure.as_ref().map(|policy| policy.bind(py)), - ), - )?; - let targets: BTreeSet = group.targets.into_iter().collect(); - if let Some(schedule) = group.schedule { - triggers.insert( - group.name.clone(), - TriggerDefinition { - targets: targets.clone(), - declared_at: declared_at.clone(), - ..compile_cron(schedule.bind(py), &group.name, &group.description)? - }, - ); - } + let (trigger, calendar) = run_schedule((path, run))?; + triggers.extend(trigger.map(|trigger| (name.to_owned(), trigger))); + calendar_requests.extend(calendar); + let controls = run_controls(run); groups.insert( - group.name, + name.to_owned(), GroupDefinition { job: JobDefinition { - description: group.description, + description: run.description.clone(), selection, settings: BTreeMap::new(), - concurrency_groups: group.concurrency_groups.into_iter().collect(), + concurrency_groups: run.concurrency_groups.iter().cloned().collect(), select: None, exclude: None, - targets, - declared_at, + targets: run.targets.iter().cloned().collect(), + declared_at: file_site(path, run.name.position), owner: controls.owner, overlap: controls.overlap, on_failure: controls.on_failure, @@ -262,8 +316,8 @@ pub(crate) fn compile_declarations( authored_sensors, compile_calendars( py, - (project_dir, &input.target), - authored_calendars, + project_dir, + (&authored_calendars, calendar_requests), Definitions { jobs, groups, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs index b69fc3bbd..ba5fb5914 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs @@ -140,6 +140,15 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ optional: &[], fixed: &[], }, + ErrorSpec { + name: "OrchestrationDeclarationError", + parent: "DiscoveryError", + doc: "Raised when an orchestration job, job group, calendar or sensor declaration is invalid.", + class_attributes: &[("code", ClassValue::Text("D017"))], + required: &[], + optional: &[], + fixed: &[], + }, ErrorSpec { name: "ProjectPythonPathError", parent: "DiscoveryError", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 77a316516..4fe23096f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -17,6 +17,8 @@ pub(crate) mod key_checks; pub(crate) mod locations; pub(crate) mod model_file_record; pub(crate) mod model_records; +pub(crate) mod orchestration_files; +pub(crate) mod orchestration_values; pub(crate) mod project_config; pub(crate) mod project_facts; pub(crate) mod project_files; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs new file mode 100644 index 000000000..91ec16535 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_files.rs @@ -0,0 +1,45 @@ +//! Orchestration declarations read natively, raised as the Python exceptions discovery raises. + +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use sqlbuild_discovery::orchestration_files::main::orchestration_inventory::orchestration_inventory; +use sqlbuild_discovery::orchestration_files::models::{ + OrchestrationFailure, OrchestrationInventory, +}; +use sqlbuild_discovery::tree::models::ProjectTree; + +use crate::bindings::_helpers::discovery::orchestration_values::check_orchestration_values; +use crate::bindings::_helpers::discovery::raised::{ + failure_error, relative_read_error, stage_error, +}; + +/// Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). +pub(crate) fn orchestration_declarations( + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = project_dir.py(); + let root: PathBuf = project_dir.extract()?; + let tree = ProjectTree::new(&root); + let version = py.version_info(); + let inventory = py.detach(|| orchestration_inventory(&tree, (version.major, version.minor))); + let inventory = + inventory.map_err(|failure| orchestration_error(&tree, project_dir, failure))?; + check_orchestration_values(py, &inventory).map_err(|failure| failure_error(py, failure))?; + Ok(inventory) +} + +fn orchestration_error( + tree: &ProjectTree, + project_dir: &Bound<'_, PyAny>, + failure: OrchestrationFailure, +) -> PyErr { + match failure { + OrchestrationFailure::Declaration(failure) => failure_error(project_dir.py(), failure), + OrchestrationFailure::Unreadable { + relative_path, + failure, + } => relative_read_error((tree, project_dir), relative_path, failure), + OrchestrationFailure::Stage(failure) => stage_error(tree, project_dir, failure), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs new file mode 100644 index 000000000..f4f0288eb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/orchestration_values.rs @@ -0,0 +1,114 @@ +//! Check the values of TOML jobs and groups at compile, with the checks publication applies. + +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use chrono_tz::Tz; +use pyo3::prelude::Python; +use serde_json::Value; +use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; +use sqlbuild_discovery::models::{DiscoveryFailure, FailureKind}; +use sqlbuild_discovery::orchestration_files::models::{ + FilePosition, OrchestrationInventory, Positioned, RunFile, ScheduleSource, +}; +use sqlbuild_orchestration::scheduler::main::parse_cron::parse_cron; +use sqlbuild_orchestration::store::errors::ControlError; + +use crate::bindings::_helpers::native_host::orchestration_declarations::run_resources; + +/// A `D017` at `position` of `path`, worded as discovery words its own. +type ValueError = (FilePosition, String); + +/// The first invalid schedule, setting, resource limit or selector, as `D017` at its file, line and column. +pub(crate) fn check_orchestration_values( + py: Python<'_>, + inventory: &OrchestrationInventory, +) -> Result<(), DiscoveryFailure> { + let runs = inventory + .jobs + .iter() + .map(|(path, job)| (path, &job.run)) + .chain( + inventory + .groups + .iter() + .map(|(path, group)| (path, &group.run)), + ); + for (path, run) in runs { + schedule_values(run).map_err(|error| failure(path, error))?; + } + for (path, job) in &inventory.jobs { + for selector in std::iter::once(&job.select).chain(&job.exclude) { + selector_syntax(selector).map_err(|error| failure(path, error))?; + } + let mut settings: BTreeMap = BTreeMap::new(); + for (key, value) in &job.settings { + settings.insert(key.clone(), value.clone()); + job_settings_arguments(&BTreeMap::from([(key.clone(), value.clone())])) + .map_err(|message| failure(path, (job.setting_positions[key], message)))?; + } + if let Some(first) = job.setting_positions.values().min() { + job_settings_arguments(&settings) + .map_err(|message| failure(path, (*first, message)))?; + } + if let Some(resources) = &job.resources { + run_resources(&job.run.name.value, &resources.value).map_err(|error| { + failure(path, (resources.position, error.value(py).to_string())) + })?; + } + } + Ok(()) +} + +fn failure(path: &str, (FilePosition { line, column }, message): ValueError) -> DiscoveryFailure { + DiscoveryFailure { + kind: FailureKind::Orchestration, + message: format!("{path}:{line}:{column}: {message}"), + help: None, + } +} + +fn schedule_values(run: &RunFile) -> Result<(), ValueError> { + let Some(schedule) = &run.schedule else { + return Ok(()); + }; + let timezone: &Positioned = &schedule.timezone; + timezone.value.parse::().map_err(|error| { + ( + timezone.position, + format!("schedule timezone is not an IANA time zone: {error}"), + ) + })?; + if let ScheduleSource::Cron(cron) = &schedule.source { + parse_cron(&cron.value, &timezone.value).map_err(|error| { + let reason: String = match error { + ControlError::Invalid(reason) => reason, + other => other.to_string(), + }; + (cron.position, format!("schedule cron: {reason}")) + })?; + } + if let Some(starts_at) = &schedule.starts_at { + serde_json::from_value::>(Value::String(starts_at.value.clone())).map_err( + |error| { + ( + starts_at.position, + format!("schedule starts_at must be an RFC 3339 timestamp: {error}"), + ) + }, + )?; + } + Ok(()) +} + +fn selector_syntax(selector: &Positioned) -> Result<(), ValueError> { + let parts = selector + .value + .split_whitespace() + .flat_map(|token| token.split(',')); + for part in parts { + parse_selector(part).map_err(|error| (selector.position, error.message))?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs index b3b1a096e..cccf046b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs @@ -8,6 +8,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::project_validation::main::validate_project_inputs::validate_project_inputs; use sqlbuild_discovery::project_validation::models::{ProjectFacts, ValidationFailure}; +use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; use crate::bindings::_helpers::discovery::project_facts::{NodeRow, project_facts}; use crate::bindings::_helpers::discovery::raised::{failure_error, read_error}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -32,7 +33,11 @@ fn validate_discovered_project_inputs( inputs: Bound<'_, PyAny>, nodes: Vec, ) -> Result<(), PyErr> { - let facts: ProjectFacts = project_facts(py, &inputs, nodes)?; + let mut facts: ProjectFacts = project_facts(py, &inputs, nodes)?; + let project_dir: Bound<'_, PyAny> = native_field(&inputs, "project_dir")?; + if !project_dir.is_none() { + facts.orchestration = orchestration_declarations(&project_dir)?.names(); + } validate_project_inputs(&facts).map_err(|failure| validation_error(py, failure)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs index aca358755..cd32e4ceb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs @@ -19,7 +19,7 @@ const OS_MODULE: &str = "os"; const DECODE_ENCODING: &str = "utf-8"; /// The exception class of each native failure kind. -const FAILURE_CLASSES: [(&str, ErrorId); 13] = [ +const FAILURE_CLASSES: [(&str, ErrorId); 14] = [ ("model_sql", ErrorId::ModelSqlParse), ("declaration", ErrorId::DeclarationParse), ("sql_test", ErrorId::SqlTestParse), @@ -33,6 +33,7 @@ const FAILURE_CLASSES: [(&str, ErrorId); 13] = [ ("conflict", ErrorId::DiscoveryConflict), ("seed", ErrorId::SeedDiscovery), ("python_path", ErrorId::ProjectPythonPath), + ("orchestration", ErrorId::OrchestrationDeclaration), ]; fn raised(result: PyResult>) -> PyErr { @@ -112,16 +113,25 @@ pub(crate) fn stage_error( StageFailure::Unlistable { relative_path, error, - } => match path_object(project_dir.py(), tree, relative_path) - .and_then(|relative| project_dir.div(relative.bind(project_dir.py()))) - { - Ok(directory) => read_error(error, &directory), - Err(error) => error, - }, + } => relative_read_error((tree, project_dir), relative_path, error), StageFailure::Internal(reason) => PyRuntimeError::new_err(reason), } } +/// `read_error` for `relative_path` below `project_dir`, or the error building that path raised. +pub(crate) fn relative_read_error( + (tree, project_dir): (&ProjectTree, &Bound<'_, PyAny>), + relative_path: String, + failure: ReadFailure, +) -> PyErr { + match path_object(project_dir.py(), tree, relative_path) + .and_then(|relative| project_dir.div(relative.bind(project_dir.py()))) + { + Ok(path) => read_error(failure, &path), + Err(error) => error, + } +} + /// The error `read_error` builds for an operating-system read failure, for the platform oracle. #[pyfunction] #[pyo3(signature = (errno, winerror, message, *, file_path))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs index 5f8cb953e..d9bbce20a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/command.rs @@ -159,11 +159,10 @@ pub(super) fn compile_snapshot<'py>( project_dir.extract()?, ( discovered.clone(), - arguments + !arguments .value("action")? .extract::>()? - .is_some_and(|action| action == EXECUTE_ACTION) - .then(Vec::new), + .is_some_and(|action| action == EXECUTE_ACTION), ), )?; Ok((snapshot, blocked_nodes(&project)?)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 40a0682a2..8a2aff78a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -10,26 +10,25 @@ use pyo3::{IntoPyObject, pyfunction}; use serde_json::{Value, json}; use sqlbuild_analysis::planner_scope::main::node_loads::node_loads; use sqlbuild_core::models::DiagnosticSeverity; +use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; use sqlbuild_discovery::project_validation::main::validate_orchestration_names::validate_orchestration_names; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compiled_project::snapshot::NativeOrchestrationSnapshot; use crate::bindings::_helpers::compiled_project::snapshot_declarations::{ - compile_declarations, discover_declarations, + compile_declarations, python_declarations, }; use crate::bindings::_helpers::compiled_project::snapshot_models::model_nodes; use crate::bindings::_helpers::compiled_project::snapshot_resources::resource_nodes; use crate::bindings::_helpers::compiled_project::snapshot_roles::compile_project_roles; use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; +use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; use crate::bindings::_helpers::discovery::project_facts::{named_node_rows, project_facts}; use crate::bindings::_helpers::discovery::project_validation::validation_error; use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; -use crate::bindings::_helpers::native_host::orchestration_declarations::{ - collect_declarations, orchestration_names, -}; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; @@ -40,9 +39,6 @@ use crate::bindings::_helpers::records::fields::field; /// The Python node kind whose functions load sources. const LOADER_KIND: &str = "loader"; -/// Where declarations passed to the snapshot directly are reported as declared. -const DECLARATIONS_ARGUMENT: &str = "declarations argument"; - /// Compile catalog nodes and identities in the location, never in the server. #[pyfunction] #[pyo3(signature = (project, metadata_json, project_dir, **options))] @@ -56,28 +52,22 @@ pub(crate) fn build_orchestration_snapshot<'py>( project.py(), "build_orchestration_snapshot", options, - (&["discovered_inputs"], &["declarations"]), + (&["discovered_inputs"], &[]), )?; - let declarations = arguments.value("declarations")?; orchestration_snapshot( project, metadata_json, project_dir, - ( - arguments.value("discovered_inputs")?, - (!declarations.is_none()) - .then(|| declarations.extract()) - .transpose()?, - ), + (arguments.value("discovered_inputs")?, true), ) } -/// Compile the snapshot from the project, its discovery, and the declarations passed directly, if any. +/// Compile the snapshot from the project, its discovery and, when `declared`, its orchestration declarations. pub(crate) fn orchestration_snapshot<'py>( project: PyRef<'py, NativeProjectView>, metadata_json: &str, project_dir: PathBuf, - (discovered_inputs, declarations): (Bound<'py, PyAny>, Option>>), + (discovered_inputs, declared): (Bound<'py, PyAny>, bool), ) -> PyResult { let py = project.py(); let mut input: SnapshotInput = serde_json::from_str(metadata_json) @@ -121,31 +111,23 @@ pub(crate) fn orchestration_snapshot<'py>( if let Some(providers) = providers_document(&discovered_inputs, &input.nodes)? { input.documents.insert("providers".into(), providers); } - let declarations = match declarations { - Some(declarations) => declarations - .into_iter() - .map(|declaration| (declaration, DECLARATIONS_ARGUMENT.to_owned())) - .collect(), - None => discover_declarations(py, &project_dir)?, - }; let graph = project.lineage_graph(py)?; - if !declarations.is_empty() { + let inventory = if declared { + orchestration_declarations(project_dir.clone().into_pyobject(py)?.as_any())? + } else { + OrchestrationInventory::default() + }; + if !inventory.is_empty() { let mut facts = project_facts(py, &discovered_inputs, named_node_rows(&discovered_inputs)?)?; - facts.orchestration = orchestration_names(py, &declarations)?; + facts.orchestration = inventory.names(); validate_orchestration_names(&facts).map_err(|failure| validation_error(py, failure))?; - let authored = collect_declarations( - py, - declarations - .into_iter() - .map(|(declaration, _)| declaration) - .collect(), - )?; + let declarations = python_declarations(py, &project_dir, &inventory)?; input = compile_declarations( py, input, (&project_dir, graph.bind(py).get().graph(), &selected), - authored, + (&inventory, declarations), )?; } input = compile_project_roles( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index d0ef09dcb..2f12728a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -3,7 +3,6 @@ /// The calendar's evaluation window is supplied by the host, not a provider. pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; -pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; pub(crate) const SOURCE_NEW_DATA_CONDITION: &str = "source_new_data"; pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; @@ -14,15 +13,9 @@ pub(crate) const AUTHORING_MODULE: &str = "sqlbuild.orchestration_declarations.m /// Frames of these modules construct declarations on the author's behalf. pub(crate) const CONSTRUCTING_MODULES: [&str; 2] = [AUTHORING_MODULE, "dataclasses"]; -/// Kinds named in the global unique-name check. -pub(crate) const JOB_LABEL: &str = "job"; -pub(crate) const GROUP_LABEL: &str = "job group"; -pub(crate) const SENSOR_LABEL: &str = "sensor"; -pub(crate) const CALENDAR_LABEL: &str = "calendar schedule"; +/// The authoring classes publication imports calendar and sensor modules for. +pub(crate) const CALENDAR_CLASS: &str = "Calendar"; +pub(crate) const SENSOR_CLASS: &str = "Sensor"; /// Pages one bounded state-history SELECT streams before the scan continues from its cursor. pub(crate) const HISTORY_STREAM_PAGES: usize = 50; -/// Inclusive bounds on a job's `on_failure` retries. -pub(crate) const MAX_RETRIES: (u64, u64) = (1, 10); -/// Inclusive bounds on a job's `on_failure` retry delay, in seconds: up to 24 hours. -pub(crate) const MAX_RETRY_DELAY: (u64, u64) = (0, 86_400); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index ae92c1946..9b4d86606 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -1,79 +1,29 @@ //! The public authoring records enter native compilation here, never scheduler execution. -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::collections::{BTreeMap, HashSet}; use std::path::{Component, Path}; -use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use pyo3::{FromPyObject, pyfunction}; use serde_json::Value; -use sqlbuild_analysis::graph::models::JobSelection; -use sqlbuild_discovery::project_validation::models::OrchestrationNameFact; +use sqlbuild_discovery::orchestration_files::models::{self as files, RunFile}; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::declarations::models::{DeclaredAt, OnFailure, Overlap, RunResources}; use crate::bindings::_helpers::native_host::constants::{ - AUTHORING_MODULE, CALENDAR_LABEL, CONSTRUCTING_MODULES, GROUP_LABEL, JOB_LABEL, MAX_RETRIES, - MAX_RETRY_DELAY, RUN_EVENT_CONDITION, SENSOR_LABEL, + AUTHORING_MODULE, CALENDAR_CLASS, CONSTRUCTING_MODULES, SENSOR_CLASS, }; use crate::bindings::_helpers::native_host::host::entry; -#[derive(FromPyObject)] -pub(crate) struct AuthoredJob { - pub(crate) name: String, - pub(crate) description: String, - pub(crate) select: String, - pub(crate) exclude: String, - #[pyo3(from_py_with = authored_setting)] - pub(crate) settings: Value, - #[pyo3(from_py_with = authored_setting)] - pub(crate) concurrency: Value, - pub(crate) concurrency_groups: Vec, - pub(crate) schedule: Option>, - pub(crate) targets: Vec, - #[pyo3(attribute("_declared_at"))] - pub(crate) declared_at: Option<(String, u32)>, - pub(crate) owner: Option, - pub(crate) overlap: String, - pub(crate) on_failure: Option>, - pub(crate) resources: Option>, -} - -#[derive(FromPyObject, PartialEq, Eq, PartialOrd, Ord)] -pub(crate) struct AuthoredJobReference { - pub(crate) name: String, -} - -#[derive(FromPyObject)] -pub(crate) struct AuthoredGroup { - pub(crate) name: String, - pub(crate) description: String, - pub(crate) jobs: Vec, - pub(crate) concurrency_groups: Vec, - pub(crate) schedule: Option>, - pub(crate) targets: Vec, - #[pyo3(attribute("_declared_at"))] - pub(crate) declared_at: Option<(String, u32)>, - pub(crate) owner: Option, - pub(crate) overlap: String, - pub(crate) on_failure: Option>, -} - -/// A job's or group's owner, overlap and failure policy, validated where they enter native code. -pub(crate) struct RunControls { - pub(crate) owner: Option, - pub(crate) overlap: Overlap, - pub(crate) on_failure: Option, -} - #[derive(FromPyObject)] pub(crate) struct AuthoredSensorCondition { pub(crate) kind: String, #[pyo3(default)] pub(crate) kinds: Vec, #[pyo3(default)] - pub(crate) job: Option, + pub(crate) job: Option, #[pyo3(default)] pub(crate) executed_successfully: Option>, #[pyo3(default)] @@ -87,8 +37,8 @@ pub(crate) struct AuthoredSensor { pub(crate) name: String, pub(crate) description: String, pub(crate) when: Option, - pub(crate) function: Option>, - pub(crate) run: AuthoredJobReference, + pub(crate) function: Option>, + pub(crate) run: String, #[pyo3(from_py_with = authored_setting)] pub(crate) minimum_interval_seconds: Value, pub(crate) default_status: String, @@ -97,11 +47,11 @@ pub(crate) struct AuthoredSensor { pub(crate) declared_at: Option<(String, u32)>, } -pub(crate) enum AuthoredDeclaration { - Job(AuthoredJob), - Group(AuthoredGroup), - Calendar(Py), - Sensor(AuthoredSensor), +/// A job's or group's owner, overlap and failure policy, as publication records them. +pub(crate) struct RunControls { + pub(crate) owner: Option, + pub(crate) overlap: Overlap, + pub(crate) on_failure: Option, } /// The file and line of the author's code constructing a declaration; `None` for code without a source file. @@ -165,70 +115,28 @@ pub(crate) fn authored_setting(value: &Bound<'_, PyAny>) -> PyResult, &str, Option<&Bound<'_, PyAny>>), -) -> PyResult { - if owner - .as_deref() - .is_some_and(|owner| owner.trim().is_empty()) - { - return Err(PyValueError::new_err(format!( - "job {name} owner must not be blank" - ))); +/// The controls of a TOML job or group; discovery has already checked the owner and the retry bounds. +pub(crate) fn run_controls(run: &RunFile) -> RunControls { + RunControls { + owner: run.owner.clone(), + overlap: match run.overlap { + files::Overlap::Queue => Overlap::Queue, + files::Overlap::Skip => Overlap::Skip, + }, + on_failure: run.on_failure.map(|policy| OnFailure { + retries: policy.retries, + retry_delay_seconds: policy.retry_delay_seconds, + }), } - let overlap = match overlap { - "queue" => Overlap::Queue, - "skip" => Overlap::Skip, - other => { - return Err(PyValueError::new_err(format!( - "job {name} overlap must be queue or skip, not {other}" - ))); - } - }; - let on_failure = if let Some(policy) = policy { - let bounded = |field: &str, (low, high): (u64, u64)| -> PyResult { - authored_setting(&policy.getattr(field)?)? - .as_u64() - .filter(|value| (low..=high).contains(value)) - .ok_or_else(|| { - PyValueError::new_err(format!( - "job {name} on_failure {field} must be a whole number from {low} to {high}" - )) - }) - }; - Some(OnFailure { - retries: u8::try_from(bounded("retries", MAX_RETRIES)?)?, - retry_delay_seconds: u32::try_from(bounded("retry_delay_seconds", MAX_RETRY_DELAY)?)?, - }) - } else { - None - }; - Ok(RunControls { - owner, - overlap, - on_failure, - }) } -/// Refuse a memory size Docker cannot read and CPUs or a process limit that is not positive. -pub(crate) fn job_resources( +/// Refuse an unknown limit, a memory size Docker cannot read, and CPUs or a process limit that is not positive. +pub(crate) fn run_resources( name: &str, - resources: Option<&Bound<'_, PyAny>>, -) -> PyResult> { - let Some(resources) = resources else { - return Ok(None); - }; - let mut written = serde_json::Map::new(); - for field in ["memory", "cpus", "pids"] { - let value = authored_setting(&resources.getattr(field)?)?; - if !value.is_null() { - written.insert(field.into(), value); - } - } + written: &serde_json::Map, +) -> PyResult { let invalid = |reason: String| PyValueError::new_err(format!("job {name} resources: {reason}")); - let resources: RunResources = serde_json::from_value(Value::Object(written)) + let resources: RunResources = serde_json::from_value(Value::Object(written.clone())) .map_err(|error| invalid(error.to_string()))?; if resources .cpus @@ -239,11 +147,50 @@ pub(crate) fn job_resources( if resources.pids == Some(0) { return Err(invalid("pids must be a positive whole number".into())); } - Ok(Some(resources)) + Ok(resources) } -/// Only module execution crosses into Python; discovery and collection stay native. -pub(crate) fn module_jobs<'py>( +/// The calendars, by name with their descriptions, and the sensors that importing the modules defines. +pub(crate) struct ImportedDeclarations<'py> { + pub(crate) calendars: BTreeMap>, + pub(crate) calendar_descriptions: BTreeMap, + pub(crate) sensors: Vec, +} + +/// Sort imported declarations into calendars and sensors, refusing a calendar name defined twice. +pub(crate) fn imported_declarations<'py>( + py: Python<'py>, + declarations: Vec>, +) -> PyResult> { + let sensor_class = py.import(AUTHORING_MODULE)?.getattr(SENSOR_CLASS)?; + let mut imported = ImportedDeclarations { + calendars: BTreeMap::new(), + calendar_descriptions: BTreeMap::new(), + sensors: Vec::new(), + }; + for declaration in declarations { + if declaration.is_instance(&sensor_class)? { + imported.sensors.push(declaration.extract()?); + continue; + } + let name: String = declaration.getattr("name")?.extract()?; + let description: String = declaration.getattr("description")?.extract()?; + if imported + .calendars + .insert(name.clone(), declaration) + .is_some() + { + return Err(PyValueError::new_err(format!( + "calendar '{name}' is declared twice" + ))); + } + imported.calendar_descriptions.insert(name, description); + } + Ok(imported) +} + +/// The calendars and sensors one module defines; only module execution crosses into Python. +pub(crate) fn module_declarations<'py>( py: Python<'py>, file_path: &Path, project_dir: &Path, @@ -252,281 +199,27 @@ pub(crate) fn module_jobs<'py>( options.set_item("file_path", file_path)?; options.set_item("project_dir", project_dir)?; let module = entry(py, UserCodeHost::LoadOrchestrationModule)?.call((), Some(&options))?; - let job_class = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("Job")?; - let group_class = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("JobGroup")?; - let calendar_class = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("CalendarSchedule")?; - let sensor_class = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("Sensor")?; + let authoring = py.import(AUTHORING_MODULE)?; + let classes = [ + authoring.getattr(CALENDAR_CLASS)?, + authoring.getattr(SENSOR_CLASS)?, + ]; let attributes = module.getattr("__dict__")?.cast_into::()?; let mut pending: Vec> = attributes.values().iter().collect(); let mut seen: HashSet = HashSet::new(); - let mut jobs: Vec> = Vec::new(); + let mut found: Vec> = Vec::new(); while let Some(value) = pending.pop() { if !seen.insert(value.as_ptr() as usize) { continue; } - if value.is_instance(&job_class)? - || value.is_instance(&group_class)? - || value.is_instance(&calendar_class)? - || value.is_instance(&sensor_class)? + if classes + .iter() + .any(|class| value.is_instance(class).unwrap_or(false)) { - jobs.push(value); + found.push(value); } else if value.is_instance_of::() || value.is_instance_of::() { pending.extend(value.try_iter()?.collect::>>()?); } } - Ok(jobs) -} - -pub(crate) fn collect_declarations<'py>( - py: Python<'py>, - declarations: Vec>, -) -> PyResult> { - let job = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("Job")?; - let group = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("JobGroup")?; - let calendar = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("CalendarSchedule")?; - let sensor = py - .import("sqlbuild.orchestration_declarations.models.authoring")? - .getattr("Sensor")?; - let mut roots: HashSet = HashSet::new(); - for declaration in &declarations { - if !roots.insert(declaration.as_ptr() as usize) { - if declaration.is_instance(&calendar)? { - return Err(PyValueError::new_err("duplicate calendar declaration")); - } - return Err(PyValueError::new_err(format!( - "duplicate job {}", - declaration.getattr("name")?.extract::()? - ))); - } - } - let mut pending = declarations; - let mut seen: HashSet = HashSet::new(); - let mut collected: Vec = Vec::new(); - while let Some(declaration) = pending.pop() { - if !seen.insert(declaration.as_ptr() as usize) { - continue; - } - let authored = if declaration.is_instance(&sensor)? { - let target = declaration.getattr("run")?; - if !target.is_instance(&job)? && !target.is_instance(&group)? { - return Err(PyTypeError::new_err( - "sensor run must be a Job or JobGroup object", - )); - } - pending.push(target); - let condition = declaration.getattr("when")?; - if !condition.is_none() - && condition.getattr("kind")?.extract::()? == RUN_EVENT_CONDITION - { - let source = condition.getattr("job")?; - if !source.is_instance(&job)? && !source.is_instance(&group)? { - return Err(PyTypeError::new_err( - "run_event job must be a Job or JobGroup object", - )); - } - pending.push(source); - } - AuthoredDeclaration::Sensor(declaration.extract()?) - } else if declaration.is_instance(&calendar)? { - let target = declaration.getattr("job")?; - if !target.is_instance(&job)? && !target.is_instance(&group)? { - return Err(PyTypeError::new_err( - "calendar job must be a Job or JobGroup object", - )); - } - pending.push(target); - AuthoredDeclaration::Calendar(declaration.unbind()) - } else if declaration.is_instance(&group)? { - for member in declaration.getattr("jobs")?.try_iter()? { - let member = member?; - if !member.is_instance(&job)? { - return Err(PyTypeError::new_err("group members must be Job objects")); - } - pending.push(member); - } - AuthoredDeclaration::Group(declaration.extract()?) - } else if !declaration.is_instance(&job)? { - return Err(PyTypeError::new_err( - "snapshot declarations must be Job, JobGroup, CalendarSchedule or Sensor objects", - )); - } else { - AuthoredDeclaration::Job(declaration.extract()?) - }; - collected.push(authored); - } - Ok(collected) -} - -/// Every declaration reachable from `roots`, once each, with its kind and the module of the root that reaches it first. -pub(crate) fn reachable_declarations<'py>( - py: Python<'py>, - roots: &[(Bound<'py, PyAny>, String)], -) -> PyResult, &'static str, String)>> { - let authoring = py.import(AUTHORING_MODULE)?; - let classes = [ - (authoring.getattr("Job")?, JOB_LABEL), - (authoring.getattr("JobGroup")?, GROUP_LABEL), - (authoring.getattr("Sensor")?, SENSOR_LABEL), - (authoring.getattr("CalendarSchedule")?, CALENDAR_LABEL), - ]; - let mut seen: HashSet = HashSet::new(); - let mut reached: Vec<(Bound<'py, PyAny>, &'static str, String)> = Vec::new(); - for (root, path) in roots { - let mut pending: Vec> = vec![root.clone()]; - while let Some(declaration) = pending.pop() { - if !seen.insert(declaration.as_ptr() as usize) { - continue; - } - let Some(kind) = classes - .iter() - .find(|(class, _)| declaration.is_instance(class).unwrap_or(false)) - .map(|(_, kind)| *kind) - else { - continue; - }; - pending.extend(nested_declarations(&declaration, kind)?); - reached.push((declaration, kind, path.clone())); - } - } - Ok(reached) -} - -/// Every named declaration reachable from `roots`, labelled with the module of the root that reaches it first. -pub(crate) fn orchestration_names( - py: Python<'_>, - roots: &[(Bound<'_, PyAny>, String)], -) -> PyResult> { - let mut names: Vec = Vec::new(); - for (declaration, kind, path) in reachable_declarations(py, roots)? { - let name: Option = if kind == CALENDAR_LABEL { - let function = declaration.getattr("function")?; - (!function.is_none()) - .then(|| function.getattr("__name__")?.extract()) - .transpose()? - } else { - Some(declaration.getattr("name")?.extract()?) - }; - names.extend(name.map(|name| OrchestrationNameFact { name, kind, path })); - } - Ok(names) -} - -/// The declarations one declaration refers to: a group's jobs, a calendar's job, a sensor's job and run-event job. -fn nested_declarations<'py>( - declaration: &Bound<'py, PyAny>, - kind: &str, -) -> PyResult>> { - Ok(match kind { - GROUP_LABEL => declaration - .getattr("jobs")? - .try_iter()? - .collect::>>()?, - CALENDAR_LABEL => vec![declaration.getattr("job")?], - SENSOR_LABEL => { - let condition = declaration.getattr("when")?; - let mut nested = vec![declaration.getattr("run")?]; - if !condition.is_none() - && condition.getattr("kind")?.extract::()? == RUN_EVENT_CONDITION - { - nested.push(condition.getattr("job")?); - } - nested - } - _ => Vec::new(), - }) -} - -/// Each reachable job's own selector tokens and each group's member names for `target`, as publication reads them. -pub(crate) fn job_selections( - py: Python<'_>, - roots: &[(Bound<'_, PyAny>, String)], - target: Option<&str>, -) -> PyResult> { - let mut catalog: BTreeMap = BTreeMap::new(); - for (declaration, kind, _) in reachable_declarations(py, roots)? { - if kind != JOB_LABEL && kind != GROUP_LABEL { - continue; - } - if !declared_for(&declaration, target)? { - continue; - } - let tokens = |field: &str| -> PyResult> { - Ok(declaration - .getattr(field)? - .extract::()? - .split_whitespace() - .map(str::to_owned) - .collect()) - }; - let selection: JobSelection = match kind { - JOB_LABEL => JobSelection::Job { - select: tokens("select")?, - exclude: tokens("exclude")?, - }, - GROUP_LABEL => JobSelection::Group { - members: declaration - .getattr("jobs")? - .try_iter()? - .map(|job| job?.getattr("name")?.extract()) - .collect::>()?, - }, - _ => continue, - }; - let name: String = declaration.getattr("name")?.extract()?; - if catalog.insert(name.clone(), selection).is_some() { - return Err(PyValueError::new_err(format!( - "job name '{name}' is declared twice" - ))); - } - } - Ok(catalog) -} - -/// Whether a declaration applies to `target`: it names no targets, or names this one. -fn declared_for(declaration: &Bound<'_, PyAny>, target: Option<&str>) -> PyResult { - let targets: Vec = declaration.getattr("targets")?.extract()?; - Ok( - targets.is_empty() - || target.is_some_and(|target| targets.iter().any(|name| name == target)), - ) -} - -/// The jobs and groups that a schedule, calendar schedule or sensor declared for `target` starts. -pub(crate) fn triggered_job_names( - py: Python<'_>, - roots: &[(Bound<'_, PyAny>, String)], - target: Option<&str>, -) -> PyResult> { - let mut names: BTreeSet = BTreeSet::new(); - for (declaration, kind, _) in reachable_declarations(py, roots)? { - if !declared_for(&declaration, target)? { - continue; - } - let started: Option> = match kind { - JOB_LABEL | GROUP_LABEL => { - (!declaration.getattr("schedule")?.is_none()).then(|| declaration.clone()) - } - CALENDAR_LABEL => Some(declaration.getattr("job")?), - SENSOR_LABEL => Some(declaration.getattr("run")?), - _ => None, - }; - if let Some(job) = started { - names.insert(job.getattr("name")?.extract()?); - } - } - Ok(names) + Ok(found) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs index 8a01736e9..edb486b35 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_sensors.rs @@ -2,6 +2,7 @@ use std::path::Path; +use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; @@ -11,9 +12,10 @@ use sqlbuild_orchestration::sensors::models::{ }; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; -use crate::bindings::_helpers::compiled_project::snapshot_declarations::discover_declarations; +use crate::bindings::_helpers::compiled_project::snapshot_declarations::python_declarations; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::discovery::orchestration_files::orchestration_declarations; use crate::bindings::_helpers::native_host::constants::SENSOR_CONTEXT_PARAMETER; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_environment::{ @@ -35,7 +37,8 @@ pub(crate) fn evaluate_sensor( let authoring = py.import("sqlbuild.orchestration_declarations.models.authoring")?; let sensor_class = authoring.getattr("Sensor")?; let mut matches: Vec> = Vec::new(); - for (declaration, _) in discover_declarations(py, project_dir)? { + let inventory = orchestration_declarations(project_dir.into_pyobject(py)?.as_any())?; + for declaration in python_declarations(py, project_dir, &inventory)? { if declaration.is_instance(&sensor_class)? && declaration.getattr("name")?.extract::()? == *name { @@ -50,11 +53,7 @@ pub(crate) fn evaluate_sensor( let targets: Vec = sensor.getattr("targets")?.extract()?; if !sensor.getattr("when")?.is_none() || (!targets.is_empty() && !targets.contains(&evaluation.target)) - || sensor - .getattr("run")? - .getattr("name")? - .extract::()? - != evaluation.definition.job_key + || sensor.getattr("run")?.extract::()? != evaluation.definition.job_key { return Err(PyValueError::new_err("custom sensor definition changed")); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 9faf31f8e..a76334202 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -1,13 +1,12 @@ //! Authored trigger controls and bounded calendars evaluated only in the location interpreter. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use chrono::{DateTime, TimeDelta, Timelike, Utc}; use chrono_tz::Tz; -use pyo3::FromPyObject; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use serde_json::{Value, json}; use sqlbuild_analysis::graph::models::ProjectGraph; @@ -27,53 +26,17 @@ use crate::bindings::_helpers::native_host::constants::{ }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::{ - AuthoredSensor, authored_setting, declared_at, + AuthoredSensor, declared_at, }; use crate::bindings::_helpers::native_host::orchestration_environment::{ callback_code_hash, orchestration_code_hash, }; -#[derive(FromPyObject)] -struct AuthoredControls { - #[pyo3(from_py_with = authored_setting)] - starts_at: Value, - default_status: String, -} - -fn compile_trigger( - authored: &Bound<'_, PyAny>, - job_key: &str, - description: &str, - schedule: Schedule, -) -> PyResult { - let controls = authored.extract::()?; - Ok(TriggerDefinition { - description: description.into(), - job_key: job_key.into(), - schedule, - starts_at: serde_json::from_value(controls.starts_at) - .map_err(|error| PyValueError::new_err(format!("starts_at: {error}")))?, - default_status: serde_json::from_value(Value::String(controls.default_status)) - .map_err(|error| PyValueError::new_err(format!("default_status: {error}")))?, - targets: BTreeSet::new(), - declared_at: None, - }) -} - -pub(crate) fn compile_cron( - authored: &Bound<'_, PyAny>, - name: &str, - description: &str, -) -> PyResult { - compile_trigger( - authored, - name, - description, - Schedule::Cron { - expression: authored.getattr("cron")?.extract()?, - timezone: authored.getattr("timezone")?.extract()?, - }, - ) +/// A job's or group's schedule on a named calendar, as its trigger before the calendar's times fill it. +pub(crate) struct CalendarRequest { + pub(crate) calendar: String, + pub(crate) timezone: String, + pub(crate) trigger: TriggerDefinition, } fn horizon(value: &str) -> PyResult { @@ -105,13 +68,12 @@ fn horizon(value: &str) -> PyResult { } fn calendar_times( - authored: &Bound<'_, PyAny>, + (authored, timezone): (&Bound<'_, PyAny>, &str), session: &Bound<'_, PyAny>, providers: &Bound<'_, PyDict>, start: DateTime, ) -> PyResult { let py = authored.py(); - let timezone: String = authored.getattr("timezone")?.extract()?; timezone .parse::() .map_err(|_| PyValueError::new_err("calendar timezone must be an IANA timezone"))?; @@ -184,42 +146,14 @@ fn calendar_times( }) } -/// The name and description of each calendar that decorates a function. -pub(crate) fn calendar_descriptions( - py: Python<'_>, - calendars: &[Py], -) -> PyResult> { - let mut described: Vec<(String, String)> = Vec::new(); - for calendar in calendars { - let calendar: &Bound<'_, PyAny> = calendar.bind(py); - let function: Bound<'_, PyAny> = calendar.getattr("function")?; - if !function.is_none() { - described.push(( - function.getattr("__name__")?.extract()?, - calendar.getattr("description")?.extract()?, - )); - } - } - Ok(described) -} - +/// Each calendar request's trigger, with each (calendar, timezone) pair evaluated once. pub(crate) fn compile_calendars( py: Python<'_>, - (project_dir, target): (&Path, &str), - authored: Vec>, + project_dir: &Path, + (calendars, requests): (&BTreeMap>, Vec), mut definitions: Definitions, ) -> PyResult { - let mut calendars: Vec> = Vec::new(); - for calendar in authored { - let targets = calendar - .bind(py) - .getattr("targets")? - .extract::>()?; - if targets.is_empty() || targets.iter().any(|value| value == target) { - calendars.push(calendar); - } - } - if calendars.is_empty() { + if requests.is_empty() { return Ok(definitions); } let providers = @@ -237,33 +171,38 @@ pub(crate) fn compile_calendars( .with_nanosecond(now.timestamp_subsec_micros() * 1_000) .ok_or_else(|| PyValueError::new_err("calendar clock is out of range"))?; within(&session, |_| { - for calendar in calendars { - let calendar = calendar.bind(py); - let name: String = calendar - .getattr("function")? - .getattr("__name__")? - .extract()?; - let job_key: String = calendar.getattr("job")?.getattr("name")?.extract()?; - let description: String = calendar.getattr("description")?.extract()?; - if description.trim().is_empty() - || (!definitions.jobs.contains_key(&job_key) - && !definitions.groups.contains_key(&job_key)) - { - return Err(PyValueError::new_err(format!( - "calendar {name} needs a description and a published job" - ))); - } - let schedule = calendar_times(calendar, &session, &by_name, start)?; - let mut trigger = compile_trigger(calendar, &job_key, &description, schedule)?; + let mut evaluated: BTreeMap<(String, String), Schedule> = BTreeMap::new(); + for request in requests { + let key = (request.calendar.clone(), request.timezone.clone()); + let schedule = match evaluated.get(&key) { + Some(schedule) => schedule.clone(), + None => { + let calendar = calendars.get(&request.calendar).ok_or_else(|| { + PyValueError::new_err(format!( + "calendar '{}' is not declared by a calendar module", + request.calendar + )) + })?; + let schedule = + calendar_times((calendar, &request.timezone), &session, &by_name, start) + .map_err(|error| { + PyValueError::new_err(format!( + "calendar '{}': {error}", + request.calendar + )) + })?; + evaluated.insert(key, schedule.clone()); + schedule + } + }; + let mut trigger = TriggerDefinition { + schedule, + ..request.trigger + }; trigger.starts_at.get_or_insert(start); - trigger.targets = calendar - .getattr("targets")? - .extract::>()? - .into_iter() - .collect(); - trigger.declared_at = - declared_at(project_dir, calendar.getattr("_declared_at")?.extract()?)?; - definitions.triggers.insert(name, trigger); + definitions + .triggers + .insert(trigger.job_key.clone(), trigger); } Ok(definitions) }) @@ -294,7 +233,7 @@ pub(crate) fn compile_sensors( condition["kinds"] = json!(when.kinds); } if let Some(job) = when.job { - condition["job_key"] = json!(job.name); + condition["job_key"] = json!(job); } if let Some(select) = when.select { let selectors: Vec = @@ -317,7 +256,7 @@ pub(crate) fn compile_sensors( }; let definition: SensorDefinition = serde_json::from_value(json!({ "description": sensor.description, - "job_key": sensor.run.name, + "job_key": sensor.run, "when": condition, "minimum_interval_seconds": sensor.minimum_interval_seconds, "default_status": sensor.default_status, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 7c48897c6..acf3e15c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -618,6 +618,7 @@ native_ids! { NonHermeticRule => "NonHermeticRuleError", OpaqueModuleState => "OpaqueModuleStateError", PlannerInput => "PlannerInputError", + OrchestrationDeclaration => "OrchestrationDeclarationError", ProjectCompile => "ProjectCompileError", ProjectConfig => "ProjectConfigError", ProjectPath => "ProjectPathError", diff --git a/crates/sqlbuild-refactor/Cargo.toml b/crates/sqlbuild-refactor/Cargo.toml index 8d992e24b..b8462bc04 100644 --- a/crates/sqlbuild-refactor/Cargo.toml +++ b/crates/sqlbuild-refactor/Cargo.toml @@ -20,3 +20,4 @@ sqlbuild-model-config.workspace = true sqlbuild-pysource.workspace = true sqlbuild-sqltext.workspace = true tempfile.workspace = true +toml.workspace = true diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs new file mode 100644 index 000000000..73a4806d0 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/job_selectors.rs @@ -0,0 +1,324 @@ +//! `select` and `exclude` in TOML job files, rewritten for renamed and moved models. + +use std::collections::BTreeMap; +use std::path::Path; + +use serde::Deserialize; +use sqlbuild_core::text::errors::TextDecodeError; +use toml::Spanned; + +use crate::refactoring::_helpers::edits::text_edits::{line_column, text_edit}; +use crate::refactoring::_helpers::files::paths::parent; +use crate::refactoring::_helpers::files::workspace::project_files_with_suffix; +use crate::refactoring::_helpers::scanning::chars::chars; +use crate::refactoring::constants::{ + JOB_FILES_DIRECTORY, MODEL_ROOT, PARTIAL_FOLDER_MOVE_REASON, TOML_SUFFIX, + UNREADABLE_JOB_REASON, UNWRITABLE_SELECTOR_REASON, +}; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::{EditKind, ManualLocation, ModelFacts, TextEdit}; + +/// The selector fields of a job file; publication checks everything else. +#[derive(Deserialize)] +struct JobSelectors { + select: Option>, + exclude: Option>, +} + +/// Selector edits by path, selectors to edit by hand or to check, and unreadable job files. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct SelectorParts { + pub(crate) edits: Vec<(String, TextEdit)>, + pub(crate) manual: Vec, + pub(crate) notes: Vec, + pub(crate) blocking: Vec, +} + +/// What the refactoring changes, as job selectors see it. +struct Changes { + /// Old model name to new, for each model whose name changes. + names: BTreeMap, + /// Old folder to new, for each folder every model below it leaves together. + folders: BTreeMap, + /// Folders some, but not all, of their models leave. + partial: BTreeMap>, +} + +/// Rewrite renamed models and wholly moved folders in job selectors; a partly moved folder becomes a note. +pub(crate) fn job_selector_parts( + project_dir: &Path, + models: &[ModelFacts], + renamed: &[(&ModelFacts, &str, &str)], +) -> Result { + let mut parts = SelectorParts::default(); + let jobs_dir = project_dir.join(JOB_FILES_DIRECTORY); + if !jobs_dir.is_dir() { + return Ok(parts); + } + let changes = changes(models, renamed); + for (relative, text) in project_files_with_suffix(&jobs_dir, TOML_SUFFIX)? { + let path = format!("{JOB_FILES_DIRECTORY}/{relative}"); + let contents = match text { + Ok(contents) => contents, + Err(TextDecodeError { .. }) => { + parts + .blocking + .push(location(path, None, UNREADABLE_JOB_REASON)); + continue; + } + }; + let Ok(job) = toml::from_str::(&contents) else { + continue; + }; + let text = chars(&contents); + for field in [job.select, job.exclude].into_iter().flatten() { + let span = field.span(); + let (start, end) = ( + contents[..span.start].chars().count(), + contents[..span.end].chars().count(), + ); + let (line, column) = line_column(&text, start); + for (folder, moved) in &changes.partial { + if selected_folders(field.get_ref()).any(|selected| selected == *folder) { + let reason = PARTIAL_FOLDER_MOVE_REASON + .replace("{folder}", folder) + .replace("{models}", &moved.join(", ")); + parts + .notes + .push(location(path.clone(), Some((line, column)), &reason)); + } + } + let rewritten = rewrite(field.get_ref(), &changes); + if rewritten == *field.get_ref() { + continue; + } + match literal(&contents[span.start..span.end], &rewritten) { + Some(replacement) => parts.edits.push(( + path.clone(), + text_edit(&text, (start, end), replacement, EditKind::Selector), + )), + None => parts.manual.push(location( + path.clone(), + Some((line, column)), + &UNWRITABLE_SELECTOR_REASON.replace("{selector}", &rewritten), + )), + } + } + } + Ok(parts) +} + +fn location(path: String, at: Option<(usize, usize)>, reason: &str) -> ManualLocation { + ManualLocation { + path, + line: at.map(|(line, _)| line), + column: at.map(|(_, column)| column), + reason: reason.to_owned(), + } +} + +fn changes(models: &[ModelFacts], renamed: &[(&ModelFacts, &str, &str)]) -> Changes { + let names: BTreeMap = renamed + .iter() + .filter(|(model, new_name, _)| model.name != *new_name) + .map(|(model, new_name, _)| (model.name.clone(), (*new_name).to_owned())) + .collect(); + let destinations: BTreeMap<&str, String> = renamed + .iter() + .map(|(model, _, destination)| (model.path.as_str(), parent(destination))) + .collect(); + let mut folders: BTreeMap = BTreeMap::new(); + let mut partial: BTreeMap> = BTreeMap::new(); + let moved_from = renamed + .iter() + .map(|(model, _, destination)| (parent(&model.path), parent(destination))) + .filter(|(from, to)| from != to) + .flat_map(|(from, _)| ancestors(&from)); + for folder in moved_from { + if folder == MODEL_ROOT || folders.contains_key(&folder) || partial.contains_key(&folder) { + continue; + } + let below: Vec<&ModelFacts> = models + .iter() + .filter(|model| within(&parent(&model.path), &folder)) + .collect(); + let moved_to = |model: &ModelFacts| -> Option { + let to = destinations.get(model.path.as_str())?; + let rest = &parent(&model.path)[folder.len()..]; + to.strip_suffix(rest).map(str::to_owned) + }; + let targets: Vec> = below.iter().map(|model| moved_to(model)).collect(); + let whole: Option = targets.first().cloned().flatten().filter(|to| { + *to != folder + && !within(to, &folder) + && all_move_to(&targets, to) + && !receives_other_models(models, &destinations, &folder, to) + }); + match whole { + Some(to) => { + folders.insert(folder, to); + } + None => { + let leaving: Vec = below + .iter() + .filter(|model| { + destinations + .get(model.path.as_str()) + .is_some_and(|to| !within(to, &folder)) + }) + .map(|model| model.name.clone()) + .collect(); + if !leaving.is_empty() { + partial.insert(folder, leaving); + } + } + } + } + Changes { + names, + folders, + partial, + } +} + +/// Whether every model below a folder moves to `to`. +fn all_move_to(targets: &[Option], to: &str) -> bool { + targets.iter().all(|target| target.as_deref() == Some(to)) +} + +/// Whether a model from outside `folder` keeps or gets a place in `to`, so selecting `to` would widen to it. +fn receives_other_models( + models: &[ModelFacts], + destinations: &BTreeMap<&str, String>, + folder: &str, + to: &str, +) -> bool { + models.iter().any(|model| { + let lands: String = destinations + .get(model.path.as_str()) + .cloned() + .unwrap_or_else(|| parent(&model.path)); + within(&lands, to) && !within(&parent(&model.path), folder) + }) +} + +/// `folder` and each folder above it, up to the model root. +fn ancestors(folder: &str) -> Vec { + let parts: Vec<&str> = folder.split('/').collect(); + (1..=parts.len()) + .map(|count| parts[..count].join("/")) + .collect() +} + +fn within(path: &str, folder: &str) -> bool { + path == folder || path.starts_with(&format!("{folder}/")) +} + +/// The folders a selector string names, normalized as path selectors match them. +fn selected_folders(selector: &str) -> impl Iterator + '_ { + selector + .split_whitespace() + .flat_map(|token| token.split(',')) + .filter_map(selected_folder) +} + +/// The normalized folder one selector part names, ignoring its graph operators. +fn selected_folder(part: &str) -> Option { + let (_, folder) = folder_of(part.trim_matches('+'))?; + Some(folder) +} + +/// The `path:` prefix (if any) and normalized folder of a folder selector part. +fn folder_of(core: &str) -> Option<(&str, String)> { + let (prefix, value) = match core.split_once(':') { + Some(("path", value)) => ("path:", value), + Some(_) => return None, + None if core.contains('/') && !core.contains('~') => ("", core), + None => return None, + }; + Some(( + prefix, + value.replace('\\', "/").trim_matches('/').to_owned(), + )) +} + +/// `selector` with renamed model names and moved folders replaced, whitespace kept. +fn rewrite(selector: &str, changes: &Changes) -> String { + let mut out = String::with_capacity(selector.len()); + let mut token = String::new(); + for character in selector.chars().chain(std::iter::once(' ')) { + if character.is_whitespace() { + if !token.is_empty() { + let parts: Vec = token + .split(',') + .map(|part| part_of(part, changes)) + .collect(); + out.push_str(&parts.join(",")); + token.clear(); + } + out.push(character); + } else { + token.push(character); + } + } + out.pop(); + out +} + +fn part_of(part: &str, changes: &Changes) -> String { + let core = part.trim_matches('+'); + if core.is_empty() { + return part.to_owned(); + } + let lead = &part[..part.len() - part.trim_start_matches('+').len()]; + let tail = &part[part.trim_end_matches('+').len()..]; + let name = |value: &str| -> String { + changes + .names + .get(value.trim()) + .map_or_else(|| value.to_owned(), |new| value.replace(value.trim(), new)) + }; + let core = if let Some((start, end)) = core.split_once('~') { + format!("{}~{}", name(start), name(end)) + } else if let Some((prefix, folder)) = folder_of(core) { + match changes.folders.get(&folder) { + Some(to) => { + let trailing = if core.ends_with('/') { "/" } else { "" }; + format!("{prefix}{to}{trailing}") + } + None => core.to_owned(), + } + } else if core.contains(':') { + core.to_owned() + } else { + name(core) + }; + format!("{lead}{core}{tail}") +} + +/// `value` written as a TOML string in the style of `raw`, or `None` for multi-line strings. +fn literal(raw: &str, value: &str) -> Option { + if raw.starts_with("'''") || raw.starts_with("\"\"\"") { + return None; + } + if raw.starts_with('\'') && !value.contains(['\'', '\n', '\r']) { + return Some(format!("'{value}'")); + } + let mut escaped = String::with_capacity(value.len() + 2); + escaped.push('"'); + for character in value.chars() { + match character { + '"' => escaped.push_str("\\\""), + '\\' => escaped.push_str("\\\\"), + '\n' => escaped.push_str("\\n"), + '\t' => escaped.push_str("\\t"), + '\r' => escaped.push_str("\\r"), + control if control.is_control() => { + escaped.push_str(&format!("\\u{:04X}", u32::from(control))); + } + other => escaped.push(other), + } + } + escaped.push('"'); + Some(escaped) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs index c363fc9d2..ea437df70 100644 --- a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod batch_planning; pub(crate) mod column_planning; pub(crate) mod column_references; pub(crate) mod declaration_moves; +pub(crate) mod job_selectors; pub(crate) mod model_planning; pub(crate) mod model_references; pub(crate) mod python_imports; diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/single_planning.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/single_planning.rs index 7bd5d7374..4485bc6da 100644 --- a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/single_planning.rs +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/single_planning.rs @@ -6,6 +6,7 @@ use crate::refactoring::_helpers::edits::text_edits::build_plan; use crate::refactoring::_helpers::planning::column_planning::{ column_rename_context, column_rename_parts, }; +use crate::refactoring::_helpers::planning::job_selectors::job_selector_parts; use crate::refactoring::_helpers::planning::model_planning::{model_parts, model_target}; use crate::refactoring::_helpers::planning::python_imports::python_import_parts; use crate::refactoring::_helpers::scanning::scan_context::ScanContext; @@ -43,11 +44,20 @@ pub(crate) fn plan_single( let mut moves = parts.moves.clone(); let destination = target.destination(); if destination != target.source_path { - moves.push((target.source_path.clone(), destination)); + moves.push((target.source_path.clone(), destination.clone())); } let imports = python_import_parts(Path::new(&facts.project_dir), &moves)?; parts.edits.extend(imports.edits); parts.blocking.extend(imports.blocking); + let selectors = job_selector_parts( + Path::new(&facts.project_dir), + &facts.models, + &[(target.model, &target.request.new_name, &destination)], + )?; + parts.edits.extend(selectors.edits); + parts.manual.extend(selectors.manual); + parts.notes.extend(selectors.notes); + parts.blocking.extend(selectors.blocking); Ok(build_plan( target.request, parts, diff --git a/crates/sqlbuild-refactor/src/refactoring/constants.rs b/crates/sqlbuild-refactor/src/refactoring/constants.rs index eea18227e..8f7ebeb91 100644 --- a/crates/sqlbuild-refactor/src/refactoring/constants.rs +++ b/crates/sqlbuild-refactor/src/refactoring/constants.rs @@ -101,3 +101,12 @@ pub const GENERATED_SOURCE_CONFLICT: &str = "source file is marked as generated" pub const SOURCE_BLOCK_CONFLICT: &str = "source declaration has no unambiguous YAML block"; /// The `ValueError` prefix of a `name_pattern` the native matcher rejects. pub const INVALID_NAME_PATTERN_PREFIX: &str = "invalid name_pattern: "; +/// Where job files live; their `select` and `exclude` follow renamed and moved models. +pub const JOB_FILES_DIRECTORY: &str = "orchestration/jobs"; +pub const TOML_SUFFIX: &str = ".toml"; +/// The folder path selectors start from. +pub const MODEL_ROOT: &str = "models"; +pub const UNREADABLE_JOB_REASON: &str = "this job file cannot be read as UTF-8, so its selectors cannot be updated; fix or remove it first"; +pub const PARTIAL_FOLDER_MOVE_REASON: &str = "this job selects folder '{folder}', and {models} moved out of it; check the job still selects what it should"; +pub const UNWRITABLE_SELECTOR_REASON: &str = + "this selector is a multi-line string; change it to '{selector}' by hand"; diff --git a/crates/sqlbuild-refactor/src/refactoring/main/plan_refactor.rs b/crates/sqlbuild-refactor/src/refactoring/main/plan_refactor.rs index b9e0b4971..dae4b7e82 100644 --- a/crates/sqlbuild-refactor/src/refactoring/main/plan_refactor.rs +++ b/crates/sqlbuild-refactor/src/refactoring/main/plan_refactor.rs @@ -4,6 +4,7 @@ use std::path::Path; use crate::refactoring::_helpers::edits::text_edits::build_plan; use crate::refactoring::_helpers::planning::batch_planning::batch_parts; +use crate::refactoring::_helpers::planning::job_selectors::job_selector_parts; use crate::refactoring::_helpers::planning::model_planning::{ModelTarget, model_target}; use crate::refactoring::_helpers::planning::python_imports::python_import_parts; use crate::refactoring::_helpers::planning::single_planning::plan_single; @@ -38,6 +39,23 @@ pub fn plan_refactor( let imports = python_import_parts(Path::new(&facts.project_dir), &moves)?; parts.edits.extend(imports.edits); parts.blocking.extend(imports.blocking); + let destinations: Vec = targets.iter().map(ModelTarget::destination).collect(); + let renamed: Vec<(&_, &str, &str)> = targets + .iter() + .zip(&destinations) + .map(|(target, destination)| { + ( + target.model, + target.request.new_name.as_str(), + destination.as_str(), + ) + }) + .collect(); + let selectors = job_selector_parts(Path::new(&facts.project_dir), &facts.models, &renamed)?; + parts.edits.extend(selectors.edits); + parts.manual.extend(selectors.manual); + parts.notes.extend(selectors.notes); + parts.blocking.extend(selectors.blocking); let batch: Vec = targets.into_iter().map(|target| target.request).collect(); let mut plan = build_plan( batch[0].clone(), diff --git a/crates/sqlbuild-refactor/src/refactoring/models.rs b/crates/sqlbuild-refactor/src/refactoring/models.rs index 1c54b51bc..08708b981 100644 --- a/crates/sqlbuild-refactor/src/refactoring/models.rs +++ b/crates/sqlbuild-refactor/src/refactoring/models.rs @@ -24,6 +24,8 @@ pub enum EditKind { Migration, /// A Python import of a moved module. Import, + /// A job's `select` or `exclude` naming a renamed or moved model. + Selector, } impl EditKind { @@ -36,6 +38,7 @@ impl EditKind { Self::Column => "column", Self::Migration => "migration", Self::Import => "import", + Self::Selector => "selector", } } } diff --git a/crates/sqlbuild-refactor/src/refactoring/tests.rs b/crates/sqlbuild-refactor/src/refactoring/tests.rs index 3c5fadaa1..a35f9949b 100644 --- a/crates/sqlbuild-refactor/src/refactoring/tests.rs +++ b/crates/sqlbuild-refactor/src/refactoring/tests.rs @@ -8,6 +8,8 @@ mod declaration_edits; mod header_edits; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_job_selectors.rs"] +mod job_selectors; #[path = "tests/test_planning.rs"] mod planning; #[path = "tests/test_sql_sites.rs"] diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_job_selectors.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_job_selectors.rs new file mode 100644 index 000000000..0c8a5f6c1 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_job_selectors.rs @@ -0,0 +1,172 @@ +use crate::refactoring::_helpers::edits::text_edits::apply_text_edits; +use crate::refactoring::_helpers::planning::job_selectors::job_selector_parts; +use crate::refactoring::_helpers::scanning::chars::chars; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::ModelFacts; +use crate::refactoring::tests::helpers::{model, write_files}; +use crate::refactoring::tests::test_types::JobSelectorTestCase; + +const JOB: &str = "orchestration/jobs/orders/refresh.toml"; + +/// A model the refactoring leaves alone keeps its name and path. +const STAYS: (&str, &str, &str, &str) = ("customers", "models/marts/customers.sql", "", ""); + +#[test] +fn given_job_selectors_when_models_are_renamed_or_moved_then_exact_parts_follow() +-> Result<(), RefactorError> { + let test_cases = [ + JobSelectorTestCase { + description: "exact names follow; lookalikes, globs, tags and other kinds stay", + models: &[ + ( + "orders", + "models/marts/orders.sql", + "sales", + "models/marts/sales.sql", + ), + STAYS, + ], + job: "select = \"+orders+ orders_x tag:orders source:orders job:orders orders* order?s orders,customers orders~customers +customers~orders+\"\n", + expected_job: "select = \"+sales+ orders_x tag:orders source:orders job:orders orders* order?s sales,customers sales~customers +customers~sales+\"\n", + expected_manual: &[], + }, + JobSelectorTestCase { + description: "literal and escaped strings keep their style", + models: &[ + ( + "orders", + "models/marts/orders.sql", + "sales", + "models/marts/sales.sql", + ), + STAYS, + ], + job: "select = 'orders'\nexclude = \"orders \\u0074ag:x\"\n", + expected_job: "select = 'sales'\nexclude = \"sales tag:x\"\n", + expected_manual: &[], + }, + JobSelectorTestCase { + description: "a folder every model leaves moves, nested folders included", + models: &[ + ( + "orders", + "models/staging/orders.sql", + "orders", + "models/core/orders.sql", + ), + ( + "lines", + "models/staging/sub/lines.sql", + "lines", + "models/core/sub/lines.sql", + ), + STAYS, + ], + job: "select = \"models/staging/ path:models/staging +models/staging/sub+ models/stagingx models\"\n", + expected_job: "select = \"models/core/ path:models/core +models/core/sub+ models/stagingx models\"\n", + expected_manual: &[], + }, + JobSelectorTestCase { + description: "a folder only some models leave stays and is reported", + models: &[ + ( + "orders", + "models/staging/orders.sql", + "orders", + "models/core/orders.sql", + ), + ("lines", "models/staging/lines.sql", "", ""), + ], + job: "name = \"x_job\"\nselect = \"models/staging orders\"\n", + expected_job: "name = \"x_job\"\nselect = \"models/staging orders\"\n", + expected_manual: &[ + "2:10 this job selects folder 'models/staging', and orders moved out of it; check the job still selects what it should", + ], + }, + JobSelectorTestCase { + description: "a folder moving into one that holds other models stays and is reported", + models: &[ + ( + "orders", + "models/staging/orders.sql", + "orders", + "models/marts/orders.sql", + ), + STAYS, + ], + job: "name = \"x_job\"\nselect = \"models/staging\"\nexclude = \"models/staging\"\n", + expected_job: "name = \"x_job\"\nselect = \"models/staging\"\nexclude = \"models/staging\"\n", + expected_manual: &[ + "2:10 this job selects folder 'models/staging', and orders moved out of it; check the job still selects what it should", + "3:11 this job selects folder 'models/staging', and orders moved out of it; check the job still selects what it should", + ], + }, + JobSelectorTestCase { + description: "a multi-line selector is reported instead of rewritten", + models: &[( + "orders", + "models/marts/orders.sql", + "sales", + "models/marts/sales.sql", + )], + job: "select = \"\"\"\norders\"\"\"\n", + expected_job: "select = \"\"\"\norders\"\"\"\n", + expected_manual: &[ + "1:10 this selector is a multi-line string; change it to 'sales' by hand", + ], + }, + ]; + for test_case in test_cases { + let project = + tempfile::tempdir().map_err(|error| RefactorError::value(error.to_string()))?; + write_files(project.path(), &[(JOB, test_case.job)])?; + let models: Vec = test_case + .models + .iter() + .map(|(name, path, _, _)| ModelFacts { + path: (*path).to_owned(), + ..model(name, Some("table"), "analytics") + }) + .collect(); + let renamed: Vec<(&ModelFacts, &str, &str)> = test_case + .models + .iter() + .zip(&models) + .filter(|((_, _, new_name, _), _)| !new_name.is_empty()) + .map(|((_, _, new_name, new_path), facts)| (facts, *new_name, *new_path)) + .collect(); + let parts = job_selector_parts(project.path(), &models, &renamed)?; + let edits = parts + .edits + .into_iter() + .map(|(_, edit)| edit) + .collect::>(); + let manual: Vec = parts + .manual + .iter() + .chain(&parts.notes) + .map(|location| { + format!( + "{}:{} {}", + location.line.unwrap_or_default(), + location.column.unwrap_or_default(), + location.reason + ) + }) + .collect(); + assert_eq!( + apply_text_edits(&chars(test_case.job), &edits)? + .into_iter() + .collect::(), + test_case.expected_job, + "{}", + test_case.description + ); + assert_eq!( + manual, test_case.expected_manual, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs index 3a74e7187..7ac01f504 100644 --- a/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs @@ -257,3 +257,13 @@ pub(super) struct StagedCommitOrderTestCase { pub(super) changes: &'static [(&'static str, &'static str)], pub(super) expected_files: &'static [(&'static str, &'static str)], } + +/// Models as `(name, path, new name, new path)`, a job's selector lines and what they become. +pub(super) struct JobSelectorTestCase { + pub(super) description: &'static str, + pub(super) models: &'static [(&'static str, &'static str, &'static str, &'static str)], + pub(super) job: &'static str, + pub(super) expected_job: &'static str, + /// `line:column reason` of each selector a person must edit or check. + pub(super) expected_manual: &'static [&'static str], +} diff --git a/deploy/project/orchestration/jobs/refresh_orders_job.toml b/deploy/project/orchestration/jobs/refresh_orders_job.toml new file mode 100644 index 000000000..b3727eca8 --- /dev/null +++ b/deploy/project/orchestration/jobs/refresh_orders_job.toml @@ -0,0 +1,5 @@ +# A small scheduled project baked into the reference location image. +name = "refresh_orders_job" +description = "Refresh orders every minute." +select = "orders" +schedule = { cron = "* * * * *", timezone = "UTC" } diff --git a/deploy/project/orchestration/orders.py b/deploy/project/orchestration/orders.py deleted file mode 100644 index 802379465..000000000 --- a/deploy/project/orchestration/orders.py +++ /dev/null @@ -1,10 +0,0 @@ -"""A small scheduled project baked into the reference location image.""" - -from sqlbuild.orchestration import job, schedule - -orders = job( - "refresh_orders", - select="orders", - description="Refresh orders every minute.", - schedule=schedule(cron="* * * * *", timezone="UTC"), -) diff --git a/python-boundary.toml b/python-boundary.toml index 7f8009a8b..5a89a1c58 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -20,7 +20,7 @@ note = "Public orchestration facade: re-exports only." [[stays]] path = "orchestration_declarations/models/authoring.py" rule = 3 -note = "Public job and schedule authoring records only; selection resolution, validation and scheduling stay native." +note = "Public calendar and sensor authoring records only; jobs, job groups and schedules are TOML read natively, and selection, validation and scheduling stay native." [[stays]] path = "orchestration_declarations/models/sensor_context.py" diff --git a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py index b420c64c0..a72a2b462 100644 --- a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py @@ -389,6 +389,17 @@ def _python_cases() -> tuple[FailureCase, ...]: for module in ("first", "second") }, ), + failure_case( + name="orchestration-python-job-module", + expected_code="D017", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "nightly = job('nightly', description='Orders.', select='stg_orders')\n" + ) + }, + expected_message="files cannot sit directly in orchestration/", + ), failure_case( name="legacy-event-exporters-directory", expected_code="D015", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 6b4a04a62..2ffdb2b67 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -57,6 +57,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [Source freshness](concepts/planning/source-freshness.md) (`concepts/planning/source-freshness`) - Observe external source changes and propagate them through planning. - [Selection and staleness](concepts/planning/selection-and-staleness.md) (`concepts/planning/selection-and-staleness`) - How selection interacts with change detection, and the stale warnings that stop silent partial rebuilds. - [Column Lineage](concepts/column-lineage.md) (`concepts/column-lineage`) - Trace individual columns through your SQL pipeline - understand where data comes from and where it goes. +- [Orchestration](concepts/orchestration.md) (`concepts/orchestration`) - Declare jobs, job groups and schedules in TOML, and calendars and sensors in Python, for a SQLBuild server to run. ## Refactor diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index f2ab366db..a716fb092 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -159,18 +159,18 @@ belongs to a scheduled job without the changed model. That job's next run refres model is held back: ``` - staging_daily (schedule) finance_daily (schedule) + staging_daily_job (schedule) finance_daily_job (schedule) stg_orders* ─▶ int_orders ✓ ──────────▶ fct_revenue ⏸ ─▶ rpt_pnl ⏸ │ └──▶ orders_export ✓ (in no scheduled job) ``` -- A job counts when a schedule, calendar schedule or sensor starts it, directly or through its +- A job counts when a schedule, calendar or sensor starts it, directly or through its group, for the target you build. A job nothing starts counts as no job. - A model held back by one job runs if another job that holds the change also contains it. - A model that reads a held-back model is held back too. Unchanged parents never hold a model back. - A job never holds back its own models while it runs: a scheduled run, or a build that selects the - job, such as `sqb build finance_daily --changed`. + job, such as `sqb build finance_daily_job --changed`. - With no scheduled jobs, everything downstream runs. `--reach all` builds everything downstream of a change. `--reach` without `--changed` is refused. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md index c80138c62..b936b25b5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md @@ -98,6 +98,7 @@ Renaming or moving a model rewrites: - `__ref__` and `__expected__` fixture CTEs in tests and scenarios; - `cursor_inputs` keys, `relationships (to ...)` audits, and `__ref("")` inside quoted SQL in `MODEL` headers and reusable `SCHEMA` declarations; - `relationships` audits (`to:` and `field:`) and `__ref("")` in source and seed YAML. +- the model's name and folder in job and job group `select` and `exclude` selectors; see [Rename models in jobs](../concepts/orchestration.md#rename-models-in-jobs). The model's own unit tests, the test files whose every `TEST` block targets only that model, follow it: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 6d1e76165..20e2e029c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -392,18 +392,18 @@ Details: Declare who owns a job, whether scheduled requests wait for a running copy, and how often a failed run retries: -```python -from sqlbuild.orchestration import job, on_failure, schedule - -nightly_orders = job( - "nightly_orders", - description="Orders and their daily totals", - select="+order_totals", - owner="Orders team", - overlap="skip", - on_failure=on_failure(retries=2, retry_delay_seconds=600), - schedule=schedule(cron="0 2 * * *", timezone="UTC"), -) +```toml +# orchestration/jobs/nightly_orders_job.toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" +owner = "Orders team" +overlap = "skip" +schedule = { cron = "0 2 * * *", timezone = "UTC" } + +[on_failure] +retries = 2 +retry_delay_seconds = 600 ``` | Setting | Values | Default | Effect | @@ -431,8 +431,8 @@ Details: - A group retries at group level, with the group's own `on_failure`: each retry is one rerun of the group's failed and skipped nodes. A member job's `on_failure` applies only when that job runs on its own. -- An invalid value fails publication with the job's name, such as - `job nightly_orders overlap must be queue or skip, not coalesce`. +- An invalid value is a `D017` error at its line, such as + ``orchestration/jobs/nightly_orders_job.toml:6:11: unknown variant `coalesce`, expected `queue` or `skip` ``. ## Limit worker containers @@ -456,15 +456,15 @@ pids = 512 max_memory = "4g" ``` -```python -from sqlbuild.orchestration import job, resources - -nightly_orders = job( - "nightly_orders", - description="Orders and their daily totals", - select="+order_totals", - resources=resources(memory="3g", cpus=2), -) +```toml +# orchestration/jobs/nightly_orders_job.toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" + +[resources] +memory = "3g" +cpus = 2 ``` | Setting | `run` table | `resources` | Value | @@ -484,7 +484,7 @@ nightly_orders = job( `no-new-privileges`. - A job in a group uses its own `resources`. Local launchers ignore `resources`. - An invalid value fails publication with the job's name, such as - `job nightly_orders resources: cpus must be a positive number`. + `job nightly_orders_job resources: cpus must be a positive number`. ## Run ad-hoc commands diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index 4add747a7..f4f1fb3ed 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -90,7 +90,7 @@ error[P010]: model 'orders_summary' has no description A description may take at most `[format] max_description_lines` lines (default `10`) once wrapped at the line width, the same limit `sqb format --check` applies to headers. A longer Python description, from a decorator or a docstring, is a `P027` compile error at its declaration. A job, -job group, sensor or calendar schedule with a longer description cannot publish. +job group, sensor or calendar with a longer description cannot publish. SQL tests, audits, macros, enums, constants, materializations and event or output sinks do not need a description. Descriptions do not change fingerprints, so adding or editing one does not rebuild a diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md new file mode 100644 index 000000000..d37f7e9a8 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -0,0 +1,230 @@ + + +# Orchestration + +> Declare jobs, job groups and schedules in TOML, and calendars and sensors in Python, for a SQLBuild server to run. + +Online: https://sqlbuild.com/docs/concepts/orchestration/ + +## Contents + +- Project layout +- Declare a job +- Group jobs +- Schedule a job +- Schedule from a calendar +- Start a job from a sensor +- Rename models in jobs +- Errors + +A job names the nodes a server builds together, and a schedule says when. Write each job as a TOML +file under `orchestration/jobs/`; `sqb compile` checks it and a code location publishes it. + +```toml +# orchestration/jobs/nightly_orders_job.toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" +schedule = { cron = "0 2 * * *", timezone = "UTC" } +``` + +```bash +sqb build nightly_orders_job # build what the job builds, locally +``` + +## Project layout + +| Folder | Holds | Name suffix | +| --- | --- | --- | +| `orchestration/jobs/` | One job per `.toml` file | `_job` | +| `orchestration/job_groups/` | One job group per `.toml` file | `_job_group` | +| `orchestration/calendars/` | One calendar per `.py` module | `_calendar` | +| `orchestration/sensors/` | One sensor per `.py` module | `_sensor` | + +- Subfolders are allowed, such as `orchestration/jobs/finance/revenue_job.toml`. +- A name must end with its folder's suffix. Names are unique across the whole project, with models, + sources and every other resource. +- A Python module whose file name starts with `_` is a private helper and declares nothing. +- Any other file under `orchestration/`, or a folder not in the table, is a `D017` error. + +## Declare a job + +A job builds its `select` minus its `exclude`, with the same [selectors](selectors.md) +as `sqb build`. + +```toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" +exclude = "tag:deprecated" +targets = ["production"] +owner = "Orders team" +overlap = "skip" +concurrency_groups = ["warehouse_heavy"] +schedule = { cron = "0 2 * * *", timezone = "Europe/London" } + +[on_failure] +retries = 2 +retry_delay_seconds = 600 + +[settings] +concurrency = 4 +vars = { batch_size = 500 } + +[resources] +memory = "3g" +cpus = 2 +``` + +| Key | Required | Value | +| --- | --- | --- | +| `name` | Yes | The job's name, ending in `_job` | +| `description` | Yes | Shown on the job's page | +| `select` | Yes | The nodes the job builds | +| `exclude` | No | Nodes removed from `select` | +| `targets` | No | The targets the job exists in; all targets when omitted | +| `owner` | No | Shown on the job's page and in the job list | +| `overlap` | No | `queue` (default) or `skip`; see [Skip overlapping runs](../cli/server.md#skip-overlapping-runs-and-retry-failures) | +| `concurrency_groups` | No | Server concurrency limits the job's runs share | +| `schedule` | No | When the server starts the job; see below | +| `[on_failure]` | No | `retries` from 1 to 10, `retry_delay_seconds` from 0 to 86400 | +| `[settings]` | No | Build options for each run, such as `concurrency`, `vars`, `full_refresh` or `changed` | +| `[resources]` | No | Docker worker limits; see [Limit worker containers](../cli/server.md#limit-worker-containers) | + +A job's `[settings]` cannot set `select` or `exclude`: the job's own selection is the run's +selection. + +## Group jobs + +A job group runs its jobs as one run. A job waits for another job in the group when it reads a node +that job builds. + +```toml +# orchestration/job_groups/daily_orders_job_group.toml +name = "daily_orders_job_group" +description = "Stage orders, then build the marts" +jobs = ["staging_orders_job", "orders_marts_job"] +schedule = { cron = "0 3 * * *", timezone = "UTC" } +``` + +A group takes `name`, `description` and `jobs`, and the same `targets`, `owner`, `overlap`, +`concurrency_groups`, `schedule` and `[on_failure]` as a job. Its members are jobs, never other +groups. + +## Schedule a job + +`schedule` takes a cron expression or the name of a calendar, and always a timezone. + +| Key | Value | +| --- | --- | +| `cron` | A five-field cron expression | +| `calendar` | The name of a calendar in `orchestration/calendars/` | +| `timezone` | An IANA timezone, such as `"UTC"` or `"America/New_York"` | +| `starts_at` | An RFC 3339 time with an offset; no ticks before it | +| `default_status` | `running` (default) or `paused` when the schedule is first published | + +```toml +schedule = { cron = "15 2 * * *", timezone = "Europe/London", starts_at = "2026-07-03T00:00:00Z" } +``` + +## Schedule from a calendar + +A calendar returns the upcoming times a job runs, such as fulfillment cut-offs that come from +another system. The code location evaluates it when it publishes and again before its times run +out. + +```python +# orchestration/calendars/fulfillment_calendar.py +from sqlbuild.orchestration import calendar + +@calendar(name="fulfillment_calendar", description="Upcoming fulfillment times", horizon="1d") +def fulfillment_calendar(window, fulfillment): + return fulfillment.upcoming(window) +``` + +```toml +schedule = { calendar = "fulfillment_calendar", timezone = "America/New_York" } +``` + +The function receives a `window` with timezone-aware `start` and `end`, in the job's timezone, and +returns timezone-aware datetimes inside it. Other parameters name +[providers](python-nodes/providers.md). `horizon` sets how far ahead the window reaches. + +## Start a job from a sensor + +A sensor starts a job or job group when something happens, named by `run`. + +```python +# orchestration/sensors/on_new_orders_sensor.py +from sqlbuild.orchestration import sensor, source_new_data + +on_new_orders = sensor( + "on_new_orders_sensor", + description="Refresh orders for each new source version", + when=source_new_data(select="source:raw_orders"), + run="orders_refresh_job", + minimum_interval_seconds=60, +) +``` + +| Condition | Starts a run for | +| --- | --- | +| `platform_event(kinds=[...])` | Server events, such as an overdue tick | +| `run_event(job="...", kinds=[...])` | Outcomes of another job or group, optionally only when `executed_successfully` nodes ran and succeeded | +| `node_event(select="...", kinds=[...])` | Each success, failure or recovery of the selected nodes in any run | +| `source_new_data(select="...")` | Each new version of the selected sources | +| `source_stale(select="...", after_seconds=...)` | Each source version older than `after_seconds` | + +A sensor without `when` decorates a function that polls and returns `RunRequest(run_key=...)` +values or a `SkipReason`. A sensor also takes `minimum_interval_seconds` (default 30), +`default_status` and `targets`. + +## Rename models in jobs + +`sqb rename` and `sqb mv` update job and job group selectors with the models they rename or move: + +```bash +sqb rename stg_orders stg_orders_clean +``` + +```text +Rename model stg_orders -> stg_orders_clean +├── models/marts/fact_orders.sql +│ └── 10:13 reference __ref("stg_orders") -> __ref("stg_orders_clean") +├── models/staging/stg_orders_clean.sql (moved from models/staging/stg_orders.sql) +│ └── 1:8 migration + migrate_from stg_orders +└── orchestration/jobs/orders_refresh_job.toml + └── 3:10 selector "+stg_orders+ fact_orders" -> "+stg_orders_clean+ fact_orders" +``` + +- Exact names are rewritten, including inside graph operators such as `+stg_orders+`, `,` and `~`. + Tags, globs and `job:` selectors stay as written. +- A folder selector, such as `models/staging` or `path:models/staging/`, moves when every model + under it moves to the same folder. When only some leave, the command lists a note at the + selector to check by hand. +- A selector in a multi-line TOML string is reported for a manual edit and blocks the refactor. + +## Errors + +Every problem with an orchestration file is a `D017` error at its file, line and column, reported by +`sqb compile` and every other command that reads the project. + +| Error | Fix | +| --- | --- | +| ``orchestration/jobs/nightly_orders_job.toml:4:11: unknown variant `coalesce`, expected `queue` or `skip` `` | Use one of the listed values | +| `files cannot sit directly in orchestration/` | Move jobs and groups to TOML files in `orchestration/jobs/` or `orchestration/job_groups/` | +| `` `job` was removed from sqlbuild.orchestration`` | Declare the job in TOML and name it in the sensor's `run` | +| `declares a second sensor; each module declares exactly one` | Split the module | +| `schedule cron: invalid cron expression: Pattern must have 5 fields when seconds are disallowed.` | Write a five-field cron, such as `0 2 * * *` | +| `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | +| `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | + +Details: + +- `sqb build` and the other commands read job and group files without running project Python, so + a job selector never imports a calendar or sensor module. +- A job or group declared with `targets` is selectable only when the command's target is one of + them, as on the server. +- Selectors are checked for syntax at compile. A selector that names a node the project does not + have fails when a location publishes, such as `job nightly_orders_job: unknown selector name + 'order_total'`. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index ddb45ff27..db0ae5c2e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -764,7 +764,7 @@ Code that extends SQLBuild itself keeps its own root: | `hooks/python/` | Python model lifecycle hooks | | `functions/python/` | Python UDFs | | `macros/python/`, `enums/`, `constants/` and scoped `_macros/python/`, `_enums/`, `_constants/` | Python macros and declarations | -| `orchestration/` | Jobs, job groups, sensors and calendar schedules. SQLBuild imports these modules only when a selector names a job or an unknown name, and when a location publishes. See [Job selectors](selectors.md#job) | +| `orchestration/` | Jobs and job groups as TOML in `jobs/` and `job_groups/`; calendars and sensors as Python in `calendars/` and `sensors/`. See [Orchestration](orchestration.md) | Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create an unofficial project structure. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index c4c2db12e..cb06ba474 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -162,7 +162,7 @@ sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` -Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendar schedules. +Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendars. ## Lifecycle: run, build, check diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index 09e66562e..9ba80be23 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -82,13 +82,13 @@ sqb build --select source:raw__orders ### Job -Select the SQL and Python nodes a job or job group declared under `orchestration/` builds. Job names are globally +Select the SQL and Python nodes a [job or job group](orchestration.md) builds. Job names are globally unique, so a bare name works; `job:` is the explicit form and accepts glob patterns: ```bash -sqb build nightly_orders -sqb build --select job:nightly_orders --exclude order_totals -sqb build --select "job:nightly_orders,tag:finance" +sqb build nightly_orders_job +sqb build --select job:nightly_orders_job --exclude order_totals +sqb build --select "job:nightly_orders_job,tag:finance" ``` A job selects its own `select` minus its own `exclude`. A job group selects the union of its jobs. @@ -97,8 +97,7 @@ them, as on the server. Job selectors compose with `+`, `,` and `--exclude` like any other selector. A job's own selection cannot name another job, and path-between selectors cannot start or end at a job. -SQLBuild imports the modules under `orchestration/` only when a selector names a job or a name no -model, source, seed or function has. +SQLBuild reads job and job group files without running project Python. ### Unit test diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b465d0857..cc6d6d1f4 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1777,7 +1777,6 @@ def build_orchestration_snapshot( project_dir: Path, *, discovered_inputs: DiscoveredProjectInputs, - declarations: Sequence[object] | None = None, ) -> NativeOrchestrationSnapshot: ... def run_location_command( *, @@ -8451,6 +8450,9 @@ class SqlHookParseError(DiscoveryError): class EventExporterDiscoveryError(DiscoveryError): """Raised when project lifecycle event exporters are invalid.""" +class OrchestrationDeclarationError(DiscoveryError): + """Raised when an orchestration job, job group, calendar or sensor declaration is invalid.""" + class ProjectPythonPathError(DiscoveryError): """Raised when project-owned Python lives outside a supported extension root.""" diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index 1b042a5e2..73abd5f7f 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -1,24 +1,17 @@ -"""Public authoring and snapshot compilation API for orchestration declarations.""" +"""Calendar and sensor authoring and snapshot compilation; jobs and job groups are TOML files.""" from sqlbuild.orchestration_declarations.main.compile_snapshot import compile_snapshot -from sqlbuild.orchestration_declarations.models.authoring import ( - CalendarSchedule as calendar_schedule, -) +from sqlbuild.orchestration_declarations.models.authoring import Calendar as calendar from sqlbuild.orchestration_declarations.models.authoring import ( CalendarWindow, RunRequest, SkipReason, ) -from sqlbuild.orchestration_declarations.models.authoring import Job as job -from sqlbuild.orchestration_declarations.models.authoring import JobGroup as job_group from sqlbuild.orchestration_declarations.models.authoring import NodeEventCondition as node_event -from sqlbuild.orchestration_declarations.models.authoring import OnFailure as on_failure from sqlbuild.orchestration_declarations.models.authoring import ( PlatformEventCondition as platform_event, ) -from sqlbuild.orchestration_declarations.models.authoring import Resources as resources from sqlbuild.orchestration_declarations.models.authoring import RunEventCondition as run_event -from sqlbuild.orchestration_declarations.models.authoring import Schedule as schedule from sqlbuild.orchestration_declarations.models.authoring import Sensor as sensor from sqlbuild.orchestration_declarations.models.authoring import ( SourceNewDataCondition as source_new_data, @@ -37,16 +30,11 @@ "RunRequest", "SensorContext", "SkipReason", - "calendar_schedule", + "calendar", "compile_snapshot", - "job", - "job_group", "node_event", - "on_failure", "platform_event", - "resources", "run_event", - "schedule", "sensor", "source_new_data", "source_stale", diff --git a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py index fb42de5d9..f901b134b 100644 --- a/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py +++ b/src/sqlbuild/orchestration_declarations/main/compile_snapshot.py @@ -1,8 +1,7 @@ -"""Delegate authoring-record compilation to the native retained snapshot owner.""" +"""Delegate snapshot compilation to the native retained snapshot owner.""" from __future__ import annotations -from collections.abc import Sequence from pathlib import Path from sqlbuild._native import ( @@ -11,12 +10,6 @@ build_orchestration_snapshot, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration_declarations.models.authoring import ( - CalendarSchedule, - Job, - JobGroup, - Sensor, -) def compile_snapshot( @@ -25,13 +18,11 @@ def compile_snapshot( metadata_json: str, project_dir: Path, discovered_inputs: DiscoveredProjectInputs, - declarations: Sequence[Job | JobGroup | CalendarSchedule | Sensor] | None = None, ) -> NativeOrchestrationSnapshot: - """Compile authoring records and retained project facts into a data-only snapshot.""" + """Compile the orchestration declarations and retained facts into a data-only snapshot.""" return build_orchestration_snapshot( project, metadata_json, project_dir, discovered_inputs=discovered_inputs, - declarations=declarations, ) diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index 0bf9122d9..a51b33d16 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -11,75 +11,6 @@ from sqlbuild.orchestration_declarations.types import NodeEventKind, PlatformEventKind, RunEventKind -@dataclass(frozen=True, kw_only=True) -class Schedule: - """Cron with an RFC 3339 start; missed times collapse into one run of the latest.""" - - cron: str - timezone: str - starts_at: str | None = None - default_status: Literal["running", "paused"] = "running" - - -@dataclass(frozen=True, kw_only=True) -class OnFailure: - """Rerun a failed or partial run from failure, `retries` times, `retry_delay_seconds` apart.""" - - retries: int - retry_delay_seconds: int = 0 - - -@dataclass(frozen=True, kw_only=True) -class Resources: - """Worker container limits for a job; memory is bytes or a size such as "512m" or "2g".""" - - memory: int | str | None = None - cpus: float | None = None - pids: int | None = None - - -@dataclass(frozen=True, eq=False) -class Job: - """A named selection to execute, optionally on a wall-clock schedule.""" - - name: str - description: str = field(kw_only=True) - select: str = field(kw_only=True) - exclude: str = field(default="", kw_only=True) - settings: dict[str, object] = field(default_factory=dict, kw_only=True) - concurrency: int | None = field(default=None, kw_only=True) - concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) - schedule: Schedule | None = field(default=None, kw_only=True) - targets: tuple[str, ...] = field(default=(), kw_only=True) - owner: str | None = field(default=None, kw_only=True) - overlap: Literal["queue", "skip"] = field(default="queue", kw_only=True) - on_failure: OnFailure | None = field(default=None, kw_only=True) - resources: Resources | None = field(default=None, kw_only=True) - _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) - - def __post_init__(self) -> None: - object.__setattr__(self, "_declared_at", declaration_site()) - - -@dataclass(frozen=True, eq=False) -class JobGroup: - """Jobs ordered by the lineage between the nodes they build.""" - - name: str - description: str = field(kw_only=True) - jobs: Sequence[Job] = field(kw_only=True) - concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) - schedule: Schedule | None = field(default=None, kw_only=True) - targets: tuple[str, ...] = field(default=(), kw_only=True) - owner: str | None = field(default=None, kw_only=True) - overlap: Literal["queue", "skip"] = field(default="queue", kw_only=True) - on_failure: OnFailure | None = field(default=None, kw_only=True) - _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) - - def __post_init__(self) -> None: - object.__setattr__(self, "_declared_at", declaration_site()) - - @dataclass(frozen=True, kw_only=True) class CalendarWindow: """Timezone-aware, half-open interval for an upcoming calendar evaluation.""" @@ -89,23 +20,19 @@ class CalendarWindow: @dataclass(frozen=True, kw_only=True, eq=False) -class CalendarSchedule: - """Decorate a location callback returning upcoming timezone-aware datetimes.""" +class Calendar: + """Decorate a location callback returning upcoming timezone-aware datetimes for schedules.""" - job: Job | JobGroup + name: str description: str horizon: str - timezone: str - starts_at: str | None = None - default_status: Literal["running", "paused"] = "running" - targets: tuple[str, ...] = () function: Callable[..., Iterable[datetime]] | None = field(default=None, repr=False) _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) def __post_init__(self) -> None: object.__setattr__(self, "_declared_at", declaration_site()) - def __call__(self, function: Callable[..., Iterable[datetime]]) -> CalendarSchedule: + def __call__(self, function: Callable[..., Iterable[datetime]]) -> Calendar: return replace(self, function=function) @@ -119,9 +46,9 @@ class PlatformEventCondition: @dataclass(frozen=True, kw_only=True) class RunEventCondition: - """Match scoped terminal outcomes, optionally requiring genuinely executed successful nodes.""" + """Match a job's or group's terminal outcomes; listed nodes must have run and succeeded.""" - job: Job | JobGroup + job: str kinds: Sequence[RunEventKind] executed_successfully: Sequence[str] | None = None kind: Literal["run_event"] = field(default="run_event", init=False) @@ -167,7 +94,7 @@ class Sensor: | SourceStaleCondition | None ) = field(default=None, kw_only=True) - run: Job | JobGroup = field(kw_only=True) + run: str = field(kw_only=True) minimum_interval_seconds: int = field(default=30, kw_only=True) default_status: Literal["running", "paused"] = field(default="running", kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 8140e645d..62caae701 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -26,6 +26,7 @@ query_duckdb, run_sqb, ) +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import duckshop_orchestration def run_replay_command( @@ -916,9 +917,7 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: 'MODEL (description "Test model order_summary.", ' 'materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' ), - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py" - ).read_text(), + **duckshop_orchestration("python_node_jobs"), "python/tasks/reports.py": ( "from python.tasks.counts import count_orders\nfrom sqlbuild.tasks import task\n\n\n" "@task(depends_on=count_orders)\n" @@ -1377,7 +1376,7 @@ def prepare_changed_project(*, tmp_path: Path, warehouse: ChangedWarehouse) -> P def add_changed_overlay(project_dir: Path, fixture: Path) -> None: """Overlay one of the DuckShop changed-builds cases on the base case.""" - for path in (*fixture.rglob("*.sql"), *fixture.rglob("*.py")): + for path in (*fixture.rglob("*.sql"), *fixture.rglob("*.py"), *fixture.rglob("*.toml")): destination: Path = project_dir / path.relative_to(fixture) destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(path.read_text(encoding="utf-8"), encoding="utf-8") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py index 54c79457f..20fedd7e7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_changed_builds.py @@ -231,7 +231,7 @@ def test_given_a_build_killed_mid_write_when_building_with_changed_then_every_in "Changed (1)", "Downstream of changes (3)", "Held back (3)", - "job finance_daily", + "job finance_daily_job", "reads fct_revenue, which is held back", ), expected_job_run=frozenset({"fct_revenue", "rpt_pnl"}), @@ -258,7 +258,7 @@ def test_given_scheduled_jobs_when_building_with_changed_then_a_change_stops_at_ redefined, build_output = changed_build_output(project_dir, ("--changed",)) held: frozenset[str] = run_changed_build(project_dir, ("--changed",)) job_run: frozenset[str] = run_changed_build( - project_dir, ("--changed", "--select", "finance_daily") + project_dir, ("--changed", "--select", "finance_daily_job") ) reach_all: frozenset[str] = run_changed_build(project_dir, ("--changed", "--reach", "all")) refused: subprocess.CompletedProcess[str] = run_sqb( @@ -268,7 +268,7 @@ def test_given_scheduled_jobs_when_building_with_changed_then_a_change_stops_at_ assert first == REACH_MODELS assert new_data == frozenset(), "new source data is not a change by default" assert redefined == test_case.expected_redefined, ( - "the change stops at finance_daily; a job without a trigger and a model in no job run" + "the change stops at finance_daily_job; a job without a trigger and a model in no job run" ) assert test_case.expected_summary in build_output assert plan.returncode == 0, plan.stdout + plan.stderr @@ -309,13 +309,15 @@ def test_given_two_jobs_each_holding_a_redefined_upstream_when_building_changed_ sales_staged: frozenset[str] = run_changed_build( project_dir, ("--changed", "--select", "stg_orders") ) - crm_run: frozenset[str] = run_changed_build(project_dir, ("--changed", "--select", "crm_daily")) + crm_run: frozenset[str] = run_changed_build( + project_dir, ("--changed", "--select", "crm_daily_job") + ) downstream: frozenset[str] = run_changed_build(project_dir, ("--changed",)) assert first == {"stg_customers", "stg_orders", "customer_orders", "customer_sales"} assert sales_staged == {"stg_orders"} assert crm_run == test_case.expected_crm_run assert downstream == test_case.expected_downstream, ( - "sales_daily holds the stg_orders change that reached customer_orders, so customer_sales " - "runs even though crm_daily holds the stg_customers change too" + "sales_daily_job holds the stg_orders change that reached customer_orders, so customer_sales " + "runs even though crm_daily_job holds the stg_customers change too" ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py index 80803f98f..9f94302b1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py @@ -106,7 +106,7 @@ ), PythonNodeSelectionE2ETestCase( description="a job selecting only Python nodes builds them", - select=("job:customer_counts",), + select=("job:customer_counts_job",), expected_exit_code=0, expected_output_patterns=(r"count_customers\s+OK", r"customers_extract\s+OK"), expected_row_counts={"raw_customers": 3}, @@ -114,7 +114,7 @@ ), PythonNodeSelectionE2ETestCase( description="a job named without job: builds its Python nodes", - select=("customer_counts",), + select=("customer_counts_job",), expected_exit_code=0, expected_output_patterns=(r"count_customers\s+OK", r"customers_extract\s+OK"), expected_row_counts={"raw_customers": 3}, @@ -122,7 +122,7 @@ ), PythonNodeSelectionE2ETestCase( description="a job selecting a task with its upstream model builds both", - select=("job:order_counts",), + select=("job:order_counts_job",), expected_exit_code=0, expected_output_patterns=(r"count_orders\s+OK",), expected_row_counts={"orders": 2}, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index d3cfbc3b5..5fba89969 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5588,3 +5588,14 @@ def truncate_sql_test_cache(project_dir: Path) -> None: for cache in sorted((project_dir / "target/cache").rglob("sql-test-artifacts.json")): text: str = cache.read_text() _ = cache.write_text(text[: len(text) // 2]) + + +def orders_job_project(job_values: str) -> dict[str, str]: + """Return a one-model project whose `orders_job` TOML file continues with `job_values`.""" + return { + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "models/orders.sql": "MODEL (description 'Orders.');\nSELECT 1 AS order_id\n", + "orchestration/jobs/orders_job.toml": ( + 'name = "orders_job"\ndescription = "Orders."\n' + job_values + ), + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py index 6fc4af0c6..d36210219 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py @@ -13,6 +13,7 @@ PythonProjectLayoutCompileTestCase, UnrelatedPythonPackageBuildTestCase, ) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import orders_job_project from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -32,6 +33,110 @@ "Move pipeline Python nodes and the helper modules they import under python/", ), ), + PythonProjectLayoutCompileTestCase( + description="Python jobs left in orchestration/ point to the TOML layout", + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "nightly = job('nightly', description='Orders.', select='orders')\n" + ), + }, + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs.py", + "files cannot sit directly in orchestration/", + "jobs, job groups and schedules are TOML files in orchestration/jobs/", + "https://sqlbuild.com/docs/concepts/orchestration/", + ), + ), + PythonProjectLayoutCompileTestCase( + description="a sensor importing the removed job API points to the TOML layout", + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "orchestration/sensors/orders_sensor.py": ( + "from sqlbuild.orchestration import job, sensor\n" + "orders = sensor('orders_sensor', description='Orders.', " + "run=job('nightly', description='Orders.', select='orders'))\n" + ), + }, + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/sensors/orders_sensor.py:1:", + "`job` was removed from sqlbuild.orchestration", + "https://sqlbuild.com/docs/concepts/orchestration/", + ), + ), + PythonProjectLayoutCompileTestCase( + description="an invalid cron is reported at its value", + repo_files=orders_job_project( + 'select = "orders"\nschedule = { cron = "not a cron", timezone = "UTC" }\n' + ), + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs/orders_job.toml:4:21:", + "schedule cron: invalid cron expression", + ), + ), + PythonProjectLayoutCompileTestCase( + description="an unknown time zone is reported at its value", + repo_files=orders_job_project( + 'select = "orders"\nschedule = { cron = "0 2 * * *", timezone = "Mars/Olympus" }\n' + ), + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs/orders_job.toml:4:45:", + "schedule timezone is not an IANA time zone", + ), + ), + PythonProjectLayoutCompileTestCase( + description="a starts_at that is not RFC 3339 is reported at its value", + repo_files=orders_job_project( + 'select = "orders"\nschedule = { cron = "0 2 * * *", timezone = "UTC", starts_at = "yesterday" }\n' + ), + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs/orders_job.toml:4:64:", + "schedule starts_at must be an RFC 3339 timestamp", + ), + ), + PythonProjectLayoutCompileTestCase( + description="an invalid build setting is reported at its value", + repo_files=orders_job_project( + 'select = "orders"\n\n[settings]\nconcurrency = "lots"\n' + ), + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs/orders_job.toml:6:15:", + "concurrency", + ), + ), + PythonProjectLayoutCompileTestCase( + description="an unknown selector type is reported at the selector", + repo_files=orders_job_project('select = "orders tagg:finance"\n'), + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs/orders_job.toml:3:10:", + "unknown selector type 'tagg' in 'tagg:finance'", + ), + ), + PythonProjectLayoutCompileTestCase( + description="an empty intersection part is reported at the selector", + repo_files=orders_job_project('select = "tag:finance,"\n'), + expected_exit_code=1, + expected_stderr_fragments=( + "error[D017]", + "orchestration/jobs/orders_job.toml:3:10:", + "empty selector", + ), + ), PythonProjectLayoutCompileTestCase( description="removed tasks Python root", repo_files={ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py index ab99b5aa0..22720cb3a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py @@ -299,3 +299,14 @@ class PythonNamedMoveE2ETestCase: destination: str expected_exit_code: int expected_model_path: str + + +@dataclass(frozen=True) +class JobSelectorRefactorE2ETestCase: + """A rename or move, the job file before and after it, and the job built afterwards.""" + + description: str + command: tuple[str, ...] + job: str + expected_job: str + expected_output: tuple[str, ...] = () diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py index 0a429d21d..da2c79c8b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/helpers.py @@ -698,3 +698,8 @@ def layout_view(name: str, source: str = '__ref("stg_orders")') -> str: for folder, name in (("fin_a", "a"), ("fin_b", "b"), ("core", "c")) }, } + + +def refresh_job(select: str) -> str: + """Return a job file whose `select` is the TOML value `select`.""" + return f'name = "orders_refresh_job"\ndescription = "Refresh orders."\nselect = {select}\n' diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py new file mode 100644 index 000000000..9202e5299 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_job_selectors.py @@ -0,0 +1,81 @@ +"""CLI e2e coverage for `sqb rename` and `sqb mv` rewriting TOML job selectors.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + JobSelectorRefactorE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.refactor.helpers import ( + refresh_job, + sqb, + write_orders_project, +) + +_JOB: str = "orchestration/jobs/orders/refresh.toml" +_UNRELATED: str = "orchestration/jobs/orders/totals.toml" +_SELECTOR_REWRITE: str = Path( + "benchmarks/duckshop/edge_cases/orchestration/selector_rewrite/jobs/orders_refresh_job.toml" +).read_text() +_UNRELATED_JOB: str = ( + 'name = "orders_totals_job"\ndescription = "Rebuild totals."\nselect = "customer_totals"\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + JobSelectorRefactorE2ETestCase( + description="a rename rewrites exact names and leaves globs and other kinds alone", + command=("rename", "stg_orders", "stg_orders_clean"), + job=_SELECTOR_REWRITE, + expected_job=_SELECTOR_REWRITE.replace( + "+stg_orders+ fact_orders,+stg_orders stg_orders~customer_totals", + "+stg_orders_clean+ fact_orders,+stg_orders_clean stg_orders_clean~customer_totals", + ).replace("'stg_orders+,customer_totals'", "'stg_orders_clean+,customer_totals'"), + expected_output=("selector", _JOB), + ), + JobSelectorRefactorE2ETestCase( + description="moving every model out of a folder moves its folder selectors", + command=("mv", "stg_orders", "models/core/"), + job=refresh_job('"models/staging +path:models/staging/ stg_orders"'), + expected_job=refresh_job('"models/core +path:models/core/ stg_orders"'), + expected_output=("selector", _JOB), + ), + JobSelectorRefactorE2ETestCase( + description="moving some models out of a folder leaves it and asks for a check", + command=("mv", "fact_orders", "models/finance/"), + job=refresh_job('"+models/marts"'), + expected_job=refresh_job('"+models/marts"'), + expected_output=( + "Notes", + f"{_JOB}:3:10", + "this job selects folder 'models/marts', and fact_orders moved out of it", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_job_selectors_when_refactoring_then_jobs_follow_and_still_build( + tmp_path: Path, test_case: JobSelectorRefactorE2ETestCase +) -> None: + project_dir: Path = write_orders_project( + tmp_path=tmp_path, files={_JOB: test_case.job, _UNRELATED: _UNRELATED_JOB} + ) + + refactored: subprocess.CompletedProcess[str] = sqb(project_dir, *test_case.command) + built: subprocess.CompletedProcess[str] = sqb( + project_dir, "build", "--select", "job:orders_refresh_job" + ) + + output: str = refactored.stdout + refactored.stderr + assert refactored.returncode == 0, output + for fragment in test_case.expected_output: + assert fragment in output, output + assert (project_dir / _JOB).read_text(encoding="utf-8") == test_case.expected_job + assert (project_dir / _UNRELATED).read_text(encoding="utf-8") == _UNRELATED_JOB + assert built.returncode == 0, built.stdout + built.stderr diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_select_by_name.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_select_by_name.py index cecb5fdc9..b92cf8593 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_select_by_name.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_select_by_name.py @@ -152,7 +152,7 @@ def test_given_test_names_when_selecting_then_exactly_those_tests_run( expected_output_fragments=( "Project resource name 'orders' is declared as both model in models/orders.sql " "and unit test in tests/unit/orders.sql", - "and calendar schedule names must be globally unique", + "and calendar names must be globally unique", ), ), ), diff --git a/tests/goldens/compiler/failure/orchestration-python-job-module.json b/tests/goldens/compiler/failure/orchestration-python-job-module.json new file mode 100644 index 000000000..b811a2c18 --- /dev/null +++ b/tests/goldens/compiler/failure/orchestration-python-job-module.json @@ -0,0 +1,19 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "D017", + "message": "orchestration/jobs.py:1:1: files cannot sit directly in orchestration/", + "help": "jobs, job groups and schedules are TOML files in orchestration/jobs/ and orchestration/job_groups/; see https://sqlbuild.com/docs/concepts/orchestration/" + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index a0984474b..c68d9cbad 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1381,5 +1381,4 @@ class JobSelectorBuildTestCase: build_args: tuple[str, ...] expected_exit_code: int expected_tables: frozenset[str] - expected_jobs_imported: bool expected_fragment: str = "" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py index 6b1d2d8bf..9a8aaf725 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py @@ -26,14 +26,27 @@ "models/customers.sql": ( "MODEL (description 'Customers.', materialized table);\nSELECT 7 AS customer_id\n" ), - "orchestration/jobs.py": ( + "orchestration/jobs/nightly_job.toml": ( + 'name = "nightly_job"\ndescription = "Orders."\nselect = "+orders"\n' + ), + "orchestration/jobs/crm_job.toml": ( + 'name = "crm_job"\ndescription = "Customers."\nselect = "customers"\n' + ), + "orchestration/jobs/restock_job.toml": ( + 'name = "restock_job"\n' + 'description = "Prod only."\n' + 'select = "customers"\n' + 'targets = ["prod"]\n' + ), + "orchestration/job_groups/daily_job_group.toml": ( + 'name = "daily_job_group"\ndescription = "Everything."\njobs = ["nightly_job", "crm_job"]\n' + ), + "orchestration/sensors/overdue_sensor.py": ( "from pathlib import Path\n" - "from sqlbuild.orchestration import job, job_group\n" - "Path(__file__).parent.parent.joinpath('jobs_imported').write_text('yes')\n" - "nightly = job('nightly', description='Orders.', select='+orders')\n" - "crm = job('crm', description='Customers.', select='customers')\n" - "daily = job_group('daily', description='Everything.', jobs=[nightly, crm])\n" - "restock = job('restock', description='Prod only.', select='customers', targets=['prod'])\n" + "from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor\n" + "Path(__file__).parents[2].joinpath('sensor_imported').write_text('yes')\n" + "overdue = sensor('overdue_sensor', description='Catch up.', " + "when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), run='nightly_job')\n" ), } @@ -43,67 +56,58 @@ [ JobSelectorBuildTestCase( description="a job prefix builds the job's nodes", - build_args=("--select", "job:nightly"), + build_args=("--select", "job:nightly_job"), expected_exit_code=0, expected_tables=frozenset({"orders", "stg_orders"}), - expected_jobs_imported=True, ), JobSelectorBuildTestCase( description="a bare job name needs no prefix", - build_args=("--select", "nightly"), + build_args=("--select", "nightly_job"), expected_exit_code=0, expected_tables=frozenset({"orders", "stg_orders"}), - expected_jobs_imported=True, ), JobSelectorBuildTestCase( description="a job group builds its jobs' nodes", - build_args=("--select", "job:daily"), + build_args=("--select", "job:daily_job_group"), expected_exit_code=0, expected_tables=frozenset({"customers", "orders", "stg_orders"}), - expected_jobs_imported=True, ), JobSelectorBuildTestCase( description="a job intersects with a tag", - build_args=("--select", "job:daily,tag:staging"), + build_args=("--select", "job:daily_job_group,tag:staging"), expected_exit_code=0, expected_tables=frozenset({"stg_orders"}), - expected_jobs_imported=True, ), JobSelectorBuildTestCase( description="a job can be excluded from a group", - build_args=("--select", "job:daily", "--exclude", "job:crm"), + build_args=("--select", "job:daily_job_group", "--exclude", "job:crm_job"), expected_exit_code=0, expected_tables=frozenset({"orders", "stg_orders"}), - expected_jobs_imported=True, ), JobSelectorBuildTestCase( description="a job declared for another target is not selectable", - build_args=("--select", "job:restock"), + build_args=("--select", "job:restock_job"), expected_exit_code=1, expected_tables=frozenset(), - expected_jobs_imported=True, - expected_fragment="unknown job name 'restock'", + expected_fragment="unknown job name 'restock_job'", ), JobSelectorBuildTestCase( description="a job declared for the selected target is selectable", - build_args=("--target", "prod", "--select", "job:restock"), + build_args=("--target", "prod", "--select", "job:restock_job"), expected_exit_code=0, expected_tables=frozenset({"customers"}), - expected_jobs_imported=True, ), JobSelectorBuildTestCase( - description="model selectors never import orchestration modules", + description="model selectors build without job declarations", build_args=("--select", "+orders"), expected_exit_code=0, expected_tables=frozenset({"orders", "stg_orders"}), - expected_jobs_imported=False, ), JobSelectorBuildTestCase( description="an unknown job suggests a known one", build_args=("--select", "job:nightyl"), expected_exit_code=1, expected_tables=frozenset(), - expected_jobs_imported=True, expected_fragment="unknown job name 'nightyl'", ), ], @@ -126,7 +130,7 @@ def test_given_declared_jobs_when_building_by_job_then_only_their_nodes_build( assert exit_code == test_case.expected_exit_code, captured assert test_case.expected_fragment in captured - assert (tmp_path / "jobs_imported").exists() == test_case.expected_jobs_imported + assert not (tmp_path / "sensor_imported").exists() with duckdb.connect(str(tmp_path / "demo.duckdb")) as warehouse: rows: list[tuple[str]] = warehouse.sql( "SELECT table_name FROM duckdb_tables() " diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index fa3f650f9..406627370 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -3,17 +3,8 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.orchestration_declarations.models.authoring import ( - CalendarSchedule, - Job, - JobGroup, - OnFailure, - Schedule, - Sensor, -) @dataclass(frozen=True) @@ -300,7 +291,8 @@ class PythonCheckJobRejectionTestCase: @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str - schedule: Schedule + schedule: str + expected_timezone: str expected_selection: tuple[str, ...] expected_controls: dict[str, object] = field( default_factory=lambda: { @@ -313,18 +305,10 @@ class SnapshotScheduleTestCase: @dataclass(frozen=True) class SnapshotDeclarationRejectionTestCase: description: str - job_description: str = "Orders." - select: str = "orders" - cron: str = "0 2 * * *" - timezone: str = "UTC" - settings: dict[str, object] = field(default_factory=dict) - concurrency: int | None = None + expected_message: str + fields: dict[str, str] = field(default_factory=dict) + tables: str = "" copies: int = 1 - expected_message: str = "" - schedule: Schedule | None = None - owner: str | None = None - overlap: Any = "queue" - on_failure: OnFailure | None = None @dataclass(frozen=True) @@ -332,7 +316,6 @@ class SnapshotDiscoveryTestCase: description: str files: dict[str, str] expected_jobs: tuple[str, ...] - declarations: tuple[Job | JobGroup | CalendarSchedule | Sensor, ...] | None = None expected_groups: dict[str, dict[str, list[str]]] = field(default_factory=dict) expected_sensors: dict[str, dict[str, object]] = field(default_factory=dict) expected_schedules: tuple[str, ...] = () diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index d114213a6..8c8165381 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -46,7 +46,6 @@ CompiledProjectView, ) from sqlbuild.orchestration import compile_snapshot -from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files if TYPE_CHECKING: @@ -429,6 +428,22 @@ def expected_summary(name: str) -> dict[str, object]: WAFFLE_SHOP_NATIVE_MODELS: int = 13 +def duckshop_orchestration(fixture: str) -> dict[str, str]: + """A DuckShop orchestration fixture tree as project files below `orchestration/`.""" + root: Path = Path("benchmarks/duckshop/edge_cases/orchestration") / fixture + paths: list[Path] = sorted([*root.rglob("*.py"), *root.rglob("*.toml")]) + return { + f"orchestration/{path.relative_to(root).as_posix()}": path.read_text() for path in paths + } + + +def edited_duckshop_orchestration(fixture: str, path: str, old: str, new: str) -> dict[str, str]: + """A DuckShop orchestration fixture with `old` replaced by `new` in the file at `path`.""" + files: dict[str, str] = duckshop_orchestration(fixture) + assert old in files[path], f"{old!r} is not in {path}" + return files | {path: files[path].replace(old, new)} + + def copy_project(*, fixtures: tuple[str, ...], files: dict[str, str], project_dir: Path) -> Path: """Copy each fixture into `project_dir`, then write `files` over them.""" @@ -754,10 +769,22 @@ def published_texts(test_case: SnapshotExpressionRangeTestCase, tmp_path: Path) def python_pipeline_snapshot( - tmp_path: Path, declarations: list[Job | JobGroup] + tmp_path: Path, jobs: dict[str, str], groups: dict[str, tuple[str, ...]] | None = None ) -> NativeOrchestrationSnapshot: - """Publish the Python pipeline project with the given job declarations.""" - write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES) + """Publish the Python pipeline project with TOML jobs, by name and selector, and groups.""" + files: dict[str, str] = { + f"orchestration/jobs/{name}.toml": ( + f'name = {json.dumps(name)}\ndescription = "Orders."\nselect = {json.dumps(select)}\n' + ) + for name, select in jobs.items() + } | { + f"orchestration/job_groups/{name}.toml": ( + f'name = {json.dumps(name)}\ndescription = "Nightly orders."\n' + f"jobs = {json.dumps(list(members))}\n" + ) + for name, members in (groups or {}).items() + } + write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES | files) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered, @@ -768,6 +795,5 @@ def python_pipeline_snapshot( project=project, metadata_json=json.dumps(METADATA), project_dir=tmp_path, - declarations=declarations, discovered_inputs=discovered, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 904cdb86d..d6321b2d2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -117,6 +117,18 @@ class WorkerExecutionTestCase: expected_node_counts: dict[str, int] | None = None expected_origin: Literal["schedule", "sensor"] = "schedule" files: dict[str, str] = field(default_factory=dict) + # The scheduled job written before `files`; cases that declare their own jobs set it to {}. + default_job_files: dict[str, str] = field( + default_factory=lambda: { + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", ' + 'starts_at = "2000-01-01T00:00:00Z" }\n' + ), + } + ) files_after_publication: dict[str, str] = field(default_factory=dict) store_setup: str = "" store_edit: Callable[[sqlite3.Connection], None] = lambda _: None diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py index b8121c3fa..3117d3bfc 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/docker/compose/helpers.py @@ -202,8 +202,8 @@ def build_projects(self) -> list[str]: for version, sql in cases.items(): project: Path = self.directory / version (project / "models").mkdir(parents=True) - (project / "orchestration").mkdir() - for relative in ("sqlbuild_project.toml", "orchestration/orders.py"): + (project / "orchestration/jobs").mkdir(parents=True) + for relative in ("sqlbuild_project.toml", "orchestration/jobs/refresh_orders_job.toml"): shutil.copyfile(Path("deploy/project") / relative, project / relative) (project / "models/orders.sql").write_text( "MODEL (description 'Example orders.', materialized table);\n" + sql + "\n" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index d29d97e19..4007281a0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -61,6 +61,7 @@ from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec from sqlbuild.microbatches.constants import MICROBATCH_COLUMNS from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchWriteResult +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import duckshop_orchestration from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ClaimedWorker, ScheduledWorker, @@ -123,7 +124,13 @@ def open_legacy_warehouse(project: Path, adapter_name: str, connection_key: str) f'[connections.local]\n{connection_key} = "warehouse.db"\n' '[targets.development]\nschema = "main"\n' ), - "orchestration/orders.py": 'raise AssertionError("unexpected declaration import")\n', + "orchestration/calendars/unexpected_calendar.py": ( + "from sqlbuild.orchestration import calendar\n" + 'raise AssertionError("unexpected declaration import")\n' + "@calendar(name='unexpected_calendar', description='Never imported.', " + "horizon='1d')\n" + "def unexpected(window):\n return []\n" + ), }, ) adapter: Any = resolve_adapter(adapter_name=adapter_name) @@ -802,17 +809,14 @@ def scheduled_worker( "models/orders.sql": ( "MODEL (description 'Orders.', materialized table);\n" + test_case.model_sql + "\n" ), - "orchestration/orders.py": ( - "import os\nfrom sqlbuild.orchestration import job, schedule\n" + "python/environment_guard.py": ( + "import os\n" "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" " assert 'SQLBUILD_PLATFORM_WEBHOOK_URL' not in os.environ\n" " assert 'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL' not in os.environ\n" - 'annual_orders = job("annual_orders", select="orders", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' ), } + | test_case.default_job_files | test_case.files, ) test_case.warehouse_setup(project / "warehouse.duckdb") @@ -907,7 +911,7 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) - assert run["node_counts_json"] == ( test_case.expected_node_counts or {test_case.expected_status: test_case.expected_node_count} ), log.read_text() - assert run["job_key"] == "annual_orders" + assert run["job_key"] == "annual_orders_job" runs_url: str = f"{api.url}/api/locations/{api.location_id}/targets/development/runs" headers: dict[str, str] = {"Authorization": f"Bearer {api.bearer}"} with urlopen( @@ -1250,11 +1254,13 @@ def scheduler_crash_history(directory: Path, fixture: dict[str, Any]) -> dict[st start: datetime = datetime.now(UTC).replace(second=0, microsecond=0) - timedelta( minutes=fixture["backlog_minutes"] ) - declarations: str = "".join( - f'{name} = job("{name}", select="orders", description="Build {name}.", ' - f'schedule=schedule(cron="* * * * *", timezone="UTC"))\n' + jobs: dict[str, str] = { + f"orchestration/jobs/{name}.toml": ( + f'name = "{name}"\ndescription = "Build {name}."\nselect = "orders"\n' + 'schedule = { cron = "* * * * *", timezone = "UTC" }\n' + ) for name in fixture["jobs"] - ) + } write_files( project_dir=project, files={ @@ -1266,9 +1272,8 @@ def scheduler_crash_history(directory: Path, fixture: dict[str, Any]) -> dict[st "models/orders.sql": ( "MODEL (description 'Orders.', materialized table);\nSELECT 1 AS order_id\n" ), - "orchestration/orders.py": "from sqlbuild.orchestration import job, schedule\n" - + declarations, - }, + } + | jobs, ) executable: str = str(Path(sys.executable).with_name("sqb")) control_store: Path = directory / "control/.sqlbuild/control.sqlite3" @@ -1452,9 +1457,10 @@ def tagged_model(name: str, tags: str) -> str: CHANGED_FIXTURE: Path = Path("benchmarks/duckshop/edge_cases/changed_builds") CHANGED_JOB: str = ( - "from sqlbuild.orchestration import job\n" - "changed_orders = job('changed_orders', description='Build what changed.', " - "select='stg_orders stg_customers customer_orders', settings={'changed': True})\n" + 'name = "changed_orders_job"\n' + 'description = "Build what changed."\n' + 'select = "stg_orders stg_customers customer_orders"\n' + "\n[settings]\nchanged = true\n" ) @@ -1475,7 +1481,10 @@ def changed_job_files(project_toml: str) -> dict[str, str]: return { str(path.relative_to(CHANGED_FIXTURE)): path.read_text() for path in sorted(CHANGED_FIXTURE.glob("*/*")) - } | {"orchestration/orders.py": CHANGED_JOB, "sqlbuild_project.toml": project_toml} + } | { + "orchestration/jobs/changed_orders_job.toml": CHANGED_JOB, + "sqlbuild_project.toml": project_toml, + } def request_job_run(worker: ScheduledWorker, job: str) -> dict[str, Any]: @@ -1600,9 +1609,7 @@ def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: "models/orders.sql": Path( "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.sql" ).read_text(), - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" - ).read_text(), + **duckshop_orchestration("loaded_orders"), } @@ -1616,9 +1623,7 @@ def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: "models/orders.sql": Path( "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql" ).read_text(), - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" - ).read_text(), + **duckshop_orchestration("loaded_orders"), } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index d269fe16c..f761f6232 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -62,7 +62,7 @@ expected_customers_tables=0, # DuckDB has no session warehouse, so the CLI refuses the job's warehouse: the # published job reached the CLI instead of being refused by the worker. - job_settings="settings={'warehouse': 'ANALYST_XS'}, ", + job_settings='\n[settings]\nwarehouse = "ANALYST_XS"\n', expected_scheduled_status="failed", expected_worker_log="error[C260]: --warehouse ANALYST_XS was given", ), @@ -84,12 +84,12 @@ def test_given_an_ad_hoc_command_when_checked_and_launched_then_the_worker_runs_ ) } | { - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", select="orders", ' - f'description="Build orders.", {test_case.job_settings}' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", ' + 'starts_at = "2000-01-01T00:00:00Z" }\n' + test_case.job_settings ) }, ) @@ -100,7 +100,7 @@ def test_given_an_ad_hoc_command_when_checked_and_launched_then_the_worker_runs_ server_arguments=("--password-origin", "https://control.example.com"), ) as worker: scheduled: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert scheduled["status"] == test_case.expected_scheduled_status, worker.log.read_text() assert scheduled["status_reason"] != WAREHOUSE_REFUSAL, "published jobs are not refused" @@ -334,7 +334,7 @@ def test_given_an_ad_hoc_verb_when_the_checkout_changes_after_verification_then_ worker_case, server_arguments=("--password-origin", "https://control.example.com"), ) as worker: - _ = wait_for_run(worker.api, worker.process, worker.log, job_key="annual_orders") + _ = wait_for_run(worker.api, worker.process, worker.log, job_key="annual_orders_job") admin: dict[str, Any] = provision_person( tmp_path / "control", "create-user", "orders-admin" ) @@ -475,7 +475,7 @@ def test_given_ad_hoc_loads_and_variables_when_launched_then_only_permitted_peop server_arguments=("--password-origin", "https://control.example.com"), ) as worker: scheduled: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert scheduled["status"] == "success", worker.log.read_text() admin: dict[str, Any] = provision_person( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py index 512db9188..b8c6978b0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_backfill.py @@ -73,13 +73,22 @@ def test_given_explicit_backfill_bounds_when_running_and_rerunning_a_group_then_ files=test_case.files | { "sources/raw_orders.yml": "sources:\n - name: raw_orders\n description: Synthetic order windows.\n schema: main\n table: raw_orders\n", - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, job_group\n" - "orders = job('windowed_orders', description='Windowed orders.', select='orders', " - "settings={'start_cursor': '0', 'end_cursor': '1'})\n" - "annual_orders = job_group('annual_orders', description='Windowed group.', jobs=[orders])\n" + "orchestration/jobs/windowed_orders_job.toml": ( + 'name = "windowed_orders_job"\n' + 'description = "Windowed orders."\n' + 'select = "orders"\n' + "\n" + "[settings]\n" + 'start_cursor = "0"\n' + 'end_cursor = "1"\n' + ), + "orchestration/job_groups/annual_orders_job_group.toml": ( + 'name = "annual_orders_job_group"\n' + 'description = "Windowed group."\n' + 'jobs = ["windowed_orders_job"]\n' ), }, + default_job_files={}, ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: @@ -95,7 +104,7 @@ def test_given_explicit_backfill_bounds_when_running_and_rerunning_a_group_then_ "SQLBUILD_SCHEDULER_TEST_STORE", "request-run", worker.api.location_id, - "annual_orders", + "annual_orders_job_group", "--target", "development", "--run-id", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py index c5325d6f5..829ce52e7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_calendar.py @@ -13,7 +13,10 @@ import duckdb import pytest -from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + TYPE_PROOF_FILES, + duckshop_orchestration, +) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( CalendarRefreshTestCase, CalendarRejectionTestCase, @@ -56,7 +59,9 @@ "positive fixed duration", horizon="999999999999999999999999999d", ), - CalendarRejectionTestCase("unknown zone", "[]", "IANA timezone", timezone="Unknown/Orders"), + CalendarRejectionTestCase( + "unknown zone", "[]", "is not an IANA time zone", timezone="Unknown/Orders" + ), ], ids=lambda case: case.description, ) @@ -71,11 +76,16 @@ def test_given_invalid_calendar_results_when_publishing_through_the_cli_then_val "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + '\n[targets.development]\nschema = "main"\n', - "orchestration/orders.py": ( + "orchestration/jobs/order_reporting_job.toml": ( + 'name = "order_reporting_job"\n' + 'description = "Order reporting."\n' + 'select = "orders"\n' + f'schedule = {{ calendar = "deliveries_calendar", timezone = {json.dumps(test_case.timezone)} }}\n' + ), + "orchestration/calendars/deliveries_calendar.py": ( "from datetime import datetime, timedelta\nfrom itertools import repeat\n" - "from sqlbuild.orchestration import calendar_schedule, job\n" - "orders = job('order_reporting', description='Order reporting.', select='orders')\n" - f"@calendar_schedule(job=orders, description='Delivery times.', horizon={test_case.horizon!r}, timezone={test_case.timezone!r})\n" + "from sqlbuild.orchestration import calendar\n" + f"@calendar(name='deliveries_calendar', description='Delivery times.', horizon={test_case.horizon!r})\n" f"def deliveries(window):\n return {test_case.expression}\n" ), }, @@ -134,18 +144,23 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + '\n[targets.development]\nschema = "main"\n', - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py" - ) - .read_text() - .replace("America/New_York", test_case.expected_timezone) - + ( - "\n@calendar_schedule(job=orders, description='Other target.', horizon='8s', timezone='UTC', targets=('production',))\n" + **{ + path: text.replace("America/New_York", test_case.expected_timezone) + for path, text in duckshop_orchestration("calendar_orders").items() + }, + "orchestration/jobs/production_reporting_job.toml": ( + 'name = "production_reporting_job"\n' + 'description = "Other target."\n' + 'select = "orders"\n' + 'targets = ["production"]\n' + 'schedule = { calendar = "excluded_calendar", timezone = "UTC" }\n' + ), + "orchestration/calendars/excluded_calendar.py": ( + "from sqlbuild.orchestration import calendar\n" + "@calendar(name='excluded_calendar', description='Other target.', horizon='8s')\n" "def excluded_calendar(window):\n raise ValueError('excluded calendar must not evaluate')\n" ), - "orchestration/platform_orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" - ).read_text(), + **duckshop_orchestration("platform_orders"), "providers/fulfillment.py": ( "import json, os\nfrom pathlib import Path\nfrom datetime import timedelta, timezone\n" "from sqlbuild.provider.classes.provider import Provider\n" @@ -156,7 +171,7 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr " def record(self, value):\n" " with Path(os.environ['SQLBUILD_CALENDAR_EVALUATIONS']).open('a') as out:\n" " out.write(json.dumps(value | {'pid': os.getpid()}) + '\\n')\n" - " def upcoming(self, window):\n" + " def calendar(self, window):\n" " self.record({'phase': 'evaluate', 'start': window.start.isoformat(), 'end': window.end.isoformat(), 'zone': window.start.tzinfo.key})\n" " data = json.loads(Path(os.environ['SQLBUILD_FULFILLMENT_DATA']).read_text())\n" " if data.get('fail'):\n raise ValueError('synthetic calendar unavailable')\n" @@ -189,23 +204,23 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr assert first["nodes"] == second["nodes"] assert second["changes"] == [] assert first["documents"]["triggers"].keys() == { - "fulfillment_calendar", - "overdue_orders", + "order_reporting_job", + "overdue_orders_sensor", } - assert first["documents"]["triggers"]["fulfillment_calendar"]["declared_at"] == { - "path": "orchestration/orders.py", - "line": 8, + assert first["documents"]["triggers"]["order_reporting_job"]["declared_at"] == { + "path": "orchestration/jobs/order_reporting_job.toml", + "line": 2, } with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as control: stored, next_due = control.execute( "SELECT definition_json, next_due_at FROM sqb_triggers JOIN sqb_trigger_state USING (trigger_id) WHERE kind = 'schedule'" ).fetchone() - original_times: list[str] = first["documents"]["triggers"]["fulfillment_calendar"][ + original_times: list[str] = first["documents"]["triggers"]["order_reporting_job"][ "schedule" ]["times"] assert json.loads(stored)["schedule"]["times"] == sorted( original_times - + second["documents"]["triggers"]["fulfillment_calendar"]["schedule"]["times"] + + second["documents"]["triggers"]["order_reporting_job"]["schedule"]["times"] ) assert datetime.fromisoformat(next_due) == datetime.fromisoformat(original_times[0]) observations: list[dict[str, Any]] = [ @@ -220,9 +235,9 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr start: datetime = datetime.fromisoformat(evaluation["start"]) end: datetime = datetime.fromisoformat(evaluation["end"]) assert end - start == timedelta(seconds=8) - schedule: dict[str, Any] = snapshot["documents"]["triggers"][ - "fulfillment_calendar" - ]["schedule"] + schedule: dict[str, Any] = snapshot["documents"]["triggers"]["order_reporting_job"][ + "schedule" + ] assert schedule["kind"] == "calendar" assert [datetime.fromisoformat(value) for value in schedule["times"]] == [ start + timedelta(seconds=offset) for offset in [0, 3] @@ -246,8 +261,7 @@ def test_given_a_served_calendar_when_time_passes_and_a_provider_fails_then_refr third: dict[str, Any] = wait_for_generation(api, "development", 3, process, log) assert "CAST(19 AS INTEGER)" in third["nodes"]["model:stg_orders"]["raw_sql"] assert ( - len(third["documents"]["triggers"]["fulfillment_calendar"]["schedule"]["times"]) - == 1 + len(third["documents"]["triggers"]["order_reporting_job"]["schedule"]["times"]) == 1 ) assert [ json.loads(line)["generation"] for line in publications.read_text().splitlines() @@ -277,16 +291,29 @@ def test_given_a_published_calendar_when_a_real_worker_compiles_then_it_does_not test_case, files=test_case.files | { - "orchestration/orders.py": ( - "from pathlib import Path\nfrom sqlbuild.orchestration import calendar_schedule, job, schedule\n" - "orders = job('order_reporting', description='Order reporting.', select='orders', schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" - "@calendar_schedule(job=orders, description='Upcoming deliveries.', horizon='1d', timezone='UTC')\n" + "orchestration/jobs/order_reporting_job.toml": ( + 'name = "order_reporting_job"\n' + 'description = "Order reporting."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", ' + 'starts_at = "2000-01-01T00:00:00Z" }\n' + ), + "orchestration/jobs/deliveries_job.toml": ( + 'name = "deliveries_job"\n' + 'description = "Upcoming deliveries."\n' + 'select = "orders"\n' + 'schedule = { calendar = "deliveries_calendar", timezone = "UTC" }\n' + ), + "orchestration/calendars/deliveries_calendar.py": ( + "from pathlib import Path\nfrom sqlbuild.orchestration import calendar\n" + "@calendar(name='deliveries_calendar', description='Upcoming deliveries.', horizon='1d')\n" "def deliveries(window):\n" f" marker = Path({str(evaluated)!r})\n" " if marker.exists():\n raise ValueError('calendar evaluated inside a run worker')\n" " marker.write_text('evaluated')\n return []\n" - ) + ), }, + default_job_files={}, ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: assert evaluated.read_text() == "evaluated" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py index c6a00930b..10f655d91 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_changed.py @@ -80,7 +80,9 @@ def test_given_a_changed_job_when_inputs_change_between_runs_then_only_their_consumers_run( test_case: ChangedJobTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - test_case = replace(test_case, files=changed_job_files(test_case.project_toml)) + test_case = replace( + test_case, files=changed_job_files(test_case.project_toml), default_job_files={} + ) observed: list[tuple[str, frozenset[str], dict[str, int]]] = [] with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: warehouse: Path = worker.project / test_case.warehouse_file @@ -95,7 +97,7 @@ def test_given_a_changed_job_when_inputs_change_between_runs_then_only_their_con time.sleep(ORDER_SKEW_SECONDS) for sql in mutation: test_case.execute(warehouse, sql) - run: dict[str, Any] = request_job_run(worker, "changed_orders") + run: dict[str, Any] = request_job_run(worker, "changed_orders_job") assert run["status"] == "success", worker.log.read_text() started: set[tuple[Any, ...]] = set( test_case.execute( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py index e6da7b8e1..6b543dd1c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_custom_sensor.py @@ -11,6 +11,7 @@ import duckdb import pytest +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import duckshop_orchestration from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( WorkerExecutionTestCase, ) @@ -43,9 +44,7 @@ [(83,)], expected_origin="sensor", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/custom_orders.py" - ).read_text(), + **duckshop_orchestration("custom_orders"), "providers/fulfillment.py": ( "import json, os, time\nfrom pathlib import Path\n" "from sqlbuild.orchestration import RunRequest, SkipReason\n" @@ -71,6 +70,7 @@ " return [RunRequest(run_key='orders-1'), RunRequest(run_key='orders-1')]\n" ), }, + default_job_files={}, ) ], ids=lambda case: case.description, @@ -168,7 +168,7 @@ def test_given_a_custom_polling_sensor_when_a_callback_fails_or_times_out_then_a os.kill(timed_call["pid"], 0) data.write_text(json.dumps({"mode": "ready"})) destination: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert destination["status"] == test_case.expected_status, log.read_text() for browser in [admin, session]: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py index 5c85fca19..69a5bcc7f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_dev.py @@ -17,7 +17,10 @@ import duckdb import pytest -from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import TYPE_PROOF_FILES +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + TYPE_PROOF_FILES, + duckshop_orchestration, +) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( DevelopmentTestCase, SnapshotApi, @@ -68,11 +71,12 @@ def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paus ) -> None: project: Path = tmp_path / "project" pidfile: Path = tmp_path / "interpreter.pid" - declaration: str = ( - "import os\nfrom pathlib import Path\n" + probe: str = ( + "import os\nfrom pathlib import Path\nfrom sqlbuild.orchestration import calendar\n" "assert 'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL' not in os.environ\n" f"Path({str(pidfile)!r}).write_text(str(os.getpid()))\n" - + Path("benchmarks/duckshop/edge_cases/orchestration/development_orders.py").read_text() + "@calendar(name='interpreter_probe_calendar', description='Probe.', horizon='1d')\n" + "def interpreter_probe(window):\n return []\n" ) write_files( project_dir=project, @@ -86,7 +90,8 @@ def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paus "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( "SELECT order_id, amount", "SELECT order_id, CAST(amount AS BIGINT) AS amount" ), - "orchestration/orders.py": declaration, + **duckshop_orchestration("development_orders"), + "orchestration/calendars/interpreter_probe_calendar.py": probe, }, ) executable: str = str(Path(sys.executable).with_name("sqb")) @@ -137,7 +142,7 @@ def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paus "admin", "request-run", identity, - "annual_orders", + "annual_orders_job", "--target", test_case.target, "--run-id", @@ -182,9 +187,12 @@ def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paus with urlopen(f"{scope}/changes", timeout=5) as stream: assert stream.readline() == b"event: changed\n" assert json.loads(stream.readline().removeprefix(b"data: ")) == {"topic": "all"} - _ = (project / "orchestration/orders.py").write_text( - declaration - + '\nadditional_orders = job("additional_orders", description="Additional orders.", select="orders", schedule=annual)\n' + _ = (project / "orchestration/jobs/additional_orders_job.toml").write_text( + 'name = "additional_orders_job"\n' + 'description = "Additional orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 1 1 *", timezone = "UTC", ' + 'starts_at = "2099-01-01T00:00:00Z" }\n' ) assert f"snapshot:{identity}" in islice( ( @@ -198,7 +206,7 @@ def test_given_development_when_reloaded_and_restarted_then_local_scope_and_paus statuses: dict[str, str] = { entry["job_key"]: entry["status"] for entry in json.load(response) } - assert statuses == {"annual_orders": "paused", "additional_orders": "running"} + assert statuses == {"annual_orders_job": "paused", "additional_orders_job": "running"} stop_location(restarted, int(pidfile.read_text()), log) with sqlite3.connect(control) as connection: assert connection.execute("SELECT count(*) FROM sqb_development_locations").fetchone() == ( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py index 69792e82d..f0e16d790 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_docker_worker.py @@ -13,6 +13,7 @@ import pytest +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import duckshop_orchestration from tests.integration.src.sqlbuild.compiler.compiled_project.publication import helpers from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ScheduledWorker, @@ -50,18 +51,11 @@ [(37,)], launcher="docker", files={ - "orchestration/orders.py": ( - "import os\nfrom sqlbuild.orchestration import job, schedule\n" - "if 'SQLBUILD_RUN_TOKEN' in os.environ:\n" - " assert os.getenv('SQLBUILD_WORKER_TEST_VALUE') == 'orders-context'\n" - " for name in ('SQLBUILD_LOCATION_TOKEN', 'SQLBUILD_CONTROL_DATABASE_URL', " - "'SQLBUILD_SCHEDULER_TEST_STORE', 'SQLBUILD_PLATFORM_WEBHOOK_URL', " - "'SQLBUILD_SECURITY_AUDIT_WEBHOOK_URL'):\n" - " assert name not in os.environ, name\n" - 'annual_orders = job("annual_orders", select="orders", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ) }, ), @@ -72,11 +66,8 @@ "success", [(41,)], launcher="docker", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py" - ).read_text() - }, + files={**duckshop_orchestration("container_resources_orders")}, + default_job_files={}, launcher_settings={ "run": {"memory": "768m", "cpus": 2, "pids": 256, "max_memory": "1g"} }, @@ -89,11 +80,8 @@ "success", [(43,)], launcher="docker", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/container_resources_orders.py" - ).read_text() - }, + files={**duckshop_orchestration("container_resources_orders")}, + default_job_files={}, expected_host_config={"Memory": 768 << 20, "NanoCpus": 500_000_000}, ), WorkerExecutionTestCase( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py index 79e8367e7..995ca03d7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_job_subsets.py @@ -32,11 +32,13 @@ files={ "models/customers.sql": "MODEL (description 'Customers.', materialized table);\nSELECT 7 AS customer_id\n", "models/products.sql": "MODEL (description 'Products.', materialized table);\nSELECT 3 AS product_id\n", - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job\n" - "nightly = job('nightly', description='Orders and customers.', select='orders customers')\n" + "orchestration/jobs/nightly_job.toml": ( + 'name = "nightly_job"\n' + 'description = "Orders and customers."\n' + 'select = "orders customers"\n' ), }, + default_job_files={}, ) ], ids=lambda case: case.description, @@ -53,7 +55,7 @@ def test_given_a_published_job_when_an_operator_excludes_a_model_then_only_the_r "SQLBUILD_SCHEDULER_TEST_STORE", "request-run", worker.api.location_id, - "nightly", + "nightly_job", "--target", "development", "--reason", @@ -82,7 +84,7 @@ def test_given_a_published_job_when_an_operator_excludes_a_model_then_only_the_r worker.api, worker.process, worker.log, run_id=run_id ) assert completed["status"] == test_case.expected_status, worker.log.read_text() - assert completed["job_key"] == "nightly" + assert completed["job_key"] == "nightly_job" with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: assert warehouse.sql( "SELECT table_name FROM duckdb_tables() " diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py index 73c839748..0feaabe88 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_local_recovery.py @@ -13,6 +13,7 @@ import pytest +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import duckshop_orchestration from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( ScheduledWorker, WorkerExecutionTestCase, @@ -113,10 +114,9 @@ def test_given_a_running_local_worker_when_scheduler_dies_then_restart_adopts_an launcher="uncooperative", concurrency_limits={"orders_limit": 1}, files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/local_recovery_orders.py" - ).read_text(), + **duckshop_orchestration("local_recovery_orders"), }, + default_job_files={}, store_setup=Path( "benchmarks/duckshop/edge_cases/orchestration/local_recovery_publication_fault.sql" ).read_text(), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py index 780abc474..5a452a73a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_person_journey.py @@ -93,7 +93,7 @@ def test_given_an_initialized_installation_when_people_run_and_reload_then_permi "run_id": str(uuid4()), "location_id": worker.api.location_id, "target": "development", - "job_key": "annual_orders", + "job_key": "annual_orders_job", "reason": "Refresh order totals", "priority": 0, } @@ -349,7 +349,7 @@ def test_given_person_run_controls_when_scopes_change_then_frozen_execution_and_ "run_id": str(uuid4()), "location_id": worker.api.location_id, "target": "development", - "job_key": "annual_orders", + "job_key": "annual_orders_job", "reason": "Build order totals", "priority": 0, } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py index e79a6f48f..349e372f5 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py @@ -43,7 +43,6 @@ 'name = "orders"\nadapter = "duckdb"\n[connections.local]\ndatabase = "warehouse.db"\n' '[targets.development]\nschema = "main"\n' ), - "orchestration/orders.py": "", "seeds/raw_orders.csv": "id,ordered_at\n1,2026-01-01 00:00:00\n2,2026-01-01 01:00:00\n", "seeds/raw_orders.yml": dedent( """ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py index 0215796c8..47eebd80a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py @@ -167,7 +167,7 @@ def test_given_a_job_loading_an_intermediate_loader_when_a_role_runs_it_then_the "run_id": str(uuid4()), "location_id": api.location_id, "target": "development", - "job_key": "annual_orders", + "job_key": "annual_orders_job", "reason": "Refresh shipments", "priority": 0, } diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index bc538509f..a4e413d3e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -62,14 +62,22 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical test_case: SnapshotPublicationTestCase, tmp_path: Path ) -> None: project_dir: Path = tmp_path / "project" - files: dict[str, str] = TYPE_PROOF_FILES | { - "orchestration/edge_cases/definitions.py": Path( - "benchmarks/duckshop/project/orchestration/edge_cases/definitions.py" - ).read_text(), - "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( - "MODEL (", "MODEL (tags ['edge_case'],", 1 - ), - } + duckshop: Path = Path("benchmarks/duckshop/project") + files: dict[str, str] = ( + TYPE_PROOF_FILES + | { + path.relative_to(duckshop).as_posix(): path.read_text() + for path in [ + *(duckshop / "orchestration").rglob("*.py"), + *(duckshop / "orchestration").rglob("*.toml"), + ] + } + | { + "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( + "MODEL (", "MODEL (tags ['edge_case'],", 1 + ), + } + ) write_files(project_dir=project_dir, files=files) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) project: CompiledProjectView = build_compiled_project( @@ -130,34 +138,36 @@ def test_given_retained_compiler_data_when_publishing_then_http_updates_atomical by_name: dict[str, dict[str, Any]] = {trigger["name"]: trigger for trigger in triggers} assert len(triggers) == 4 assert set(by_name) == { - "edge_snapshot_orders", - "edge_platform_orders", - "edge_job_outcomes", - "edge_group_outcomes", + "edge_snapshot_orders_job", + "edge_platform_orders_sensor", + "edge_job_outcomes_sensor", + "edge_group_outcomes_sensor", } - scheduled: dict[str, Any] = by_name["edge_snapshot_orders"] + scheduled: dict[str, Any] = by_name["edge_snapshot_orders_job"] assert scheduled["status"] == "paused" assert scheduled["definition_json"]["starts_at"] == "2026-07-02T19:47:23.456789Z" assert scheduled["definition_json"]["default_status"] == "paused" - assert by_name["edge_platform_orders"]["status"] == "paused" - assert by_name["edge_platform_orders"]["definition_json"]["default_status"] == "paused" - assert by_name["edge_platform_orders"]["definition_json"]["when"] == { + assert by_name["edge_platform_orders_sensor"]["status"] == "paused" + assert ( + by_name["edge_platform_orders_sensor"]["definition_json"]["default_status"] == "paused" + ) + assert by_name["edge_platform_orders_sensor"]["definition_json"]["when"] == { "kind": "platform_event", "kinds": ["tick_overdue", "location_broken", "launch_failed"], } - assert by_name["edge_job_outcomes"]["definition_json"]["when"] == { + assert by_name["edge_job_outcomes_sensor"]["definition_json"]["when"] == { "kind": "run_event", - "job_key": "edge_snapshot_orders", + "job_key": "edge_snapshot_orders_job", "kinds": ["warn", "partial"], } - assert by_name["edge_group_outcomes"]["definition_json"]["when"] == { + assert by_name["edge_group_outcomes_sensor"]["definition_json"]["when"] == { "kind": "run_event", - "job_key": "edge_snapshot_group", + "job_key": "edge_snapshot_job_group", "kinds": ["completed"], "executed_successfully": ["model:orders"], } - assert by_name["edge_job_outcomes"]["status"] == "paused" - assert by_name["edge_group_outcomes"]["status"] == "paused" + assert by_name["edge_job_outcomes_sensor"]["status"] == "paused" + assert by_name["edge_group_outcomes_sensor"]["status"] == "paused" @pytest.mark.parametrize( @@ -189,9 +199,8 @@ def test_given_a_code_location_when_publishing_from_cli_then_compilation_isolate "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job\n" - 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' + "orchestration/jobs/daily_orders_job.toml": ( + 'name = "daily_orders_job"\ndescription = "Build orders."\nselect = "orders"\n' ), } write_files(project_dir=project_dir, files=files) @@ -238,7 +247,7 @@ def test_given_a_code_location_when_publishing_from_cli_then_compilation_isolate assert "Location snapshot published." in first_output.err initial: dict[str, Any] = current_snapshot(api, test_case.target) assert frozenset(initial["nodes"]) == test_case.expected_nodes - assert initial["documents"]["jobs"]["daily_orders"]["selection"] == ["model:orders"] + assert initial["documents"]["jobs"]["daily_orders_job"]["selection"] == ["model:orders"] assert not (project_dir / ".sqlbuild/control.sqlite3").exists() assert not (project_dir / "warehouse.duckdb").exists() with sqlite3.connect(tmp_path / "control/.sqlbuild/control.sqlite3") as connection: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py index 84c65833b..e2562c2f3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_reload.py @@ -58,18 +58,25 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ ) -> None: project: Path = tmp_path / "project" compiler_pid: Path = tmp_path / "compiler.pid" - declaration: str = ( - "from sqlbuild.orchestration import job\n" - "import os\nfrom pathlib import Path\n" - f"Path({str(compiler_pid)!r}).write_text(str(os.getpid()))\n" - 'print("Orders declaration loaded.")\n' - 'daily_orders = job("daily_orders", select="orders", description="Build orders.")\n' - ) + calendar: str = "orchestration/calendars/compiler_probe_calendar.py" + orchestration: dict[str, str] = { + calendar: ( + "from sqlbuild.orchestration import calendar\n" + "import os\nfrom pathlib import Path\n" + f"Path({str(compiler_pid)!r}).write_text(str(os.getpid()))\n" + 'print("Orders declaration loaded.")\n' + "@calendar(name='compiler_probe_calendar', description='Probe.', horizon='1d')\n" + "def compiler_probe(window):\n return []\n" + ), + "orchestration/jobs/daily_orders_job.toml": ( + 'name = "daily_orders_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + } files: dict[str, str] = TYPE_PROOF_FILES | { "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + f"\n[connections.local]\ndatabase = {json.dumps(test_case.database)}\n" + f'\n[targets.{json.dumps(test_case.target)}]\nschema = "main"\n', - "orchestration/orders.py": declaration, + **orchestration, "sinks/orders.py": 'print("Loading orders sink", flush=True)\n', } write_files(project_dir=project, files=files) @@ -134,20 +141,27 @@ def test_given_a_served_location_when_inputs_change_or_restart_then_generations_ assert int(compiler_pid.read_text()) == first_interpreter assert "CAST(14 AS INTEGER)" in recovered["nodes"]["model:stg_orders"]["raw_sql"] - _ = (project / "orchestration/orders.py").write_text("invalid Python declaration !\n") + _ = (project / calendar).write_text("invalid Python declaration !\n") wait_for_log(process, log, "Location reload failed; retaining last good snapshot.") _ = wait_for_location_health( api, test_case.target, {"kind": "compile_failed", "generation": 4}, process, log ) assert current_snapshot(api, test_case.target)["record"] == recovered["record"] - _ = (project / "orchestration/orders.py").write_text( - declaration.replace("Build orders.", "Refresh order totals.") + write_files( + project_dir=project, + files=orchestration + | { + "orchestration/jobs/daily_orders_job.toml": orchestration[ + "orchestration/jobs/daily_orders_job.toml" + ].replace("Build orders.", "Refresh order totals.") + }, ) third: dict[str, Any] = wait_for_generation(api, test_case.target, 4, process, log) replacement_interpreter: int = int(compiler_pid.read_text()) assert replacement_interpreter != first_interpreter assert ( - third["documents"]["jobs"]["daily_orders"]["description"] == "Refresh order totals." + third["documents"]["jobs"]["daily_orders_job"]["description"] + == "Refresh order totals." ) _ = subprocess.run( [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py index 4682b6bb2..9c5caf7b7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_rerun.py @@ -59,20 +59,29 @@ def test_given_a_partial_group_when_retrying_from_failure_then_successful_nodes_ "MODEL (description 'Independent inventory.', materialized table);\n" "SELECT 29 AS amount\n" ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, job_group, schedule\n" - "staging = job('staging', description='Stage orders.', select='orders broken_orders')\n" - "marts = job('marts', description='Build totals.', select='totals')\n" - "inventory = job('stock', description='Build inventory.', select='inventory')\n" - "annual_orders = job_group('annual_orders', description='Retry boundaries.', " - "jobs=[inventory, marts, staging], schedule=schedule(cron='0 0 * * *', " - "timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "orchestration/jobs/staging_job.toml": ( + 'name = "staging_job"\n' + 'description = "Stage orders."\n' + 'select = "orders broken_orders"\n' + ), + "orchestration/jobs/marts_job.toml": ( + 'name = "marts_job"\ndescription = "Build totals."\nselect = "totals"\n' + ), + "orchestration/jobs/stock_job.toml": ( + 'name = "stock_job"\ndescription = "Build inventory."\nselect = "inventory"\n' + ), + "orchestration/job_groups/annual_orders_job_group.toml": ( + 'name = "annual_orders_job_group"\n' + 'description = "Retry boundaries."\n' + 'jobs = ["stock_job", "marts_job", "staging_job"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, + default_job_files={}, ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: source: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job_group" ) assert source["status"] == "partial", worker.log.read_text() with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: @@ -112,8 +121,8 @@ def test_given_a_partial_group_when_retrying_from_failure_then_successful_nodes_ (run_id,), ).fetchall() assert [(name, json.loads(selection)) for name, selection in children] == [ - ("marts", ["model:totals"]), - ("staging", ["model:broken_orders"]), + ("marts_job", ["model:totals"]), + ("staging_job", ["model:broken_orders"]), ] assert control.execute( "SELECT count(*) FROM sqb_security_audit WHERE action='run.rerun_requested'" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py index 18be7d027..54e7ee6a1 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_source_sensor.py @@ -18,6 +18,7 @@ write_source_freshness_records, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import duckshop_orchestration from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( WorkerExecutionTestCase, ) @@ -41,13 +42,12 @@ [(73,)], expected_origin="sensor", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" - ).read_text(), + **duckshop_orchestration("source_orders"), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, + default_job_files={}, ), WorkerExecutionTestCase( "stale external timestamp versions", @@ -57,9 +57,7 @@ [(73,)], expected_origin="sensor", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" - ).read_text(), + **duckshop_orchestration("stale_orders"), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/managed_orders.yml" ).read_text(), @@ -70,6 +68,7 @@ "benchmarks/duckshop/edge_cases/orchestration/source_stale_journey.json" ).read_text(), }, + default_job_files={}, ), WorkerExecutionTestCase( "a changed: True job still rebuilds on new source data", @@ -79,13 +78,12 @@ [(73,)], expected_origin="sensor", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/changed_source_orders.py" - ).read_text(), + **duckshop_orchestration("changed_source_orders"), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, + default_job_files={}, warehouse_setup=create_raw_orders, ), ], @@ -190,7 +188,7 @@ def test_given_external_source_versions_when_a_location_resyncs_then_only_change worker.api, worker.process, worker.log, run_id=runs[-1][0] ) assert result["status"] == test_case.expected_status - assert result["job_key"] == "orders_refresh" + assert result["job_key"] == "orders_refresh_job" assert result["node_counts_json"] == {"success": 1} with duckdb.connect( str(worker.project / "warehouse.duckdb"), read_only=True diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py index a379eb200..b581d4661 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_statistics.py @@ -52,7 +52,7 @@ def test_given_relation_dropped_during_sampling_when_read_fails_then_catalog_can adapter, connection = open_legacy_warehouse(project, "sqlite", "path") _ = adapter.execute(connection=connection, sql=fixture["create_sql"]) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() (project / "models").mkdir() (project / "models/orders.sql").write_text(fixture["model"]) hook: Path = tmp_path / "sampling-hook" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py index 429858e78..2885a202c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_catalog.py @@ -96,7 +96,12 @@ def test_given_case_distinct_postgres_relations_when_reading_catalog_then_column '[connections.local]\nconninfo = "${ENV:SQLBUILD_CATALOG_TEST_URL}"\n' '[targets.development]\nschema = "main"\n' ), - "orchestration/orders.py": 'raise AssertionError("unexpected declaration import")\n', + "orchestration/calendars/unexpected_calendar.py": ( + "from sqlbuild.orchestration import calendar\n" + 'raise AssertionError("unexpected declaration import")\n' + "@calendar(name='unexpected_calendar', description='Never imported.', horizon='1d')\n" + "def unexpected(window):\n return []\n" + ), }, ) with PostgresContainer(POSTGRES_IMAGE, driver=None) as postgres: @@ -192,7 +197,7 @@ def test_given_served_location_when_catalog_changes_then_facts_and_current_proje ) _ = adapter.execute(connection=connection, sql=fixture["create_sql"]) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() write_files(project_dir=project, files={"models/orders.sql": fixture["model"]}) expected: dict[str, Any] = fixture["record"] | {"relation_type": test_case.expected_table_type} control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 94c53c35f..a20a7e800 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -397,7 +397,7 @@ def test_given_a_served_location_when_state_event_history_is_rebuilt_and_retaine ) test_case.append_history(adapter, connection, fixture) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() write_files(project_dir=project, files={"models/orders.sql": fixture["model"]}) with snapshot_api(tmp_path / "control") as api: log: Path = tmp_path / "location.log" @@ -942,7 +942,7 @@ def test_given_a_served_location_when_warehouse_history_changes_then_periodic_re "wide fixture exceeds one API request before any synchronization" ) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() write_files( project_dir=project, files={"models/orders.sql": fixture["model"]}, @@ -1052,7 +1052,7 @@ def test_given_durable_mirror_when_late_rows_arrive_then_incremental_overlap_pre ) append_legacy_history(adapter, connection, fixture["initial"], fixture["insert_sql"]) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() write_files( project_dir=project, files={"models/orders.sql": "MODEL (description 'Orders.'); SELECT 17 AS amount"}, @@ -1352,7 +1352,7 @@ def test_given_change_tokens_when_a_served_location_resyncs_then_only_changed_or project, test_case.adapter_name, test_case.connection_key ) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() fixture: dict[str, Any] = json.loads( Path( "benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json" @@ -1522,7 +1522,7 @@ def test_given_a_served_location_when_a_sync_has_many_pages_then_pages_run_back_ project, test_case.adapter_name, test_case.connection_key ) adapter.close(connection) - (project / "orchestration/orders.py").write_text("") + (project / "orchestration/calendars/unexpected_calendar.py").unlink() fixture: dict[str, Any] = json.loads( Path( "benchmarks/duckshop/edge_cases/orchestration/automatic_warehouse_history.json" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index b1c6be952..4b469b733 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -27,6 +27,7 @@ from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( METADATA, TYPE_PROOF_FILES, + duckshop_orchestration, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( CheckExecutionTestCase, @@ -63,7 +64,7 @@ "SELECT amount FROM orders", "partial", [(11,)], - expected_child_statuses={"staging": ("partial",), "marts": ("partial",)}, + expected_child_statuses={"staging_job": ("partial",), "marts_job": ("partial",)}, ), GroupExecutionTestCase( "successful jobs honor compiled boundaries", @@ -71,7 +72,7 @@ "SELECT amount FROM totals", "success", [(17,)], - expected_child_statuses={"staging": ("success",), "marts": ("success",)}, + expected_child_statuses={"staging_job": ("success",), "marts_job": ("success",)}, expected_totals=("success", None), ), GroupExecutionTestCase( @@ -81,18 +82,20 @@ "success", [(17,)], expected_origin="sensor", - expected_child_statuses={"staging": ("success",), "marts": ("success",)}, + expected_child_statuses={"staging_job": ("success",), "marts_job": ("success",)}, expected_totals=("success", None), files={ - "orchestration/orders.py": ( - "from sqlbuild.orchestration import PlatformEventKind, job, job_group, platform_event, sensor\n" - "staging = job('staging', description='Stage orders.', select='stg_orders broken_orders')\n" - "marts = job('marts', description='Build independent marts.', select='orders totals')\n" - "annual_orders = job_group('annual_orders', description='Stage then build marts.', jobs=[marts, staging])\n" - "health_orders = sensor('health_orders', description='Refresh on platform heartbeat.', " - "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run=annual_orders, " + "orchestration/job_groups/annual_orders_job_group.toml": ( + 'name = "annual_orders_job_group"\n' + 'description = "Stage then build marts."\n' + 'jobs = ["marts_job", "staging_job"]\n' + ), + "orchestration/sensors/health_orders_sensor.py": ( + "from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor\n" + "health_orders = sensor('health_orders_sensor', description='Refresh on platform heartbeat.', " + "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run='annual_orders_job_group', " "minimum_interval_seconds=60)\n" - ) + ), }, ), GroupExecutionTestCase( @@ -101,7 +104,10 @@ "SELECT count(*) FROM information_schema.tables WHERE table_name IN ('orders','totals')", "failed", [(0,)], - expected_child_statuses={"staging": ("partial", "cancelled"), "marts": ("cancelled",)}, + expected_child_statuses={ + "staging_job": ("partial", "cancelled"), + "marts_job": ("cancelled",), + }, expected_totals=("cancelled", None), expected_marts_started=False, files={ @@ -138,22 +144,29 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail "MODEL (description 'Dependent totals.', materialized table);\n" 'SELECT amount FROM __ref("broken_orders")\n' ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, job_group, schedule\n" - 'staging = job("staging", description="Stage orders.", ' - 'select="stg_orders broken_orders")\n' - 'marts = job("marts", description="Build independent marts.", select="orders totals")\n' - 'annual_orders = job_group("annual_orders", description="Stage then build marts.", ' - "jobs=[marts, staging], " - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/staging_job.toml": ( + 'name = "staging_job"\n' + 'description = "Stage orders."\n' + 'select = "stg_orders broken_orders"\n' + ), + "orchestration/jobs/marts_job.toml": ( + 'name = "marts_job"\n' + 'description = "Build independent marts."\n' + 'select = "orders totals"\n' + ), + "orchestration/job_groups/annual_orders_job_group.toml": ( + 'name = "annual_orders_job_group"\n' + 'description = "Stage then build marts."\n' + 'jobs = ["marts_job", "staging_job"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), } | test_case.files, + default_job_files={}, ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: parent: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job_group" ) assert parent["status"] == test_case.expected_status, worker.log.read_text() request: Request = Request( @@ -162,14 +175,14 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail ) with urlopen(request, timeout=5) as response: runs: dict[str, Any] = {run["job_key"]: run for run in json.load(response)} - assert set(runs) == {"annual_orders", *test_case.expected_child_statuses} + assert set(runs) == {"annual_orders_job_group", *test_case.expected_child_statuses} assert all( runs[name]["status"] in statuses for name, statuses in test_case.expected_child_statuses.items() ) - assert (runs["marts"]["started_at"] is not None) == test_case.expected_marts_started - assert datetime.fromisoformat(runs["staging"]["ended_at"]) <= datetime.fromisoformat( - runs["marts"]["started_at"] or runs["staging"]["ended_at"] + assert (runs["marts_job"]["started_at"] is not None) == test_case.expected_marts_started + assert datetime.fromisoformat(runs["staging_job"]["ended_at"]) <= datetime.fromisoformat( + runs["marts_job"]["started_at"] or runs["staging_job"]["ended_at"] ) with sqlite3.connect(worker.control_store) as control: assert control.execute( @@ -179,18 +192,18 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail control.execute( "SELECT parent_run_id, root_run_id FROM sqb_runs WHERE kind='job' ORDER BY job_key" ).fetchall() - == [(runs["annual_orders"]["run_id"],) * 2] * 2 + == [(runs["annual_orders_job_group"]["run_id"],) * 2] * 2 ) assert ( control.execute( "SELECT status, skipped_because_of FROM sqb_run_nodes WHERE run_id=? AND node_key='model:totals'", - (runs["marts"]["run_id"],), + (runs["marts_job"]["run_id"],), ).fetchone() == test_case.expected_totals ) assert control.execute( "SELECT status FROM sqb_run_nodes WHERE run_id=? AND node_key='model:broken_orders'", - (runs["staging"]["run_id"],), + (runs["staging_job"]["run_id"],), ).fetchone() == ({"success": "success"}.get(test_case.expected_status, "failed"),) assert control.execute("SELECT count(*) FROM sqb_write_claims").fetchone() == (0,) assert control.execute( @@ -225,13 +238,11 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail "tests/unit/test_stg_orders.sql": Path( "benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test.sql" ).read_text(), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders raw_orders stg_orders order_totals", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders raw_orders stg_orders order_totals"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, expected_run_nodes=[ @@ -269,13 +280,11 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail "tests/unit/test_stg_orders.sql": Path( "benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_cases.sql" ).read_text(), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders raw_orders stg_orders order_totals", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders raw_orders stg_orders order_totals"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, expected_run_nodes=[ @@ -313,13 +322,11 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail "tests/unit/test_stg_orders.sql": Path( "benchmarks/duckshop/edge_cases/orchestration/worker_failed_unit_test_assert.sql" ).read_text(), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders raw_orders stg_orders order_totals", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders raw_orders stg_orders order_totals"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, expected_run_nodes=[ @@ -362,13 +369,11 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail "tests/unit/test_udf__order_band.sql": Path( "benchmarks/duckshop/edge_cases/orchestration/worker_failed_function_test.sql" ).read_text(), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders raw_orders udf__order_band stg_orders order_totals", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders raw_orders udf__order_band stg_orders order_totals"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, expected_run_nodes=[ @@ -415,13 +420,11 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail ) .read_text() .replace("(5, 'large')", "(5, 'small')"), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders raw_orders udf__order_band stg_orders order_totals", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders raw_orders udf__order_band stg_orders order_totals"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, expected_run_nodes=[ @@ -440,7 +443,7 @@ def test_given_a_failing_unit_test_when_a_worker_builds_then_its_model_fails_and ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: run: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert run["status"] == test_case.expected_status, worker.log.read_text() with sqlite3.connect(worker.control_store) as control: @@ -482,13 +485,11 @@ def test_given_a_failing_unit_test_when_a_worker_builds_then_its_model_fails_and "MODEL (description 'Returns.', materialized table);\n" 'SELECT return_id FROM __source("raw_returns")\n' ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders stale_orders late_orders returns", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders stale_orders late_orders returns"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, ) @@ -500,7 +501,7 @@ def test_given_a_stale_source_when_a_worker_builds_then_blocked_nodes_finish_ski ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: run: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert run["status"] == test_case.expected_status, worker.log.read_text() with sqlite3.connect(worker.control_store) as control: @@ -540,13 +541,11 @@ def test_given_a_stale_source_when_a_worker_builds_then_blocked_nodes_finish_ski "MODEL (description 'Coded orders.', materialized table);\n" 'SELECT code FROM __seed("order_codes")\n' ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - 'annual_orders = job("annual_orders", ' - 'select="orders order_codes coded_orders", ' - 'description="Build orders.", ' - 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' - 'starts_at="2000-01-01T00:00:00Z"))\n' + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders order_codes coded_orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, ) @@ -558,7 +557,7 @@ def test_given_a_failing_seed_audit_when_a_worker_builds_then_its_seed_fails_wit ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: run: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert run["status"] == test_case.expected_status, worker.log.read_text() with sqlite3.connect(worker.control_store) as control: @@ -598,20 +597,34 @@ def test_given_a_failing_seed_audit_when_a_worker_builds_then_its_seed_fails_wit "models/inventory.sql": ( "MODEL (description 'Inventory.', materialized table);\nSELECT 23 AS amount\n" ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, job_group, schedule\n" - "orders = job('orders_job', description='Build orders.', select='orders', " - "concurrency_groups=('inventory_limit',))\n" - "inventory = job('inventory_job', description='Build inventory.', select='inventory', " - "concurrency_groups=('orders_limit',))\n" - "annual_orders = job_group('annual_orders', description='Orders group.', jobs=[orders], " - "concurrency_groups=('orders_limit',), " - "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" - "annual_inventory = job_group('annual_inventory', description='Inventory group.', " - "jobs=[inventory], concurrency_groups=('inventory_limit',), " - "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" + "orchestration/jobs/orders_job.toml": ( + 'name = "orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'concurrency_groups = ["inventory_limit"]\n' + ), + "orchestration/jobs/inventory_job.toml": ( + 'name = "inventory_job"\n' + 'description = "Build inventory."\n' + 'select = "inventory"\n' + 'concurrency_groups = ["orders_limit"]\n' + ), + "orchestration/job_groups/annual_orders_job_group.toml": ( + 'name = "annual_orders_job_group"\n' + 'description = "Orders group."\n' + 'jobs = ["orders_job"]\n' + 'concurrency_groups = ["orders_limit"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + ), + "orchestration/job_groups/annual_inventory_job_group.toml": ( + 'name = "annual_inventory_job_group"\n' + 'description = "Inventory group."\n' + 'jobs = ["inventory_job"]\n' + 'concurrency_groups = ["inventory_limit"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, + default_job_files={}, store_setup=( "CREATE TABLE test_slot_claims (run_id TEXT, group_key TEXT, occupied INTEGER);\n" "CREATE TRIGGER test_observe_slots AFTER INSERT ON sqb_concurrency_slots BEGIN " @@ -627,10 +640,10 @@ def test_given_opposing_group_limits_when_scheduled_then_each_group_holds_every_ ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: orders: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job_group" ) inventory: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_inventory" + worker.api, worker.process, worker.log, job_key="annual_inventory_job_group" ) assert orders["status"] == inventory["status"] == "success", worker.log.read_text() with sqlite3.connect(worker.control_store) as control: @@ -638,10 +651,10 @@ def test_given_opposing_group_limits_when_scheduled_then_each_group_holds_every_ "SELECT r.job_key, c.group_key, max(c.occupied) FROM test_slot_claims c " "JOIN sqb_runs r ON r.run_id=c.run_id GROUP BY r.job_key, c.group_key ORDER BY r.job_key" ).fetchall() == [ - ("annual_inventory", "inventory_limit", 1), - ("annual_inventory", "orders_limit", 1), - ("annual_orders", "inventory_limit", 1), - ("annual_orders", "orders_limit", 1), + ("annual_inventory_job_group", "inventory_limit", 1), + ("annual_inventory_job_group", "orders_limit", 1), + ("annual_orders_job_group", "inventory_limit", 1), + ("annual_orders_job_group", "orders_limit", 1), ] assert control.execute( "SELECT count(*) FROM sqb_runs WHERE status='success'" @@ -804,13 +817,15 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ [(47,)], expected_origin="sensor", files={ - "orchestration/orders.py": ( - "from sqlbuild.orchestration import PlatformEventKind, job, platform_event, sensor\n" - "annual_orders = job('annual_orders', select='orders', description='Build orders.')\n" - "health_orders = sensor('health_orders', description='Refresh on platform heartbeat.', " - "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run=annual_orders, " + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + "orchestration/sensors/health_orders_sensor.py": ( + "from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor\n" + "health_orders = sensor('health_orders_sensor', description='Refresh on platform heartbeat.', " + "when=platform_event(kinds=[PlatformEventKind.HEARTBEAT]), run='annual_orders_job', " "minimum_interval_seconds=60)\n" - ) + ), }, ), WorkerExecutionTestCase( @@ -825,12 +840,16 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ '[connections.local]\ndatabase = "warehouse.duckdb"\n' '[targets.development]\nschema = "main"\n' ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, schedule\n" - "annual_orders = job('annual_orders', select='orders', description='Build orders.', " - "settings={'vars': {'batch': 23}, 'reload': True, 'run_tests': False}, " - "schedule=schedule(cron='0 0 * * *', timezone='UTC', " - "starts_at='2000-01-01T00:00:00Z'))\n" + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + "\n" + "[settings]\n" + "vars = { batch = 23 }\n" + "reload = true\n" + "run_tests = false\n" ), }, ), @@ -845,10 +864,9 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ "sources/raw_orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders.py" - ).read_text(), + **duckshop_orchestration("upstream_source_orders"), }, + default_job_files={}, warehouse_setup=lambda path: ( duckdb.connect(str(path)) .execute("CREATE TABLE raw_orders AS SELECT CAST(23 AS BIGINT) AS order_id") @@ -867,10 +885,9 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ "sources/raw_orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/upstream_source_orders.py" - ).read_text(), + **duckshop_orchestration("upstream_source_orders"), }, + default_job_files={}, warehouse_setup=lambda path: ( duckdb.connect(str(path)) .execute("CREATE TABLE raw_orders AS SELECT CAST(23 AS BIGINT) AS order_id") @@ -954,12 +971,8 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re "SELECT amount FROM orders", "success", [(9,)], - files=INTERMEDIATE_LOADER_FILES - | { - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py" - ).read_text() - }, + files=INTERMEDIATE_LOADER_FILES | {**duckshop_orchestration("upstream_loaded_orders")}, + default_job_files={}, expected_node_count=3, expected_write_set=frozenset( {"model:orders", "source:raw_shipments", "source:stage_shipments"} @@ -1188,7 +1201,9 @@ def test_given_an_attached_check_when_a_worker_executes_then_its_own_outcome_is_ [(59,)], files={ "python/checks/orders.py": "failed", - "orchestration/orders.py": "on_failure", + "orchestration/jobs/annual_orders_job.toml": "on_failure_job", + "orchestration/jobs/refresh_inventory_job.toml": "on_failure_destination", + "orchestration/sensors/failed_orders_sensor.py": "on_failure_sensor", "models/inventory.sql": "inventory", }, ), @@ -1206,11 +1221,11 @@ def test_given_an_attached_check_failure_when_a_sensor_observes_it_then_its_dest ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: source: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert source["status"] == "failed", worker.log.read_text() destination: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="refresh_inventory" + worker.api, worker.process, worker.log, job_key="refresh_inventory_job" ) assert destination["status"] == test_case.expected_status, worker.log.read_text() assert destination["selection_json"] == ["model:inventory"] @@ -1231,7 +1246,9 @@ def test_given_an_attached_check_failure_when_a_sensor_observes_it_then_its_dest [(17,)], files={ "python/checks/orders.py": "cancelled", - "orchestration/orders.py": "on_failure", + "orchestration/jobs/annual_orders_job.toml": "on_failure_job", + "orchestration/jobs/refresh_inventory_job.toml": "on_failure_destination", + "orchestration/sensors/failed_orders_sensor.py": "on_failure_sensor", "models/inventory.sql": "inventory", }, ), @@ -1248,9 +1265,9 @@ def test_given_an_attached_check_query_when_cancelled_then_it_cannot_trigger_fai test_case, files={path: fixture[key] for path, key in test_case.files.items()} ) with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: - cancel_running_statement(worker, job_key="annual_orders") + cancel_running_statement(worker, job_key="annual_orders_job") run: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert run["status"] == test_case.expected_status, worker.log.read_text() with sqlite3.connect(worker.control_store) as control: @@ -1291,16 +1308,27 @@ def test_given_an_attached_check_query_when_cancelled_then_it_cannot_trigger_fai [(59,)], expected_origin="sensor", files={ - "orchestration/orders.py": ( - "from sqlbuild.orchestration import RunEventKind, job, job_group, run_event, schedule, sensor\n" - "source = job('source_orders', select='orders', description='Source orders.')\n" - "source_group = job_group('source_group', description='Source group.', jobs=[source], " - "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" - "annual_orders = job('annual_orders', select='orders', description='Refresh orders.', settings={'reload': True})\n" - "on_group = sensor('on_group', description='Refresh after group completion.', " - "when=run_event(job=source_group, kinds=[RunEventKind.COMPLETED, RunEventKind.SUCCESS], executed_successfully=('orders',)), " - "run=annual_orders, minimum_interval_seconds=1)\n" - ) + "orchestration/jobs/source_orders_job.toml": ( + 'name = "source_orders_job"\ndescription = "Source orders."\nselect = "orders"\n' + ), + "orchestration/job_groups/source_job_group.toml": ( + 'name = "source_job_group"\n' + 'description = "Source group."\n' + 'jobs = ["source_orders_job"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + ), + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Refresh orders."\n' + 'select = "orders"\n' + "\n[settings]\nreload = true\n" + ), + "orchestration/sensors/on_group_sensor.py": ( + "from sqlbuild.orchestration import RunEventKind, run_event, sensor\n" + "on_group = sensor('on_group_sensor', description='Refresh after group completion.', " + "when=run_event(job='source_job_group', kinds=[RunEventKind.COMPLETED, RunEventKind.SUCCESS], executed_successfully=('orders',)), " + "run='annual_orders_job', minimum_interval_seconds=1)\n" + ), }, ) ], @@ -1311,7 +1339,7 @@ def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destin ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: destination: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert destination["status"] == test_case.expected_status, worker.log.read_text() result: dict[str, Any] = worker_result(worker, test_case) @@ -1320,7 +1348,7 @@ def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destin assert control.execute("SELECT run_key,run_id FROM sqb_run_requests").fetchall() == [ ( control.execute( - "SELECT run_id FROM sqb_runs WHERE job_key='source_group'" + "SELECT run_id FROM sqb_runs WHERE job_key='source_job_group'" ).fetchone()[0], destination["run_id"], ) @@ -1328,9 +1356,9 @@ def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destin assert control.execute( "SELECT job_key,status FROM sqb_runs ORDER BY job_key" ).fetchall() == [ - ("annual_orders", "success"), - ("source_group", "success"), - ("source_orders", "success"), + ("annual_orders_job", "success"), + ("source_job_group", "success"), + ("source_orders_job", "success"), ] @@ -1346,15 +1374,26 @@ def test_given_a_real_group_completion_when_a_run_sensor_matches_then_one_destin expected_origin="sensor", files={ "models/stg_orders.sql": "MODEL (description 'Staging orders.');\nSELECT 71 AS amount\n", - "orchestration/orders.py": ( - "from sqlbuild.orchestration import NodeEventKind, job, job_group, node_event, schedule, sensor\n" - "source = job('source_orders', select='orders', description='Source orders.')\n" - "source_group = job_group('source_group', description='Source group.', jobs=[source], " - "schedule=schedule(cron='0 0 * * *', timezone='UTC', starts_at='2000-01-01T00:00:00Z'))\n" - "annual_orders = job('annual_orders', select='stg_orders', description='Refresh staging.', settings={'reload': True})\n" - "on_node = sensor('on_node', description='React to actual orders rebuilds.', " + "orchestration/jobs/source_orders_job.toml": ( + 'name = "source_orders_job"\ndescription = "Source orders."\nselect = "orders"\n' + ), + "orchestration/job_groups/source_job_group.toml": ( + 'name = "source_job_group"\n' + 'description = "Source group."\n' + 'jobs = ["source_orders_job"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + ), + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Refresh staging."\n' + 'select = "stg_orders"\n' + "\n[settings]\nreload = true\n" + ), + "orchestration/sensors/on_node_sensor.py": ( + "from sqlbuild.orchestration import NodeEventKind, node_event, sensor\n" + "on_node = sensor('on_node_sensor', description='React to actual orders rebuilds.', " "when=node_event(select='orders', kinds=[NodeEventKind.SUCCESS]), " - "run=annual_orders, minimum_interval_seconds=1)\n" + "run='annual_orders_job', minimum_interval_seconds=1)\n" ), }, ) @@ -1366,23 +1405,23 @@ def test_given_a_real_group_child_rebuild_when_a_node_sensor_matches_then_only_t ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: destination: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job" ) assert destination["status"] == test_case.expected_status, worker.log.read_text() with sqlite3.connect(worker.control_store) as control: assert control.execute( "SELECT job_key,status FROM sqb_runs ORDER BY job_key" ).fetchall() == [ - ("annual_orders", "success"), - ("source_group", "success"), - ("source_orders", "success"), + ("annual_orders_job", "success"), + ("source_job_group", "success"), + ("source_orders_job", "success"), ] assert control.execute( "SELECT q.run_key,q.run_id FROM sqb_run_requests q" ).fetchall() == [ ( control.execute( - "SELECT run_id FROM sqb_runs WHERE job_key='source_orders'" + "SELECT run_id FROM sqb_runs WHERE job_key='source_orders_job'" ).fetchone()[0] + "/model:orders", destination["run_id"], @@ -1429,15 +1468,22 @@ def test_given_an_executing_scheduled_worker_when_cancelled_then_the_statement_s "MODEL (description 'Downstream totals.', materialized table);\n" 'SELECT amount FROM __ref("orders")\n' ), - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, job_group, schedule\n" - "orders = job('long_orders', description='Long query.', select='orders')\n" - "totals = job('waiting_totals', description='Waiting query.', select='totals')\n" - "annual_orders = job_group('annual_orders', description='Cancellable group.', " - "jobs=[totals, orders], schedule=schedule(cron='0 0 * * *', timezone='UTC', " - "starts_at='2000-01-01T00:00:00Z'))\n" + "orchestration/jobs/long_orders_job.toml": ( + 'name = "long_orders_job"\ndescription = "Long query."\nselect = "orders"\n' + ), + "orchestration/jobs/waiting_totals_job.toml": ( + 'name = "waiting_totals_job"\n' + 'description = "Waiting query."\n' + 'select = "totals"\n' + ), + "orchestration/job_groups/annual_orders_job_group.toml": ( + 'name = "annual_orders_job_group"\n' + 'description = "Cancellable group."\n' + 'jobs = ["waiting_totals_job", "long_orders_job"]\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' ), }, + default_job_files={}, ), ], ids=lambda case: case.description, @@ -1451,21 +1497,21 @@ def test_given_an_executing_group_when_parent_cancelled_then_all_children_stop_b worker.process, worker.log, pending=frozenset({"queued"}), - job_key="annual_orders", + job_key="annual_orders_job_group", ) assert parent["status"] == "running", worker.log.read_text() - cancel_running_statement(worker, job_key="long_orders", run_id=parent["run_id"]) + cancel_running_statement(worker, job_key="long_orders_job", run_id=parent["run_id"]) completed: dict[str, Any] = wait_for_run( - worker.api, worker.process, worker.log, job_key="annual_orders" + worker.api, worker.process, worker.log, job_key="annual_orders_job_group" ) assert completed["status"] == test_case.expected_status, worker.log.read_text() with sqlite3.connect(worker.control_store) as control: assert control.execute( "SELECT job_key, status FROM sqb_runs ORDER BY job_key" ).fetchall() == [ - ("annual_orders", "cancelled"), - ("long_orders", "cancelled"), - ("waiting_totals", "cancelled"), + ("annual_orders_job_group", "cancelled"), + ("long_orders_job", "cancelled"), + ("waiting_totals_job", "cancelled"), ] assert control.execute( "SELECT count(*) FROM sqb_runs WHERE parent_run_id=? AND ended_at>?", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 06e61dc87..aefebb42f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -4,7 +4,6 @@ import json import re -from dataclasses import replace from pathlib import Path from typing import Any @@ -22,17 +21,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.orchestration import ( - RunEventKind, - compile_snapshot, - job, - job_group, - on_failure, - run_event, - schedule, - sensor, -) -from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup +from sqlbuild.orchestration import compile_snapshot from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( PythonCheckJobRejectionTestCase, PythonCheckJobTestCase, @@ -55,8 +44,9 @@ FUNCTION_PROJECT_FILES, METADATA, PARITY_FILES, - PYTHON_PIPELINE_FILES, TYPE_PROOF_FILES, + duckshop_orchestration, + edited_duckshop_orchestration, published_texts, python_pipeline_snapshot, ) @@ -64,12 +54,22 @@ ORCHESTRATION_NAME_CONFLICT: str = ( "; model, source, seed, function, loader, task, asset, check, provider, hook, unit test, " - "scenario, job, job group, sensor, and calendar schedule names must be globally unique; " - "rename one of them (a job, job group or sensor takes its name from its first argument, and " - "a calendar schedule from the function it decorates)" + "scenario, job, job group, sensor, and calendar names must be globally unique; rename one of " + "them (a job or job group takes its name from `name` in its TOML file, and a calendar or " + "sensor from its `name` argument)" ) _DUCKSHOP: Path = Path("benchmarks/duckshop/project") + + +_DAILY_BUILD: dict[str, str] = { + "name": '"daily_build_job"', + "description": '"Orders."', + "select": '"orders"', + "schedule": '{ cron = "0 2 * * *", timezone = "UTC" }', +} + + _PROVIDER_PROJECT: dict[str, str] = { "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', "models/orders.sql": "MODEL (description 'Orders.');\n\nSELECT 1 AS order_id\n", @@ -179,9 +179,7 @@ "DuckShop polling callback with nested code is relocatable", TYPE_PROOF_FILES | { - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/custom_orders.py" - ).read_text(), + **duckshop_orchestration("custom_orders"), }, frozenset({"model"}), ), @@ -836,22 +834,23 @@ def test_given_a_dependency_lock_when_it_changes_then_environment_identity_chang [ SnapshotScheduleTestCase( description="daylight saving zone", - schedule=schedule(cron="15 2 * * *", timezone="Europe/London"), + schedule='{ cron = "15 2 * * *", timezone = "Europe/London" }', + expected_timezone="Europe/London", expected_selection=("model:orders",), ), SnapshotScheduleTestCase( description="fixed offset zone", - schedule=schedule(cron="15 2 * * *", timezone="UTC"), + schedule='{ cron = "15 2 * * *", timezone = "UTC" }', + expected_timezone="UTC", expected_selection=("model:orders",), ), SnapshotScheduleTestCase( description="paused with a zoned start", - schedule=schedule( - cron="15 2 * * *", - timezone="Europe/London", - starts_at="2026-07-03T01:17:23.456789+05:30", - default_status="paused", + schedule=( + '{ cron = "15 2 * * *", timezone = "Europe/London", ' + 'starts_at = "2026-07-03T01:17:23.456789+05:30", default_status = "paused" }' ), + expected_timezone="Europe/London", expected_selection=("model:orders",), expected_controls={ "starts_at": "2026-07-02T19:47:23.456789Z", @@ -871,32 +870,46 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select adapter=DuckDbAdapter(), resolved_connection={"database": "warehouse", "schema": "main"}, ) - nightly: Job = job( - "nightly_orders", - description="Refresh order reporting.", - select="+orders", - exclude="stg_orders", - settings={"run_tests": False, "vars": {"batch": 3}, "warehouse": "reporting"}, - concurrency=4, - concurrency_groups=("order_reporting",), - schedule=test_case.schedule, - targets=("development",), - owner="Orders team", - overlap="skip", - on_failure=on_failure(retries=2, retry_delay_seconds=600), - ) - excluded: Job = replace(nightly, name="other_target", targets=("production",)) - first: dict[str, Any] = json.loads( - compile_snapshot( - project=project, - metadata_json=json.dumps(METADATA), + + def snapshot(schedule: str) -> dict[str, Any]: + write_files( project_dir=tmp_path, - declarations=[nightly, excluded], - discovered_inputs=discovered, - ).to_json() - ) + files={ + "orchestration/jobs/nightly_orders_job.toml": ( + 'name = "nightly_orders_job"\n' + 'description = "Refresh order reporting."\n' + 'select = "+orders"\n' + 'exclude = "stg_orders"\n' + 'targets = ["development"]\n' + 'owner = "Orders team"\n' + 'overlap = "skip"\n' + 'concurrency_groups = ["order_reporting"]\n' + f"schedule = {schedule}\n" + "\n[on_failure]\nretries = 2\nretry_delay_seconds = 600\n" + '\n[settings]\nrun_tests = false\nvars = { batch = 3 }\nwarehouse = "reporting"\n' + "concurrency = 4\n" + ), + "orchestration/jobs/other_target_job.toml": ( + 'name = "other_target_job"\n' + 'description = "Refresh order reporting."\n' + 'select = "+orders"\n' + 'targets = ["production"]\n' + f"schedule = {schedule}\n" + ), + }, + ) + return json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + ).to_json() + ) + + first: dict[str, Any] = snapshot(test_case.schedule) assert first["documents"]["jobs"] == { - "nightly_orders": { + "nightly_orders_job": { "description": "Refresh order reporting.", "selection": list(test_case.expected_selection), "settings": { @@ -912,29 +925,21 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select "owner": "Orders team", "overlap": "skip", "on_failure": {"retries": 2, "retry_delay_seconds": 600}, + "declared_at": {"path": "orchestration/jobs/nightly_orders_job.toml", "line": 1}, } } - trigger: dict[str, Any] = first["documents"]["triggers"]["nightly_orders"] - assert trigger["job_key"] == "nightly_orders" + trigger: dict[str, Any] = first["documents"]["triggers"]["nightly_orders_job"] + assert trigger["job_key"] == "nightly_orders_job" assert trigger["targets"] == ["development"] assert trigger["schedule"] == { "kind": "cron", "expression": "15 2 * * *", - "timezone": test_case.schedule.timezone, + "timezone": test_case.expected_timezone, } assert {key: trigger[key] for key in test_case.expected_controls} == test_case.expected_controls - edited: Job = replace(nightly, schedule=schedule(cron="30 3 * * *", timezone="UTC")) - second: dict[str, Any] = json.loads( - compile_snapshot( - project=project, - metadata_json=json.dumps(METADATA), - project_dir=tmp_path, - declarations=[edited], - discovered_inputs=discovered, - ).to_json() - ) + second: dict[str, Any] = snapshot('{ cron = "30 3 * * *", timezone = "UTC" }') assert second["documents"]["triggers"].keys() == first["documents"]["triggers"].keys() - assert second["documents"]["triggers"]["nightly_orders"]["schedule"]["expression"] == ( + assert second["documents"]["triggers"]["nightly_orders_job"]["schedule"]["expression"] == ( "30 3 * * *" ) assert second["nodes"] == first["nodes"] @@ -945,102 +950,115 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select [ SnapshotDeclarationRejectionTestCase( description="blank description", - job_description=" ", - expected_message="needs a description", + fields={"description": '" "'}, + expected_message="2:15: description must not be blank", ), SnapshotDeclarationRejectionTestCase( - description="blank selection", select="", expected_message="needs a selection" + description="blank selection", + fields={"select": '""'}, + expected_message="3:10: select must not be blank", ), SnapshotDeclarationRejectionTestCase( - description="missing model", select="missing", expected_message="missing" + description="missing model", + fields={"select": '"missing"'}, + expected_message="missing", ), SnapshotDeclarationRejectionTestCase( - description="invalid cron", cron="not a cron", expected_message="cron" + description="invalid cron", + fields={"schedule": '{ cron = "not a cron", timezone = "UTC" }'}, + expected_message="cron", ), SnapshotDeclarationRejectionTestCase( - description="invalid timezone", timezone="Unknown/Zone", expected_message="time zone" + description="invalid timezone", + fields={"schedule": '{ cron = "0 2 * * *", timezone = "Unknown/Zone" }'}, + expected_message="time zone", ), SnapshotDeclarationRejectionTestCase( description="start needs an explicit offset", - schedule=schedule(cron="0 2 * * *", timezone="UTC", starts_at="2026-07-03T01:17:23"), + fields={ + "schedule": '{ cron = "0 2 * * *", timezone = "UTC", starts_at = "2026-07-03T01:17:23" }' + }, expected_message="starts_at", ), SnapshotDeclarationRejectionTestCase( - description="duplicate names", copies=2, expected_message="duplicate job daily_build" + description="duplicate names", + copies=2, + expected_message="name 'daily_build_job' is declared as both job", ), SnapshotDeclarationRejectionTestCase( description="unknown setting", - settings={"concurreny": 4}, + tables="[settings]\nconcurreny = 4\n", expected_message="unknown build setting concurreny", ), SnapshotDeclarationRejectionTestCase( description="selection cannot escape its pinned run", - settings={"select": "customers"}, + tables='[settings]\nselect = "customers"\n', expected_message="owned by the run request", ), SnapshotDeclarationRejectionTestCase( description="zero concurrency", - concurrency=0, + tables="[settings]\nconcurrency = 0\n", expected_message="must be positive", ), SnapshotDeclarationRejectionTestCase( description="boolean is not concurrency", - concurrency=True, + tables="[settings]\nconcurrency = true\n", expected_message="concurrency must be an integer", ), SnapshotDeclarationRejectionTestCase( description="integer is not a boolean flag", - settings={"full_refresh": 1}, + tables="[settings]\nfull_refresh = 1\n", expected_message="must be boolean", ), SnapshotDeclarationRejectionTestCase( description="integer strings are not typed settings", - settings={"concurrency": "3"}, + tables='[settings]\nconcurrency = "3"\n', expected_message="must be an integer", ), SnapshotDeclarationRejectionTestCase( description="variables need an object", - settings={"vars": ["orders"]}, + tables='[settings]\nvars = ["orders"]\n', expected_message="must be a JSON object", ), SnapshotDeclarationRejectionTestCase( description="mutually exclusive load settings", - settings={"load_sources": False, "reload": True}, + tables="[settings]\nload_sources = false\nreload = true\n", expected_message="mutually exclusive", ), SnapshotDeclarationRejectionTestCase( description="conflicting cursor families", - settings={"start_cursor": "2", "end_cursor_int": "4"}, + tables='[settings]\nstart_cursor = "2"\nend_cursor_int = "4"\n', expected_message="cannot be combined", ), SnapshotDeclarationRejectionTestCase( - description="blank owner", owner=" ", expected_message="owner must not be blank" + description="blank owner", + fields={"owner": '" "'}, + expected_message="owner must not be blank", ), SnapshotDeclarationRejectionTestCase( description="unknown overlap", - overlap="coalesce", - expected_message="overlap must be queue or skip, not coalesce", + fields={"overlap": '"coalesce"'}, + expected_message="unknown variant `coalesce`, expected `queue` or `skip`", ), SnapshotDeclarationRejectionTestCase( description="no retries", - on_failure=on_failure(retries=0), + tables="[on_failure]\nretries = 0\n", expected_message="retries must be a whole number from 1 to 10", ), SnapshotDeclarationRejectionTestCase( description="too many retries", - on_failure=on_failure(retries=11), + tables="[on_failure]\nretries = 11\n", expected_message="retries must be a whole number from 1 to 10", ), SnapshotDeclarationRejectionTestCase( description="retry delay over a day", - on_failure=on_failure(retries=1, retry_delay_seconds=86_401), + tables="[on_failure]\nretries = 1\nretry_delay_seconds = 86401\n", expected_message="retry_delay_seconds must be a whole number from 0 to 86400", ), SnapshotDeclarationRejectionTestCase( - description="duplicate concurrency declarations", - concurrency=3, - settings={"concurrency": 4}, - expected_message="declared twice", + description="concurrency belongs under settings", + fields={"concurrency": "3"}, + expected_message="unknown job key 'concurrency'", ), ], ids=lambda case: case.description, @@ -1048,33 +1066,24 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select def test_given_invalid_declarations_when_compiling_then_native_validation_rejects_them( tmp_path: Path, test_case: SnapshotDeclarationRejectionTestCase ) -> None: - write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discovered, - adapter=DuckDbAdapter(), - resolved_connection={"database": "warehouse", "schema": "main"}, + text: str = "".join( + f"{key} = {value}\n" for key, value in (_DAILY_BUILD | test_case.fields).items() ) - declarations: list[Job] = [ - job( - "daily_build", - description=test_case.job_description, - select=test_case.select, - settings=test_case.settings, - concurrency=test_case.concurrency, - schedule=test_case.schedule - or schedule(cron=test_case.cron, timezone=test_case.timezone), - owner=test_case.owner, - overlap=test_case.overlap, - on_failure=test_case.on_failure, - ) - ] * test_case.copies + copies: dict[str, str] = { + f"orchestration/jobs/{'copy/' * copy}daily_build_job.toml": text + "\n" + test_case.tables + for copy in range(test_case.copies) + } + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | copies) with pytest.raises(ValueError, match=test_case.expected_message): + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) compile_snapshot( - project=project, + project=build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ), metadata_json=json.dumps(METADATA), project_dir=tmp_path, - declarations=declarations, discovered_inputs=discovered, ) @@ -1083,53 +1092,38 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject "test_case", [ SnapshotDiscoveryTestCase( - description="relative imports aliases and factory containers", + description="group-only references retain lineage waits", files={ - "orchestration/shared.py": ( - "from sqlbuild.orchestration import job\n" - "staging = job('staging', description='Stage orders.', select='stg_orders')\n" + "orchestration/jobs/staging_job.toml": ( + 'name = "staging_job"\ndescription = "Stage orders."\nselect = "stg_orders"\n' ), - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job\n" - "from .shared import staging\n" - "alias = staging\n" - "def factory():\n" - " return [job('daily', description='Daily orders.', select='orders')]\n" - "jobs = [factory(), staging]\n" - "jobs.append(jobs)\n" + "orchestration/jobs/inventory_job.toml": ( + 'name = "inventory_job"\n' + 'description = "Repeat staging."\n' + 'select = "stg_orders"\n' ), - }, - expected_jobs=("daily", "staging"), - ), - SnapshotDiscoveryTestCase( - description="group-only references retain lineage waits", - files={ - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job, job_group\n" - "def factory():\n" - " staging = job('staging', description='Stage orders.', select='stg_orders')\n" - " marts = job('marts', description='Build orders.', select='orders')\n" - " inventory = job('inventory', description='Repeat staging.', select='stg_orders')\n" - " return job_group('daily', description='Ordered jobs.', " - "jobs=[marts, inventory, staging])\n" - "daily = factory()\n" + "orchestration/jobs/marts_job.toml": ( + 'name = "marts_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + "orchestration/job_groups/daily_job_group.toml": ( + 'name = "daily_job_group"\n' + 'description = "Ordered jobs."\n' + 'jobs = ["marts_job", "inventory_job", "staging_job"]\n' ), }, - expected_jobs=("inventory", "marts", "staging"), + expected_jobs=("inventory_job", "marts_job", "staging_job"), expected_groups={ - "daily": { - "inventory": [], - "marts": ["inventory", "staging"], - "staging": [], + "daily_job_group": { + "inventory_job": [], + "marts_job": ["inventory_job", "staging_job"], + "staging_job": [], } }, ), SnapshotDiscoveryTestCase( description="DuckShop layer jobs that share a function wait only by lineage", files={ - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py" - ).read_text(), + **duckshop_orchestration("shared_function_layers"), "functions/sql/order_cents.sql": ( "FUNCTION (description 'Amount in cents.'," " arguments (amount INTEGER), returns INTEGER);\nSELECT amount * 100\n" @@ -1148,142 +1142,154 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject 'SELECT order_id, __udf("order_cents")(cents) AS cents\nFROM __ref("stg_orders")\n' ), }, - expected_jobs=("marts_layer", "staging_layer"), + expected_jobs=("marts_layer_job", "staging_layer_job"), expected_groups={ - "layered_orders": {"marts_layer": ["staging_layer"], "staging_layer": []} + "layered_orders_job_group": { + "marts_layer_job": ["staging_layer_job"], + "staging_layer_job": [], + } }, ), - SnapshotDiscoveryTestCase( - description="explicit empty declarations do not execute project files", - files={"orchestration/jobs.py": "raise RuntimeError('must not import')\n"}, - expected_jobs=(), - declarations=(), - ), SnapshotDiscoveryTestCase( description="DuckShop edge-case declaration containers", files={ - "orchestration/edge_cases/definitions.py": Path( - "benchmarks/duckshop/project/orchestration/edge_cases/definitions.py" - ).read_text(), + **{ + path.relative_to(_DUCKSHOP).as_posix(): path.read_text() + for path in (_DUCKSHOP / "orchestration").rglob("*") + if path.is_file() + }, "models/orders.sql": TYPE_PROOF_FILES["models/orders.sql"].replace( "MODEL (", "MODEL (tags ['edge_case'],", 1 ), }, expected_jobs=( - "edge_snapshot_inventory", - "edge_snapshot_orders", - "edge_snapshot_products", + "edge_snapshot_inventory_job", + "edge_snapshot_orders_job", + "edge_snapshot_products_job", ), expected_groups={ - "edge_snapshot_group": {"edge_snapshot_orders": []}, - "edge_inventory_group": {"edge_snapshot_inventory": []}, - "edge_products_group": {"edge_snapshot_products": []}, + "edge_snapshot_job_group": {"edge_snapshot_orders_job": []}, + "edge_inventory_job_group": {"edge_snapshot_inventory_job": []}, + "edge_products_job_group": {"edge_snapshot_products_job": []}, }, expected_sensors={ - "edge_platform_orders": { + "edge_platform_orders_sensor": { "description": "Catch up retained platform observations with a stopped group sensor.", - "job_key": "edge_snapshot_group", + "job_key": "edge_snapshot_job_group", "when": { "kind": "platform_event", "kinds": ["tick_overdue", "location_broken", "launch_failed"], }, "minimum_interval_seconds": 7, "default_status": "paused", - "declared_at": {"path": "orchestration/edge_cases/definitions.py", "line": 61}, + "declared_at": { + "path": "orchestration/sensors/edge_cases/edge_platform_orders_sensor.py", + "line": 5, + }, }, - "edge_job_outcomes": { + "edge_job_outcomes_sensor": { "description": "Keep warning and partial outcomes distinct from success.", - "job_key": "edge_snapshot_inventory", + "job_key": "edge_snapshot_inventory_job", "when": { "kind": "run_event", - "job_key": "edge_snapshot_orders", + "job_key": "edge_snapshot_orders_job", "kinds": ["warn", "partial"], }, "minimum_interval_seconds": 30, "default_status": "paused", - "declared_at": {"path": "orchestration/edge_cases/definitions.py", "line": 77}, + "declared_at": { + "path": "orchestration/sensors/edge_cases/edge_job_outcomes_sensor.py", + "line": 5, + }, }, - "edge_group_outcomes": { + "edge_group_outcomes_sensor": { "description": "Observe group completion including cancellation and presumed failure.", - "job_key": "edge_snapshot_inventory", + "job_key": "edge_snapshot_inventory_job", "when": { "kind": "run_event", - "job_key": "edge_snapshot_group", + "job_key": "edge_snapshot_job_group", "kinds": ["completed"], "executed_successfully": ["model:orders"], }, "minimum_interval_seconds": 30, "default_status": "paused", - "declared_at": {"path": "orchestration/edge_cases/definitions.py", "line": 84}, + "declared_at": { + "path": "orchestration/sensors/edge_cases/edge_group_outcomes_sensor.py", + "line": 5, + }, }, }, - expected_schedules=("edge_snapshot_orders",), + expected_schedules=("edge_snapshot_orders_job",), expected_trigger_declared_at={ - "edge_snapshot_orders": { - "path": "orchestration/edge_cases/definitions.py", - "line": 14, + "edge_snapshot_orders_job": { + "path": "orchestration/jobs/edge_cases/edge_snapshot_orders_job.toml", + "line": 2, }, }, expected_declared_at={ - "edge_snapshot_orders": { - "path": "orchestration/edge_cases/definitions.py", - "line": 14, + "edge_snapshot_orders_job": { + "path": "orchestration/jobs/edge_cases/edge_snapshot_orders_job.toml", + "line": 2, }, - "edge_snapshot_group": { - "path": "orchestration/edge_cases/definitions.py", - "line": 30, + "edge_snapshot_job_group": { + "path": "orchestration/job_groups/edge_cases/edge_snapshot_job_group.toml", + "line": 1, }, }, ), SnapshotDiscoveryTestCase( - description="sensor-only roots collect source groups and destination jobs natively", - files={}, - declarations=( - sensor( - "on_orders", - description="Build inventory after group completion.", - when=run_event( - job=job_group( - "source_group", - description="Source orders.", - jobs=[job("source_orders", description="Orders.", select="orders")], - ), - kinds=[RunEventKind.COMPLETED], - executed_successfully=("orders", "orders*"), - ), - run=job("inventory", description="Inventory.", select="stg_orders"), + description="a run_event on a group freezes its members' execution selectors", + files={ + "orchestration/jobs/source_orders_job.toml": ( + 'name = "source_orders_job"\ndescription = "Orders."\nselect = "orders"\n' ), - ), - expected_jobs=("inventory", "source_orders"), - expected_declared_at={"inventory": None, "source_group": None}, - expected_groups={"source_group": {"source_orders": []}}, + "orchestration/jobs/inventory_job.toml": ( + 'name = "inventory_job"\ndescription = "Inventory."\nselect = "stg_orders"\n' + ), + "orchestration/job_groups/source_job_group.toml": ( + 'name = "source_job_group"\ndescription = "Source orders."\n' + 'jobs = ["source_orders_job"]\n' + ), + "orchestration/sensors/on_orders_sensor.py": ( + "from sqlbuild.orchestration import RunEventKind, run_event, sensor\n" + "on_orders = sensor(\n" + " 'on_orders_sensor',\n" + " description='Build inventory after group completion.',\n" + " when=run_event(\n" + " job='source_job_group',\n" + " kinds=[RunEventKind.COMPLETED],\n" + " executed_successfully=('orders', 'orders*'),\n" + " ),\n" + " run='inventory_job',\n" + ")\n" + ), + }, + expected_jobs=("inventory_job", "source_orders_job"), + expected_groups={"source_job_group": {"source_orders_job": []}}, expected_sensors={ - "on_orders": { + "on_orders_sensor": { "description": "Build inventory after group completion.", - "job_key": "inventory", + "job_key": "inventory_job", "when": { "kind": "run_event", - "job_key": "source_group", + "job_key": "source_job_group", "kinds": ["completed"], "executed_successfully": ["model:orders"], }, "minimum_interval_seconds": 30, "default_status": "running", + "declared_at": {"path": "orchestration/sensors/on_orders_sensor.py", "line": 2}, } }, ), SnapshotDiscoveryTestCase( description="any-run node selectors freeze canonical keys without a source job reference", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" - ).read_text() - }, - expected_jobs=("inventory", "source_orders"), + files={**duckshop_orchestration("node_orders")}, + expected_jobs=("inventory_job", "source_orders_job"), expected_sensors={ - "on_orders": { + "on_orders_sensor": { "description": "Build inventory for every genuine orders rebuild.", - "job_key": "inventory", + "job_key": "inventory_job", "when": { "kind": "node_event", "selection": ["model:orders"], @@ -1291,24 +1297,25 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "running", - "declared_at": {"path": "orchestration/orders.py", "line": 7}, + "declared_at": {"path": "orchestration/sensors/on_orders_sensor.py", "line": 5}, } }, ), SnapshotDiscoveryTestCase( description="failure and recovery selectors preserve their individual transition kinds", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + **edited_duckshop_orchestration( + "node_orders", + "orchestration/sensors/on_orders_sensor.py", + "NodeEventKind.SUCCESS", + "NodeEventKind.FAILED, NodeEventKind.RECOVERED", ) - .read_text() - .replace("NodeEventKind.SUCCESS", "NodeEventKind.FAILED, NodeEventKind.RECOVERED") }, - expected_jobs=("inventory", "source_orders"), + expected_jobs=("inventory_job", "source_orders_job"), expected_sensors={ - "on_orders": { + "on_orders_sensor": { "description": "Build inventory for every genuine orders rebuild.", - "job_key": "inventory", + "job_key": "inventory_job", "when": { "kind": "node_event", "selection": ["model:orders"], @@ -1316,47 +1323,46 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 30, "default_status": "running", - "declared_at": {"path": "orchestration/orders.py", "line": 7}, + "declared_at": {"path": "orchestration/sensors/on_orders_sensor.py", "line": 5}, } }, ), SnapshotDiscoveryTestCase( description="source_new_data selectors freeze only source nodes without event kinds", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" - ).read_text(), + **duckshop_orchestration("source_orders"), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, - expected_jobs=("orders_refresh",), + expected_jobs=("orders_refresh_job",), expected_sensors={ - "on_source_orders": { + "on_source_orders_sensor": { "description": "Refresh orders for the first source version and each changed version.", - "job_key": "orders_refresh", + "job_key": "orders_refresh_job", "when": {"kind": "source_new_data", "selection": ["source:raw_orders"]}, "minimum_interval_seconds": 1, "default_status": "running", - "declared_at": {"path": "orchestration/orders.py", "line": 10}, + "declared_at": { + "path": "orchestration/sensors/on_source_orders_sensor.py", + "line": 5, + }, } }, ), SnapshotDiscoveryTestCase( description="source_stale freezes source nodes and a positive timestamp-age threshold", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" - ).read_text(), + **duckshop_orchestration("stale_orders"), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, - expected_jobs=("orders_refresh",), + expected_jobs=("orders_refresh_job",), expected_sensors={ - "on_stale_orders": { + "on_stale_orders_sensor": { "description": "Refresh orders once for each timestamp version older than one minute.", - "job_key": "orders_refresh", + "job_key": "orders_refresh_job", "when": { "kind": "source_stale", "selection": ["source:raw_orders"], @@ -1364,51 +1370,49 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject }, "minimum_interval_seconds": 1, "default_status": "running", - "declared_at": {"path": "orchestration/orders.py", "line": 7}, + "declared_at": { + "path": "orchestration/sensors/on_stale_orders_sensor.py", + "line": 5, + }, } }, ), SnapshotDiscoveryTestCase( description="source-job execution selectors compile to frozen canonical keys", - files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" - ).read_text() - }, - expected_jobs=("inventory", "source_orders"), + files={**duckshop_orchestration("run_orders")}, + expected_jobs=("inventory_job", "source_orders_job"), expected_sensors={ - "on_orders": { + "on_orders_sensor": { "description": "Build inventory only after orders actually succeeds.", - "job_key": "inventory", + "job_key": "inventory_job", "when": { "kind": "run_event", - "job_key": "source_orders", + "job_key": "source_orders_job", "kinds": ["completed"], "executed_successfully": ["model:orders"], }, "minimum_interval_seconds": 30, "default_status": "running", - "declared_at": {"path": "orchestration/orders.py", "line": 7}, + "declared_at": {"path": "orchestration/sensors/on_orders_sensor.py", "line": 5}, } }, ), SnapshotDiscoveryTestCase( description="sensor-only roots preserve referenced jobs and filter targets", - files={ - "orchestration/health.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" - ).read_text() - }, - expected_jobs=("health_orders",), + files={**duckshop_orchestration("platform_orders")}, + expected_jobs=("health_orders_job",), expected_sensors={ - "overdue_orders": { + "overdue_orders_sensor": { "description": "Catch up overdue order reporting.", - "job_key": "health_orders", + "job_key": "health_orders_job", "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, "minimum_interval_seconds": 7, "default_status": "paused", "targets": ["development"], - "declared_at": {"path": "orchestration/health.py", "line": 9}, + "declared_at": { + "path": "orchestration/sensors/overdue_orders_sensor.py", + "line": 8, + }, } }, ), @@ -1435,7 +1439,6 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects project=project, metadata_json=json.dumps(METADATA), project_dir=tmp_path, - declarations=test_case.declarations, discovered_inputs=discovered, ).to_json() ) @@ -1470,21 +1473,40 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects [ SnapshotDiscoveryRejectionTestCase( description="group members are jobs, not nested groups", + files=duckshop_orchestration("nested_groups"), + expected_error=ValueError, + expected_message=( + "names 'daily_orders_job_group', which is not a job in orchestration/jobs/" + ), + ), + SnapshotDiscoveryRejectionTestCase( + description="a sensor import does not define is refused, not dropped", files={ - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/nested_groups.py" - ).read_text(), + "orchestration/jobs/refresh_orders_job.toml": ( + 'name = "refresh_orders_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + "orchestration/sensors/overdue_sensor.py": ( + "from sqlbuild.orchestration import PlatformEventKind, platform_event, sensor\n" + "SENSORS = {'overdue': sensor('overdue_sensor', description='Catch up.', " + "when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]), " + "run='refresh_orders_job')}\n" + ), }, - expected_error=TypeError, - expected_message="group members must be Job objects", + expected_error=ValueError, + expected_message=( + "sensor 'overdue_sensor' is declared here, but importing the module does not " + "define it" + ), ), SnapshotDiscoveryRejectionTestCase( description="a polling sensor needs an actual callback", files={ - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, sensor\n" - "orders = job('refresh_orders', description='Build orders.', select='orders')\n" - "poll = sensor('poll', description='Poll orders.', run=orders)\n" + "orchestration/jobs/refresh_orders_job.toml": ( + 'name = "refresh_orders_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + "orchestration/sensors/poll_sensor.py": ( + "from sqlbuild.orchestration import sensor\n" + "poll = sensor('poll_sensor', description='Poll orders.', run='refresh_orders_job')\n" ), }, expected_error=ValueError, @@ -1493,10 +1515,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="callbacks with event conditions remain explicitly unsupported", files={ - "orchestration/orders.py": ( - "from sqlbuild.orchestration import job, sensor, platform_event, PlatformEventKind\n" - "orders = job('refresh_orders', description='Build orders.', select='orders')\n" - "@sensor('poll', description='Poll orders.', run=orders, " + "orchestration/jobs/refresh_orders_job.toml": ( + 'name = "refresh_orders_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + "orchestration/sensors/poll_sensor.py": ( + "from sqlbuild.orchestration import sensor, platform_event, PlatformEventKind\n" + "@sensor('poll_sensor', description='Poll orders.', run='refresh_orders_job', " "when=platform_event(kinds=[PlatformEventKind.TICK_OVERDUE]))\n" "def poll(ctx):\n return []\n" ), @@ -1507,113 +1531,93 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="a job description past max_description_lines cannot publish", files={ - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job\n" - "daily = job('daily', description='Build orders. ' * 100, select='orders')\n" + "orchestration/jobs/daily_job.toml": ( + 'name = "daily_job"\n' + 'description = """Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders. Build orders."""\n' + 'select = "orders"\n' ), }, expected_error=ValueError, expected_message=re.escape( - "job 'daily' description exceeds 10 lines; shorten it to 10 lines" + "job 'daily_job' description exceeds 10 lines; shorten it to 10 lines" ), ), SnapshotDiscoveryRejectionTestCase( - description="a group declaring after is refused at its module", + description="a group cannot declare an order between its jobs", files={ - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job, job_group\n" - "staging = job('staging', description='Stage orders.', select='stg_orders')\n" - "marts = job('marts', description='Build orders.', select='orders')\n" - "daily = job_group('daily', description='Ordered jobs.', " - "jobs=[marts, staging], after={staging: [marts]})\n" + "orchestration/jobs/staging_job.toml": ( + 'name = "staging_job"\ndescription = "Stage orders."\nselect = "stg_orders"\n' + ), + "orchestration/jobs/marts_job.toml": ( + 'name = "marts_job"\ndescription = "Build orders."\nselect = "orders"\n' + ), + "orchestration/job_groups/daily_job_group.toml": ( + 'name = "daily_job_group"\n' + 'description = "Ordered jobs."\n' + 'jobs = ["marts_job", "staging_job"]\n' + 'after = { staging_job = ["marts_job"] }\n' ), }, expected_error=ValueError, - expected_message=re.escape( - "Failed to import orchestration file orchestration/jobs.py: JobGroup.__init__() " - "got an unexpected keyword argument 'after'" - ), + expected_message="after", ), SnapshotDiscoveryRejectionTestCase( - description="distinct objects cannot share a job name", + description="two job files cannot share a name", files={ - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job\n" - "jobs = [job('daily', description='Orders.', select='orders') for _ in range(2)]\n" + "orchestration/jobs/daily_job.toml": ( + 'name = "daily_job"\ndescription = "Orders."\nselect = "orders"\n' + ), + "orchestration/jobs/copy/daily_job.toml": ( + 'name = "daily_job"\ndescription = "Orders."\nselect = "orders"\n' ), }, expected_error=DiscoveryConflictError, expected_message=re.escape( - "Project resource name 'daily' is declared as both job in orchestration/jobs.py " - "and job in orchestration/jobs.py" + ORCHESTRATION_NAME_CONFLICT + "Project resource name 'daily_job' is declared as both job in " + "orchestration/jobs/copy/daily_job.toml and job in orchestration/jobs/daily_job.toml" + + ORCHESTRATION_NAME_CONFLICT ), ), SnapshotDiscoveryRejectionTestCase( description="DuckShop jobs cannot reuse a model name", - files={ - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/name_collision.py" - ).read_text() - }, - expected_error=DiscoveryConflictError, - expected_message=re.escape( - "Project resource name 'orders' is declared as both model in models/orders.sql " - "and job in orchestration/jobs.py" + ORCHESTRATION_NAME_CONFLICT - ), - ), - SnapshotDiscoveryRejectionTestCase( - description="a sensor cannot reuse its job's name", - files={ - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job, sensor\n" - "daily = job('daily', description='Orders.', select='orders')\n" - "poll = sensor('daily', description='Poll orders.', run=daily)\n" - ), - }, + files=duckshop_orchestration("name_collision") + | {"models/orders_job.sql": "MODEL (description 'Orders.');\nSELECT 1 AS order_id\n"}, expected_error=DiscoveryConflictError, expected_message=re.escape( - "Project resource name 'daily' is declared as both sensor in orchestration/jobs.py " - "and job in orchestration/jobs.py" + ORCHESTRATION_NAME_CONFLICT + "Project resource name 'orders_job' is declared as both model in " + "models/orders_job.sql and job in orchestration/jobs/orders_job.toml" + + ORCHESTRATION_NAME_CONFLICT ), ), SnapshotDiscoveryRejectionTestCase( description="broken imports fail on every retry", - files={ - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/failed_import.py" - ).read_text() - }, + files=duckshop_orchestration("failed_import"), expected_error=PythonNodeDiscoveryError, - expected_message="Failed to import orchestration file orchestration/jobs.py", + expected_message=( + "Failed to import orchestration file orchestration/sensors/broken_orders_sensor.py" + ), ), SnapshotDiscoveryRejectionTestCase( description="DuckShop settings cannot replace the selection", - files={ - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/invalid_settings.py" - ).read_text() - }, + files={**duckshop_orchestration("invalid_settings")}, expected_error=ValueError, expected_message="owned by the run request", ), SnapshotDiscoveryRejectionTestCase( description="DuckShop mutually exclusive settings fail on every retry", - files={ - "orchestration/jobs.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/conflicting_settings.py" - ).read_text() - }, + files={**duckshop_orchestration("conflicting_settings")}, expected_error=ValueError, expected_message="mutually exclusive", ), SnapshotDiscoveryRejectionTestCase( description="a zero sensor interval cannot publish", files={ - "orchestration/health.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + **edited_duckshop_orchestration( + "platform_orders", + "orchestration/sensors/overdue_orders_sensor.py", + "minimum_interval_seconds=7", + "minimum_interval_seconds=0", ) - .read_text() - .replace("minimum_interval_seconds=7", "minimum_interval_seconds=0") }, expected_error=ValueError, expected_message="nonzero u32", @@ -1621,11 +1625,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="empty platform conditions cannot publish", files={ - "orchestration/health.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + **edited_duckshop_orchestration( + "platform_orders", + "orchestration/sensors/overdue_orders_sensor.py", + "kinds=[PlatformEventKind.TICK_OVERDUE]", + "kinds=[]", ) - .read_text() - .replace("kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=[]") }, expected_error=ValueError, expected_message="nonempty event kinds", @@ -1633,11 +1638,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="platform kinds are a closed vocabulary", files={ - "orchestration/health.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + **edited_duckshop_orchestration( + "platform_orders", + "orchestration/sensors/overdue_orders_sensor.py", + "kinds=[PlatformEventKind.TICK_OVERDUE]", + "kinds=['unknown_event']", ) - .read_text() - .replace("kinds=[PlatformEventKind.TICK_OVERDUE]", "kinds=['unknown_event']") }, expected_error=ValueError, expected_message="unknown variant", @@ -1645,11 +1651,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="empty node selectors cannot broaden matching to all nodes", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + **edited_duckshop_orchestration( + "node_orders", + "orchestration/sensors/on_orders_sensor.py", + 'select="orders orders*"', + 'select=" "', ) - .read_text() - .replace('select="orders orders*"', 'select=" "') }, expected_error=ValueError, expected_message="node_event needs nonempty selectors", @@ -1657,11 +1664,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="unknown node event kinds cannot publish", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/node_orders.py" + **edited_duckshop_orchestration( + "node_orders", + "orchestration/sensors/on_orders_sensor.py", + "kinds=[NodeEventKind.SUCCESS]", + "kinds=['unknown_event']", ) - .read_text() - .replace("kinds=[NodeEventKind.SUCCESS]", "kinds=['unknown_event']") }, expected_error=ValueError, expected_message="unknown variant", @@ -1669,11 +1677,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="source_new_data rejects model selections", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" - ) - .read_text() - .replace('select="source:raw_orders"', 'select="orders"'), + **edited_duckshop_orchestration( + "source_orders", + "orchestration/sensors/on_source_orders_sensor.py", + 'select="source:raw_orders"', + 'select="orders"', + ), }, expected_error=ValueError, expected_message="source_new_data must select only source nodes", @@ -1681,11 +1690,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="source_new_data rejects blank selections", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/source_orders.py" - ) - .read_text() - .replace('select="source:raw_orders"', 'select=" "'), + **edited_duckshop_orchestration( + "source_orders", + "orchestration/sensors/on_source_orders_sensor.py", + 'select="source:raw_orders"', + 'select=" "', + ), }, expected_error=ValueError, expected_message="source_new_data needs nonempty selectors", @@ -1693,56 +1703,60 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="source_stale rejects a zero threshold", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" - ) - .read_text() - .replace("after_seconds=60", "after_seconds=0"), + **edited_duckshop_orchestration( + "stale_orders", + "orchestration/sensors/on_stale_orders_sensor.py", + "after_seconds=60", + "after_seconds=0", + ), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, expected_error=ValueError, - expected_message="sensor on_stale_orders", + expected_message="sensor on_stale_orders_sensor", ), SnapshotDiscoveryRejectionTestCase( description="source_stale rejects a boolean threshold", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" - ) - .read_text() - .replace("after_seconds=60", "after_seconds=True"), + **edited_duckshop_orchestration( + "stale_orders", + "orchestration/sensors/on_stale_orders_sensor.py", + "after_seconds=60", + "after_seconds=True", + ), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, expected_error=ValueError, - expected_message="sensor on_stale_orders", + expected_message="sensor on_stale_orders_sensor", ), SnapshotDiscoveryRejectionTestCase( description="source_stale rejects an overflowing threshold", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" - ) - .read_text() - .replace("after_seconds=60", "after_seconds=2**32"), + **edited_duckshop_orchestration( + "stale_orders", + "orchestration/sensors/on_stale_orders_sensor.py", + "after_seconds=60", + "after_seconds=2**32", + ), "sources/orders.yml": Path( "benchmarks/duckshop/edge_cases/orchestration/source_orders.yml" ).read_text(), }, expected_error=ValueError, - expected_message="sensor on_stale_orders", + expected_message="sensor on_stale_orders_sensor", ), SnapshotDiscoveryRejectionTestCase( description="source_stale rejects model selections", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/stale_orders.py" - ) - .read_text() - .replace('select="source:raw_orders"', 'select="orders"'), + **edited_duckshop_orchestration( + "stale_orders", + "orchestration/sensors/on_stale_orders_sensor.py", + 'select="source:raw_orders"', + 'select="orders"', + ), }, expected_error=ValueError, expected_message="source_stale must select only source nodes", @@ -1750,11 +1764,9 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="explicit empty execution selectors cannot broaden matching", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + **edited_duckshop_orchestration( + "run_orders", "orchestration/sensors/on_orders_sensor.py", '("orders",)', "()" ) - .read_text() - .replace('("orders",)', "()") }, expected_error=ValueError, expected_message="executed_successfully needs nonempty selectors", @@ -1762,11 +1774,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="blank execution selectors cannot broaden matching", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + **edited_duckshop_orchestration( + "run_orders", + "orchestration/sensors/on_orders_sensor.py", + '("orders",)', + '(" ",)', ) - .read_text() - .replace('("orders",)', '(" ",)') }, expected_error=ValueError, expected_message="executed_successfully needs nonempty selectors", @@ -1774,11 +1787,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="unresolved execution selectors fail closed", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + **edited_duckshop_orchestration( + "run_orders", + "orchestration/sensors/on_orders_sensor.py", + '("orders",)', + '("unknown_orders",)', ) - .read_text() - .replace('("orders",)', '("unknown_orders",)') }, expected_error=ValueError, expected_message="unknown selector name 'unknown_orders'", @@ -1786,11 +1800,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="unmatched execution patterns cannot become an omitted filter", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + **edited_duckshop_orchestration( + "run_orders", + "orchestration/sensors/on_orders_sensor.py", + '("orders",)', + '("unknown_*",)', ) - .read_text() - .replace('("orders",)', '("unknown_*",)') }, expected_error=ValueError, expected_message="executed_successfully: unknown selector name 'unknown_\\*'", @@ -1798,11 +1813,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="invalid execution selector syntax fails closed", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + **edited_duckshop_orchestration( + "run_orders", + "orchestration/sensors/on_orders_sensor.py", + '("orders",)', + '("tag:",)', ) - .read_text() - .replace('("orders",)', '("tag:",)') }, expected_error=ValueError, expected_message="has empty value after ':'", @@ -1810,11 +1826,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="execution selectors outside the source job fail closed", files={ - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/run_orders.py" + **edited_duckshop_orchestration( + "run_orders", + "orchestration/sensors/on_orders_sensor.py", + '("orders",)', + '("stg_orders",)', ) - .read_text() - .replace('("orders",)', '("stg_orders",)') }, expected_error=ValueError, expected_message="executed_successfully must select nonempty source job/group nodes", @@ -1822,11 +1839,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects SnapshotDiscoveryRejectionTestCase( description="sensor code defaults use running or paused", files={ - "orchestration/health.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/platform_orders.py" + **edited_duckshop_orchestration( + "platform_orders", + "orchestration/sensors/overdue_orders_sensor.py", + 'default_status="paused"', + 'default_status="stopped"', ) - .read_text() - .replace('default_status="paused"', 'default_status="stopped"') }, expected_error=ValueError, expected_message="unknown variant", @@ -1838,16 +1856,15 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho tmp_path: Path, test_case: SnapshotDiscoveryRejectionTestCase ) -> None: write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES | test_case.files) - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discovered, - adapter=DuckDbAdapter(), - resolved_connection={"database": "warehouse", "schema": "main"}, - ) for _ in range(2): with pytest.raises(test_case.expected_error, match=test_case.expected_message): + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) compile_snapshot( - project=project, + project=build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ), metadata_json=json.dumps(METADATA), project_dir=tmp_path, discovered_inputs=discovered, @@ -1903,22 +1920,8 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_cli( tmp_path: Path, test_case: PythonJobSelectionTestCase ) -> None: - write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES) - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discovered, - adapter=DuckDbAdapter(), - resolved_connection={"database": "warehouse", "schema": "main"}, - ) - snapshot: dict[str, Any] = json.loads( - compile_snapshot( - project=project, - metadata_json=json.dumps(METADATA), - project_dir=tmp_path, - declarations=[job("orders_job", description="Orders.", select=test_case.select)], - discovered_inputs=discovered, - ).to_json() + python_pipeline_snapshot(tmp_path, {"orders_job": test_case.select}).to_json() ) assert snapshot["documents"]["jobs"]["orders_job"]["selection"] == list( @@ -1936,7 +1939,7 @@ def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_c ("carrier_sync_job", "python/fulfillment/carrier_sync"), ), expected_groups={ - "orders_nightly": { + "orders_nightly_job_group": { "carrier_sync_job": ["orders_export_job"], "orders_export_job": [], } @@ -1946,7 +1949,7 @@ def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_c description="a Python node's SQL dependency orders it after the model's job", jobs=(("orders_export_job", "asset:partner_orders_file"), ("models_job", "+orders")), expected_groups={ - "orders_nightly": {"models_job": [], "orders_export_job": ["models_job"]} + "orders_nightly_job_group": {"models_job": [], "orders_export_job": ["models_job"]} }, ), ], @@ -1955,13 +1958,11 @@ def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_c def test_given_python_dependencies_across_jobs_when_publishing_a_group_then_jobs_wait_for_them( tmp_path: Path, test_case: PythonJobGroupTestCase ) -> None: - jobs: list[Job] = [ - job(name, description="Orders.", select=select) for name, select in test_case.jobs - ] - group: JobGroup = job_group("orders_nightly", description="Nightly orders.", jobs=jobs) - + jobs: dict[str, str] = dict(test_case.jobs) snapshot: dict[str, Any] = json.loads( - python_pipeline_snapshot(tmp_path, [*jobs, group]).to_json() + python_pipeline_snapshot( + tmp_path, jobs, {"orders_nightly_job_group": tuple(jobs)} + ).to_json() ) assert { @@ -1979,7 +1980,7 @@ def test_given_python_dependencies_across_jobs_when_publishing_a_group_then_jobs ("export_job", "asset:partner_orders_file"), ), expected_message=( - "group dependencies contain a cycle in job group orders_nightly: " + "group dependencies contain a cycle in job group orders_nightly_job_group: " "export_job waits for staging_job (model:orders reads model:stg_orders); " "staging_job waits for export_job " "(task:request_carrier_collection reads asset:partner_orders_file)" @@ -1991,13 +1992,9 @@ def test_given_python_dependencies_across_jobs_when_publishing_a_group_then_jobs def test_given_a_python_dependency_cycle_across_jobs_when_publishing_a_group_then_it_is_rejected( tmp_path: Path, test_case: PythonJobGroupCycleTestCase ) -> None: - jobs: list[Job] = [ - job(name, description="Orders.", select=select) for name, select in test_case.jobs - ] - group: JobGroup = job_group("orders_nightly", description="Nightly orders.", jobs=jobs) - + jobs: dict[str, str] = dict(test_case.jobs) with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): - python_pipeline_snapshot(tmp_path, [*jobs, group]) + python_pipeline_snapshot(tmp_path, jobs, {"orders_nightly_job_group": tuple(jobs)}) @pytest.mark.parametrize( @@ -2014,11 +2011,11 @@ def test_given_a_python_dependency_cycle_across_jobs_when_publishing_a_group_the def test_given_a_job_matching_a_check_and_its_node_when_publishing_then_the_check_runs_with_it( tmp_path: Path, test_case: PythonCheckJobTestCase ) -> None: - nightly: Job = job("nightly_orders", description="Orders.", select=test_case.select) - - snapshot: dict[str, Any] = json.loads(python_pipeline_snapshot(tmp_path, [nightly]).to_json()) + snapshot: dict[str, Any] = json.loads( + python_pipeline_snapshot(tmp_path, {"nightly_orders_job": test_case.select}).to_json() + ) - assert snapshot["documents"]["jobs"]["nightly_orders"]["selection"] == ( + assert snapshot["documents"]["jobs"]["nightly_orders_job"]["selection"] == ( test_case.expected_selection ) @@ -2030,7 +2027,7 @@ def test_given_a_job_matching_a_check_and_its_node_when_publishing_then_the_chec description="a job matching a check without the model it checks names both", select="tag:order_checks", expected_message=( - "job nightly_orders: Python check 'orders_present' checks model:orders, which " + "job nightly_orders_job: Python check 'orders_present' checks model:orders, which " "the selection does not include; select it too, or narrow the selector so it no " "longer matches the check." ), @@ -2041,7 +2038,5 @@ def test_given_a_job_matching_a_check_and_its_node_when_publishing_then_the_chec def test_given_a_job_matching_a_check_without_its_node_when_publishing_then_it_is_rejected( tmp_path: Path, test_case: PythonCheckJobRejectionTestCase ) -> None: - nightly: Job = job("nightly_orders", description="Orders.", select=test_case.select) - with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): - python_pipeline_snapshot(tmp_path, [nightly]) + python_pipeline_snapshot(tmp_path, {"nightly_orders_job": test_case.select}) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py index 79ec931f6..70a63b7df 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py @@ -1059,7 +1059,7 @@ def test_given_parameterized_test_when_discovering_inputs_then_cases_are_not_dup expected_error_fragment=( "Project resource name 'orders' is declared as both model in " "models/marts/orders.sql and unit test in tests/unit/orders.sql; .*unit test, " - "scenario, job, job group, sensor, and calendar schedule names must be globally unique" + "scenario, job, job group, sensor, and calendar names must be globally unique" ), ), DiscoverProjectInputsErrorTestCase( diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index c8956b692..d323e30c6 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -143,18 +143,18 @@ belongs to a scheduled job without the changed model. That job's next run refres model is held back: ``` - staging_daily (schedule) finance_daily (schedule) + staging_daily_job (schedule) finance_daily_job (schedule) stg_orders* ─▶ int_orders ✓ ──────────▶ fct_revenue ⏸ ─▶ rpt_pnl ⏸ │ └──▶ orders_export ✓ (in no scheduled job) ``` -- A job counts when a schedule, calendar schedule or sensor starts it, directly or through its +- A job counts when a schedule, calendar or sensor starts it, directly or through its group, for the target you build. A job nothing starts counts as no job. - A model held back by one job runs if another job that holds the change also contains it. - A model that reads a held-back model is held back too. Unchanged parents never hold a model back. - A job never holds back its own models while it runs: a scheduled run, or a build that selects the - job, such as `sqb build finance_daily --changed`. + job, such as `sqb build finance_daily_job --changed`. - With no scheduled jobs, everything downstream runs. `--reach all` builds everything downstream of a change. `--reach` without `--changed` is refused. diff --git a/website/src/content/docs/docs/cli/rename.mdx b/website/src/content/docs/docs/cli/rename.mdx index a1bfdc847..24b828d4e 100644 --- a/website/src/content/docs/docs/cli/rename.mdx +++ b/website/src/content/docs/docs/cli/rename.mdx @@ -82,6 +82,7 @@ Renaming or moving a model rewrites: - `__ref__` and `__expected__` fixture CTEs in tests and scenarios; - `cursor_inputs` keys, `relationships (to ...)` audits, and `__ref("")` inside quoted SQL in `MODEL` headers and reusable `SCHEMA` declarations; - `relationships` audits (`to:` and `field:`) and `__ref("")` in source and seed YAML. +- the model's name and folder in job and job group `select` and `exclude` selectors; see [Rename models in jobs](/docs/concepts/orchestration#rename-models-in-jobs). The model's own unit tests, the test files whose every `TEST` block targets only that model, follow it: diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index b92bd99a3..f1190bf0c 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -374,18 +374,18 @@ Details: Declare who owns a job, whether scheduled requests wait for a running copy, and how often a failed run retries: -```python -from sqlbuild.orchestration import job, on_failure, schedule - -nightly_orders = job( - "nightly_orders", - description="Orders and their daily totals", - select="+order_totals", - owner="Orders team", - overlap="skip", - on_failure=on_failure(retries=2, retry_delay_seconds=600), - schedule=schedule(cron="0 2 * * *", timezone="UTC"), -) +```toml +# orchestration/jobs/nightly_orders_job.toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" +owner = "Orders team" +overlap = "skip" +schedule = { cron = "0 2 * * *", timezone = "UTC" } + +[on_failure] +retries = 2 +retry_delay_seconds = 600 ``` | Setting | Values | Default | Effect | @@ -413,8 +413,8 @@ Details: - A group retries at group level, with the group's own `on_failure`: each retry is one rerun of the group's failed and skipped nodes. A member job's `on_failure` applies only when that job runs on its own. -- An invalid value fails publication with the job's name, such as - `job nightly_orders overlap must be queue or skip, not coalesce`. +- An invalid value is a `D017` error at its line, such as + ``orchestration/jobs/nightly_orders_job.toml:6:11: unknown variant `coalesce`, expected `queue` or `skip` ``. ## Limit worker containers @@ -438,15 +438,15 @@ pids = 512 max_memory = "4g" ``` -```python -from sqlbuild.orchestration import job, resources - -nightly_orders = job( - "nightly_orders", - description="Orders and their daily totals", - select="+order_totals", - resources=resources(memory="3g", cpus=2), -) +```toml +# orchestration/jobs/nightly_orders_job.toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" + +[resources] +memory = "3g" +cpus = 2 ``` | Setting | `run` table | `resources` | Value | @@ -466,7 +466,7 @@ nightly_orders = job( `no-new-privileges`. - A job in a group uses its own `resources`. Local launchers ignore `resources`. - An invalid value fails publication with the job's name, such as - `job nightly_orders resources: cpus must be a positive number`. + `job nightly_orders_job resources: cpus must be a positive number`. ## Run ad-hoc commands diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index 74edd28c5..303297b71 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -74,7 +74,7 @@ error[P010]: model 'orders_summary' has no description A description may take at most `[format] max_description_lines` lines (default `10`) once wrapped at the line width, the same limit `sqb format --check` applies to headers. A longer Python description, from a decorator or a docstring, is a `P027` compile error at its declaration. A job, -job group, sensor or calendar schedule with a longer description cannot publish. +job group, sensor or calendar with a longer description cannot publish. SQL tests, audits, macros, enums, constants, materializations and event or output sinks do not need a description. Descriptions do not change fingerprints, so adding or editing one does not rebuild a diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx new file mode 100644 index 000000000..3aee7cb79 --- /dev/null +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -0,0 +1,217 @@ +--- +title: "Orchestration" +description: "Declare jobs, job groups and schedules in TOML, and calendars and sensors in Python, for a SQLBuild server to run." +--- + +A job names the nodes a server builds together, and a schedule says when. Write each job as a TOML +file under `orchestration/jobs/`; `sqb compile` checks it and a code location publishes it. + +```toml +# orchestration/jobs/nightly_orders_job.toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" +schedule = { cron = "0 2 * * *", timezone = "UTC" } +``` + +```bash +sqb build nightly_orders_job # build what the job builds, locally +``` + +## Project layout + +| Folder | Holds | Name suffix | +| --- | --- | --- | +| `orchestration/jobs/` | One job per `.toml` file | `_job` | +| `orchestration/job_groups/` | One job group per `.toml` file | `_job_group` | +| `orchestration/calendars/` | One calendar per `.py` module | `_calendar` | +| `orchestration/sensors/` | One sensor per `.py` module | `_sensor` | + +- Subfolders are allowed, such as `orchestration/jobs/finance/revenue_job.toml`. +- A name must end with its folder's suffix. Names are unique across the whole project, with models, + sources and every other resource. +- A Python module whose file name starts with `_` is a private helper and declares nothing. +- Any other file under `orchestration/`, or a folder not in the table, is a `D017` error. + +## Declare a job + +A job builds its `select` minus its `exclude`, with the same [selectors](/docs/concepts/selectors) +as `sqb build`. + +```toml +name = "nightly_orders_job" +description = "Orders and their daily totals" +select = "+order_totals" +exclude = "tag:deprecated" +targets = ["production"] +owner = "Orders team" +overlap = "skip" +concurrency_groups = ["warehouse_heavy"] +schedule = { cron = "0 2 * * *", timezone = "Europe/London" } + +[on_failure] +retries = 2 +retry_delay_seconds = 600 + +[settings] +concurrency = 4 +vars = { batch_size = 500 } + +[resources] +memory = "3g" +cpus = 2 +``` + +| Key | Required | Value | +| --- | --- | --- | +| `name` | Yes | The job's name, ending in `_job` | +| `description` | Yes | Shown on the job's page | +| `select` | Yes | The nodes the job builds | +| `exclude` | No | Nodes removed from `select` | +| `targets` | No | The targets the job exists in; all targets when omitted | +| `owner` | No | Shown on the job's page and in the job list | +| `overlap` | No | `queue` (default) or `skip`; see [Skip overlapping runs](/docs/cli/server#skip-overlapping-runs-and-retry-failures) | +| `concurrency_groups` | No | Server concurrency limits the job's runs share | +| `schedule` | No | When the server starts the job; see below | +| `[on_failure]` | No | `retries` from 1 to 10, `retry_delay_seconds` from 0 to 86400 | +| `[settings]` | No | Build options for each run, such as `concurrency`, `vars`, `full_refresh` or `changed` | +| `[resources]` | No | Docker worker limits; see [Limit worker containers](/docs/cli/server#limit-worker-containers) | + +A job's `[settings]` cannot set `select` or `exclude`: the job's own selection is the run's +selection. + +## Group jobs + +A job group runs its jobs as one run. A job waits for another job in the group when it reads a node +that job builds. + +```toml +# orchestration/job_groups/daily_orders_job_group.toml +name = "daily_orders_job_group" +description = "Stage orders, then build the marts" +jobs = ["staging_orders_job", "orders_marts_job"] +schedule = { cron = "0 3 * * *", timezone = "UTC" } +``` + +A group takes `name`, `description` and `jobs`, and the same `targets`, `owner`, `overlap`, +`concurrency_groups`, `schedule` and `[on_failure]` as a job. Its members are jobs, never other +groups. + +## Schedule a job + +`schedule` takes a cron expression or the name of a calendar, and always a timezone. + +| Key | Value | +| --- | --- | +| `cron` | A five-field cron expression | +| `calendar` | The name of a calendar in `orchestration/calendars/` | +| `timezone` | An IANA timezone, such as `"UTC"` or `"America/New_York"` | +| `starts_at` | An RFC 3339 time with an offset; no ticks before it | +| `default_status` | `running` (default) or `paused` when the schedule is first published | + +```toml +schedule = { cron = "15 2 * * *", timezone = "Europe/London", starts_at = "2026-07-03T00:00:00Z" } +``` + +## Schedule from a calendar + +A calendar returns the upcoming times a job runs, such as fulfillment cut-offs that come from +another system. The code location evaluates it when it publishes and again before its times run +out. + +```python +# orchestration/calendars/fulfillment_calendar.py +from sqlbuild.orchestration import calendar + + +@calendar(name="fulfillment_calendar", description="Upcoming fulfillment times", horizon="1d") +def fulfillment_calendar(window, fulfillment): + return fulfillment.upcoming(window) +``` + +```toml +schedule = { calendar = "fulfillment_calendar", timezone = "America/New_York" } +``` + +The function receives a `window` with timezone-aware `start` and `end`, in the job's timezone, and +returns timezone-aware datetimes inside it. Other parameters name +[providers](/docs/concepts/python-nodes/providers). `horizon` sets how far ahead the window reaches. + +## Start a job from a sensor + +A sensor starts a job or job group when something happens, named by `run`. + +```python +# orchestration/sensors/on_new_orders_sensor.py +from sqlbuild.orchestration import sensor, source_new_data + +on_new_orders = sensor( + "on_new_orders_sensor", + description="Refresh orders for each new source version", + when=source_new_data(select="source:raw_orders"), + run="orders_refresh_job", + minimum_interval_seconds=60, +) +``` + +| Condition | Starts a run for | +| --- | --- | +| `platform_event(kinds=[...])` | Server events, such as an overdue tick | +| `run_event(job="...", kinds=[...])` | Outcomes of another job or group, optionally only when `executed_successfully` nodes ran and succeeded | +| `node_event(select="...", kinds=[...])` | Each success, failure or recovery of the selected nodes in any run | +| `source_new_data(select="...")` | Each new version of the selected sources | +| `source_stale(select="...", after_seconds=...)` | Each source version older than `after_seconds` | + +A sensor without `when` decorates a function that polls and returns `RunRequest(run_key=...)` +values or a `SkipReason`. A sensor also takes `minimum_interval_seconds` (default 30), +`default_status` and `targets`. + +## Rename models in jobs + +`sqb rename` and `sqb mv` update job and job group selectors with the models they rename or move: + +```bash +sqb rename stg_orders stg_orders_clean +``` + +```text +Rename model stg_orders -> stg_orders_clean +├── models/marts/fact_orders.sql +│ └── 10:13 reference __ref("stg_orders") -> __ref("stg_orders_clean") +├── models/staging/stg_orders_clean.sql (moved from models/staging/stg_orders.sql) +│ └── 1:8 migration + migrate_from stg_orders +└── orchestration/jobs/orders_refresh_job.toml + └── 3:10 selector "+stg_orders+ fact_orders" -> "+stg_orders_clean+ fact_orders" +``` + +- Exact names are rewritten, including inside graph operators such as `+stg_orders+`, `,` and `~`. + Tags, globs and `job:` selectors stay as written. +- A folder selector, such as `models/staging` or `path:models/staging/`, moves when every model + under it moves to the same folder. When only some leave, the command lists a note at the + selector to check by hand. +- A selector in a multi-line TOML string is reported for a manual edit and blocks the refactor. + +## Errors + +Every problem with an orchestration file is a `D017` error at its file, line and column, reported by +`sqb compile` and every other command that reads the project. + +| Error | Fix | +| --- | --- | +| ``orchestration/jobs/nightly_orders_job.toml:4:11: unknown variant `coalesce`, expected `queue` or `skip` `` | Use one of the listed values | +| `files cannot sit directly in orchestration/` | Move jobs and groups to TOML files in `orchestration/jobs/` or `orchestration/job_groups/` | +| `` `job` was removed from sqlbuild.orchestration`` | Declare the job in TOML and name it in the sensor's `run` | +| `declares a second sensor; each module declares exactly one` | Split the module | +| `schedule cron: invalid cron expression: Pattern must have 5 fields when seconds are disallowed.` | Write a five-field cron, such as `0 2 * * *` | +| `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | +| `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | + +Details: + +- `sqb build` and the other commands read job and group files without running project Python, so + a job selector never imports a calendar or sensor module. +- A job or group declared with `targets` is selectable only when the command's target is one of + them, as on the server. +- Selectors are checked for syntax at compile. A selector that names a node the project does not + have fails when a location publishes, such as `job nightly_orders_job: unknown selector name + 'order_total'`. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 4b86231cd..5951af7c2 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -739,7 +739,7 @@ Code that extends SQLBuild itself keeps its own root: | `hooks/python/` | Python model lifecycle hooks | | `functions/python/` | Python UDFs | | `macros/python/`, `enums/`, `constants/` and scoped `_macros/python/`, `_enums/`, `_constants/` | Python macros and declarations | -| `orchestration/` | Jobs, job groups, sensors and calendar schedules. SQLBuild imports these modules only when a selector names a job or an unknown name, and when a location publishes. See [Job selectors](/docs/concepts/selectors#job) | +| `orchestration/` | Jobs and job groups as TOML in `jobs/` and `job_groups/`; calendars and sensors as Python in `calendars/` and `sensors/`. See [Orchestration](/docs/concepts/orchestration) | Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create an unofficial project structure. diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 9d215b702..1c575fb56 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -146,7 +146,7 @@ sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` -Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendar schedules. +Names are globally unique across models, sources, seeds, functions, loaders, tasks, assets, checks, providers, hooks, unit tests, scenarios, jobs, job groups, sensors, and calendars. ## Lifecycle: run, build, check diff --git a/website/src/content/docs/docs/concepts/selectors.mdx b/website/src/content/docs/docs/concepts/selectors.mdx index 7f598d7a3..2d7e7e2a1 100644 --- a/website/src/content/docs/docs/concepts/selectors.mdx +++ b/website/src/content/docs/docs/concepts/selectors.mdx @@ -69,13 +69,13 @@ sqb build --select source:raw__orders ### Job -Select the SQL and Python nodes a job or job group declared under `orchestration/` builds. Job names are globally +Select the SQL and Python nodes a [job or job group](/docs/concepts/orchestration) builds. Job names are globally unique, so a bare name works; `job:` is the explicit form and accepts glob patterns: ```bash -sqb build nightly_orders -sqb build --select job:nightly_orders --exclude order_totals -sqb build --select "job:nightly_orders,tag:finance" +sqb build nightly_orders_job +sqb build --select job:nightly_orders_job --exclude order_totals +sqb build --select "job:nightly_orders_job,tag:finance" ``` A job selects its own `select` minus its own `exclude`. A job group selects the union of its jobs. @@ -84,8 +84,7 @@ them, as on the server. Job selectors compose with `+`, `,` and `--exclude` like any other selector. A job's own selection cannot name another job, and path-between selectors cannot start or end at a job. -SQLBuild imports the modules under `orchestration/` only when a selector names a job or a name no -model, source, seed or function has. +SQLBuild reads job and job group files without running project Python. ### Unit test diff --git a/website/src/sidebar.json b/website/src/sidebar.json index cee743e40..30e1b55f5 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -94,7 +94,8 @@ "label": "Review a branch", "link": "/docs/cli/review/" }, - "docs/concepts/column-lineage" + "docs/concepts/column-lineage", + "docs/concepts/orchestration" ] }, {