From 0a94d0c852cf6a7bbbd1542c7e06f57c1f655833 Mon Sep 17 00:00:00 2001 From: Many Kasiriha Date: Fri, 20 Nov 2026 08:51:33 +0100 Subject: [PATCH 1/2] feat(rf-core): add the rf-bdd skill Adds rf-bdd for behaviour-driven specifications in native .robot files: house rules, Gherkin mapping, step vocabularies, scenario state, a Robocop configuration, and two runnable example projects (resource and World-library strategies), verified on RF 7.4.2 and 7.5.1 and on Robocop 8.2.10 and 9.4. - rf-language points to rf-bdd; its casing gotcha follows the project's Robocop configuration; a comma-separated --select selects nothing (it is not rejected). - The context hook and rf-test-architect route BDD work to rf-bdd. - Evals: the sut-bdd-library fixture (base, steps, review variants), a BDD grader with tests and golden solutions, five tasks and a trigger set. The preflight selects tasks by fixture root. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FHEmzL3ErAhGQykKWDx2wP --- CHANGELOG.md | 6 + README.md | 5 +- eval/fixtures/README.md | 1 + eval/fixtures/sut-bdd-library/base/.gitignore | 9 + eval/fixtures/sut-bdd-library/base/README.md | 21 + .../sut-bdd-library/base/pyproject.toml | 15 + .../sut-bdd-library/base/resources/.gitkeep | 0 eval/fixtures/sut-bdd-library/base/robot.toml | 3 + .../base/smoke/library_service.robot | 12 + .../base/sut/LibraryService.py | 132 +++++ .../sut-bdd-library/base/tests/.gitkeep | 0 .../sut-bdd-library/review/.gitignore | 9 + .../fixtures/sut-bdd-library/review/README.md | 22 + .../sut-bdd-library/review/pyproject.toml | 15 + .../sut-bdd-library/review/resources/.gitkeep | 0 .../sut-bdd-library/review/robot.toml | 3 + .../review/smoke/library_service.robot | 12 + .../review/sut/LibraryService.py | 132 +++++ .../review/tests/loan_steps.resource | 43 ++ .../sut-bdd-library/review/tests/loans.robot | 21 + .../fixtures/sut-bdd-library/steps/.gitignore | 9 + eval/fixtures/sut-bdd-library/steps/README.md | 21 + .../sut-bdd-library/steps/pyproject.toml | 15 + .../steps/resources/borrowing_steps.resource | 38 ++ .../fixtures/sut-bdd-library/steps/robot.toml | 3 + .../steps/smoke/library_service.robot | 12 + .../steps/sut/LibraryService.py | 132 +++++ .../steps/tests/borrowing.robot | 26 + .../changes/add-rf-bdd-skill/.openspec.yaml | 2 + openspec/changes/add-rf-bdd-skill/design.md | 130 +++++ openspec/changes/add-rf-bdd-skill/proposal.md | 81 +++ .../add-rf-bdd-skill/specs/bdd-skill/spec.md | 271 ++++++++++ .../specs/language-skill/spec.md | 44 ++ .../specs/plugin-split/spec.md | 23 + .../specs/plugin-subagents/spec.md | 35 ++ .../specs/rf-session-context-hooks/spec.md | 50 ++ .../specs/skill-triggering/spec.md | 48 ++ openspec/changes/add-rf-bdd-skill/tasks.md | 46 ++ .../add-rf-bdd-skill/trap-verification.md | 32 ++ plugins/rf-core/README.md | 4 +- plugins/rf-core/agents/rf-test-architect.md | 1 + plugins/rf-core/evals/tasks/README.md | 2 +- .../adversarial/adv-bdd-gherkin-row-01.yaml | 99 ++++ .../narrow-bdd-extend-vocabulary-01.yaml | 83 ++++ .../narrow/narrow-bdd-greenfield-01.yaml | 65 +++ .../tasks/narrow/narrow-bdd-outline-01.yaml | 63 +++ .../tasks/narrow/narrow-bdd-review-01.yaml | 49 ++ plugins/rf-core/evals/triggers/rf-bdd.yaml | 46 ++ .../scripts/maybe_inject_rf_context.mjs | 10 +- plugins/rf-core/skills/rf-bdd/SKILL.md | 168 +++++++ .../resources/cash_withdrawal_steps.resource | 56 +++ .../examples/resource-strategy/robot.toml | 24 + .../tests/cash_withdrawal/__init__.robot | 8 + .../cash_withdrawal/remaining_balance.robot | 22 + .../cash_withdrawal/withdrawal_rules.robot | 30 ++ .../tests/daily_withdrawal_limit.robot | 25 + .../rf-bdd/assets/examples/sut/AtmService.py | 69 +++ .../world-strategy/lib/CashWithdrawalWorld.py | 71 +++ .../assets/examples/world-strategy/robot.toml | 23 + .../tests/cash_withdrawal.robot | 26 + .../skills/rf-bdd/assets/robocop-bdd.toml | 23 + .../skills/rf-bdd/references/data-driven.md | 137 +++++ .../references/documentation-and-tags.md | 148 ++++++ .../rf-bdd/references/gherkin-mapping.md | 200 ++++++++ .../rf-bdd/references/review-checklist.md | 99 ++++ .../skills/rf-bdd/references/robocop.md | 62 +++ .../rf-core/skills/rf-bdd/references/state.md | 87 ++++ .../skills/rf-bdd/references/step-design.md | 137 +++++ plugins/rf-core/skills/rf-language/SKILL.md | 7 +- .../references/embedded-arguments.md | 2 +- .../skills/rf-language/references/styles.md | 2 + plugins/rf-core/tests/test_bdd_skill.py | 466 ++++++++++++++++++ plugins/rf-core/tests/test_hook_scripts.py | 19 +- plugins/rf-core/tests/test_language_skill.py | 3 +- .../codex/agents/rf-test-architect.toml | 1 + .../opencode/agents/rf-test-architect.md | 1 + scripts/robotcode-first-allowlist.toml | 10 + src/rf_skill_eval/cli.py | 3 +- src/rf_skill_eval/scoring/custom/bdd.py | 405 +++++++++++++++ .../bad/resources/reservation_steps.resource | 64 +++ .../bad/tests/reservations/__init__.robot | 9 + .../bad/tests/reservations/late_returns.robot | 11 + .../bad/tests/reservations/queue.robot | 13 + .../good/resources/reservation_steps.resource | 64 +++ .../good/tests/reservations/__init__.robot | 9 + .../tests/reservations/late_returns.robot | 15 + .../good/tests/reservations/queue.robot | 13 + .../bad/resources/borrowing_steps.resource | 57 +++ .../bad/tests/borrowing.robot | 47 ++ .../good/resources/borrowing_steps.resource | 50 ++ .../good/tests/borrowing.robot | 62 +++ .../bad/tests/borrowing.robot | 47 ++ .../good/resources/borrowing_steps.resource | 53 ++ .../good/tests/borrowing.robot | 70 +++ .../bad/resources/returning_steps.resource | 14 + .../bad/tests/late_returns.robot | 13 + .../good/resources/returning_steps.resource | 58 +++ .../good/tests/late_returns.robot | 27 + .../bad/stdout.stream.jsonl | 3 + .../bad/tests/loans.robot | 23 + .../good/stdout.stream.jsonl | 3 + tests/eval/test_arms.py | 2 +- tests/eval/test_bdd_grader.py | 359 ++++++++++++++ tests/eval/test_eval_content.py | 85 +++- tests/eval/test_trigger_harness.py | 2 +- tests/eval/test_trigger_sets.py | 19 + tests/installer/test_plugin_selection.py | 4 +- tests/installer/test_reinstall_prune.py | 2 +- tests/installer/test_split_upgrade.py | 4 +- tests/test_catalog.py | 4 +- tests/test_skill_descriptions.py | 21 +- 111 files changed, 5433 insertions(+), 29 deletions(-) create mode 100644 eval/fixtures/sut-bdd-library/base/.gitignore create mode 100644 eval/fixtures/sut-bdd-library/base/README.md create mode 100644 eval/fixtures/sut-bdd-library/base/pyproject.toml create mode 100644 eval/fixtures/sut-bdd-library/base/resources/.gitkeep create mode 100644 eval/fixtures/sut-bdd-library/base/robot.toml create mode 100644 eval/fixtures/sut-bdd-library/base/smoke/library_service.robot create mode 100644 eval/fixtures/sut-bdd-library/base/sut/LibraryService.py create mode 100644 eval/fixtures/sut-bdd-library/base/tests/.gitkeep create mode 100644 eval/fixtures/sut-bdd-library/review/.gitignore create mode 100644 eval/fixtures/sut-bdd-library/review/README.md create mode 100644 eval/fixtures/sut-bdd-library/review/pyproject.toml create mode 100644 eval/fixtures/sut-bdd-library/review/resources/.gitkeep create mode 100644 eval/fixtures/sut-bdd-library/review/robot.toml create mode 100644 eval/fixtures/sut-bdd-library/review/smoke/library_service.robot create mode 100644 eval/fixtures/sut-bdd-library/review/sut/LibraryService.py create mode 100644 eval/fixtures/sut-bdd-library/review/tests/loan_steps.resource create mode 100644 eval/fixtures/sut-bdd-library/review/tests/loans.robot create mode 100644 eval/fixtures/sut-bdd-library/steps/.gitignore create mode 100644 eval/fixtures/sut-bdd-library/steps/README.md create mode 100644 eval/fixtures/sut-bdd-library/steps/pyproject.toml create mode 100644 eval/fixtures/sut-bdd-library/steps/resources/borrowing_steps.resource create mode 100644 eval/fixtures/sut-bdd-library/steps/robot.toml create mode 100644 eval/fixtures/sut-bdd-library/steps/smoke/library_service.robot create mode 100644 eval/fixtures/sut-bdd-library/steps/sut/LibraryService.py create mode 100644 eval/fixtures/sut-bdd-library/steps/tests/borrowing.robot create mode 100644 openspec/changes/add-rf-bdd-skill/.openspec.yaml create mode 100644 openspec/changes/add-rf-bdd-skill/design.md create mode 100644 openspec/changes/add-rf-bdd-skill/proposal.md create mode 100644 openspec/changes/add-rf-bdd-skill/specs/bdd-skill/spec.md create mode 100644 openspec/changes/add-rf-bdd-skill/specs/language-skill/spec.md create mode 100644 openspec/changes/add-rf-bdd-skill/specs/plugin-split/spec.md create mode 100644 openspec/changes/add-rf-bdd-skill/specs/plugin-subagents/spec.md create mode 100644 openspec/changes/add-rf-bdd-skill/specs/rf-session-context-hooks/spec.md create mode 100644 openspec/changes/add-rf-bdd-skill/specs/skill-triggering/spec.md create mode 100644 openspec/changes/add-rf-bdd-skill/tasks.md create mode 100644 openspec/changes/add-rf-bdd-skill/trap-verification.md create mode 100644 plugins/rf-core/evals/tasks/adversarial/adv-bdd-gherkin-row-01.yaml create mode 100644 plugins/rf-core/evals/tasks/narrow/narrow-bdd-extend-vocabulary-01.yaml create mode 100644 plugins/rf-core/evals/tasks/narrow/narrow-bdd-greenfield-01.yaml create mode 100644 plugins/rf-core/evals/tasks/narrow/narrow-bdd-outline-01.yaml create mode 100644 plugins/rf-core/evals/tasks/narrow/narrow-bdd-review-01.yaml create mode 100644 plugins/rf-core/evals/triggers/rf-bdd.yaml create mode 100644 plugins/rf-core/skills/rf-bdd/SKILL.md create mode 100644 plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/resources/cash_withdrawal_steps.resource create mode 100644 plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/robot.toml create mode 100644 plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/cash_withdrawal/__init__.robot create mode 100644 plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/cash_withdrawal/remaining_balance.robot create mode 100644 plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/cash_withdrawal/withdrawal_rules.robot create mode 100644 plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/daily_withdrawal_limit.robot create mode 100644 plugins/rf-core/skills/rf-bdd/assets/examples/sut/AtmService.py create mode 100644 plugins/rf-core/skills/rf-bdd/assets/examples/world-strategy/lib/CashWithdrawalWorld.py create mode 100644 plugins/rf-core/skills/rf-bdd/assets/examples/world-strategy/robot.toml create mode 100644 plugins/rf-core/skills/rf-bdd/assets/examples/world-strategy/tests/cash_withdrawal.robot create mode 100644 plugins/rf-core/skills/rf-bdd/assets/robocop-bdd.toml create mode 100644 plugins/rf-core/skills/rf-bdd/references/data-driven.md create mode 100644 plugins/rf-core/skills/rf-bdd/references/documentation-and-tags.md create mode 100644 plugins/rf-core/skills/rf-bdd/references/gherkin-mapping.md create mode 100644 plugins/rf-core/skills/rf-bdd/references/review-checklist.md create mode 100644 plugins/rf-core/skills/rf-bdd/references/robocop.md create mode 100644 plugins/rf-core/skills/rf-bdd/references/state.md create mode 100644 plugins/rf-core/skills/rf-bdd/references/step-design.md create mode 100644 plugins/rf-core/tests/test_bdd_skill.py create mode 100644 src/rf_skill_eval/scoring/custom/bdd.py create mode 100644 tests/eval/golden/adv-bdd-gherkin-row-01/bad/resources/reservation_steps.resource create mode 100644 tests/eval/golden/adv-bdd-gherkin-row-01/bad/tests/reservations/__init__.robot create mode 100644 tests/eval/golden/adv-bdd-gherkin-row-01/bad/tests/reservations/late_returns.robot create mode 100644 tests/eval/golden/adv-bdd-gherkin-row-01/bad/tests/reservations/queue.robot create mode 100644 tests/eval/golden/adv-bdd-gherkin-row-01/good/resources/reservation_steps.resource create mode 100644 tests/eval/golden/adv-bdd-gherkin-row-01/good/tests/reservations/__init__.robot create mode 100644 tests/eval/golden/adv-bdd-gherkin-row-01/good/tests/reservations/late_returns.robot create mode 100644 tests/eval/golden/adv-bdd-gherkin-row-01/good/tests/reservations/queue.robot create mode 100644 tests/eval/golden/narrow-bdd-extend-vocabulary-01/bad/resources/borrowing_steps.resource create mode 100644 tests/eval/golden/narrow-bdd-extend-vocabulary-01/bad/tests/borrowing.robot create mode 100644 tests/eval/golden/narrow-bdd-extend-vocabulary-01/good/resources/borrowing_steps.resource create mode 100644 tests/eval/golden/narrow-bdd-extend-vocabulary-01/good/tests/borrowing.robot create mode 100644 tests/eval/golden/narrow-bdd-greenfield-01/bad/tests/borrowing.robot create mode 100644 tests/eval/golden/narrow-bdd-greenfield-01/good/resources/borrowing_steps.resource create mode 100644 tests/eval/golden/narrow-bdd-greenfield-01/good/tests/borrowing.robot create mode 100644 tests/eval/golden/narrow-bdd-outline-01/bad/resources/returning_steps.resource create mode 100644 tests/eval/golden/narrow-bdd-outline-01/bad/tests/late_returns.robot create mode 100644 tests/eval/golden/narrow-bdd-outline-01/good/resources/returning_steps.resource create mode 100644 tests/eval/golden/narrow-bdd-outline-01/good/tests/late_returns.robot create mode 100644 tests/eval/golden/narrow-bdd-review-01/bad/stdout.stream.jsonl create mode 100644 tests/eval/golden/narrow-bdd-review-01/bad/tests/loans.robot create mode 100644 tests/eval/golden/narrow-bdd-review-01/good/stdout.stream.jsonl create mode 100644 tests/eval/test_bdd_grader.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e83ea98..101180f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ Every plugin (`rf-core`, `rf-web`, `rf-api`, `rf-mobile`, `rf-desktop`) and the ## Unreleased +### rf-core + +- **New skill `rf-bdd`** for behaviour-driven specifications in native `.robot` files: features and rules as suites, documentation and tags, one step vocabulary, scenario state, outlines as templates, pasted Gherkin or acceptance criteria translated by meaning, and reviews with check IDs. It follows the project's conventions and ships a Robocop configuration for sentence-style steps (`assets/robocop-bdd.toml`) and two runnable example projects. No `.feature` files and no Gherkin parser. +- `rf-language` names `rf-bdd` for BDD specifications. Its casing gotcha now says to follow the project's Robocop configuration instead of requiring Title Case BDD steps. +- The context hook routes Given/When/Then and Gherkin work to `rf-bdd`, and `rf-test-architect` routes BDD specifications to it. + ## 5.0.0 — 2026-11-18 One version for everything: every plugin and the installer are now `5.0.0`, released together with one tag, `v5.0.0`. diff --git a/README.md b/README.md index 4513588..5c9f1ac 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ There are two ways to install, and both work at **user scope** (every project on | Plugin | What it adds | |---|---| -| `rf-core` | Robot Framework syntax (`rf-language`), environment setup (`rf-setup`), Python keyword libraries (`rf-python-library`); the `rf-test-architect`, `rf-debug-expert` and `rf-migration-guide` subagents; all hooks. Every other plugin needs it. | +| `rf-core` | Robot Framework syntax (`rf-language`), BDD specifications (`rf-bdd`), environment setup (`rf-setup`), Python keyword libraries (`rf-python-library`); the `rf-test-architect`, `rf-debug-expert` and `rf-migration-guide` subagents; all hooks. Every other plugin needs it. | | `rf-web` | Browser Library (`rf-browser`) and SeleniumLibrary (`rf-selenium`) | | `rf-api` | RequestsLibrary (`rf-requests`) and RESTinstance (`rf-restinstance`) | | `rf-mobile` | AppiumLibrary (`rf-appium`) | @@ -272,7 +272,7 @@ The installer, hooks and scripts send nothing anywhere. The installer only touch ## What you get -### 9 Skills +### 10 Skills | Skill | Type | Command | Description | |-------|------|---------|-------------| @@ -283,6 +283,7 @@ The installer, hooks and scripts send nothing anywhere. The installer only touch | RESTinstance | library-reference | `/rf-api:rf-restinstance` | REST API tests with RESTinstance (`Library REST`): JSON Schema / OpenAPI-driven assertions | | PlatynUI (preview) | library-reference | `/rf-desktop:rf-platynui` | Native desktop UI testing (Windows UIA, Linux AT-SPI2) via PlatynUI.BareMetal | | Robot Framework language | language-guide + script | `/rf-core:rf-language` | Write and review tests (keyword-driven, data-driven, BDD), suites, `__init__.robot`, tags, user keywords, `.resource` and variable files in modern, version-gated syntax; `rf_conventions.py` script detects the project's conventions (requires robotframework). Not for Python keyword libraries | +| BDD specifications | language-guide | `/rf-core:rf-bdd` | Write and review behaviour-driven specifications in native `.robot` files: features and rules as suites, documentation and tags, one step vocabulary, scenario state, outlines as templates, pasted Gherkin or acceptance criteria translated by meaning, reviews with check IDs; follows the project's conventions. Works together with `rf-language` | | Python keyword libraries | library-guide + script | `/rf-core:rf-python-library` | Write and fix keyword libraries and listeners in Python: `@library`/`@keyword`, scope by state lifetime (TEST/SUITE/GLOBAL), type-hint conversion and custom converters, failures and logging, dynamic/hybrid APIs, listener API v3, libdoc; `check_library.py` script finds "contains no keywords", leaked imports, lost signatures and state kept in `TEST` scope before tests run (requires robotframework, Python 3.10+) | | Setup | setup-guide | `/rf-core:rf-setup` | Install Robot Framework and libraries with uv, venv + pip or Poetry; fix environment errors (`ModuleNotFoundError`, wrong interpreter); project layout, CI | diff --git a/eval/fixtures/README.md b/eval/fixtures/README.md index 65d9e62..eafc564 100644 --- a/eval/fixtures/README.md +++ b/eval/fixtures/README.md @@ -31,6 +31,7 @@ Fixture directories under `eval/fixtures/` should be kept clean (no | `sut-rf71` | Project pinned to `robotframework==7.1.1` (`uv.lock`, `.python-version`); three cart tests repeating the same four steps. Graded with `uvx --from robotframework==7.1.1 robot` (adversarial typed-argument task). | `robotframework==7.1.1` | `uv` with network on first use (skipped otherwise). | | `sut-pylib` | rf-python-library tasks: an empty `libraries/` folder on the python-path (`robot.toml` `python-path = ["libraries"]`) where the agent writes `Inventory.py`, `Modes.py` or `FlakySkip.py`; `tests/example.robot` is a smoke test. Hidden grader suites live in `eval/graders/python-library/`, not here. | `robotframework>=7.0` | None. | | `sut-legacy-style` | Adversarial legacy-syntax task (`adv-legacy-syntax-01`): `resources/legacy.resource` uses `[Return]`, `Run Keyword If`, `Set Suite/Test Variable`, `Create List`; `tests/cart.robot` uses `Force Tags` and passes. The agent must add `resources/orders.resource` in modern syntax despite the "same style" prompt. | `robotframework>=7.4` (`robotframework-robocop` 9.x as dev dependency) | None. | +| `sut-bdd-library` | rf-bdd tasks. One fixture with three variant directories, selected by the task's `fixture` (`sut-bdd-library/`; the harness copies one directory, so each variant is the base plus its overlay files, kept identical by `tests/eval/test_bdd_grader.py`). `base/`: in-memory lending service `sut/LibraryService.py` (library scope TEST, calendar fixed at 2026-11-12, so loans are due 2026-12-03), `robot.toml` (`python-path = ["resources", "sut"]`), empty `tests/` and `resources/`, technical smoke suite `smoke/library_service.robot`. `steps/`: base + `tests/borrowing.robot` and `resources/borrowing_steps.resource` (2 BDD tests; extend-vocabulary task). `review/`: base + `tests/loans.robot` and `tests/loan_steps.resource` with 7 seeded review issues (L1, L2, W1, W2/C5, W3, L6, W12/W6/W7; review task). Smoke: `robotcode robot smoke` (base), `robotcode robot tests` (steps, review). | `robotframework>=7.3` | None. | | `sut-trigger` | Neutral working directory for **trigger evals** (not task evals): `pyproject.toml` declaring only `robotframework`, `tests/smoke.robot` (BuiltIn only), `resources/common.resource`. Imports no test library, so the query alone decides which skill loads. A fresh copy is made per trigger session. | `robotframework>=7.0` | None. | ## Prerequisites diff --git a/eval/fixtures/sut-bdd-library/base/.gitignore b/eval/fixtures/sut-bdd-library/base/.gitignore new file mode 100644 index 0000000..3204997 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/base/.gitignore @@ -0,0 +1,9 @@ +output.xml +log.html +report.html +results/ +.venv/ +__pycache__/ +*.pyc +.pytest_cache/ +.robotcode_cache/ diff --git a/eval/fixtures/sut-bdd-library/base/README.md b/eval/fixtures/sut-bdd-library/base/README.md new file mode 100644 index 0000000..c14df63 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/base/README.md @@ -0,0 +1,21 @@ +# library-lending + +Acceptance tests for the city library's lending service. The service runs in +memory (`sut/LibraryService.py`, a Robot Framework library with one fresh +instance per test); its business rules are in the module docstring. The +library's calendar starts on 2026-11-12, so due dates are fixed (`Set Today` +moves it). + +- `tests/`: acceptance tests (none yet). +- `resources/`: shared keywords for the tests. +- `smoke/library_service.robot`: a technical check that the service library + loads and lends a book. + +`robot.toml` puts `resources/` and `sut/` on the python-path, so both are +imported by name. Run from this directory with robotcode, which reads +`robot.toml`: + +```bash +robotcode robot # the acceptance tests in tests/ +robotcode robot smoke +``` diff --git a/eval/fixtures/sut-bdd-library/base/pyproject.toml b/eval/fixtures/sut-bdd-library/base/pyproject.toml new file mode 100644 index 0000000..300da31 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/base/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "library-lending" +version = "0.0.1" +description = "Acceptance tests for the city library's lending service." +requires-python = ">=3.10" +dependencies = [ + "robotframework>=7.3", +] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = [] diff --git a/eval/fixtures/sut-bdd-library/base/resources/.gitkeep b/eval/fixtures/sut-bdd-library/base/resources/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/eval/fixtures/sut-bdd-library/base/robot.toml b/eval/fixtures/sut-bdd-library/base/robot.toml new file mode 100644 index 0000000..458a7b4 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/base/robot.toml @@ -0,0 +1,3 @@ +paths = ["tests"] +output-dir = "results" +python-path = ["resources", "sut"] diff --git a/eval/fixtures/sut-bdd-library/base/smoke/library_service.robot b/eval/fixtures/sut-bdd-library/base/smoke/library_service.robot new file mode 100644 index 0000000..a7872af --- /dev/null +++ b/eval/fixtures/sut-bdd-library/base/smoke/library_service.robot @@ -0,0 +1,12 @@ +*** Settings *** +Documentation Technical smoke check: the service library loads and lends a book. +Library LibraryService + + +*** Test Cases *** +Service lends an available copy + Register Member Smoke Reader + Add Book Smoke Test Title 1 + ${loan} Borrow Smoke Reader Smoke Test Title + Should Be Equal ${loan}[status] lent + Should Be Equal As Strings ${loan}[due_date] 2026-12-03 diff --git a/eval/fixtures/sut-bdd-library/base/sut/LibraryService.py b/eval/fixtures/sut-bdd-library/base/sut/LibraryService.py new file mode 100644 index 0000000..331d802 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/base/sut/LibraryService.py @@ -0,0 +1,132 @@ +"""Lending service of the city library (in-memory backend). + +Rules: +- A member borrows a book if a copy is available, the member is not blocked, + has no unpaid fines and is below the loan limit (standard: 3, premium: 10). +- Loans are due 21 days after the lending day. +- Returning late costs 0.50 EUR per day; the fine is added to the member's account. +- A member may reserve a book that has no copy available; reservations queue in order. +- A reserved returned copy is held for the first member in the queue. +""" + +from datetime import date, timedelta +from decimal import Decimal + +from robot.api.deco import keyword, library + +LOAN_LIMITS = {"standard": 3, "premium": 10} +LOAN_DAYS = 21 +FINE_PER_DAY = Decimal("0.50") + + +@library(scope="TEST") +class LibraryService: + """Fake library backend. A fresh instance is created for every test.""" + + def __init__(self) -> None: + self._today = date(2026, 11, 12) + self._copies: dict[str, int] = {} + self._members: dict[str, dict] = {} + self._loans: dict[str, dict[str, date]] = {} + self._reservations: dict[str, list[str]] = {} + self._held_for: dict[str, list[str]] = {} + self._last_receipt = "" + + @keyword + def set_today(self, day: date) -> None: + """Sets the library's current date (ISO format, e.g. 2026-11-12).""" + self._today = day + + @keyword + def add_book(self, title: str, copies: int = 1) -> None: + """Adds ``copies`` copies of ``title`` to the shelf.""" + self._copies[title] = self._copies.get(title, 0) + copies + + @keyword + def register_member(self, name: str, membership: str = "standard") -> None: + """Registers a member with a ``standard`` or ``premium`` membership.""" + if membership not in LOAN_LIMITS: + raise ValueError(f"Unknown membership '{membership}'") + self._members[name] = {"membership": membership, "fines": Decimal("0"), "blocked": False} + self._loans[name] = {} + + @keyword + def block_member(self, name: str) -> None: + """Blocks a member; blocked members cannot borrow.""" + self._members[name]["blocked"] = True + + @keyword + def add_fine(self, name: str, amount: Decimal) -> None: + """Adds an unpaid fine of ``amount`` EUR to a member's account.""" + self._members[name]["fines"] += Decimal(amount) + + @keyword + def borrow(self, name: str, title: str) -> dict: + """Requests a loan. Returns ``{status, reason, due_date}``.""" + member = self._members[name] + if member["blocked"]: + return self._refused("Member is blocked") + if member["fines"] > 0: + return self._refused("Unpaid fines") + if len(self._loans[name]) >= LOAN_LIMITS[member["membership"]]: + return self._refused("Loan limit reached") + held = self._held_for.get(title, []) + if held and held[0] != name: + return self._refused("Copy is reserved for another member") + if held: + held.pop(0) + elif self._copies.get(title, 0) <= 0: + return self._refused("No copy available") + else: + self._copies[title] -= 1 + due = self._today + timedelta(days=LOAN_DAYS) + self._loans[name][title] = due + self._last_receipt = f"Title: {title}\nDue: {due.isoformat()}" + return {"status": "lent", "reason": None, "due_date": due} + + @keyword + def return_book(self, name: str, title: str) -> Decimal: + """Returns a loan on today's date. Returns the fine charged (EUR).""" + due = self._loans[name].pop(title) + days_late = max((self._today - due).days, 0) + fine = FINE_PER_DAY * days_late + self._members[name]["fines"] += fine + queue = self._reservations.get(title, []) + if queue: + self._held_for.setdefault(title, []).append(queue.pop(0)) + else: + self._copies[title] = self._copies.get(title, 0) + 1 + return fine + + @keyword + def reserve(self, name: str, title: str) -> int: + """Reserves ``title``; returns the member's position in the queue (1-based).""" + if self._copies.get(title, 0) > 0: + raise ValueError(f"'{title}' is available; reserve only unavailable books") + queue = self._reservations.setdefault(title, []) + queue.append(name) + return len(queue) + + @keyword + def get_loans(self, name: str) -> list[str]: + """Returns the titles ``name`` currently has on loan, sorted.""" + return sorted(self._loans[name]) + + @keyword + def get_fines(self, name: str) -> Decimal: + """Returns the unpaid fines of ``name`` in EUR.""" + return self._members[name]["fines"] + + @keyword + def get_available_copies(self, title: str) -> int: + """Returns how many copies of ``title`` are on the shelf.""" + return self._copies.get(title, 0) + + @keyword + def get_last_receipt(self) -> str: + """Returns the text of the last loan receipt.""" + return self._last_receipt + + @staticmethod + def _refused(reason: str) -> dict: + return {"status": "refused", "reason": reason, "due_date": None} diff --git a/eval/fixtures/sut-bdd-library/base/tests/.gitkeep b/eval/fixtures/sut-bdd-library/base/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/eval/fixtures/sut-bdd-library/review/.gitignore b/eval/fixtures/sut-bdd-library/review/.gitignore new file mode 100644 index 0000000..3204997 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/review/.gitignore @@ -0,0 +1,9 @@ +output.xml +log.html +report.html +results/ +.venv/ +__pycache__/ +*.pyc +.pytest_cache/ +.robotcode_cache/ diff --git a/eval/fixtures/sut-bdd-library/review/README.md b/eval/fixtures/sut-bdd-library/review/README.md new file mode 100644 index 0000000..6ec798a --- /dev/null +++ b/eval/fixtures/sut-bdd-library/review/README.md @@ -0,0 +1,22 @@ +# library-lending + +Acceptance tests for the city library's lending service. The service runs in +memory (`sut/LibraryService.py`, a Robot Framework library with one fresh +instance per test); its business rules are in the module docstring. The +library's calendar starts on 2026-11-12, so due dates are fixed (`Set Today` +moves it). + +- `tests/loans.robot`: loan tests. +- `tests/loan_steps.resource`: the keywords the loan tests use. +- `resources/`: shared keywords (none yet). +- `smoke/library_service.robot`: a technical check that the service library + loads and lends a book. + +`robot.toml` puts `resources/` and `sut/` on the python-path, so both are +imported by name. Run from this directory with robotcode, which reads +`robot.toml`: + +```bash +robotcode robot # the tests in tests/ +robotcode robot smoke +``` diff --git a/eval/fixtures/sut-bdd-library/review/pyproject.toml b/eval/fixtures/sut-bdd-library/review/pyproject.toml new file mode 100644 index 0000000..300da31 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/review/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "library-lending" +version = "0.0.1" +description = "Acceptance tests for the city library's lending service." +requires-python = ">=3.10" +dependencies = [ + "robotframework>=7.3", +] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = [] diff --git a/eval/fixtures/sut-bdd-library/review/resources/.gitkeep b/eval/fixtures/sut-bdd-library/review/resources/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/eval/fixtures/sut-bdd-library/review/robot.toml b/eval/fixtures/sut-bdd-library/review/robot.toml new file mode 100644 index 0000000..458a7b4 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/review/robot.toml @@ -0,0 +1,3 @@ +paths = ["tests"] +output-dir = "results" +python-path = ["resources", "sut"] diff --git a/eval/fixtures/sut-bdd-library/review/smoke/library_service.robot b/eval/fixtures/sut-bdd-library/review/smoke/library_service.robot new file mode 100644 index 0000000..a7872af --- /dev/null +++ b/eval/fixtures/sut-bdd-library/review/smoke/library_service.robot @@ -0,0 +1,12 @@ +*** Settings *** +Documentation Technical smoke check: the service library loads and lends a book. +Library LibraryService + + +*** Test Cases *** +Service lends an available copy + Register Member Smoke Reader + Add Book Smoke Test Title 1 + ${loan} Borrow Smoke Reader Smoke Test Title + Should Be Equal ${loan}[status] lent + Should Be Equal As Strings ${loan}[due_date] 2026-12-03 diff --git a/eval/fixtures/sut-bdd-library/review/sut/LibraryService.py b/eval/fixtures/sut-bdd-library/review/sut/LibraryService.py new file mode 100644 index 0000000..331d802 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/review/sut/LibraryService.py @@ -0,0 +1,132 @@ +"""Lending service of the city library (in-memory backend). + +Rules: +- A member borrows a book if a copy is available, the member is not blocked, + has no unpaid fines and is below the loan limit (standard: 3, premium: 10). +- Loans are due 21 days after the lending day. +- Returning late costs 0.50 EUR per day; the fine is added to the member's account. +- A member may reserve a book that has no copy available; reservations queue in order. +- A reserved returned copy is held for the first member in the queue. +""" + +from datetime import date, timedelta +from decimal import Decimal + +from robot.api.deco import keyword, library + +LOAN_LIMITS = {"standard": 3, "premium": 10} +LOAN_DAYS = 21 +FINE_PER_DAY = Decimal("0.50") + + +@library(scope="TEST") +class LibraryService: + """Fake library backend. A fresh instance is created for every test.""" + + def __init__(self) -> None: + self._today = date(2026, 11, 12) + self._copies: dict[str, int] = {} + self._members: dict[str, dict] = {} + self._loans: dict[str, dict[str, date]] = {} + self._reservations: dict[str, list[str]] = {} + self._held_for: dict[str, list[str]] = {} + self._last_receipt = "" + + @keyword + def set_today(self, day: date) -> None: + """Sets the library's current date (ISO format, e.g. 2026-11-12).""" + self._today = day + + @keyword + def add_book(self, title: str, copies: int = 1) -> None: + """Adds ``copies`` copies of ``title`` to the shelf.""" + self._copies[title] = self._copies.get(title, 0) + copies + + @keyword + def register_member(self, name: str, membership: str = "standard") -> None: + """Registers a member with a ``standard`` or ``premium`` membership.""" + if membership not in LOAN_LIMITS: + raise ValueError(f"Unknown membership '{membership}'") + self._members[name] = {"membership": membership, "fines": Decimal("0"), "blocked": False} + self._loans[name] = {} + + @keyword + def block_member(self, name: str) -> None: + """Blocks a member; blocked members cannot borrow.""" + self._members[name]["blocked"] = True + + @keyword + def add_fine(self, name: str, amount: Decimal) -> None: + """Adds an unpaid fine of ``amount`` EUR to a member's account.""" + self._members[name]["fines"] += Decimal(amount) + + @keyword + def borrow(self, name: str, title: str) -> dict: + """Requests a loan. Returns ``{status, reason, due_date}``.""" + member = self._members[name] + if member["blocked"]: + return self._refused("Member is blocked") + if member["fines"] > 0: + return self._refused("Unpaid fines") + if len(self._loans[name]) >= LOAN_LIMITS[member["membership"]]: + return self._refused("Loan limit reached") + held = self._held_for.get(title, []) + if held and held[0] != name: + return self._refused("Copy is reserved for another member") + if held: + held.pop(0) + elif self._copies.get(title, 0) <= 0: + return self._refused("No copy available") + else: + self._copies[title] -= 1 + due = self._today + timedelta(days=LOAN_DAYS) + self._loans[name][title] = due + self._last_receipt = f"Title: {title}\nDue: {due.isoformat()}" + return {"status": "lent", "reason": None, "due_date": due} + + @keyword + def return_book(self, name: str, title: str) -> Decimal: + """Returns a loan on today's date. Returns the fine charged (EUR).""" + due = self._loans[name].pop(title) + days_late = max((self._today - due).days, 0) + fine = FINE_PER_DAY * days_late + self._members[name]["fines"] += fine + queue = self._reservations.get(title, []) + if queue: + self._held_for.setdefault(title, []).append(queue.pop(0)) + else: + self._copies[title] = self._copies.get(title, 0) + 1 + return fine + + @keyword + def reserve(self, name: str, title: str) -> int: + """Reserves ``title``; returns the member's position in the queue (1-based).""" + if self._copies.get(title, 0) > 0: + raise ValueError(f"'{title}' is available; reserve only unavailable books") + queue = self._reservations.setdefault(title, []) + queue.append(name) + return len(queue) + + @keyword + def get_loans(self, name: str) -> list[str]: + """Returns the titles ``name`` currently has on loan, sorted.""" + return sorted(self._loans[name]) + + @keyword + def get_fines(self, name: str) -> Decimal: + """Returns the unpaid fines of ``name`` in EUR.""" + return self._members[name]["fines"] + + @keyword + def get_available_copies(self, title: str) -> int: + """Returns how many copies of ``title`` are on the shelf.""" + return self._copies.get(title, 0) + + @keyword + def get_last_receipt(self) -> str: + """Returns the text of the last loan receipt.""" + return self._last_receipt + + @staticmethod + def _refused(reason: str) -> dict: + return {"status": "refused", "reason": reason, "due_date": None} diff --git a/eval/fixtures/sut-bdd-library/review/tests/loan_steps.resource b/eval/fixtures/sut-bdd-library/review/tests/loan_steps.resource new file mode 100644 index 0000000..25f224a --- /dev/null +++ b/eval/fixtures/sut-bdd-library/review/tests/loan_steps.resource @@ -0,0 +1,43 @@ +*** Settings *** +Documentation Steps for loans. +Library LibraryService + + +*** Variables *** +${LAST_LOAN} ${None} + + +*** Keywords *** +${member} is registered and the library has ${title} + [Documentation] Registers nobody new; the member and book already exist. + No Operation + +${member} opens the catalogue and selects ${title} and presses borrow + [Documentation] Borrows the book. + ${result} Borrow ${member} ${title} + VAR ${LAST_LOAN} ${result} scope=TEST + +Everything is fine + [Documentation] Nothing to check. + Log ok + +${member} is a standard member + [Documentation] Registers a standard member. + Register Member ${member} + +The library has 1 copy of "${title}" + [Documentation] Adds one copy. + Add Book ${title} 1 + +${member} borrows "${title}" + [Documentation] Borrows a book. + ${result} Borrow ${member} ${title} + VAR ${LAST_LOAN} ${result} scope=TEST + +The loan works + [Documentation] The loan was granted. + Should Be Equal ${LAST_LOAN}[status] lent + +The loan is refused with "${reason}" + [Documentation] The loan was refused with the given reason. + Should Be Equal ${LAST_LOAN}[reason] ${reason} diff --git a/eval/fixtures/sut-bdd-library/review/tests/loans.robot b/eval/fixtures/sut-bdd-library/review/tests/loans.robot new file mode 100644 index 0000000..1d05d96 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/review/tests/loans.robot @@ -0,0 +1,21 @@ +*** Settings *** +Library LibraryService +Resource loan_steps.resource + + +*** Test Cases *** +Test loan 1 + Register Member user1 + Add Book book1 1 + Given user1 is registered and the library has book1 + When user1 opens the catalogue and selects book1 and presses borrow + Then everything is fine + +Check stuff + Given Mia is a standard member + And Noah is a standard member + And the library has 1 copy of "Dune" + When Mia borrows "Dune" + Then the loan works + When Noah borrows "Dune" + Then the loan is refused with "No copy available" diff --git a/eval/fixtures/sut-bdd-library/steps/.gitignore b/eval/fixtures/sut-bdd-library/steps/.gitignore new file mode 100644 index 0000000..3204997 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/steps/.gitignore @@ -0,0 +1,9 @@ +output.xml +log.html +report.html +results/ +.venv/ +__pycache__/ +*.pyc +.pytest_cache/ +.robotcode_cache/ diff --git a/eval/fixtures/sut-bdd-library/steps/README.md b/eval/fixtures/sut-bdd-library/steps/README.md new file mode 100644 index 0000000..42bab1b --- /dev/null +++ b/eval/fixtures/sut-bdd-library/steps/README.md @@ -0,0 +1,21 @@ +# library-lending + +Acceptance tests for the city library's lending service. The service runs in +memory (`sut/LibraryService.py`, a Robot Framework library with one fresh +instance per test); its business rules are in the module docstring. The +library's calendar starts on 2026-11-12, so due dates are fixed (`Set Today` +moves it). + +- `tests/borrowing.robot`: the borrowing feature. +- `resources/borrowing_steps.resource`: the step keywords of the borrowing tests. +- `smoke/library_service.robot`: a technical check that the service library + loads and lends a book. + +`robot.toml` puts `resources/` and `sut/` on the python-path, so both are +imported by name. Run from this directory with robotcode, which reads +`robot.toml`: + +```bash +robotcode robot # the acceptance tests in tests/ +robotcode robot smoke +``` diff --git a/eval/fixtures/sut-bdd-library/steps/pyproject.toml b/eval/fixtures/sut-bdd-library/steps/pyproject.toml new file mode 100644 index 0000000..300da31 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/steps/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "library-lending" +version = "0.0.1" +description = "Acceptance tests for the city library's lending service." +requires-python = ">=3.10" +dependencies = [ + "robotframework>=7.3", +] + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = [] diff --git a/eval/fixtures/sut-bdd-library/steps/resources/borrowing_steps.resource b/eval/fixtures/sut-bdd-library/steps/resources/borrowing_steps.resource new file mode 100644 index 0000000..2245522 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/steps/resources/borrowing_steps.resource @@ -0,0 +1,38 @@ +*** Settings *** +Documentation Step definitions for borrowing books. +Library LibraryService + + +*** Variables *** +${LAST_LOAN} ${None} + + +*** Keywords *** +The library has ${copies: int} copies of "${title}" + [Documentation] Puts ``copies`` copies of ``title`` on the shelf. + Add Book ${title} ${copies} + +${member} is a ${membership:standard|premium} member + [Documentation] Registers ``member`` with a standard or premium membership. + Register Member ${member} ${membership} + +${member} borrows "${title}" + [Documentation] ``member`` asks to borrow ``title`` today. + ${result} Borrow ${member} ${title} + VAR ${LAST_LOAN} ${result} scope=TEST + +The loan is granted until ${due} + [Documentation] The last loan was granted and is due on ``due`` (YYYY-MM-DD). + Should Be Equal ${LAST_LOAN}[status] lent + ... msg=Expected the loan to be granted, but it was refused: ${LAST_LOAN}[reason] values=False + Should Be Equal As Strings ${LAST_LOAN}[due_date] ${due} + +The loan is refused with "${reason}" + [Documentation] The last loan was refused, and the library gave ``reason``. + Should Be Equal ${LAST_LOAN}[status] refused + Should Be Equal ${LAST_LOAN}[reason] ${reason} + +${member} has "${title}" on loan + [Documentation] ``title`` is among the books ``member`` currently has on loan. + ${loans} Get Loans ${member} + Should Contain ${loans} ${title} diff --git a/eval/fixtures/sut-bdd-library/steps/robot.toml b/eval/fixtures/sut-bdd-library/steps/robot.toml new file mode 100644 index 0000000..458a7b4 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/steps/robot.toml @@ -0,0 +1,3 @@ +paths = ["tests"] +output-dir = "results" +python-path = ["resources", "sut"] diff --git a/eval/fixtures/sut-bdd-library/steps/smoke/library_service.robot b/eval/fixtures/sut-bdd-library/steps/smoke/library_service.robot new file mode 100644 index 0000000..a7872af --- /dev/null +++ b/eval/fixtures/sut-bdd-library/steps/smoke/library_service.robot @@ -0,0 +1,12 @@ +*** Settings *** +Documentation Technical smoke check: the service library loads and lends a book. +Library LibraryService + + +*** Test Cases *** +Service lends an available copy + Register Member Smoke Reader + Add Book Smoke Test Title 1 + ${loan} Borrow Smoke Reader Smoke Test Title + Should Be Equal ${loan}[status] lent + Should Be Equal As Strings ${loan}[due_date] 2026-12-03 diff --git a/eval/fixtures/sut-bdd-library/steps/sut/LibraryService.py b/eval/fixtures/sut-bdd-library/steps/sut/LibraryService.py new file mode 100644 index 0000000..331d802 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/steps/sut/LibraryService.py @@ -0,0 +1,132 @@ +"""Lending service of the city library (in-memory backend). + +Rules: +- A member borrows a book if a copy is available, the member is not blocked, + has no unpaid fines and is below the loan limit (standard: 3, premium: 10). +- Loans are due 21 days after the lending day. +- Returning late costs 0.50 EUR per day; the fine is added to the member's account. +- A member may reserve a book that has no copy available; reservations queue in order. +- A reserved returned copy is held for the first member in the queue. +""" + +from datetime import date, timedelta +from decimal import Decimal + +from robot.api.deco import keyword, library + +LOAN_LIMITS = {"standard": 3, "premium": 10} +LOAN_DAYS = 21 +FINE_PER_DAY = Decimal("0.50") + + +@library(scope="TEST") +class LibraryService: + """Fake library backend. A fresh instance is created for every test.""" + + def __init__(self) -> None: + self._today = date(2026, 11, 12) + self._copies: dict[str, int] = {} + self._members: dict[str, dict] = {} + self._loans: dict[str, dict[str, date]] = {} + self._reservations: dict[str, list[str]] = {} + self._held_for: dict[str, list[str]] = {} + self._last_receipt = "" + + @keyword + def set_today(self, day: date) -> None: + """Sets the library's current date (ISO format, e.g. 2026-11-12).""" + self._today = day + + @keyword + def add_book(self, title: str, copies: int = 1) -> None: + """Adds ``copies`` copies of ``title`` to the shelf.""" + self._copies[title] = self._copies.get(title, 0) + copies + + @keyword + def register_member(self, name: str, membership: str = "standard") -> None: + """Registers a member with a ``standard`` or ``premium`` membership.""" + if membership not in LOAN_LIMITS: + raise ValueError(f"Unknown membership '{membership}'") + self._members[name] = {"membership": membership, "fines": Decimal("0"), "blocked": False} + self._loans[name] = {} + + @keyword + def block_member(self, name: str) -> None: + """Blocks a member; blocked members cannot borrow.""" + self._members[name]["blocked"] = True + + @keyword + def add_fine(self, name: str, amount: Decimal) -> None: + """Adds an unpaid fine of ``amount`` EUR to a member's account.""" + self._members[name]["fines"] += Decimal(amount) + + @keyword + def borrow(self, name: str, title: str) -> dict: + """Requests a loan. Returns ``{status, reason, due_date}``.""" + member = self._members[name] + if member["blocked"]: + return self._refused("Member is blocked") + if member["fines"] > 0: + return self._refused("Unpaid fines") + if len(self._loans[name]) >= LOAN_LIMITS[member["membership"]]: + return self._refused("Loan limit reached") + held = self._held_for.get(title, []) + if held and held[0] != name: + return self._refused("Copy is reserved for another member") + if held: + held.pop(0) + elif self._copies.get(title, 0) <= 0: + return self._refused("No copy available") + else: + self._copies[title] -= 1 + due = self._today + timedelta(days=LOAN_DAYS) + self._loans[name][title] = due + self._last_receipt = f"Title: {title}\nDue: {due.isoformat()}" + return {"status": "lent", "reason": None, "due_date": due} + + @keyword + def return_book(self, name: str, title: str) -> Decimal: + """Returns a loan on today's date. Returns the fine charged (EUR).""" + due = self._loans[name].pop(title) + days_late = max((self._today - due).days, 0) + fine = FINE_PER_DAY * days_late + self._members[name]["fines"] += fine + queue = self._reservations.get(title, []) + if queue: + self._held_for.setdefault(title, []).append(queue.pop(0)) + else: + self._copies[title] = self._copies.get(title, 0) + 1 + return fine + + @keyword + def reserve(self, name: str, title: str) -> int: + """Reserves ``title``; returns the member's position in the queue (1-based).""" + if self._copies.get(title, 0) > 0: + raise ValueError(f"'{title}' is available; reserve only unavailable books") + queue = self._reservations.setdefault(title, []) + queue.append(name) + return len(queue) + + @keyword + def get_loans(self, name: str) -> list[str]: + """Returns the titles ``name`` currently has on loan, sorted.""" + return sorted(self._loans[name]) + + @keyword + def get_fines(self, name: str) -> Decimal: + """Returns the unpaid fines of ``name`` in EUR.""" + return self._members[name]["fines"] + + @keyword + def get_available_copies(self, title: str) -> int: + """Returns how many copies of ``title`` are on the shelf.""" + return self._copies.get(title, 0) + + @keyword + def get_last_receipt(self) -> str: + """Returns the text of the last loan receipt.""" + return self._last_receipt + + @staticmethod + def _refused(reason: str) -> dict: + return {"status": "refused", "reason": reason, "due_date": None} diff --git a/eval/fixtures/sut-bdd-library/steps/tests/borrowing.robot b/eval/fixtures/sut-bdd-library/steps/tests/borrowing.robot new file mode 100644 index 0000000..d94c8b5 --- /dev/null +++ b/eval/fixtures/sut-bdd-library/steps/tests/borrowing.robot @@ -0,0 +1,26 @@ +*** Settings *** +Documentation As a library member +... I want to borrow books +... So that I can read them at home +Metadata Feature Borrowing books +Resource borrowing_steps.resource +Test Tags feature:borrowing-books + + +*** Test Cases *** +Member borrows an available book + [Documentation] Rule: an available copy is lent for 21 days. + [Tags] rule:loan-period + Given the library has 2 copies of "Dune" + And Mia is a standard member + When Mia borrows "Dune" + Then the loan is granted until 2026-12-03 + And Mia has "Dune" on loan + +Book without an available copy cannot be borrowed + [Documentation] Rule: a book can only be lent while a copy is on the shelf. + [Tags] rule:copy-available + Given the library has 0 copies of "Dune" + And Mia is a standard member + When Mia borrows "Dune" + Then the loan is refused with "No copy available" diff --git a/openspec/changes/add-rf-bdd-skill/.openspec.yaml b/openspec/changes/add-rf-bdd-skill/.openspec.yaml new file mode 100644 index 0000000..41484f3 --- /dev/null +++ b/openspec/changes/add-rf-bdd-skill/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-11-18 diff --git a/openspec/changes/add-rf-bdd-skill/design.md b/openspec/changes/add-rf-bdd-skill/design.md new file mode 100644 index 0000000..a542c04 --- /dev/null +++ b/openspec/changes/add-rf-bdd-skill/design.md @@ -0,0 +1,130 @@ +## Context + +The source is `bdd-agentskill/skills/robotframework-bdd` (archived change `2026-11-12-add-robotframework-bdd-skill` in that project): `SKILL.md` (151 lines), seven references (1,188 lines), a Robocop asset and two example projects. It was measured with a custom harness (`claude -p`, an LLM judge, Sonnet) against Robot Framework 7.5.1, robotcode 2.9.0.dev186 and Robocop 9.4.0. See proposal.md (Why) for the results. + +Constraints in this repository that shape the port: +- **Pinned tools.** The lockfile pins Robot Framework 7.4.2, robotcode 2.8.0 and Robocop 8.2.10. The `rf-language` tests use Robocop 9.x through `uvx` when it isn't installed. +- **Compact descriptions.** At most 160 characters, a "Use"/"Load" cue, and a listing total of at most 1,210 characters; about 890 are used today. The `## When to use` block must come within the first 20 body lines. +- **Overlapping guidance.** `rf-language` already has BDD basics in `references/styles.md` and `references/embedded-arguments.md`. Its gotcha requires Title Case BDD steps (NAME18 default), and it imports by relative path or `${CURDIR}`. +- **Hook budget.** The context hook message is 449 of its 450-character budget. +- **Tier.** `rf-core` is `verified`: a new skill needs a trigger set, eval tasks and baselines (GOVERNANCE.md), and the PR needs a Foundation reviewer. +- **Evals.** Their format is `rf-skill-eval` YAML with typed checks, gating flags and `custom_python` graders in `src/rf_skill_eval/scoring/custom/`. Fixtures live in `eval/fixtures/` and hidden graders in `eval/graders/`. +- **Shipped content.** It must pass `check_content_rules.py`, `check-docs-voice.py`, `check-plugin-references.py` and the security scanners. Plugins contain no symlinks. + +## Goals / Non-Goals + +**Goals:** +- `rf-bdd` that works next to `rf-language` and doesn't contradict it: the agent loads both and gets one answer per question. +- The measured behaviour of the source skill survives the port: deterministic checks pass in greenfield cases, the review finds the seeded issues, and the no-parser rule holds. +- Every example and code block runs on the pinned Robot Framework 7.4.2, and also on 7.5.1. + +**Non-Goals:** +- Porting the `bdd-agentskill` harness (`run_evals.py`, LLM judge, rubric). Only its cases are ported, as `rf-skill-eval` tasks. +- A Gherkin parser, `.feature` support, or a BDD-specific hook. +- A Python script for the skill. Step 0 uses `rf-language`'s conventions report and grep fallbacks, so no second detector is shipped. +- Changing `rf-language`'s examples or its default Title Case style for non-BDD keywords. + +## Decisions + +**D1. A skill in `rf-core`, not a new plugin.** +`plugin-split` gives test design to core, and "Sibling skills share a plugin" asks siblings to ship together. A user with `rf-language` therefore always gets the boundary text and the BDD skill. +- *Alternative:* a community plugin `rf-bdd`. It would be easier to enter, but the sibling pair would be split across two plugins, users would need a second install for a technology-independent skill, and the catalogue would gain a plugin with one skill and no library. + +**D2. The skill name is `rf-bdd`.** +It follows the `rf-` rule and is short in the listing. +- *Alternatives:* `rf-bdd-specs` or `rf-specification`. They are longer, and "BDD" is what users type. + +**D3. Project conventions win; sentence style is the greenfield default.** +Step 0 reads the casing of existing step keywords and the import style. +- If steps exist, rf-bdd follows them and leaves the Robocop configuration alone. +- If there are none, it writes sentence-style steps and merges `assets/robocop-bdd.toml` into the project configuration. The hook runs Robocop with the project's configured rule set, so the merged settings also silence the hook's NAME18 warnings. + +`rf-language`'s casing gotcha is reworded to "follow the project's configuration", which makes the two skills agree. Imports follow the same rule: by name through `python-path` by default, relative where the project already imports that way. (User decision, 2026-11-18.) +- *Alternative:* Title Case everywhere. The source evals preferred sentence style for readability, and its Robocop settings were verified to work. +- *Alternative:* sentence style repo-wide. That would rewrite `rf-language`'s examples and evals, which is out of scope. + +**D4. Port, don't copy: rf-bdd owns the specification, rf-language keeps the mechanics.** +Each source reference section is sorted into one of two kinds: +- *BDD design*, which stays: feature/rule mapping, vocabulary, state strategies, the outline-fits-every-row rule, living docs, the review checklist, Gherkin translation. +- *Robot Framework mechanics*, which becomes one line naming `rf-language`: embedded-argument matching and conflicts, template status rules, tag patterns, `__init__.robot` visibility. + +The source's robotcode command table is cut to the BDD-specific uses (`doc keywords` on step resources, `discover tests` for tags), and the rest is left to the robotcode skill. + +References follow `rf-language`'s house rules: no links between references, a table of contents above 100 lines, and every reference in the reference table. The SKILL.md skeleton is rf-bdd's own (spec `bdd-skill`), because `rf-language`'s skeleton sections (keyword anatomy, tags and selection) don't apply. + +**D5. Version gate: floor RF 7.0, examples RF 7.3+.** +The examples use typed embedded arguments (`${amount: int}`), which need 7.3. That covers the pinned 7.4.2 and the CI model's fixtures. +- The gate gives `${n:\d+}` with explicit conversion as the form for older versions. +- Test-level `[Metadata]` (7.5) appears only as a gated note. The examples use suite `Metadata`, so they run on 7.4.2. + +Every code block is verified on 7.4.2 (the lock) and 7.5.1. + +**D6. The Robocop asset must work on Robocop 8.2.10 and 9.x.** +The source was verified on 9.4.0 only. Implementation runs the asset under both versions. +- Where a setting differs (for example `per-file-ignores`, or the formatter's `skip_keyword_call_pattern`), `references/robocop.md` states the version. +- If a setting has no 8.x form, it states the fallback: an inline `# robocop: off=VAR06` in the steps resource. + +The structure test lints the examples with the installed Robocop, falling back to `uvx` like the `rf-language` tests do. + +**D7. Examples are self-contained and library-neutral.** +- They keep the fake ATM SUT (`AtmService.py`) and the two strategy projects, under `assets/examples/{sut,resource-strategy,world-strategy}/`. +- They contain no Browser or Requests keywords, as `plugin-split` requires for core. +- Each `robot.toml` uses relative `python-path` entries inside the example tree. There are no symlinks. +- The scratch tool `check_examples.py` from the source project becomes the code-block layer of `test_bdd_skill.py`, which extracts `` blocks, dry-runs them and runs them for real. + +**D8. Fit the hook message by tightening existing wording.** +The router line gains `Given/When/Then specs -> rf-bdd;`. To make room, existing phrases are shortened without dropping a route, for example "Robot Framework context detected. Load the matching skill before writing RF:" becomes "RF context: load the matching skill first:". The 450-character test stays unchanged. +- *Alternative:* raising the budget. Rejected, because the budget is a spec requirement chosen for token cost. + +**D9. Co-loading is expected; rf-language's trigger set stays as it is.** +The BDD queries in `rf-language.yaml` (`lang-t02`, `lang-t14`) stay should-trigger, because rf-bdd tells the agent to load rf-language for syntax. Validation queries are never edited. + +`rf-bdd.yaml` gets: +- BDD positives: specification wording, pasted Gherkin, review for readability; +- negatives: plain RF syntax, Python library, results, Cucumber/Java, pytest-bdd, SpecFlow. + +The re-baseline of all rf-core descriptions runs before rf-bdd is staged, so any recall change in rf-language from the new neighbour is visible. A drop below the stored baseline blocks the change, per skill-triggering. + +**D10. Evals: port five source cases, grade with a custom module.** + +| Source case | Task | Tier | +|---|---|---| +| `author-greenfield` | `narrow-bdd-greenfield-01` | narrow | +| `author-outline` | `narrow-bdd-outline-01` | narrow | +| `author-extend-vocabulary` | `narrow-bdd-extend-vocabulary-01` | narrow | +| `author-gherkin-request` | `adv-bdd-gherkin-row-01` ("1 days" row) | adversarial | +| `review-procedural` | `narrow-bdd-review-01` | narrow | + +The fixture is `eval/fixtures/sut-bdd-library`: the source `library-base` (a lending service with fixed dates), plus `library-steps` for the vocabulary task and `review-procedural` for the review task, as overlays selected by task. + +Gating uses existing checks where they fit: `robot_pass` with `expected_tests`, `file_not_contains` for `.feature` and `GherkinParser`, and `file_exists`. A new `src/rf_skill_eval/scoring/custom/bdd.py` ports the source graders: +- `structure`: AST check that test bodies hold only BDD-prefixed calls or template rows; +- `living_docs`: suite documentation and `Metadata Feature` or feature tags; +- `review_findings`: no file changed, and the final message names at least N of the seeded check IDs or their issue keywords. + +No LLM judge gates a task. The remaining source cases (DataDriver, German, World library, table/docstring, background, the two other review cases) are listed as follow-up candidates, not ported now. That keeps CI cost in line with the other rf-core skills, which have 5 to 7 tasks each. + +**D11. Versions.** +Since 5.0.0 every plugin and the installer share one version, set only at release time (`release-train`). +- The PR bumps no version, and `rf-bdd` carries no `metadata.version` (the validator refuses it). +- It adds a line under `## Unreleased` in `CHANGELOG.md`, in the rf-core group. New content makes the next release a minor one; the release manager runs `scripts/bump-version.sh minor`. +- `build-catalog.py` regenerates the marketplace and the catalogues. There is no bundle any more. + +## Risks / Trade-offs + +- **[Risk]** Adding a tenth description reduces the room for the others, and a future Claude Code release shrinks the room further. → **Mitigation:** keep rf-bdd's description to about 100 characters, so the total stays under 1,000 of the 1,210 allowed. The trigger gate reports a skill that is listed by name only. +- **[Risk]** rf-language starts losing BDD-flavoured queries to rf-bdd, or rf-bdd fires on plain "write a robot test". → **Mitigation:** run the re-baseline first, include negatives aimed at rf-language in rf-bdd's set, and apply the acceptance rule that neither skill's validation result may drop below its baseline. +- **[Risk]** Robocop 8.2.10 lacks a setting the asset uses. → **Mitigation (D6):** state the version in the reference and document the inline fallback. Test on both versions. +- **[Risk]** The source results were measured on Sonnet with an LLM judge, while CI here uses GLM 5.3 Flash on OpenRouter with deterministic checks, so the numbers aren't comparable. → **Mitigation:** store fresh baselines here, and don't quote the source numbers as this repo's results. The proposal names them only as motivation. +- **[Trade-off]** Five tasks instead of twelve cases cover less (DataDriver, localisation, World library). → These are recorded as follow-up candidates. The skill content for them is still checked by the structure and example tests. +- **[Risk]** The review grader can match keywords in the final message that the agent didn't mean as findings. → **Mitigation:** require the findings format from `review-checklist.md` (`ID — file:line — …`) and count only lines in that format. + +## Migration Plan + +This adds content only: +- Users of `rf-core` get `rf-bdd` with the next release after 5.0.0. Nothing is removed or renamed, so no installer migration is needed. +- Rollback is a revert of the PR, or pinning the marketplace to `v5.0.0`. + +## Open Questions + +- Whether a second maintainer for `rf-core` is available before the governance exemption expires (2027-01-31). This doesn't block this change, because the existing dated entry covers it. diff --git a/openspec/changes/add-rf-bdd-skill/proposal.md b/openspec/changes/add-rf-bdd-skill/proposal.md new file mode 100644 index 0000000..61dfc8a --- /dev/null +++ b/openspec/changes/add-rf-bdd-skill/proposal.md @@ -0,0 +1,81 @@ +## Why + +`rf-language` teaches the mechanics of BDD in Robot Framework: steps are defined without a prefix, values go into embedded arguments, there is one `When` per scenario. It doesn't teach how to write a *specification*: +- how a feature, a rule and its examples map onto suites, documentation, metadata and tags; +- how to keep one step vocabulary across suites, and where scenario state lives; +- when an outline becomes a `Test Template` and when it becomes a separate test; +- how to translate pasted Gherkin or acceptance criteria into native `.robot`; +- how to review a suite for procedural steps, leaking state and drifting vocabulary. + +A standalone `robotframework-bdd` skill was built and measured for this in the `bdd-agentskill` project, with 12 eval cases and two arms per case: +- **with the skill:** all 9 authoring cases passed every deterministic check, review recall was 1.00 and the rubric score 3.74 of 4; +- **without it:** 3 of 9 cases passed, recall was 0.89 and the rubric score 3.57. + +The skill's largest effect was in greenfield cases, where nothing in the project sets the conventions. This change brings that skill into the catalogue under the repository's conventions, as `rf-bdd` in `rf-core`. Behaviour-driven test design applies to every system under test, and `plugin-split` puts test design in core. + +## What Changes + +- **New guidance skill `rf-bdd`** at `plugins/rf-core/skills/rf-bdd/`, ported from `bdd-agentskill/skills/robotframework-bdd`: + - a compact description (≤ 160 characters, "Use" cue); + - a `## When to use` block within the first 20 body lines, naming `rf-language` as its sibling; + - a version gate; + - Step 0, which matches the project's conventions; + - house rules: three keyword layers; one behaviour per test; domain language; sentence-style step names (the default when the project has no convention); one vocabulary; the Gherkin-to-Robot Framework mapping; scenario state; independence; living documentation; imports; + - **Write**, **Review** (no edits) and **Gherkin input** modes; + - gotchas, a reference table and Companion Skills. +- **Seven references**, rewritten so that they don't duplicate `rf-language`: + - `gherkin-mapping.md`, `step-design.md`, `state.md`, `data-driven.md`, `documentation-and-tags.md`, `robocop.md`, `review-checklist.md`; + - generic embedded-argument matching, template mechanics and tag selection stay in `rf-language`; `rf-bdd` states the one-line rule and names `rf-language`. +- **Assets:** + - `assets/robocop-bdd.toml`: NAME02/NAME18 set to `first_word_capitalized` with the possessive pattern, VAR06 ignored in `*_steps.resource`, and the format settings for template alignment and BDD-prefix separators; + - runnable example projects in `assets/examples/`: a resource strategy (the default) and a World-library strategy, over a fake ATM service. They contain no library-specific keywords. +- **Project conventions win.** In a project that already writes Title Case steps, or imports resources by relative path, `rf-bdd` follows that convention. Sentence style plus the Robocop asset, and imports by name through `python-path`, are its defaults for greenfield projects only. +- **Native `.robot` only.** No `.feature` files, no Gherkin parser, nothing Robot Framework itself doesn't support. Pasted Gherkin is translated by meaning, and every wording change is listed for the author to confirm. +- **`rf-language` edits (content):** + - its gotcha that BDD steps must be Title Case (robocop NAME18) becomes: follow the project's configured casing, and use `rf-bdd` for BDD specifications; + - its BDD sections and the "When to use" block name `rf-bdd` as the sibling; + - its Companion Skills table gets a `bdd` row. +- **Routing:** + - the `UserPromptSubmit` context hook routes Given/When/Then and Gherkin work to `rf-bdd`, within its 450-character budget; + - `rf-test-architect` routes BDD specifications to `rf-bdd`. +- **Evals in `rf-skill-eval` format** under `plugins/rf-core/evals/`: + - a trigger set `triggers/rf-bdd.yaml` with ≥ 8 should-trigger and ≥ 8 should-not-trigger queries in both splits; + - narrow, adversarial and realistic tasks ported from the `bdd-agentskill` cases; + - a fixture `eval/fixtures/sut-bdd-library` and hidden graders `eval/graders/bdd/`; + - baselines measured on the current CI model. +- **Tests:** `plugins/rf-core/tests/test_bdd_skill.py` checks structure, the reference table, dry runs and real runs of every example, and that the Robocop asset lints the examples cleanly on the pinned Robocop. +- **Changelog and docs:** + - no version bump: every plugin shares the release version, and the PR adds a line under `## Unreleased` in `CHANGELOG.md`, in the `rf-core` group (new content, so the next release is a minor one); + - the `rf-core` README, the repository README's plugin table and the catalogue summary. + +No breaking change: no skill is removed or renamed. + +## Capabilities + +### New Capabilities +- `bdd-skill`: the `rf-bdd` agent skill for writing and reviewing behaviour-driven Robot Framework specifications in native `.robot` files. It defines frontmatter and layout, the description and sibling boundary, the SKILL.md skeleton and size budget, the reference set, the version gate, Step 0 convention matching and greenfield defaults, the house rules, the Write/Review/Gherkin-input modes, the Robocop asset, runnable examples, gotchas, the no-parser rule, and the trigger set, eval tasks and tests. + +### Modified Capabilities +- `language-skill`: the scope-boundary and Companion Skills requirements name `rf-bdd`. BDD specification design (feature/rule mapping, vocabulary, state, review) is out of scope and belongs to `rf-bdd`. +- `skill-triggering`: `rf-language` ↔ `rf-bdd` becomes a sibling pair, with its signal, in "Sibling skills state their boundary", and both are listed in "Sibling skills share a plugin". +- `plugin-split`: the `rf-core` row of the plugin table lists `rf-bdd`, and the skill count becomes ten. +- `rf-session-context-hooks`: the injection routes Given/When/Then and Gherkin work to `rf-bdd`, and the trigger matches the `rf-bdd` identifier. +- `plugin-subagents`: the routing table sends BDD specifications (Given/When/Then suites, step vocabularies, Gherkin translation) to `rf-bdd`. + +## Impact + +- **New**: + - `plugins/rf-core/skills/rf-bdd/`: `SKILL.md`, 7 references, `assets/robocop-bdd.toml`, `assets/examples/`; + - `plugins/rf-core/tests/test_bdd_skill.py`; + - `plugins/rf-core/evals/triggers/rf-bdd.yaml`, task YAMLs under `plugins/rf-core/evals/tasks/{narrow,adversarial,realistic}/`, baselines; + - `eval/fixtures/sut-bdd-library/` and `eval/graders/bdd/`, plus a custom grader module if the existing check types don't cover the BDD structure checks. +- **Edited**: + - `plugins/rf-core/skills/rf-language/`: `SKILL.md` (When to use, Gotchas, Companion Skills) and `references/styles.md`, `references/embedded-arguments.md` (one-line pointers); + - `plugins/rf-core/scripts/maybe_inject_rf_context.mjs` and its tests; + - `plugins/rf-core/agents/rf-test-architect.md`, then `scripts/build-agent-variants.py` for the generated variants; + - `tests/test_skill_descriptions.py` (catalogue key `bdd`, required rows), with listing room checked: about 890 of 1210 characters are used today; + - `CHANGELOG.md` (`## Unreleased`, rf-core group), `plugins/rf-core/README.md`; + - `.claude-plugin/marketplace.json` and the README catalogue, if `build-catalog.py` changes them; + - `security/baselines/rf-core.yaml`, only if the scanners flag the new example Python files. +- **Review**: `rf-core` is `verified`, so the PR needs a Foundation reviewer. The `tools`/`allowed-tools` fields are unchanged, but the hook script changes, so it also needs a security reviewer. +- **Dependencies**: none new at run time. DataDriver appears only in a reference; installing it is `rf-setup`'s job. The examples need Robot Framework ≥ 7.3 (typed embedded arguments) and are verified against the pinned 7.4.2 and against 7.5.1. diff --git a/openspec/changes/add-rf-bdd-skill/specs/bdd-skill/spec.md b/openspec/changes/add-rf-bdd-skill/specs/bdd-skill/spec.md new file mode 100644 index 0000000..ce4cb04 --- /dev/null +++ b/openspec/changes/add-rf-bdd-skill/specs/bdd-skill/spec.md @@ -0,0 +1,271 @@ +## Purpose + +Provide the `rf-bdd` agent skill. It teaches agents to write and review behaviour-driven Robot Framework specifications in native `.robot` and `.resource` files. Features, rules and examples map onto suites, documentation, metadata and tags. One step vocabulary serves every suite, and state lives in a declared place. Outlines become templates only when one sentence fits every row, and pasted Gherkin is translated by meaning. It relies on `rf-language` for Robot Framework syntax and on the robotcode plugin for running and analysing. + +## ADDED Requirements + +### Requirement: Skill exists with conformant frontmatter and layout + +The skill SHALL live at `plugins/rf-core/skills/rf-bdd/` with `SKILL.md` whose `name` is `rf-bdd`. Its frontmatter SHALL satisfy the skill-metadata-conformance rules: +- `license: Apache-2.0`; +- a `compatibility` field naming Python 3.8+, `robotframework>=7` and `robotcode[all]>=2.8` in the project environment; +- `metadata.author`, and no `metadata.version` (a skill's version is its plugin's). + +The skill SHALL contain `SKILL.md`, a `references/` folder and an `assets/` folder. It SHALL contain no `scripts/` folder and no `allowed-tools` field. + +#### Scenario: Frontmatter validates +- **WHEN** `python scripts/validate-skills.py --channel all` runs +- **THEN** `rf-bdd` passes every frontmatter rule and has no `metadata.version` + +#### Scenario: Skill is shipped by rf-core +- **WHEN** `python scripts/build-catalog.py --check` runs +- **THEN** `rf-bdd` is listed once, in `rf-core` + +### Requirement: Description is compact and draws the BDD boundary + +The `description` SHALL: +- be at most 160 characters; +- name Robot Framework (or RF); +- name BDD or Given/When/Then; +- carry a load cue starting with "Use" or "Load"; +- name no more than 2 keyword or API names; +- contain no meta-phrasing about the agent or the skill. + +All shipped rf-* descriptions together SHALL stay within the listing room that `tests/test_skill_descriptions.py` enforces. + +#### Scenario: Compact contract passes +- **WHEN** `tests/test_skill_descriptions.py` runs +- **THEN** rf-bdd passes the compact description checks, and the listing total stays within the limit + +### Requirement: When to use block names the sibling and the trigger terms + +Within the first 20 lines of the `SKILL.md` body, a `## When to use` block SHALL: +- list the trigger terms that don't fit in the description: "BDD style", "Given/When/Then", "executable specification", "acceptance criteria", "Gherkin" or "Cucumber" pasted as input, "scenario outline", "Background", and reviewing the readability or consistency of tests or step keywords; +- name `rf-language` as the sibling, with its signal: plain Robot Framework syntax, keyword-driven or data-driven tests without a specification, resources and variables; +- say that `rf-bdd` relies on `rf-language` for syntax details; +- name the robotcode plugin's `robotcode` skill for running, discovery, keyword lookup and result reading. + +#### Scenario: Sibling named with signal +- **WHEN** the first 20 body lines of `SKILL.md` are read +- **THEN** they contain a `## When to use` heading, name `rf-language` with its signal, and name the `robotcode` skill of the robotcode plugin + +### Requirement: SKILL.md skeleton and size budget + +`SKILL.md` SHALL contain these level-2 sections in this order: +- `When to use`; +- `Version gate`; +- `Step 0: Match the project`; +- `House rules`; +- `Write a specification`; +- `Review a specification`; +- `Gherkin input`; +- `Gotchas`; +- `When to read the references`; +- `Companion Skills`. + +An orientation of at most about 5 lines before `When to use` SHALL end with one line `Verified against Robot Framework `. `SKILL.md` SHALL be at most 500 lines including frontmatter. It SHOULD be at most 300 lines; the structure test warns above 300. + +#### Scenario: Sections ordered +- **WHEN** the structure test parses the level-2 headings +- **THEN** every required heading is present, in the order given + +#### Scenario: Budget enforced +- **WHEN** the structure test counts the lines of `SKILL.md` +- **THEN** it fails above 500 lines and warns above 300 + +### Requirement: References are gated, shallow and do not duplicate rf-language + +The skill SHALL ship exactly these references: +- `references/gherkin-mapping.md`: feature, rule, scenario, background, outline, examples, step arguments (`key=value` and list arguments), and a mapping table for Gherkin-only constructs; +- `references/step-design.md`: sentence rules, naming, declarative steps, embedded-argument design for steps, catch-all and ambiguous steps, vocabulary reuse, `robot:private`; +- `references/state.md`: the resource strategy (variables declared in `*** Variables ***`, never mutated in place) and the World-library strategy (`@library(scope="TEST")`), with the leak example; +- `references/data-driven.md`: the ladder from single scenario to `Test Template` to `[Template]` to DataDriver, the rule that an outline sentence fits every row, and the data-driven smells; +- `references/documentation-and-tags.md`: suite and test documentation, `Metadata`, feature and rule tags, `tag-doc`/`tag-stat-link`, and localisation (`Language:`) with natural word order; +- `references/robocop.md`: the Robocop asset, what each setting does, how to merge it into an existing configuration, and the repeated-`--select` rule; +- `references/review-checklist.md`: the review checks with IDs and the findings format. + +Every reference SHALL be linked from the `When to read the references` table with a "read when" condition. References SHALL NOT link to other reference files or to paths in another skill. Every reference over 100 lines SHALL list its level-2 sections within its first 15 lines. + +Generic Robot Framework mechanics SHALL be stated in `rf-bdd` only as a one-line rule that names `rf-language`. These are embedded-argument matching and conflicts, template mechanics and status rules, tag patterns and selection, and `__init__.robot` visibility. + +#### Scenario: Reference set matches the table +- **WHEN** the structure test compares `references/*` with the reference table +- **THEN** the two sets are equal and contain exactly the seven files named above + +#### Scenario: No duplicated syntax tutorial +- **WHEN** the references are searched for the generic topics listed above +- **THEN** each occurrence is a one-line rule that names `rf-language`, not a full explanation + +### Requirement: Version gate before version-dependent syntax + +The `Version gate` section SHALL tell the agent to establish the project's Robot Framework version before writing version-dependent syntax. It SHALL list at least: +- typed embedded arguments `${n: int}` and typed `[Arguments]` (7.3), with a regex pattern plus explicit conversion as the form for older versions; +- BDD prefix stripping before a leading embedded argument (7.1); +- `VAR … scope=TEST` (7.0); +- test-level `[Metadata]` (7.5), with suite `Metadata` or documentation as the form for older versions. + +#### Scenario: Older version gets compatible forms +- **WHEN** the agent works in a project pinned to Robot Framework 7.1 +- **THEN** the skill directs it to use `${n:\d+}` with explicit conversion instead of `${n: int}`, and no test-level `[Metadata]` + +### Requirement: Step 0 follows the project's conventions before the skill's defaults + +`Step 0: Match the project` SHALL tell the agent to detect the project's conventions before writing, with `rf-language`'s Step 0 or the documented fallback searches. The agent SHALL then follow what it finds: +- step-keyword casing; +- the import style (by name through `python-path`, or relative paths); +- the state strategy (resource variables or a World library); +- the existing step vocabulary (`robotcode doc keywords` on the steps resources); +- the tag vocabulary. + +Only when the project has no convention for an item SHALL the skill's default apply: +- sentence-style step names, with only the first word capitalized, plus `assets/robocop-bdd.toml` merged into the project's Robocop configuration; +- imports by name, with the folder on `python-path`; +- the resource strategy. + +The skill SHALL NOT reference a path into another skill's directory. + +#### Scenario: Title Case project is respected +- **WHEN** the project's existing step keywords are Title Case (`The Cart Contains ${count} Items`) +- **THEN** the new step keywords the agent writes are Title Case too, and the agent doesn't change the project's Robocop configuration + +#### Scenario: Greenfield project gets the defaults +- **WHEN** the project has no step keywords and no Robocop configuration +- **THEN** the agent writes sentence-style step names, adds the Robocop settings from `assets/robocop-bdd.toml` to the project configuration, and imports the steps resource by name with its folder on `python-path` + +### Requirement: House rules state the specification design + +The `House rules` section SHALL state these rules, each with its reason: +1. three keyword layers (steps, domain keywords, technical keywords) and `robot:private` for helpers; +2. one behaviour per test, with one `When`; +3. domain language, not UI or implementation detail; +4. step naming (casing from Step 0; values as quoted or pattern-restricted embedded arguments); +5. one vocabulary across suites, extended rather than duplicated; +6. the Gherkin mapping: feature to suite or feature directory, rule to documentation and tag, outline to template only when one sentence fits every row, no `IF` in a template keyword; +7. state in the declared place, never mutating a declared container in place; +8. independent tests; +9. living documentation: suite documentation, `Metadata Feature`, feature and rule tags; +10. imports by the project's convention; +11. style tooling with Robocop, using the project's configuration. + +#### Scenario: Rules present +- **WHEN** the structure test parses the `House rules` section +- **THEN** it finds 11 numbered rules, which mention `robot:private`, one `When`, `Test Template`, `IF`, `Metadata` and Robocop + +### Requirement: Write, Review and Gherkin-input modes + +The skill SHALL define three modes: +- **Write**: discover the vocabulary, draft the specification (feature, rules, scenarios), then the steps and lower layers. Validate with `robotcode analyze code`, a dry run, a real run of the affected tests by longname, and Robocop. Report the files and any convention the agent followed or added. +- **Review**: read the suites and steps, apply `references/review-checklist.md`, and report findings with check IDs, file and line. The agent SHALL NOT edit files in review mode unless the user asks. +- **Gherkin input**: translate pasted Gherkin or acceptance criteria into native `.robot` suites. The agent SHALL keep every rule and every example. It SHALL split an Examples row whose sentence doesn't fit the outline (for example "1 days") into its own named test. It SHALL list every wording change in its report for the author to confirm. + +#### Scenario: Review does not edit +- **WHEN** the user asks to review `tests/` for BDD quality +- **THEN** the agent reports findings with check IDs and changes no file + +#### Scenario: Wording changes are listed +- **WHEN** pasted Gherkin has an outline row whose sentence reads "1 days" +- **THEN** that row becomes its own named test, and the report lists the change + +### Requirement: Native Robot Framework only + +The skill SHALL produce only `.robot`, `.resource`, Python library and data files that standard Robot Framework runs. It SHALL NOT create `.feature` files, configure a Gherkin parser, or use constructs that need a parser or plugin outside Robot Framework and the project's own libraries. DataDriver MAY be used for external data sets when the project has it installed or the user asks for it, with its install delegated to `rf-setup`. + +#### Scenario: Pasted feature file becomes a suite +- **WHEN** the user pastes a Gherkin feature with a Background, two Rules and a Scenario Outline +- **THEN** the result is `.robot` suites (a feature directory with `__init__.robot` when there are several files), and no `.feature` file or parser configuration exists in the project + +### Requirement: Robocop asset matches the default style + +`assets/robocop-bdd.toml` SHALL configure, for both the pinned Robocop version and Robocop 9.x: +- NAME02 and NAME18 to `first_word_capitalized`, with a pattern that accepts possessive `'s`; +- VAR06 ignored for `*_steps.resource`; +- template alignment that keeps test names next to their rows; +- a separator rule that skips keyword calls starting with a BDD prefix. + +`references/robocop.md` SHALL explain how to merge these settings into an existing `robot.toml` or `pyproject.toml`. It SHALL say that the nearest configuration file wins, and that multiple rules are selected by repeating `--select`. + +#### Scenario: Examples lint clean with the asset +- **WHEN** `robocop check` runs with the asset's settings on the resource-strategy example +- **THEN** it reports no NAME02, NAME18 or VAR06 finding + +### Requirement: Runnable, library-neutral examples + +`assets/examples/` SHALL contain: +- a fake system under test in Python; +- a resource-strategy project, with `robot.toml`, a feature directory with `__init__.robot`, a scenarios file and an outline file, and a steps resource; +- a World-library project, with `robot.toml`, a `@library(scope="TEST")` World library and one suite. + +The examples SHALL call only BuiltIn and standard-library keywords and keywords of their own libraries and resources. Every `robotframework` code block in `SKILL.md` and the references SHALL either be extracted from an example file or pass a dry run against a copy of the examples. A block whose first line starts with `# Wrong` is a counter-example and is excluded. + +#### Scenario: Examples run +- **WHEN** `plugins/rf-core/tests/test_bdd_skill.py` runs with Robot Framework installed +- **THEN** both example projects pass `robotcode analyze code`, a dry run and a real run + +#### Scenario: Code blocks are valid +- **WHEN** the test extracts the `robotframework` code blocks from `SKILL.md` and the references +- **THEN** every block that isn't a counter-example passes a dry run + +### Requirement: Gotchas state the verified BDD traps + +The `Gotchas` section SHALL contain at least these items, each with behaviour, cause and what to write instead: +1. a step keyword defined with a prefix cannot be called with another prefix; +2. a passing dry run doesn't prove that a step matched the intended keyword, so a real run is needed; +3. adjacent embedded arguments in a step need quotes or patterns; +4. literal text in a step name is matched exactly (`day(s)` is not optional text); +5. an outline sentence that doesn't fit every row needs its own test, not an `IF` in the template keyword; +6. mutating a declared `@{EMPTY}`/`&{EMPTY}` container leaks state into later tests, so rebuild it with `VAR … scope=TEST`; +7. a `Test Setup` in `__init__.robot` needs the steps resource imported in every child suite; +8. Robocop reads only the nearest configuration file, so the BDD settings go into the file the project uses; +9. a comma-separated `--select` selects nothing; +10. `tag-doc` and `tag-stat-link` patterns use `?` instead of `:`; +11. DataDriver needs a template keyword with normal `[Arguments]`, and discovery sees only its placeholder test. + +Every trap SHALL be verified against the pinned Robot Framework, robotcode and Robocop versions before it is published. + +#### Scenario: Minimum gotchas present +- **WHEN** the structure test parses the `Gotchas` section +- **THEN** it finds at least 11 list items, including items that mention dry run, `IF`, `scope=TEST`, `__init__.robot`, `--select` and DataDriver + +### Requirement: Companion Skills integration for rf-bdd + +`SKILL.md` SHALL end with a `## Companion Skills` `Need | Skill` table with rows for `rf-language`, `rf-setup` and `rf-python-library`, using the shared catalogue wording. A sentence after the table SHALL name the robotcode plugin's `robotcode` skill. The shared catalogue SHALL gain a `bdd` key, with the Need "Write or review BDD (Given/When/Then) specifications and their step keywords", and `rf-language`'s table SHALL contain that row. + +#### Scenario: Rows present +- **WHEN** `tests/test_skill_descriptions.py` parses the Companion Skills tables +- **THEN** rf-bdd has the `language`, `setup` and `python-library` rows, and rf-language has the `bdd` row + +### Requirement: Trigger query set for rf-bdd + +`plugins/rf-core/evals/triggers/rf-bdd.yaml` SHALL contain at least 8 should-trigger and 8 should-not-trigger queries. Both polarities SHALL appear in the train and validation splits. Should-not-trigger queries SHALL include: +- plain Robot Framework syntax tasks that belong to `rf-language`; +- a Python library task; +- a robotcode results task; +- Cucumber step definitions in Java; +- pytest-bdd. + +`rf-language` loading alongside `rf-bdd` on a BDD query SHALL NOT count as a failure of either set. The description SHALL meet the trigger-accuracy acceptance rules of the skill-triggering spec on the validation split. + +#### Scenario: Set loads +- **WHEN** the trigger-set loader test runs +- **THEN** `rf-bdd.yaml` passes the shape rules, and no query names a skill id + +#### Scenario: Accuracy gate +- **WHEN** the trigger evaluation runs on the validation split under the default listing budget +- **THEN** should-trigger recall is at least 0.80 and should-not-trigger accuracy is at least 0.90, or the shortfall is recorded under rule (b) + +### Requirement: Eval tasks show the skill helps + +`plugins/rf-core/evals/tasks/` SHALL contain rf-bdd tasks on fixture `sut-bdd-library`: +- at least three narrow tasks: greenfield feature, outline with a template, extending an existing vocabulary; +- at least one adversarial task: pasted Gherkin with a row that doesn't fit the outline; +- one review task with seeded issues, graded on no edits and on the reported check IDs. + +Gating checks SHALL be deterministic: tests pass with an expected count, only BDD-prefixed calls or template rows in test bodies, suite documentation and feature metadata or tags present, no `.feature` file, and no edits in the review task. Baselines SHALL be stored for the current CI model. + +#### Scenario: Tasks load and gate +- **WHEN** `rf-skill-eval` lists the tasks for plugin `rf-core` +- **THEN** the rf-bdd tasks load, each names `skill: rf-bdd` and fixture `sut-bdd-library`, and each has at least one gating deterministic check + +#### Scenario: Baselines recorded +- **WHEN** the change is complete +- **THEN** `plugins/rf-core/evals/baselines/` contains results for the rf-bdd tasks and trigger set on the current CI model diff --git a/openspec/changes/add-rf-bdd-skill/specs/language-skill/spec.md b/openspec/changes/add-rf-bdd-skill/specs/language-skill/spec.md new file mode 100644 index 0000000..134a0ea --- /dev/null +++ b/openspec/changes/add-rf-bdd-skill/specs/language-skill/spec.md @@ -0,0 +1,44 @@ +## MODIFIED Requirements + +### Requirement: Scope boundaries with sibling skills + +The skill SHALL NOT teach these topics, which belong elsewhere: +- Python keyword libraries or listeners (rf-python-library); +- library-specific keywords, locators or waits (the library skills); +- installation (rf-setup); +- BDD specification design (rf-bdd): mapping features, rules and examples onto suites, documentation and tags; step vocabularies and scenario state; translating Gherkin; reviewing BDD readability; +- driving the robotcode CLI: running, discovery, keyword documentation, result analysis, debugging. + +The BDD mechanics stay in this skill: prefix stripping, prefix-free step definitions and embedded arguments. Where a task crosses into a topic listed above, the skill SHALL name the sibling skill, or for robotcode CLI work the robotcode plugin's `robotcode` skill. Its code blocks SHALL call only BuiltIn or standard-library keywords and keywords defined in the example resources. + +The gotcha about keyword-call casing SHALL tell the agent to follow the casing the project's Robocop configuration and existing keywords use, and SHALL NOT require Title Case for BDD steps. + +#### Scenario: No library keyword catalogue +- **WHEN** the skill's code blocks are checked +- **THEN** they call only BuiltIn or standard-library keywords and keywords defined in the example resources + +#### Scenario: BDD specification work is handed to rf-bdd +- **WHEN** the `When to use` block, `references/styles.md` and the BDD part of `references/embedded-arguments.md` are read +- **THEN** each names `rf-bdd` for writing or reviewing BDD specifications, in one line + +#### Scenario: Casing follows the project +- **WHEN** the `Gotchas` section is read +- **THEN** no item requires Title Case for `Given`/`When`/`Then` steps, and the casing item says to follow the project's Robocop configuration + +### Requirement: Companion Skills integration + +`SKILL.md` SHALL end with a `## Companion Skills` `Need | Skill` table with rows for: +- rf-python-library (only once shipped in the same release); +- at least rf-browser and rf-requests as library skills; +- rf-setup; +- rf-bdd. + +A sentence after the table SHALL name the robotcode plugin's `robotcode` skill for running, keyword lookup, result analysis and debugging. The Companion Skills tables of rf-setup, rf-python-library, rf-bdd, rf-browser, rf-selenium, rf-appium, rf-requests, rf-restinstance and rf-platynui SHALL each get one row naming `rf-language`, with identical Need wording from the shared catalogue key `language`. + +#### Scenario: Table rows present +- **WHEN** the rf-language Companion Skills table is parsed +- **THEN** it has rows naming rf-python-library, rf-browser, rf-requests, rf-setup and rf-bdd, no row names rf-libdoc, rf-results or rf-robotcode, and every named skill is shipped in the same release + +#### Scenario: Other skills point back +- **WHEN** the Companion Skills tables of the nine skills listed above are parsed +- **THEN** each has a row naming `rf-language` with identical Need wording, and no table names `rf-test-design`, `rf-keyword-design` or a retired skill diff --git a/openspec/changes/add-rf-bdd-skill/specs/plugin-split/spec.md b/openspec/changes/add-rf-bdd-skill/specs/plugin-split/spec.md new file mode 100644 index 0000000..194506b --- /dev/null +++ b/openspec/changes/add-rf-bdd-skill/specs/plugin-split/spec.md @@ -0,0 +1,23 @@ +## MODIFIED Requirements + +### Requirement: The content is split into a core and technology plugins + +The catalogue SHALL contain these plugins, and every skill and subagent SHALL belong to exactly one of them: + +| Plugin | Skills | Subagents | Hooks | +|---|---|---|---| +| `rf-core` | `rf-language`, `rf-bdd`, `rf-setup`, `rf-python-library` | `rf-test-architect`, `rf-debug-expert`, `rf-migration-guide` | all | +| `rf-web` | `rf-browser`, `rf-selenium` | none | none | +| `rf-api` | `rf-requests`, `rf-restinstance` | none | none | +| `rf-mobile` | `rf-appium` | none | none | +| `rf-desktop` | `rf-platynui` | none | none | + +A technology plugin SHALL NOT be added to the catalogue without content and at least one maintainer. + +#### Scenario: Every skill has one home +- **WHEN** the skill directories of all plugins are listed +- **THEN** the ten skills appear once each, in the plugins the table names + +#### Scenario: Empty plugin refused +- **WHEN** a PR adds `plugins/rf-database/` with a `catalog.yaml` but no skill, subagent or hook +- **THEN** the catalogue check fails and says a plugin needs content diff --git a/openspec/changes/add-rf-bdd-skill/specs/plugin-subagents/spec.md b/openspec/changes/add-rf-bdd-skill/specs/plugin-subagents/spec.md new file mode 100644 index 0000000..eee3a47 --- /dev/null +++ b/openspec/changes/add-rf-bdd-skill/specs/plugin-subagents/spec.md @@ -0,0 +1,35 @@ +## MODIFIED Requirements + +### Requirement: Subagents route to the owning skills + +Each subagent SHALL contain a routing table. The table maps the kinds of work the subagent delegates to the skill that owns them, or to the robotcode command that answers them, using the skill identifiers of the shipped catalog. Routes: +- test-case and suite design, user keywords, resources, variables and legacy-syntax migration SHALL route to `rf-language`; +- BDD specifications (Given/When/Then suites, step vocabularies, translating Gherkin or acceptance criteria) SHALL route to `rf-bdd`; +- Python keyword libraries and listeners SHALL route to `rf-python-library`; +- keyword name and argument lookup SHALL route to `robotcode doc keywords` / `robotcode doc keyword`; +- reading results SHALL route to `robotcode results`; +- step debugging SHALL route to `robotcode robot-debug -bl ""`; +- library-specific behavior, and migration from one library to another, SHALL route to the matching library skill, naming its plugin; +- environment problems SHALL route to `rf-setup`. + +When a routed library skill is not available, the subagent SHALL name the plugin to install and SHALL NOT continue from memory for library-specific details. + +#### Scenario: Test architect delegates design details +- **WHEN** `rf-test-architect` has decided the project layout and must write test suites and shared keywords +- **THEN** its instructions direct it to load `rf-language` for the suites and the resource files, and not to follow an inline syntax guide + +#### Scenario: Test architect delegates BDD specifications +- **WHEN** `rf-test-architect` is asked to set up acceptance tests written as Given/When/Then specifications +- **THEN** its routing table sends the suites and step keywords to `rf-bdd` + +#### Scenario: Debug expert routes a library-side failure +- **WHEN** `rf-debug-expert` diagnoses "library contains no keywords" or state that resets between tests +- **THEN** its routing table sends that failure class to `rf-python-library` + +#### Scenario: Every routed skill exists +- **WHEN** the subagent lint test compares the skill identifiers named in all subagent files with the skill directories of every catalogue plugin +- **THEN** every named skill exists, each one outside `rf-core` is named with its plugin, and no retired or merged-away name appears (`rf-keyword-builder`, `rf-testcase-builder`, `rf-resource-architect`, `rf-libdoc-search`, `rf-libdoc-explain`, `rf-libdoc`, `rf-results`, `rf-robotcode`, `rf-keyword-consultant`, and any `robotframework-*` identifier) + +#### Scenario: Missing library skill +- **WHEN** only `rf-core` is installed and `rf-migration-guide` is asked to move a suite from SeleniumLibrary to Browser +- **THEN** it says that the steps come from `rf-browser` in the `rf-web` plugin and suggests installing that plugin, before changing any library-specific code diff --git a/openspec/changes/add-rf-bdd-skill/specs/rf-session-context-hooks/spec.md b/openspec/changes/add-rf-bdd-skill/specs/rf-session-context-hooks/spec.md new file mode 100644 index 0000000..7f6d727 --- /dev/null +++ b/openspec/changes/add-rf-bdd-skill/specs/rf-session-context-hooks/spec.md @@ -0,0 +1,50 @@ +## MODIFIED Requirements + +### Requirement: Context injection routes by task + +When the user prompt contains a Robot Framework signal, the `UserPromptSubmit` hook SHALL inject one additional-context message. The message SHALL route by kind of work: +- tests, suites, user keywords, resources and variables to `rf-language`; +- Given/When/Then specifications and Gherkin input to `rf-bdd`; +- Python libraries and listeners to `rf-python-library`; +- library usage to the library's own skill, without naming it, and to installing the technology plugin (`rf-web`, `rf-api`, `rf-mobile`, `rf-desktop`) when that skill is not available; +- keyword names and arguments to `robotcode doc keyword`, instead of memory; +- run results to `robotcode results`, instead of reading `output.xml`. + +The message SHALL end with a one-line reminder to write RF 7 syntax (`RETURN`, `VAR`, `IF`, `Test Tags`). + +#### Scenario: RF prompt gets the routing message +- **WHEN** the prompt is "add a data-driven login test to tests/login.robot" +- **THEN** the hook emits exactly one `UserPromptSubmit` additional-context message that names `rf-language`, `rf-bdd`, `rf-python-library`, `robotcode doc` and `robotcode results` + +#### Scenario: Non-RF prompt gets nothing +- **WHEN** the prompt contains no Robot Framework signal (e.g. "refactor this React component") +- **THEN** the hook emits nothing and exits 0 + +### Requirement: Injection trigger covers current skills and RF source + +The trigger SHALL match case-insensitively: +- existing RF signals: the framework name, the `.robot` and `.resource` extensions, library names, and tooling names such as libdoc, robocop and robotcode; +- the identifiers of the shipped skills and subagents, including `rf-language`, `rf-bdd`, `rf-python-library` and `rf-setup`; +- RF section headers pasted into the prompt (`*** Settings ***`, `*** Variables ***`, `*** Test Cases ***`, `*** Tasks ***`, `*** Keywords ***`). + +It SHALL NOT match bare "test", bare "RF", or Gherkin keywords such as "Given" or "Scenario" on their own. + +#### Scenario: New skill name triggers +- **WHEN** the prompt is "use rf-python-library to build a listener" +- **THEN** the hook injects the message + +#### Scenario: BDD skill name triggers +- **WHEN** the prompt is "use rf-bdd to turn these acceptance criteria into tests" +- **THEN** the hook injects the message + +#### Scenario: Pasted RF source triggers +- **WHEN** the prompt contains a line `*** Keywords ***` +- **THEN** the hook injects the message + +#### Scenario: Bare words do not trigger +- **WHEN** the prompt is "run the unit tests and fix the RF amplifier model" +- **THEN** the hook emits nothing + +#### Scenario: Plain Gherkin does not trigger +- **WHEN** the prompt is "write Cucumber step definitions in Java for: Given a cart, When I check out" +- **THEN** the hook emits nothing diff --git a/openspec/changes/add-rf-bdd-skill/specs/skill-triggering/spec.md b/openspec/changes/add-rf-bdd-skill/specs/skill-triggering/spec.md new file mode 100644 index 0000000..734af14 --- /dev/null +++ b/openspec/changes/add-rf-bdd-skill/specs/skill-triggering/spec.md @@ -0,0 +1,48 @@ +## MODIFIED Requirements + +### Requirement: Sibling skills state their boundary + +Each skill that has a sibling covering a similar task SHALL name that sibling, based on an observable signal, in a "When to use" block within the first 20 lines of its `SKILL.md` body; compact descriptions have no room for it. The block SHALL also list the concrete trigger terms (import lines, file names, CLI names, pasted errors and symptoms) that no longer fit in the description. + +The sibling pairs and signals are: +- rf-browser ↔ rf-selenium: the imported library (`Browser` vs `SeleniumLibrary`), or Playwright vs WebDriver/Selenium Grid. +- rf-requests ↔ rf-restinstance: the imported library (`RequestsLibrary` vs `REST`), or JSON Schema/OpenAPI-driven assertions. +- rf-language ↔ the robotcode plugin's `robotcode` skill: writing Robot Framework syntax vs running, discovering, looking up, analysing or debugging with the robotcode CLI. rf-language names the robotcode skill by its plugin, outside its Companion Skills table. +- rf-language ↔ rf-bdd: Robot Framework syntax, keyword-driven and data-driven tests vs a BDD specification. The BDD signals are Given/When/Then scenarios written as a specification, pasted Gherkin or acceptance criteria, step-vocabulary design, and reviewing the readability of BDD suites. rf-bdd says that it relies on rf-language for syntax, so loading both is expected. +- rf-setup ↔ the library skills: installing and fixing the environment vs writing and fixing test code. + +#### Scenario: Web siblings reference each other +- **WHEN** the "When to use" blocks of rf-browser and rf-selenium are read +- **THEN** rf-browser names rf-selenium for SeleniumLibrary/WebDriver projects, and rf-selenium names rf-browser for Browser Library/Playwright projects + +#### Scenario: API siblings reference each other +- **WHEN** the "When to use" blocks of rf-requests and rf-restinstance are read +- **THEN** each names the other together with the library import that selects it + +#### Scenario: Script skills defer to robotcode +- **WHEN** the "When to use" blocks of all shipped skills are read +- **THEN** none names `rf-results`, `rf-libdoc` or `rf-robotcode` as a sibling or fallback + +#### Scenario: Language skill defers CLI work to robotcode +- **WHEN** the "When to use" block of rf-language is read +- **THEN** it says that running, discovering, keyword lookup, result reading and debugging with the robotcode CLI are covered by the robotcode plugin's `robotcode` skill + +#### Scenario: Language and BDD siblings reference each other +- **WHEN** the "When to use" blocks of rf-language and rf-bdd are read +- **THEN** rf-language names rf-bdd for BDD specifications, and rf-bdd names rf-language for Robot Framework syntax + +#### Scenario: Setup is separated from test writing +- **WHEN** the "When to use" block of rf-setup is read +- **THEN** it lists installation and environment error triggers and says that writing tests belongs to the library skills + +### Requirement: Sibling skills share a plugin + +Skills that the "Sibling skills state their boundary" requirement pairs as siblings for the same task SHALL ship in the same plugin, so that a user who has one also has the boundary text of the other. These pairs are rf-browser and rf-selenium, rf-requests and rf-restinstance, and rf-language and rf-bdd. + +#### Scenario: Web siblings together +- **WHEN** the catalogue is listed +- **THEN** `rf-browser` and `rf-selenium` are both in `rf-web`, and `rf-requests` and `rf-restinstance` are both in `rf-api` + +#### Scenario: Language siblings together +- **WHEN** the catalogue is listed +- **THEN** `rf-language` and `rf-bdd` are both in `rf-core` diff --git a/openspec/changes/add-rf-bdd-skill/tasks.md b/openspec/changes/add-rf-bdd-skill/tasks.md new file mode 100644 index 0000000..b3d1fec --- /dev/null +++ b/openspec/changes/add-rf-bdd-skill/tasks.md @@ -0,0 +1,46 @@ +## 1. Baselines before any change + +- [ ] 1.1 Re-baseline the rf-core trigger sets (rf-language, rf-setup, rf-python-library) on train and validation under the default listing budget, before rf-bdd is staged. Store the results as the comparison baseline; verify that the files exist under `plugins/rf-core/evals/baselines/`. +- [x] 1.2 Verify the source skill's traps on the pinned tools: in a scratch project, run the 11 gotchas from spec `bdd-skill` against RF 7.4.2 / robotcode 2.8.0 / Robocop 8.2.10, and against RF 7.5.1 / Robocop 9.x. Record per trap the scratch suite, the versions and the outcome in a task note in this change folder. + +## 2. Skill content + +- [x] 2.1 Create `plugins/rf-core/skills/rf-bdd/` with `SKILL.md` frontmatter (`name: rf-bdd`, compact description ≤ 160 characters with a "Use" cue, `license`, `compatibility`, `metadata.author`, no `metadata.version`). Verify with `python scripts/validate-skills.py --channel all` and `python -m pytest tests/test_skill_descriptions.py`. +- [x] 2.2 Write the `SKILL.md` body in the spec skeleton: orientation with the `Verified against Robot Framework 7.4.2` line, `When to use` within 20 lines (naming rf-language with its signal and the robotcode skill), Version gate, Step 0 (conventions first, greenfield defaults), 11 house rules, the Write/Review/Gherkin-input modes, Gotchas (≥ 11), the reference table, Companion Skills. Port from `bdd-agentskill/skills/robotframework-bdd/SKILL.md` and cut the robotcode command table to the BDD uses. Verify ≤ 300 lines (≤ 500 hard). +- [x] 2.3 Port the seven references, applying D4: drop generic RF mechanics (keep a one-line rule naming `rf-language`), remove links between references, add contents lists to files over 100 lines, and state Robocop version differences in `robocop.md` (D6). Verify with `python scripts/check-docs-voice.py` and `python scripts/check-plugin-references.py`. +- [x] 2.4 Port `assets/robocop-bdd.toml` and the examples to `assets/examples/{sut,resource-strategy,world-strategy}/`, with relative `python-path` inside the tree, no symlinks and suite `Metadata` instead of test-level `[Metadata]`. Verify that both projects pass `robotcode analyze code`, `robot --dryrun` and a real run on RF 7.4.2 and 7.5.1, and that Robocop with the asset reports no NAME02/NAME18/VAR06 finding on 8.2.10 and 9.x. +- [x] 2.5 Run `python scripts/check_content_rules.py` and `scripts/security-scan.sh rf-core out/`. Add a reasoned baseline entry to `security/baselines/rf-core.yaml` only for confirmed false positives in the example Python files; verify that the gate passes. + +## 3. Skill tests + +- [x] 3.1 Write `plugins/rf-core/tests/test_bdd_skill.py`. The structure layer checks the frontmatter, the H2 order, the size budget, the reference set equal to the table, contents lists, no reference-to-reference links, no path into another skill, 11 house rules and ≥ 11 gotchas with the required terms. Verify with `python -m pytest plugins/rf-core/tests/test_bdd_skill.py`. +- [x] 3.2 Add the Robot Framework layer (skipped without `robot`): extract the `robotframework` blocks and `` blocks from `SKILL.md` and the references, dry-run them against a copy of the examples (excluding `# Wrong` blocks), and dry-run and run both example projects for real. Verify that the tests pass locally with RF 7.4.2. +- [x] 3.3 Add the Robocop layer (installed Robocop, else `uvx --from "robotframework-robocop>=9.1,<10"`): the asset lints the examples cleanly, and the rule IDs cited in `robocop.md` exist. Verify that the tests pass. + +## 4. rf-language and cross-references + +- [x] 4.1 Edit rf-language: the `When to use` block names `rf-bdd` for BDD specifications; the casing gotcha says to follow the project's Robocop configuration instead of requiring Title Case BDD steps; `references/styles.md` and the BDD part of `references/embedded-arguments.md` get a one-line pointer to `rf-bdd`; the Companion Skills table gets the `bdd` row. Verify with `python -m pytest plugins/rf-core/tests/test_language_skill.py`. +- [x] 4.2 In `tests/test_skill_descriptions.py`: add catalogue key `bdd`, `REQUIRED_ROWS["rf-bdd"] = ("language", "setup", "python-library")`, `bdd` in rf-language's required rows, and rf-language ↔ rf-bdd in the sibling checks. Verify with `python -m pytest tests/test_skill_descriptions.py`; the listing total must stay ≤ `MAX_DESCRIPTION_TEXT`. +- [x] 4.3 Update `plugins/rf-core/scripts/maybe_inject_rf_context.mjs`: add `Given/When/Then specs -> rf-bdd;`, tighten the existing wording so the message stays ≤ 450 characters (D8), and add `rf-bdd` to the trigger identifiers. Add tests for the BDD skill-name trigger and for plain Cucumber/Java Gherkin not triggering. Verify with `python -m pytest plugins/rf-core/tests/test_hook_scripts.py`. +- [x] 4.4 Add a `rf-bdd` row to the routing table of `plugins/rf-core/agents/rf-test-architect.md` (BDD specifications, step vocabularies, Gherkin translation), keeping the body ≤ 120 lines. Run `python scripts/build-agent-variants.py`, then verify with `bash scripts/check-drift.sh` and `python -m pytest plugins/rf-core/tests/test_subagents.py plugins/rf-core/tests/test_agent_variants.py`. + +## 5. Evals + +- [x] 5.1 Create `eval/fixtures/sut-bdd-library/` from the source `library-base` (lending service with fixed dates, `robot.toml`), with overlays for the vocabulary task (`library-steps`) and the review task (`review-procedural`, 7 seeded issues). Add it to `eval/fixtures/README.md`. Verify that the fixture's own smoke suite passes with `robot`. +- [x] 5.2 Write `src/rf_skill_eval/scoring/custom/bdd.py` with `structure`, `living_docs` and `review_findings` (D10), plus unit tests under `tests/eval/`. Verify with `python -m pytest tests/eval -k bdd`. +- [x] 5.3 Write the five task YAMLs (`narrow-bdd-greenfield-01`, `narrow-bdd-outline-01`, `narrow-bdd-extend-vocabulary-01`, `narrow-bdd-review-01`, `adv-bdd-gherkin-row-01`), each with `skill: rf-bdd`, fixture `sut-bdd-library` and gating deterministic checks. Verify with `uv run rf-skill-eval doctor` and the task-loader tests. +- [x] 5.4 Write `plugins/rf-core/evals/triggers/rf-bdd.yaml`: ≥ 8 should-trigger and ≥ 8 should-not-trigger queries, both polarities in train and validation, negatives for rf-language syntax, Python library, results, Cucumber/Java, pytest-bdd and SpecFlow, and no skill ids in the queries. Verify with `python -m pytest tests/eval/test_trigger_sets.py tests/test_skill_descriptions.py`. +- [ ] 5.5 Tune rf-bdd's description on the train split (at most 5 iterations), then measure validation for rf-bdd and the rf-core siblings under the default budget. Verify the acceptance rule: rf-bdd recall ≥ 0.80 and should-not accuracy ≥ 0.90 (or rule (b) recorded), and no sibling below its 1.1 baseline. Run holdout and the 40000-character condition once each and report them. +- [ ] 5.6 Run the five tasks on the CI model, with and without rf-bdd installed. Store the baselines under `plugins/rf-core/evals/baselines/`, and verify that the eval gate passes with the new baselines. + +## 6. Version, catalogue and docs + +- [x] 6.1 Bump no version (D11). Run `python scripts/build-catalog.py`, then verify with `python scripts/build-catalog.py --check`, and that `rf-bdd` has no `metadata.version`. +- [x] 6.2 Add the change under `## Unreleased` in `CHANGELOG.md`, rf-core group (new `rf-bdd`, rf-language casing gotcha, hook and subagent routing); update `plugins/rf-core/README.md` (skill list) and the repository README's "Which plugins" row for `rf-core`. Verify with `python scripts/check-docs-voice.py`. +- [x] 6.3 Check the Claude Desktop and standalone export: `python scripts/build-skill-zips.py --out dist/skill-zips` and `python scripts/export-skills.py`. Verify that `rf-bdd` is exported with its assets and has no `${CLAUDE_SKILL_DIR}` left in plain copies. + +## 7. Final verification and PR + +- [x] 7.1 Run every CI check from CONTRIBUTING.md locally: `build-catalog.py --check`, `check-governance.py`, `check_content_rules.py`, `validate-skills.py --channel all`, `check-plugin-references.py`, `check-drift.sh`, `check-docs-voice.py`, `pytest tests/ --ignore=tests/eval` and `pytest plugins/rf-core/tests`. Verify that all of them pass. +- [x] 7.2 Try it in a live session: `claude --plugin-dir ./plugins/rf-core` in a fresh copy of `sut-bdd-library`, with the prompt "Write BDD acceptance tests for returning a book late". Verify that rf-bdd and rf-language load, the suite passes, and no `.feature` file is created. Done on 2026-11-19 with Claude Code and Haiku 5.5 in fresh copies of `sut-bdd-library/base`. The renewal prompt doesn't fit the fixture: the lending service has no renewal, and the agent (correctly) wrote failing tests and refused to invent the feature. With the late-return prompt, two runs: rf-bdd loaded in both, rf-language in one of them; the suites passed (5/5 and 10/10); no `.feature` file. +- [ ] 7.3 Commit on a branch from the current `main` and open a PR against `main` with the PR-template checklist filled in. Request a Foundation reviewer (verified plugin) and a security reviewer (hook script change). diff --git a/openspec/changes/add-rf-bdd-skill/trap-verification.md b/openspec/changes/add-rf-bdd-skill/trap-verification.md new file mode 100644 index 0000000..21bf15f --- /dev/null +++ b/openspec/changes/add-rf-bdd-skill/trap-verification.md @@ -0,0 +1,32 @@ +# Trap Verification for rf-bdd Gotchas + +Checked on 2026-11-19 with tiny scratch suites, on two tool sets: +- **A (pinned):** Robot Framework 7.4.2, robotcode 2.8.0, Robocop 8.2.10 (worktree `.venv`). +- **B (newer):** Robot Framework 7.5.1 (`uvx --from "robotframework==7.5.1" robot`), Robocop 9.4.0 (`uvx --from "robotframework-robocop>=9.1,<10" robocop`); robotcode 2.8.0 on RF 7.5.1 through `uvx --from "robotcode[all]==2.8.0" --with "robotframework==7.5.1"` where noted. +- DataDriver 1.11.2, installed into a scratch venv (RF 7.4.2, robotcode 2.8.0) and through `uvx --with robotframework-datadriver` for RF 7.5.1. + +"Same on A and B" means the output matched line for line. + +| # | Trap | Scratch suite | Versions | Outcome | Notes | +|---|---|---|---|---|---| +| 1 | A step keyword defined with a prefix can't be called with another prefix | Keyword `Given the app is open`; tests call `Given the app is open`, `When the app is open`, `The app is open` | A, B | TRUE. Only the `Given` call passes. The others fail with `No keyword with name 'When the app is open' found.` (dry run and run). robotcode reports `KeywordNotFound` for both. | Same on A and B. | +| 2 | A passing dry run doesn't prove the intended keyword matched | (a) catch-all `The order "${id}" is ${state}` called with the typo `is payed`; (b) `${user} selects team ${city} ${team}` called with `Los Angeles Lakers` | A, B | TRUE. The dry run passes 2/2 and robotcode analyze reports nothing. The real run fails both: (a) binds the typo, `Order state is "payed", expected "paid"`; (b) binds `city=Los`, `Los != Los Angeles`. | Same on A and B. | +| 3 | Adjacent embedded arguments need quotes or patterns | `${fan} supports ${city} ${team}`, `${fan} cheers for "${city}" "${team}"`, `${fan} follows ${city} ${team:\S+}`, each called with Los Angeles Lakers | A, B | TRUE. The unquoted form binds `city=Los` (run fails, dry run passes); the quoted and pattern forms bind `Los Angeles`. | Same on A and B. | +| 4 | Literal text in a step name is matched exactly | Keyword `The book is ${days} day(s) late`; calls `3 days late`, `1 day late`, `1 day(s) late` | A, B | TRUE. Only `1 day(s) late` matches; the others fail with `No keyword with name 'Then the book is 3 days late' found.` in the dry run, the run and robotcode analyze. | Same on A and B. | +| 5 | An outline sentence that doesn't fit every row | `Test Template Returning a book ${days} days late costs ${fee} EUR` with rows 1 and 3 | A, B | TRUE. Both rows pass, and the log shows the keyword `Returning a book 1 days late costs 0.50 EUR`. Nothing warns. | An `IF` in the template keyword is stated as the anti-pattern; no runtime check needed. | +| 6 | Mutating a declared `@{EMPTY}`/`&{EMPTY}` container leaks; `VAR … scope=TEST` fixes it | Resource declares `&{NOTES} &{EMPTY}` and `@{ITEMS} @{EMPTY}`; steps use `Set To Dictionary` / `Append To List`, or `VAR &{NOTES} &{NOTES} ${value}=${count} scope=TEST` / `VAR @{ITEMS} @{ITEMS} ${item} scope=TEST`; the second test expects empty containers | A, B | TRUE. In place: the second test fails with `Length of '{'20': '10'}' should be 0 but is 1.` (dictionary) and `Length of '['receipt']' should be 0 but is 1.` (list). Rebuilt with `VAR`: both tests pass. | The leak stays within the suite that imports the resource: a second suite in the same run starts empty. Same on A and B. | +| 7 | A `Test Setup` in `__init__.robot` that calls a step needs the steps resource in every child suite | `feature/__init__.robot` imports `steps.resource` and sets `Test Setup Given the ATM holds 500 EUR`; one child imports the resource, one doesn't | A, B | TRUE. The child without the import fails: `Setup failed: No keyword with name 'Given the ATM holds 500 EUR' found.` (dry run and run). The child with the import passes. | robotcode 2.8.0 `analyze code` reports 0 errors for this (on RF 7.4.2 and 7.5.1); only the dry run or a run catches it. | +| 8 | Robocop reads only the nearest configuration file | `resource-strategy` example with the BDD settings in its `robot.toml`; a second config added in `tests/` | A, B (Robocop 8.2.10, 9.4.0) | TRUE, with a nuance. A `tests/robot.toml`, `tests/pyproject.toml` or `tests/robocop.toml` that has a Robocop section hides the root settings: 29 NAME18 and 1 NAME02 findings come back under `tests/`. A `robot.toml` or `pyproject.toml` without a Robocop section is skipped, and the root settings still apply. | No merging between files. Same on both Robocop versions. | +| 9 | A comma-separated `--select` selects nothing | `robocop check --ignore-file-config --select NAME02,NAME18 .` on the example (which has 10 NAME02 and 29 NAME18 findings without config) | A, B | TRUE. Both versions print `Option value 'NAME02,NAME18' from '--select' did not match with any rule name or id.`, then `No rule selected …`, then `No issues found.`, and exit 0. `-s NAME02,NAME18` and comma-separated rule names behave the same. Repeated `--select NAME02 --select NAME18` reports 39 findings and exits 1. | It isn't "rejected" (no error exit); it warns and passes. rf-language's workflow step says "a comma list is rejected"; consider "selects nothing". | +| 10 | `tag-doc` and `tag-stat-link` patterns use `?` instead of `:` | Test tagged `rule:no-overdraft`; `robot --tagdoc "rule:*:Business rule" --tagstatlink "rule:*:https://wiki/rules/%1:Rule"` versus `"rule?*:…"` with `%2`; also `[tag-doc]` / `[tag-stat-link]` tables in `robot.toml` run through `robotcode robot` | A, B (robotcode only on A) | TRUE. With `:` the `` element in output.xml has no `doc` and no `links`. With `?` it has `doc="Business rule"` and `links="Rule:https://wiki/rules/no-overdraft"` (`%2` is the slug). The `robot.toml` tables behave the same: `"rule:*"` gives no doc, `"feature?*"` and `"rule?*"` work. | Same on A and B. | +| 11 | DataDriver needs a template keyword with normal `[Arguments]`, and discovery sees only its placeholder | CSV with 2–3 rows; template with `[Arguments] ${amount} ${balance} ${rest}` versus an embedded-argument template; placeholder test `Withdrawal example 0 0 0` | A (scratch venv), B (uvx) | TRUE, with corrections. Normal arguments: all rows pass. Embedded template: the run fails every row with `Variable '${amount}' not found.`, but the dry run passes. `robotcode discover tests` lists only `Withdrawal example` (1 test), on RF 7.4.2 and 7.5.1. | Corrections to the source wording: (1) the dry run does see the generated tests (DataDriver runs as a listener in the dry run too); only robotcode discovery and editors see just the placeholder. (2) A comma-separated CSV doesn't fail silently: DataDriver logs `[ ERROR ] … ValueError: Unassigned requiered argument detected: ${amount}.` and the placeholder fails with `Test cannot be empty.` (3) `--test "Partial withdrawal"` and `--include boundary` fail with `contains no tests matching …`; `--variable "DYNAMICTEST:Withdrawal Examples.Partial withdrawal"` runs one row, both when running the file and the folder. | + +## Related checks + +- Test-level `[Metadata]`: RF 7.4.2 fails the test with `Setting 'Metadata' is not allowed with tests or tasks.`; RF 7.5.1 passes and writes ``. +- robotcode 2.8.0 diagnostics named in the references exist: `VariableNotFound` (undeclared `VAR … scope=TEST` state), `VariableNotReplaced` (hint, `${n}` in a templated test name), `PrivateKeyword` (warning), `MultipleKeywords` (a stricter pattern alone doesn't make a keyword win), `KeywordNotFound`. +- Robocop asset (`assets/robocop-bdd.toml`) on the resource-strategy example: `robocop check` is clean on 8.2.10 and 9.4.0. Removing each lint setting brings back its findings on both versions: `per-file-ignores` (1 VAR06), `pattern='s` (1 NAME02), the NAME18 convention (29 NAME18). So `per-file-ignores` works on 8.2.10. +- Robocop asset, format section: on 8.2.10, `robocop format` stops with `InvalidParameterError: AlignTemplatedTestCases: Failed to import … accepts following arguments: enabled, only_with_headers, min_width`, because `args_with_test` is 9.x only. Without that line, 8.2.10 formats and keeps template values on the test line. `NormalizeSeparators.skip_keyword_call_pattern` works on both versions. On 9.4.0 without `args_with_test=keep`, each example's values move to their own line. +- The inline disabler `# robocop: off=VAR06` on the first line of a resource silences VAR06 on 8.2.10 and 9.4.0. +- MISC04 counts `VAR` as "no `=`" on 9.4.0 only; 8.2.10 doesn't flag a `VAR` next to `${x}=` assignments. +- KW06 exists only in Robocop 9.x. diff --git a/plugins/rf-core/README.md b/plugins/rf-core/README.md index 46bb050..3235237 100644 --- a/plugins/rf-core/README.md +++ b/plugins/rf-core/README.md @@ -1,6 +1,6 @@ # rf-core plugin -What every coding agent needs to automate Robot Framework, whatever the system under test: the `rf-language`, `rf-setup` and `rf-python-library` skills, the `rf-test-architect`, `rf-debug-expert` and `rf-migration-guide` subagents, and the validation and context hooks. The library skills come in the technology plugins `rf-web`, `rf-api`, `rf-mobile` and `rf-desktop`, which depend on this one. Install instructions for every agent are in the [repository README](https://github.com/MarketSquare/robotframework-plugins#install). +What every coding agent needs to automate Robot Framework, whatever the system under test: the `rf-language`, `rf-bdd`, `rf-setup` and `rf-python-library` skills, the `rf-test-architect`, `rf-debug-expert` and `rf-migration-guide` subagents, and the validation and context hooks. The library skills come in the technology plugins `rf-web`, `rf-api`, `rf-mobile` and `rf-desktop`, which depend on this one. Install instructions for every agent are in the [repository README](https://github.com/MarketSquare/robotframework-plugins#install). ## Install @@ -35,6 +35,6 @@ claude plugin install robotcode@robotframework-agent-plugins | The robotcode plugin owns | rf-core owns | |---|---| -| running, discovery, keyword documentation, result analysis, static analysis, `robot.toml` configuration, the REPL and the debugger | Robot Framework syntax (`rf-language`), Python keyword libraries (`rf-python-library`), environment setup (`rf-setup`), the validation hooks, and the `rf-test-architect`, `rf-debug-expert` and `rf-migration-guide` subagents | +| running, discovery, keyword documentation, result analysis, static analysis, `robot.toml` configuration, the REPL and the debugger | Robot Framework syntax (`rf-language`), BDD specifications (`rf-bdd`), Python keyword libraries (`rf-python-library`), environment setup (`rf-setup`), the validation hooks, and the `rf-test-architect`, `rf-debug-expert` and `rf-migration-guide` subagents | Changes: [CHANGELOG.md](../../CHANGELOG.md) at the repository root. diff --git a/plugins/rf-core/agents/rf-test-architect.md b/plugins/rf-core/agents/rf-test-architect.md index cf09bca..b6d5853 100644 --- a/plugins/rf-core/agents/rf-test-architect.md +++ b/plugins/rf-core/agents/rf-test-architect.md @@ -58,6 +58,7 @@ python-path belong to `rf-setup`. | Work | Skill | |------|-------| | Suites, test cases, `__init__.robot`, tags, templates, user keywords, resource and variable files | `rf-language` | +| BDD specifications: Given/When/Then suites, step vocabularies, translating Gherkin or acceptance criteria | `rf-bdd` | | Project keyword libraries and listeners in Python (`libraries/*.py`), checked with its library checker | `rf-python-library` | | Installing RF and libraries, project layout mechanics, `robot.toml`, python-path | `rf-setup` | | Web tests with Browser / SeleniumLibrary | `rf-browser` / `rf-selenium` (plugin `rf-web`) | diff --git a/plugins/rf-core/evals/tasks/README.md b/plugins/rf-core/evals/tasks/README.md index fd0f21e..160becd 100644 --- a/plugins/rf-core/evals/tasks/README.md +++ b/plugins/rf-core/evals/tasks/README.md @@ -64,7 +64,7 @@ does not have. | `no_deprecated_keywords` | `target` | Rejects keywords their library marks deprecated (`robotcode analyze code`, `DeprecatedKeyword`) and legacy syntax from Robocop's hard-deprecation rules (DEPR03/04/07/08/09/10/11: `WITH NAME`, singular headers, `Force Tags`, `Run Keyword If`/`Unless`, `Exit For Loop`, `Return From Keyword`, `[Return]`). Skipped when robotcode or robocop is missing. | | `lint_clean` | `target` | `robocop` reports no violations; robocop missing → `skipped`. | | `import_resolves` | `path` or `module` | `Library`/`Resource` imports resolve. | -| `custom_python` | `func_ref` | `module:function` returning bool/dict/Verdict; an exception → `error`. Task-specific checks live in `rf_skill_eval.scoring.custom` (e.g. `custom.language`: `hidden_suite`, `file_unchanged`, `any_file_contains`, `no_typed_variables`, `pinned_robot_pass`, `robocop_clean`; `custom.python_library`: `hidden_suite` with `listener` / `expected_rc` / `expected_statuses`, `checker_findings_absent`; `custom.keywords`: `no_undefined_keyword_calls`); hidden grader suites are kept in `eval/graders//`, outside the staged fixture. | +| `custom_python` | `func_ref` | `module:function` returning bool/dict/Verdict; an exception → `error`. Task-specific checks live in `rf_skill_eval.scoring.custom` (e.g. `custom.language`: `hidden_suite`, `file_unchanged`, `any_file_contains`, `no_typed_variables`, `pinned_robot_pass`, `robocop_clean`; `custom.python_library`: `hidden_suite` with `listener` / `expected_rc` / `expected_statuses`, `checker_findings_absent`; `custom.keywords`: `no_undefined_keyword_calls`; `custom.bdd`: `structure`, `living_docs`, `native_only`, `step_names_absent`, `review_findings`); hidden grader suites are kept in `eval/graders//`, outside the staged fixture. | | `tool_call_count` / `tool_result_count` / `tool_call_sequence` | see `scoring/session_based.py` | Transcript checks; no transcript → `skipped`. | ## Verdicts and gate results diff --git a/plugins/rf-core/evals/tasks/adversarial/adv-bdd-gherkin-row-01.yaml b/plugins/rf-core/evals/tasks/adversarial/adv-bdd-gherkin-row-01.yaml new file mode 100644 index 0000000..5a6a241 --- /dev/null +++ b/plugins/rf-core/evals/tasks/adversarial/adv-bdd-gherkin-row-01.yaml @@ -0,0 +1,99 @@ +id: adv-bdd-gherkin-row-01 +skill: rf-bdd +tier: adversarial +description: | + Translate a pasted Gherkin ticket (Background, two Rules, a Scenario + Outline) into native Robot Framework suites (ported from + author-gherkin-request). The outline's " days" sentence reads "1 days" + for its first Examples row; the temptation is to keep the ticket's wording + (or to add a .feature file and a Gherkin parser). Gated by exactly 4 + passing tests (one scenario + three examples), no executed test or step + name reading "1 days" (so the 1-day row became its own test or got a + sentence that fits), step-only bodies, living documentation and no + .feature file or parser. +prompt: | + Our product owner wrote the acceptance criteria for ticket LIB-214 in + Gherkin. We use Robot Framework (see robot.toml) and the library service is + in sut/LibraryService.py. Please turn these into our Robot Framework tests: + + ```gherkin + Feature: Reservations + As a library member + I want to reserve a book that is on loan + So that I get it as soon as it is returned + + Background: + Given the library owns one copy of "Dune" + And Mia has borrowed "Dune" + + Rule: Reservations are served first come, first served + + Scenario: The returned copy is held for the first member in the queue + Given Noah has reserved "Dune" + And Lea has reserved "Dune" + When Mia returns "Dune" + Then Noah can borrow "Dune" + But Lea cannot borrow "Dune" + + Rule: Late returns are fined per day + + Scenario Outline: Fine for a late return + When Mia returns "Dune" days after the due date + Then Mia is charged a fine of EUR + + Examples: + | days | fine | + | 1 | 0.50 | + | 7 | 3.50 | + | 30 | 15.00 | + ``` +model: claude-sonnet-5-5 +max_turns: 30 +timeout_seconds: 600 +allowed_tools: + - Read + - Write + - Edit + - Bash + - Glob + - Grep + - Skill +grader_checks: + - type: robot_pass + name: every_scenario_and_example_passes + target: tests + expected_tests: 4 + expected_tests_exact: true + gating: true + - type: custom_python + name: no_step_reads_1_days + func_ref: rf_skill_eval.scoring.custom.bdd:step_names_absent + path: tests + regex: '(? rf-language; ", + "Given/When/Then specs, Gherkin -> rf-bdd; ", "Python libraries/listeners -> rf-python-library; installs -> rf-setup; ", "a library -> its own skill; if missing, suggest its plugin (rf-web, rf-api, rf-mobile, rf-desktop). ", "Keywords: `robotcode doc keyword`, not memory; results: `robotcode results`, not output.xml. ", diff --git a/plugins/rf-core/skills/rf-bdd/SKILL.md b/plugins/rf-core/skills/rf-bdd/SKILL.md new file mode 100644 index 0000000..b0066eb --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/SKILL.md @@ -0,0 +1,168 @@ +--- +name: rf-bdd +description: "Use for BDD in Robot Framework: Given/When/Then specs, step vocabularies, Gherkin or acceptance criteria to .robot, reviews." +license: Apache-2.0 +compatibility: "Requires Python 3.8+ with robotframework>=7 and robotcode[all]>=2.8 (which includes Robocop) in the project environment; the examples need RF 7.3+." +metadata: + author: manykarim +--- + +# Robot Framework BDD + +Write and review behaviour-driven specifications in native `.robot` and `.resource` files: features, rules and examples as suites, documentation and tags, one step vocabulary, and scenario state in a declared place. +Robot Framework syntax (embedded arguments, templates, tags, imports) comes from `rf-language`; load both. +Verified against Robot Framework 7.4.2. + +## When to use + +Load this skill together with `rf-language` when tests are written or reviewed as a specification: + +- "BDD style", "Given/When/Then", "executable specification", "specification by example", acceptance tests; +- acceptance criteria, user stories or Gherkin (Cucumber, a `.feature` text) pasted as input for Robot Framework tests; +- a scenario outline or Examples table, a Background, rules and features; +- reviewing the readability or consistency of tests or step keywords, or a step vocabulary that drifts. + +Not this skill: plain Robot Framework syntax, keyword-driven or data-driven tests without a specification, resources and variables use `rf-language`; `rf-bdd` relies on `rf-language` for syntax details. Running, discovery, keyword lookup and result reading are covered by the `robotcode` skill of the robotcode plugin. + +## Version gate + +Establish the project's Robot Framework version before writing version-dependent syntax (`rf-language` shows how). + +| Feature | Min RF | On older versions | +|---|---|---| +| `VAR … scope=TEST` | 7.0 | `Set Test Variable` | +| BDD prefix before a leading embedded argument (`Given ${holder} has …`) | 7.1 | start the step with literal text | +| Typed embedded arguments `${amount: int}`, typed `[Arguments]` | 7.3 | `${amount:\d+}` and `Convert To Integer` | +| Test-level `[Metadata]` | 7.5 | suite `Metadata` or `[Documentation]` | + +The examples in `assets/examples/` need RF 7.3. They use suite `Metadata` only. + +## Step 0: Match the project + +Detect the project's conventions before writing, and follow them. Use `rf-language`'s Step 0 (its conventions report lists the BDD steps, templates and tags), or search: + +```bash +grep -rhoE '^\s+(Given|When|Then|And|But) .*' tests | head -20 +grep -rlE '^\*\*\* Keywords \*\*\*' --include='*.resource' . +grep -rhE '^(Resource|Library)\s' tests | sort | uniq -c +grep -nE 'robocop|python-path' robot.toml pyproject.toml 2>/dev/null +``` + +Follow what you find: + +- **Step casing.** Title Case steps (`The Cart Contains ${count} Items`) stay Title Case. Leave the project's Robocop configuration alone. +- **Imports.** By name with the folder on `python-path`, or relative paths, as the project does. +- **State.** Variables in a steps resource, or a World library in Python. +- **Vocabulary.** `robotcode doc keywords ` lists the existing sentences. +- **Tags.** `robotcode discover tags` lists the tags in use. + +Only when the project has no convention for an item, use the default: + +- sentence-style step names with only the first word capitalized, plus the settings of `assets/robocop-bdd.toml` merged into the project's Robocop configuration; +- imports by name, with the folder on `python-path` in `robot.toml`; +- the resource strategy for state. + +## House rules + +Every suite has this shape (the steps are defined in `assets/examples/resource-strategy/resources/cash_withdrawal_steps.resource`): + +```robotframework +*** Settings *** +Documentation As an account holder +... I want to withdraw cash from an ATM +... So that I can get money when the bank is closed +Metadata Feature Cash withdrawal +Resource cash_withdrawal_steps.resource +Test Setup Given the ATM holds 500 EUR +Test Tags feature:cash-withdrawal + + +*** Test Cases *** +Withdrawal above the balance is refused + [Documentation] Rule: an account cannot be overdrawn at the ATM. + [Tags] rule:no-overdraft + Given Alice has a balance of 10 EUR + When Alice withdraws 20 EUR + Then the withdrawal is refused with "Insufficient funds" + And Alice's balance is 10 EUR +``` + +1. **Three layers.** Test bodies hold only `Given`/`When`/`Then`/`And`/`But` steps or template rows. A step keyword states one domain sentence and calls domain keywords, which call technical keywords (UI, API, database). Tag helpers in a steps resource `robot:private`, so robotcode warns when a test calls them. *Why:* a business reader can read the test, and a UI change touches one layer. +2. **One behaviour per test.** Given, one `When`, then Then, in fewer than 10 steps. The test name states the behaviour, not the procedure. *Why:* a failure names one broken rule. +3. **Domain language.** Third person, present tense, one idea per step, concrete data (`Alice`, `100 EUR`), no locators, URLs, HTTP verbs or SQL. A `Then` names an observable outcome. *Why:* the specification outlives the implementation. +4. **Step naming.** Casing from Step 0. Values are embedded arguments, strings in quotes (`"${reason}"`), numbers typed or restricted by a pattern. Every step keyword has `[Documentation]`. *Why:* quotes and patterns keep steps from matching each other. +5. **One vocabulary.** List the existing steps before writing one, and reuse or extend them. Never add a synonym ("logged in" next to "signed in"). *Why:* two sentences for one fact drift apart. +6. **Gherkin mapping.** Feature → suite, or a feature directory with `__init__.robot`. Rule → `[Documentation] Rule: …` plus a `rule:` tag. Background → `Test Setup Given …`, shared by two or more tests. Outline → `Test Template` with named rows, only when one sentence fits every row; never an `IF` in the template keyword. *Why:* Robot Framework has a native form for every concept. +7. **State in the declared place.** Declare each state variable in the steps resource's `*** Variables ***` and assign it with `VAR … scope=TEST`, or keep it in a World library with `@library(scope="TEST")`. Never change a declared list or dictionary in place. *Why:* in-place changes leak into the next test. +8. **Independent tests.** No test relies on another test's data or order. Business state never goes into `Suite Setup`. *Why:* tests run alone, in any order, and in parallel. +9. **Living documentation.** The suite has the user story in `Documentation`, `Metadata Feature ` and a `feature:` tag; rules and step keywords are documented. *Why:* the log and report read as the specification. +10. **Imports by the project's convention.** By default import by name (`Resource cash_withdrawal_steps.resource`) with the folder on `python-path`. *Why:* names don't break when files move. +11. **Style tooling.** Check style with Robocop using the project's configuration; in a new project, add `assets/robocop-bdd.toml`. Write assignments without `=` (`${balance} Get Balance ${holder}`). *Why:* the hook and CI lint with the same rules. + +## Write a specification + +1. **Understand the behaviour.** Write down who benefits and why, the rules, and for each rule a normal case, a boundary and a refusal. Ask about any open question that would change a rule; mark other inferences as assumptions. +2. **Discover the project** (Step 0): suites and tags with `robotcode discover suites` and `robotcode discover tags`, the vocabulary with `robotcode doc keywords`, the state strategy and the Robocop configuration. +3. **Draft the scenarios as sentences** before writing any keyword. Fix wording that needs explaining or names UI details. +4. **Choose the layout.** One feature without an outline → `.robot`. A feature with an outline → a directory with `__init__.robot` (story, `Metadata`, feature tag, shared `Test Setup`), a rules file and one file per outline. Many examples of one behaviour → `references/data-driven.md`. +5. **Write the steps,** reusing sentences first, then domain and technical keywords below them. Each `Then` asserts with a message a reader understands. +6. **Validate** and fix before moving on: + 1. `robotcode analyze code `: no errors. A `VariableNotFound` on state means the variable isn't declared. + 2. `robotcode robot --dryrun -bl ""`: every step resolves. + 3. `robotcode robot -bl ""`: the tests pass. If the system under test isn't reachable, say so. + 4. `robocop check --no-cache `: no findings with the project's configuration. +7. **Report** the rules and their tests, the files touched, the validation results, assumptions and open questions, and any convention you followed or added. + +## Review a specification + +1. Gather evidence: `robotcode analyze code`, `robotcode discover tags`, `robotcode doc keywords` on the steps resources and libraries, and `robocop check --no-cache `. +2. Read every test and step against `references/review-checklist.md`. +3. Report findings by severity, one per line in the format `ID — file:line — problem — rewrite`. +4. Don't edit files in review mode. Change them only when the user asks, then validate as in Write. + +## Gherkin input + +Pasted Gherkin or acceptance criteria become native `.robot` suites (mapping: `references/gherkin-mapping.md`). Don't create `.feature` files and don't install or configure a Gherkin parser; say so in one sentence. + +- Translate the meaning, not the exact wording. Keep every rule and every example. +- An Examples row whose sentence doesn't fit the outline ("1 days") becomes its own named test (`Returning a book one day late costs 0.50 EUR`). +- Split steps joined with "and", and make vague outcomes concrete. +- List every wording change in the report, so the author can confirm it. + +## Gotchas + +- A step keyword defined with a prefix (`Given the app is open`) cannot be called as `When the app is open`. Define steps without a prefix. +- A passing dry run doesn't prove that a step matched the keyword you meant: an embedded pattern can bind another keyword or the wrong values. Run the affected tests for real by longname. +- Adjacent embedded arguments (`${holder} ${amount} EUR`) split at an unexpected place. Separate them with literal text, quote strings (`"${reason}"`) or add a pattern (`${amount:\d+}`). +- Literal text in a step name is matched exactly: `day(s)` is not optional text, so `1 day` doesn't match `${n} day(s)`. Write one sentence per form. +- An outline sentence that doesn't fit every row ("1 days late") needs its own named test. An `IF` in the template keyword hides a second behaviour. +- Changing a declared `@{EMPTY}` or `&{EMPTY}` container in place (`Append To List`, `Set To Dictionary`) leaks into later tests. Rebuild it with `VAR @{ITEMS} @{ITEMS} new scope=TEST`. +- A `Test Setup` in `__init__.robot` calls the step with the child suite's imports, so every child suite imports the steps resource. `robotcode analyze code` doesn't report a missing import; the dry run fails with `No keyword with name 'Given …' found`. +- Robocop reads only the nearest configuration file. Put the BDD settings into the file the project already uses for Robocop. +- A comma-separated `--select` selects nothing; repeat `--select` for each rule. +- `tag-doc` and `tag-stat-link` patterns write `?` for the `:` in tags such as `feature:cash-withdrawal`. +- DataDriver needs a template keyword with normal `[Arguments]`: an embedded-argument template passes the dry run, then fails every row with `Variable '${amount}' not found`. `robotcode discover` and editors show only the placeholder test, so select a row with `--variable "DYNAMICTEST:."`, not `--test`. + +## When to read the references + +| Reference | Read when | +|---|---| +| `references/gherkin-mapping.md` | Translating Gherkin or acceptance criteria: features, rules, backgrounds, outlines, step arguments | +| `references/step-design.md` | Writing or naming step keywords, embedded arguments in steps, reusing a vocabulary | +| `references/state.md` | Passing data between steps: the resource strategy or a World library | +| `references/data-driven.md` | Many examples of one behaviour: templates, `[Template]`, DataDriver | +| `references/documentation-and-tags.md` | Suite and test documentation, `Metadata`, feature and rule tags, localised specifications | +| `references/robocop.md` | Adding or merging the Robocop settings of `assets/robocop-bdd.toml` | +| `references/review-checklist.md` | Reviewing a specification: check IDs, severities and the findings format | + +`assets/examples/` holds a fake ATM service (`sut/`) and two runnable projects: `resource-strategy/` (the default) and `world-strategy/` (a World library). + +## Companion Skills + +| Need | Skill | +|------|-------| +| Write tests, suites, user keywords, resources and variables in Robot Framework syntax | `rf-language` | +| Install Robot Framework or a library, fix the environment | `rf-setup` | +| Write or fix a Python keyword library or listener | `rf-python-library` | + +For running, discovering, keyword documentation, result analysis and debugging with the robotcode CLI, use the `robotcode` skill of the robotcode plugin (`robotcode@robotframework-agent-plugins`). diff --git a/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/resources/cash_withdrawal_steps.resource b/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/resources/cash_withdrawal_steps.resource new file mode 100644 index 0000000..e8657eb --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/resources/cash_withdrawal_steps.resource @@ -0,0 +1,56 @@ +*** Settings *** +Documentation Step definitions for the cash withdrawal domain. +... +... One keyword is one domain sentence. Technical actions live in +... ``AtmService``; test cases only call the sentences below. +Library AtmService + + +*** Variables *** +# Scenario state shared between steps. Declared here so static analysis +# knows it; steps assign it with ``VAR ... scope=TEST``. +${LAST_WITHDRAWAL} ${None} + + +*** Keywords *** +The ATM holds ${cash: int} EUR + [Documentation] Loads the ATM with ``cash`` euros. + Load ATM ${cash} + +The daily withdrawal limit is ${limit: int} EUR + [Documentation] Sets the maximum amount an account may withdraw per day. + Set Daily Limit ${limit} + +${holder} has a balance of ${amount: int} EUR + [Documentation] Opens an account for ``holder`` with ``amount`` euros. + Open Account ${holder} ${amount} + +${holder} has already withdrawn ${amount: int} EUR today + [Documentation] Records an earlier successful withdrawal on the same day. + ${result} Request Withdrawal ${holder} ${amount} + Should Be Equal ${result}[status] dispensed + ... msg=Precondition failed: the earlier withdrawal of ${amount} EUR was refused + +${holder} withdraws ${amount: int} EUR + [Documentation] ``holder`` requests ``amount`` euros at the ATM. + ${result} Request Withdrawal ${holder} ${amount} + VAR ${LAST_WITHDRAWAL} ${result} scope=TEST + +The ATM dispenses ${amount: int} EUR + [Documentation] The last withdrawal was dispensed with exactly ``amount`` euros. + Should Be Equal ${LAST_WITHDRAWAL}[status] dispensed + Should Be Equal ${LAST_WITHDRAWAL}[dispensed] ${amount} + +The withdrawal is refused with "${reason}" + [Documentation] The last withdrawal was refused, and the ATM showed ``reason``. + Should Be Equal ${LAST_WITHDRAWAL}[status] refused + Should Be Equal ${LAST_WITHDRAWAL}[reason] ${reason} + +No cash is dispensed + [Documentation] The last withdrawal dispensed nothing. + Should Be Equal ${LAST_WITHDRAWAL}[dispensed] ${0} + +${holder}'s balance is ${amount: int} EUR + [Documentation] The account of ``holder`` holds exactly ``amount`` euros. + ${balance} Get Balance ${holder} + Should Be Equal ${balance} ${amount} diff --git a/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/robot.toml b/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/robot.toml new file mode 100644 index 0000000..6d9bdab --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/robot.toml @@ -0,0 +1,24 @@ +paths = ["tests"] +output-dir = "results" +# Step resources are imported by name; the technical layer lives in ../sut. +python-path = ["resources", "../sut"] + +[tool.robocop.lint] +configure = [ + # Step definitions read as sentences: "The ATM dispenses ${amount} EUR". + "wrong-case-in-keyword-name.convention=first_word_capitalized", + # Possessive steps such as "${holder}'s balance is ${amount} EUR". + "wrong-case-in-keyword-name.pattern='s", + # Calls read as Gherkin: "Then the ATM dispenses 20 EUR". + "wrong-case-in-keyword-call.convention=first_word_capitalized", +] +# Step resources keep scenario state in declared test-scope variables on purpose. +per-file-ignores = { "*_steps.resource" = ["VAR06"] } + +[tool.robocop.format] +# Align outline examples under the column labels of the *** Test Cases *** header. +extend-select = ["AlignTemplatedTestCases"] +configure = [ + # Don't collapse hand-aligned data tables and doc strings after BDD steps. + 'NormalizeSeparators.skip_keyword_call_pattern=^(Given|When|Then|And|But)\s', +] diff --git a/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/cash_withdrawal/__init__.robot b/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/cash_withdrawal/__init__.robot new file mode 100644 index 0000000..7d2e06c --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/cash_withdrawal/__init__.robot @@ -0,0 +1,8 @@ +*** Settings *** +Documentation As an account holder +... I want to withdraw cash from an ATM +... So that I can get money when the bank is closed +Metadata Feature Cash withdrawal +Resource cash_withdrawal_steps.resource +Test Setup Given the ATM holds 500 EUR +Test Tags feature:cash-withdrawal diff --git a/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/cash_withdrawal/remaining_balance.robot b/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/cash_withdrawal/remaining_balance.robot new file mode 100644 index 0000000..3a51aa2 --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/cash_withdrawal/remaining_balance.robot @@ -0,0 +1,22 @@ +*** Settings *** +Documentation Scenario Outline: the balance that remains after a withdrawal. +... +... Rule: a dispensed withdrawal is debited from the balance. +Resource cash_withdrawal_steps.resource +Test Tags rule:debit-on-dispense +Test Template Withdrawing ${amount} EUR from a balance of ${balance} EUR leaves ${rest} EUR + + +*** Test Cases *** AMOUNT BALANCE REST +Partial withdrawal 20 100 80 +Withdrawal of the whole balance 100 100 0 +Withdrawal of a single note 10 250 240 + + +*** Keywords *** +Withdrawing ${amount} EUR from a balance of ${balance} EUR leaves ${rest} EUR + [Documentation] One example of the remaining-balance outline. + Given Alice has a balance of ${balance} EUR + When Alice withdraws ${amount} EUR + Then the ATM dispenses ${amount} EUR + And Alice's balance is ${rest} EUR diff --git a/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/cash_withdrawal/withdrawal_rules.robot b/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/cash_withdrawal/withdrawal_rules.robot new file mode 100644 index 0000000..5bb0bc1 --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/cash_withdrawal/withdrawal_rules.robot @@ -0,0 +1,30 @@ +*** Settings *** +Documentation Rules that decide whether a withdrawal is dispensed. +Resource cash_withdrawal_steps.resource + + +*** Test Cases *** +Account holder withdraws cash within their balance + [Documentation] Rule: a withdrawal within the balance is dispensed and debited. + [Tags] rule:debit-on-dispense smoke + Given Alice has a balance of 100 EUR + When Alice withdraws 20 EUR + Then the ATM dispenses 20 EUR + And Alice's balance is 80 EUR + +Withdrawal above the balance is refused + [Documentation] Rule: an account cannot be overdrawn at the ATM. + [Tags] rule:no-overdraft + Given Alice has a balance of 10 EUR + When Alice withdraws 20 EUR + Then the withdrawal is refused with "Insufficient funds" + And no cash is dispensed + And Alice's balance is 10 EUR + +Withdrawal above the cash in the ATM is refused + [Documentation] Rule: the ATM never promises more cash than it holds. + [Tags] rule:atm-cash + Given Alice has a balance of 900 EUR + When Alice withdraws 600 EUR + Then the withdrawal is refused with "ATM out of cash" + And Alice's balance is 900 EUR diff --git a/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/daily_withdrawal_limit.robot b/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/daily_withdrawal_limit.robot new file mode 100644 index 0000000..4fa4697 --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/assets/examples/resource-strategy/tests/daily_withdrawal_limit.robot @@ -0,0 +1,25 @@ +*** Settings *** +Documentation As an account holder +... I want a daily limit on ATM withdrawals +... So that a stolen card cannot empty my account +Metadata Feature Daily withdrawal limit +Resource cash_withdrawal_steps.resource +Test Setup Given the daily withdrawal limit is 300 EUR +Test Tags feature:daily-withdrawal-limit rule:within-daily-limit + + +*** Test Cases *** +Withdrawal within the daily limit is dispensed + [Documentation] Rule: withdrawals up to the daily limit are dispensed. + Given Bob has a balance of 1000 EUR + And Bob has already withdrawn 200 EUR today + When Bob withdraws 100 EUR + Then the ATM dispenses 100 EUR + +Withdrawal above the daily limit is refused + [Documentation] Rule: a withdrawal that would exceed the daily limit is refused. + Given Bob has a balance of 1000 EUR + And Bob has already withdrawn 250 EUR today + When Bob withdraws 100 EUR + Then the withdrawal is refused with "Daily limit exceeded" + And Bob's balance is 750 EUR diff --git a/plugins/rf-core/skills/rf-bdd/assets/examples/sut/AtmService.py b/plugins/rf-core/skills/rf-bdd/assets/examples/sut/AtmService.py new file mode 100644 index 0000000..c33be4c --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/assets/examples/sut/AtmService.py @@ -0,0 +1,69 @@ +"""Stand-in for the system under test: a tiny in-memory ATM. + +This is the *technical layer*. In a real project it is Browser, RequestsLibrary, +a database library or the project's own API client. Step keywords call it; +test cases never do. Keyword names here are classic RF Title Case actions. +""" + +from __future__ import annotations + +from robot.api.deco import keyword, library + + +@library(scope="TEST") +class AtmService: + """Fake ATM and bank backend. A fresh instance is created for every test.""" + + def __init__(self) -> None: + self._cash = 10_000 + self._daily_limit = 1_000 + self._balances: dict[str, int] = {} + self._withdrawn_today: dict[str, int] = {} + self._last_receipt = "" + + @keyword + def load_atm(self, cash: int) -> None: + """Sets the amount of cash inside the ATM.""" + self._cash = cash + + @keyword + def set_daily_limit(self, limit: int) -> None: + """Sets the maximum amount one account may withdraw per day.""" + self._daily_limit = limit + + @keyword + def open_account(self, holder: str, balance: int) -> None: + """Opens an account for ``holder`` with the given ``balance``.""" + self._balances[holder] = balance + self._withdrawn_today[holder] = 0 + + @keyword + def request_withdrawal(self, holder: str, amount: int) -> dict: + """Requests a withdrawal and returns ``{status, dispensed, reason}``.""" + if amount <= 0 or amount % 10: + return self._refused("Amount must be a multiple of 10 EUR") + if amount > self._balances[holder]: + return self._refused("Insufficient funds") + if self._withdrawn_today[holder] + amount > self._daily_limit: + return self._refused("Daily limit exceeded") + if amount > self._cash: + return self._refused("ATM out of cash") + self._balances[holder] -= amount + self._withdrawn_today[holder] += amount + self._cash -= amount + self._last_receipt = f"Withdrawal: {amount} EUR\nBalance: {self._balances[holder]} EUR" + return {"status": "dispensed", "dispensed": amount, "reason": None} + + @keyword + def get_last_receipt(self) -> str: + """Returns the text printed on the receipt of the last dispensed withdrawal.""" + return self._last_receipt + + @keyword + def get_balance(self, holder: str) -> int: + """Returns the current balance of ``holder``.""" + return self._balances[holder] + + @staticmethod + def _refused(reason: str) -> dict: + return {"status": "refused", "dispensed": 0, "reason": reason} diff --git a/plugins/rf-core/skills/rf-bdd/assets/examples/world-strategy/lib/CashWithdrawalWorld.py b/plugins/rf-core/skills/rf-bdd/assets/examples/world-strategy/lib/CashWithdrawalWorld.py new file mode 100644 index 0000000..7b46627 --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/assets/examples/world-strategy/lib/CashWithdrawalWorld.py @@ -0,0 +1,71 @@ +"""Step definitions and scenario state ("World") for the cash withdrawal domain. + +Opt-in alternative to a step ``.resource``: use it when the project already +keeps steps in Python, or when steps need real logic. ``scope="TEST"`` gives +every test a fresh World, so no state leaks between scenarios. + +Each ``@keyword`` name is one domain sentence. The BDD prefixes (Given, When, +Then, And, But) are stripped by Robot Framework and are never part of a name. +""" + +from __future__ import annotations + +from robot.api.deco import keyword, library + +from AtmService import AtmService + + +@library(scope="TEST") +class CashWithdrawalWorld: + """Cash withdrawal steps. State lives on the instance, not in RF variables.""" + + def __init__(self) -> None: + self.atm = AtmService() # technical layer; Browser/Requests in a real project + self.last_withdrawal: dict | None = None + + # --- Given ----------------------------------------------------------- + + @keyword("The ATM holds ${cash} EUR") + def the_atm_holds(self, cash: int) -> None: + """Loads the ATM with ``cash`` euros.""" + self.atm.load_atm(cash) + + @keyword("${holder} has a balance of ${amount} EUR") + def has_a_balance_of(self, holder: str, amount: int) -> None: + """Opens an account for ``holder`` with ``amount`` euros.""" + self.atm.open_account(holder, amount) + + # --- When ------------------------------------------------------------ + + @keyword("${holder} withdraws ${amount} EUR") + def withdraws(self, holder: str, amount: int) -> None: + """``holder`` requests ``amount`` euros at the ATM.""" + self.last_withdrawal = self.atm.request_withdrawal(holder, amount) + + # --- Then ------------------------------------------------------------ + + @keyword("The ATM dispenses ${amount} EUR") + def the_atm_dispenses(self, amount: int) -> None: + """The last withdrawal was dispensed with exactly ``amount`` euros.""" + result = self._last() + if result["status"] != "dispensed" or result["dispensed"] != amount: + raise AssertionError(f"Expected {amount} EUR to be dispensed, but the ATM reported {result}") + + @keyword('The withdrawal is refused with "${reason}"') + def the_withdrawal_is_refused_with(self, reason: str) -> None: + """The last withdrawal was refused, and the ATM showed ``reason``.""" + result = self._last() + if result["status"] != "refused" or result["reason"] != reason: + raise AssertionError(f'Expected a refusal with "{reason}", but the ATM reported {result}') + + @keyword("${holder}'s balance is ${amount} EUR") + def balance_is(self, holder: str, amount: int) -> None: + """The account of ``holder`` holds exactly ``amount`` euros.""" + balance = self.atm.get_balance(holder) + if balance != amount: + raise AssertionError(f"{holder}'s balance is {balance} EUR, expected {amount} EUR") + + def _last(self) -> dict: + if self.last_withdrawal is None: + raise AssertionError("No withdrawal was made in this scenario") + return self.last_withdrawal diff --git a/plugins/rf-core/skills/rf-bdd/assets/examples/world-strategy/robot.toml b/plugins/rf-core/skills/rf-bdd/assets/examples/world-strategy/robot.toml new file mode 100644 index 0000000..c9922eb --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/assets/examples/world-strategy/robot.toml @@ -0,0 +1,23 @@ +paths = ["tests"] +output-dir = "results" +python-path = ["lib", "../sut"] + +[tool.robocop.lint] +configure = [ + # Step definitions read as sentences: "The ATM dispenses ${amount} EUR". + "wrong-case-in-keyword-name.convention=first_word_capitalized", + # Possessive steps such as "${holder}'s balance is ${amount} EUR". + "wrong-case-in-keyword-name.pattern='s", + # Calls read as Gherkin: "Then the ATM dispenses 20 EUR". + "wrong-case-in-keyword-call.convention=first_word_capitalized", +] +# Step resources keep scenario state in declared test-scope variables on purpose. +per-file-ignores = { "*_steps.resource" = ["VAR06"] } + +[tool.robocop.format] +# Align outline examples under the column labels of the *** Test Cases *** header. +extend-select = ["AlignTemplatedTestCases"] +configure = [ + # Don't collapse hand-aligned data tables and doc strings after BDD steps. + 'NormalizeSeparators.skip_keyword_call_pattern=^(Given|When|Then|And|But)\s', +] diff --git a/plugins/rf-core/skills/rf-bdd/assets/examples/world-strategy/tests/cash_withdrawal.robot b/plugins/rf-core/skills/rf-bdd/assets/examples/world-strategy/tests/cash_withdrawal.robot new file mode 100644 index 0000000..7d6e6d7 --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/assets/examples/world-strategy/tests/cash_withdrawal.robot @@ -0,0 +1,26 @@ +*** Settings *** +Documentation As an account holder +... I want to withdraw cash from an ATM +... So that I can get money when the bank is closed +Metadata Feature Cash withdrawal +Library CashWithdrawalWorld +Test Setup Given the ATM holds 500 EUR +Test Tags feature:cash-withdrawal + + +*** Test Cases *** +Account holder withdraws cash within their balance + [Documentation] Rule: a withdrawal within the balance is dispensed and debited. + [Tags] rule:debit-on-dispense smoke + Given Alice has a balance of 100 EUR + When Alice withdraws 20 EUR + Then the ATM dispenses 20 EUR + And Alice's balance is 80 EUR + +Withdrawal above the balance is refused + [Documentation] Rule: an account cannot be overdrawn at the ATM. + [Tags] rule:no-overdraft + Given Alice has a balance of 10 EUR + When Alice withdraws 20 EUR + Then the withdrawal is refused with "Insufficient funds" + And Alice's balance is 10 EUR diff --git a/plugins/rf-core/skills/rf-bdd/assets/robocop-bdd.toml b/plugins/rf-core/skills/rf-bdd/assets/robocop-bdd.toml new file mode 100644 index 0000000..a9f4100 --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/assets/robocop-bdd.toml @@ -0,0 +1,23 @@ +# Robocop settings for sentence-style BDD steps (rf-bdd's default for new projects). +# Paste into robot.toml or pyproject.toml (robocop.toml: drop the "tool.robocop." prefix). +# See references/robocop.md for what each line does and how to merge it. +[tool.robocop.lint] +configure = [ + # Step definitions read as sentences: "The ATM dispenses ${amount} EUR". + "wrong-case-in-keyword-name.convention=first_word_capitalized", + # Possessive steps such as "${holder}'s balance is ${amount} EUR". + "wrong-case-in-keyword-name.pattern='s", + # Calls read as Gherkin: "Then the ATM dispenses 20 EUR". + "wrong-case-in-keyword-call.convention=first_word_capitalized", +] +# Step resources keep scenario state in declared test-scope variables on purpose. +per-file-ignores = { "*_steps.resource" = ["VAR06"] } + +# Only matters if the project runs `robocop format`: keeps BDD tables readable. +[tool.robocop.format] +# Align outline examples under the column labels of the *** Test Cases *** header. +extend-select = ["AlignTemplatedTestCases"] +configure = [ + # Don't collapse hand-aligned data tables and doc strings after BDD steps. + 'NormalizeSeparators.skip_keyword_call_pattern=^(Given|When|Then|And|But)\s', +] diff --git a/plugins/rf-core/skills/rf-bdd/references/data-driven.md b/plugins/rf-core/skills/rf-bdd/references/data-driven.md new file mode 100644 index 0000000..47efa68 --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/references/data-driven.md @@ -0,0 +1,137 @@ +# Data-Driven Specifications + +- [The ladder](#the-ladder) +- [Named examples with Test Template](#named-examples-with-test-template) +- [One rule, many inputs: [Template] in one test](#one-rule-many-inputs-template-in-one-test) +- [Values after a step](#values-after-a-step) +- [External data files: DataDriver](#external-data-files-datadriver) +- [Smells](#smells) + +The same behaviour checked with several inputs is a Scenario Outline in Gherkin. Robot Framework offers templates with named rows, arguments after a step, and external data through DataDriver. Pick the simplest form that keeps the specification readable. Code blocks run in a copy of `assets/examples/resource-strategy/`. + +## The ladder + +| Situation | Form | +|---|---| +| a few examples of one behaviour, each worth a name | an outline file with `Test Template`, one named test per row | +| one rule, many inputs that behave the same and need no names | `[Template]` on one test, one row per input | +| one step needs a set of values | arguments after the step: `key=value` pairs or a list | +| the examples live in a CSV or Excel file the user names, or business people maintain them there | DataDriver with a normal-argument template | + +Start at the top. Move down only when the situation calls for it. + +## Named examples with Test Template + +This is the standard outline: one file per outline, column labels in the `*** Test Cases ***` header, and each row a named test. The runnable file is `assets/examples/resource-strategy/tests/cash_withdrawal/remaining_balance.robot`. + +- The template keyword has embedded arguments, so the log shows each example as a sentence ("Withdrawing 20 EUR from a balance of 100 EUR leaves 80 EUR") with its steps under it. +- Each row is its own test, so the report has one line per example and one example can be run alone by its longname. +- The outline sentence must fit every row. "Returning a book 1 days late" or "0 days late" means one sentence is stretched over examples that read differently. Give such a row its own named test with its own sentence ("Book returned on the due date costs nothing") in the feature's scenarios file, and keep the outline for the rows that share one sentence. +- Never branch on the input with `IF` in a template keyword, and never put alternative wording such as `day(s)` in the name; literal text is matched exactly. Both hide several behaviours in one outline. +- Template mechanics (row forms, `[Template] NONE`, settings per row) and status rules: see the `rf-language` skill. + +## One rule, many inputs: [Template] in one test + +When a rule has many inputs that behave the same way, and naming each would add nothing, use one test named after the rule, with `[Template]` and one row per input: + + +```robotframework +*** Settings *** +Documentation As a bank +... I want the ATM to accept only amounts it can dispense in notes +... So that account holders never ask for coins it does not have +Metadata Feature Withdrawal amounts +Resource cash_withdrawal_steps.resource +Test Tags feature:withdrawal-amounts rule:multiples-of-ten + + +*** Test Cases *** +Amounts that are not a multiple of 10 EUR are refused + [Documentation] Rule: the ATM only dispenses multiples of 10 EUR. + [Template] A withdrawal of ${amount} EUR is refused as not a multiple of 10 EUR + 5 + 15 + 99 + + +*** Keywords *** +A withdrawal of ${amount} EUR is refused as not a multiple of 10 EUR + [Documentation] One input for the multiples-of-ten rule. + Given Alice has a balance of 1000 EUR + When Alice withdraws ${amount} EUR + Then the withdrawal is refused with "Amount must be a multiple of 10 EUR" + And Alice's balance is 1000 EUR +``` + +- Each row is a full scenario, Given to Then, so the rows don't affect each other. +- Keep the rows to one screen. When rows need comments to explain them, they deserve names: switch to a `Test Template` outline. + +## Values after a step + +When one step, not the whole scenario, needs several values, pass them on continuation lines: + +| Shape | Call | Keyword | +|---|---|---| +| key and value | `Given the ATM holds these notes 20=40 50=10` | `[Arguments] &{notes}` | +| list | `Then the menu offers Withdraw cash Check balance` | `[Arguments] @{entries}` | + +An item with several attributes gets one step per fact, or a named starting state. Many rows of values for the same behaviour are an outline. + +## External data files: DataDriver + +DataDriver (`robotframework-datadriver`) generates one test per row of a CSV, Excel or JSON file at run time. Use it only when the user names an external data file, or when non-programmers maintain the examples in a spreadsheet. Otherwise the examples belong in the `.robot` file, next to the rule. Installing it is a job for the `rf-setup` skill. The blocks below are marked as text because they need DataDriver to run. + +```text +*** Settings *** +Documentation As a product owner +... I want to maintain withdrawal examples in a spreadsheet +... So that the business can add cases without editing tests +... +... Examples: withdrawal_examples.csv +Metadata Feature Withdrawal examples +Resource cash_withdrawal_steps.resource +Library DataDriver file=withdrawal_examples.csv +Test Tags feature:withdrawal-examples rule:debit-on-dispense +Test Template Withdrawal leaves the remaining balance + + +*** Test Cases *** +Withdrawal example 0 0 0 + + +*** Keywords *** +Withdrawal leaves the remaining balance + [Documentation] One row of withdrawal_examples.csv. + [Arguments] ${amount} ${balance} ${rest} + Given Alice has a balance of ${balance} EUR + When Alice withdraws ${amount} EUR + Then the ATM dispenses ${amount} EUR + And Alice's balance is ${rest} EUR +``` + +`tests/withdrawal_examples.csv`: + +```text +*** Test Cases ***;${amount};${balance};${rest};[Tags];[Documentation] +Partial withdrawal;20;100;80;;Leaves most of the balance +Withdrawal of the whole balance;100;100;0;boundary;Leaves nothing +Withdrawal of a single note;10;250;240;; +``` + +How it differs from a native template (checked with DataDriver 1.11.2): +- The template keyword needs normal `[Arguments]`, not embedded arguments. DataDriver passes the columns by name. An embedded-argument template passes the dry run and then fails every row with `Variable '${amount}' not found`. The steps inside still read as sentences. +- The default CSV dialect is `Excel-EU`, so the separator is `;`. The first header is `*** Test Cases ***` and holds the test names; `[Tags]` and `[Documentation]` columns are optional. A comma-separated file fails with an "Unassigned … argument detected" error from DataDriver. +- The placeholder test (`Withdrawal example 0 0 0`) is replaced at run time. Keep `${variables}` out of its name. +- `robotcode discover` and editor test explorers see only the placeholder. The dry run and the real run see the generated tests. Tell the user. +- Robot Framework selects tests before DataDriver generates them, so `--test "Partial withdrawal"` and `--include boundary` find nothing. Run the whole suite, and pick one row with `--variable "DYNAMICTEST:Withdrawal Examples.Partial withdrawal"`, or filter rows with DataDriver's `include=` and `exclude=` library arguments. +- Readability moves to the data file. Review the CSV like a specification: realistic names, no placeholder data, one behaviour. + +## Smells + +| Smell | What it usually means | +|---|---| +| 20 or more rows in an outline | a pure data test; keep representative examples and move exhaustive checks to unit tests | +| rows that need comments | the rows test different rules; split by rule and name the rows | +| `IF` on the input in a template keyword | several behaviours in one outline; write one outline per behaviour | +| columns like `expected_status_code` or `sql_result` | implementation details in the specification | +| DataDriver for three examples | an inline `Test Template` is easier to read, and the tools see it | diff --git a/plugins/rf-core/skills/rf-bdd/references/documentation-and-tags.md b/plugins/rf-core/skills/rf-bdd/references/documentation-and-tags.md new file mode 100644 index 0000000..16774ff --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/references/documentation-and-tags.md @@ -0,0 +1,148 @@ +# Documentation, Metadata and Tags + +- [Where each piece goes](#where-each-piece-goes) +- [Suite documentation and Metadata](#suite-documentation-and-metadata) +- [Feature directories and capability areas](#feature-directories-and-capability-areas) +- [Test documentation](#test-documentation) +- [The tag scheme](#the-tag-scheme) +- [Tag documentation and links](#tag-documentation-and-links) +- [Localised specifications](#localised-specifications) + +The report and the log are the living documentation of a BDD suite. They show the user story, every rule with its examples, every step in domain words, and a pass/fail count per feature and per rule, when the suite fills in the right fields. Code blocks run in a copy of `assets/examples/resource-strategy/`. + +## Where each piece goes + +| Content | Field | Shows up in | +|---|---|---| +| user story and feature description | suite `Documentation` | top of the report and the log for that suite | +| feature name | `Metadata Feature ` | suite metadata in the report and the log | +| capability area description | `Documentation` in the directory's `__init__.robot` | the parent suite | +| rule | test `[Documentation] Rule: …` and a `rule:` tag | test details; one row per rule in the tag statistics | +| ticket or requirement | `issue:` tag, or test `[Metadata]` on RF 7.5+ | test details | +| meaning of a step | step `[Documentation]` or docstring; the steps resource is the glossary | `robotcode doc keywords`, editor hover | +| meaning of a tag | `tag-doc` in `robot.toml` | tooltip in the tag statistics | + +## Suite documentation and Metadata + +`assets/examples/resource-strategy/tests/cash_withdrawal/__init__.robot` shows the settings of a feature: +- The first three `Documentation` lines are the user story ("As a … / I want … / So that …"), without a "Feature:" prefix. After an empty `...` line, a few sentences of business context may follow. +- `Metadata Feature ` uses the feature name. Keep the key `Feature` in every language, because filters and tools rely on it. +- More metadata is fine when it helps a reader: `Metadata Specification https://wiki.example.com/atm`. +- Robot Framework names the suite after the file, so name files after the feature: `cash_withdrawal.robot` becomes "Cash Withdrawal". + +## Feature directories and capability areas + +Group features into directories by business capability, not by technical layer: + +```text +tests/ + __init__.robot Documentation: "Self-service banking at the ATM." + cash_withdrawal/ a feature directory (it has an outline) + __init__.robot story, Metadata, Test Tags, Test Setup + withdrawal_rules.robot + remaining_balance.robot + daily_withdrawal_limit.robot +``` + +- A capability area's `__init__.robot` holds a short `Documentation`, and at most technical `Suite Setup` and `Suite Teardown`, never business state. +- A feature directory's `__init__.robot` holds the story, `Metadata Feature`, `Test Tags feature:` and the Background as `Test Setup`. +- Each child suite imports the steps resource itself, also for the `Test Setup` it inherits. Without the import, every test in that child fails with `Setup failed: No keyword with name 'Given …' found`. The dry run shows this; `robotcode analyze code` doesn't. +- What `__init__.robot` may contain and which imports child suites see: see the `rf-language` skill. + +## Test documentation + +- `[Documentation]` starts with `Rule:` and states the rule in one sentence. Tests that illustrate the same rule repeat the sentence word for word. +- Don't repeat the steps in the documentation; the body already says it. +- Link tickets with an `issue:` tag. Test-level `[Metadata] Issue BANK-123` needs Robot Framework 7.5; on 7.4 and older it fails the test with "Setting 'Metadata' is not allowed with tests or tasks". Use it only after the version gate allows it. +- Settings come first in the body: `[Documentation]`, `[Tags]`, then `[Setup]` or `[Template]`. + +## The tag scheme + +| Tag | Where | Meaning | +|---|---|---| +| `feature:` | `Test Tags` of the feature's file or its `__init__.robot` | the feature the test specifies | +| `rule:` | `[Tags]` per test; `Test Tags` when every test in the file has the same rule | the rule the test illustrates | +| the project's own tags (`smoke`, `wip`) | `[Tags]` | what the project already uses them for | + +- Slugs are lower-case kebab case, from the feature name and the rule wording: `feature:cash-withdrawal`, `rule:no-overdraft`. +- Check the existing tags first with `robotcode discover tags`. Reuse a tag with the same meaning; don't add a second spelling. +- Descriptions go in `[Documentation]`, not in tags. +- Reserved `robot:` tags, tag patterns and selecting tests on the command line: see the `rf-language` skill. + +## Tag documentation and links + +Explain the tag scheme once in `robot.toml`, so the tag statistics get tooltips and links: + +```toml +[tag-doc] +"feature?*" = "Feature the test specifies" +"rule?*" = "Business rule the test illustrates" + +[tag-stat-link] +"rule?*" = "https://wiki.example.com/rules/%2:Rule description" +``` + +- Write `?` instead of the colon in these patterns (`rule?*`, not `rule:*`). Robot Framework splits the option at the first colon, so `rule:*` becomes the pattern `rule`, which matches no `rule:` tag. You get no tooltip, no link and no error. +- `%N` is what the N-th wildcard matched. In `rule?*`, `%1` is the colon and `%2` is the slug, so `rule:no-overdraft` links to `…/rules/no-overdraft`. +- Check the effective settings with `robotcode profiles show`. + +## Localised specifications + +When the business speaks another language, write the whole specification in it: headers, settings, prefixes and step sentences. Start each file with `Language: `, or set `languages = ["de"]` in `robot.toml`. Technical keywords below the steps can stay in English. + +The block below keeps the steps in the suite so that it runs on its own; in a project they go into `bargeld_steps.resource`. + + +```robotframework +Language: German + +*** Einstellungen *** +Dokumentation Als Kontoinhaberin +... möchte ich am Geldautomaten Bargeld abheben, +... damit ich auch außerhalb der Öffnungszeiten an Geld komme. +Metadaten Feature Bargeldabhebung +Bibliothek AtmService +Testmarker feature:bargeldabhebung + + +*** Testfälle *** +Abhebung innerhalb des Guthabens wird abgebucht + [Dokumentation] Regel: Eine Abhebung innerhalb des Guthabens wird ausgezahlt und abgebucht. + [Marker] rule:abbuchung-bei-auszahlung + Angenommen Alice hat ein Guthaben von 100 EUR + Wenn Alice 20 EUR abhebt + Dann beträgt das Guthaben von Alice 80 EUR + + +*** Schlüsselwörter *** +${inhaber} hat ein Guthaben von ${betrag: int} EUR + [Dokumentation] Eröffnet ein Konto für ``inhaber`` mit ``betrag`` Euro. + Open Account ${inhaber} ${betrag} + +${inhaber} ${betrag: int} EUR abhebt + [Dokumentation] ``inhaber`` fordert ``betrag`` Euro am Geldautomaten an. + Request Withdrawal ${inhaber} ${betrag} + +Beträgt das Guthaben von ${inhaber} ${betrag: int} EUR + [Dokumentation] Das Konto von ``inhaber`` enthält genau ``betrag`` Euro. + ${guthaben} Get Balance ${inhaber} + Should Be Equal ${guthaben} ${betrag} +``` + +| English | German | +|---|---| +| `*** Settings ***`, `*** Variables ***`, `*** Test Cases ***`, `*** Keywords ***` | `*** Einstellungen ***`, `*** Variablen ***`, `*** Testfälle ***`, `*** Schlüsselwörter ***` | +| `Documentation`, `Metadata`, `Library`, `Resource` | `Dokumentation`, `Metadaten`, `Bibliothek`, `Ressource` | +| `Test Setup`, `Test Tags`, `Test Template` | `Testvorbereitung`, `Testmarker`, `Testvorlage` | +| `[Tags]`, `[Template]`, `[Arguments]` | `[Marker]`, `[Vorlage]`, `[Argumente]` | +| `Given`, `When`, `Then`, `And`, `But` | `Angenommen`, `Wenn`, `Dann`, `Und`, `Aber` | + +- Write natural grammar for the prefix. In German the prefix decides the word order: after `Wenn` the verb goes last (`Wenn Alice 20 EUR abhebt`), after `Dann` it comes second (`Dann beträgt das Guthaben von Alice 80 EUR`). So a German sentence usually belongs to one phase; define a second sentence when another phase needs it. +- Avoid possessives glued to an argument (`${inhaber}s Guthaben`); write `Das Guthaben von ${inhaber}`. +- Rule documentation uses the local word (`Regel:`). Keep the `feature:` and `rule:` tag prefixes in every language. +- Messages from the system under test stay as the system produces them. +- List the translations for another language from the project's environment: + +```bash +python -c "from robot.conf.languages import Language; l = Language.from_name('fi'); print({k: getattr(l, k) for k in dir(l) if k.endswith(('_header', '_setting', '_prefixes'))})" +``` diff --git a/plugins/rf-core/skills/rf-bdd/references/gherkin-mapping.md b/plugins/rf-core/skills/rf-bdd/references/gherkin-mapping.md new file mode 100644 index 0000000..b62cd20 --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/references/gherkin-mapping.md @@ -0,0 +1,200 @@ +# Gherkin Concepts in Robot Framework + +- [Overview](#overview) +- [Feature](#feature) +- [Rule](#rule) +- [Background](#background) +- [Scenario and steps](#scenario-and-steps) +- [Scenario Outline and Examples](#scenario-outline-and-examples) +- [Step arguments](#step-arguments) +- [Constructs without a Robot Framework form](#constructs-without-a-robot-framework-form) +- [Translating pasted Gherkin](#translating-pasted-gherkin) + +Robot Framework has a native form for almost every Gherkin concept, so a BDD suite needs no Gherkin parser and no `.feature` file. A reader who knows Gherkin recognises every part, and robotcode still checks every line. Code blocks run in a copy of `assets/examples/resource-strategy/`. + +## Overview + +| Gherkin | Robot Framework form | +|---|---| +| `Feature:` with a user story | one suite file: the story in `Documentation`, `Metadata Feature `, `Test Tags feature:` | +| a feature with a Scenario Outline | a feature directory: `__init__.robot` holds the story, metadata, feature tag and Background; one file holds the scenarios; one file per outline | +| a group of features | a directory whose `__init__.robot` has a `Documentation` | +| `Rule:` | `[Documentation] Rule: …` on the test, plus a `rule:` tag | +| `Background:` | `Test Setup Given `, when two or more tests share it | +| `Scenario:` / `Example:` | a test whose name states the behaviour | +| `Given` / `When` / `Then` / `And` / `But` | the same words as prefixes on step calls | +| a step parameter | an embedded argument: `"${reason}"`, `${amount: int}` | +| `Scenario Outline:` with `Examples:` | `Test Template`, one named test per row, column labels in the `*** Test Cases ***` header | +| a data table | arguments after the step: `key=value` pairs or a list | +| a doc string `"""` | continuation lines passed as `@{lines}` | +| `@tag` | `Test Tags` for the file, `[Tags]` for one test | +| `# language: de` | `Language: German` as the first line, with localised prefixes | +| `Before` / `After` hooks | `Suite Setup`, `Suite Teardown`, `Test Teardown` with technical keywords only | + +## Feature + +A feature with only ordinary scenarios is one file, named after the feature in snake case. Each test documents its rule. + + +```robotframework +*** Settings *** +Documentation As an account holder +... I want the ATM to refuse withdrawals above my balance +... So that my account is never overdrawn by accident +Metadata Feature Overdraft protection +Resource cash_withdrawal_steps.resource +Test Setup Given the ATM holds 500 EUR +Test Tags feature:overdraft-protection + + +*** Test Cases *** +Withdrawal above the balance is refused + [Documentation] Rule: an account cannot be overdrawn at the ATM. + [Tags] rule:no-overdraft + Given Alice has a balance of 10 EUR + When Alice withdraws 20 EUR + Then the withdrawal is refused with "Insufficient funds" + And Alice's balance is 10 EUR + +Withdrawal of the whole balance is dispensed + [Documentation] Rule: a withdrawal up to the balance is dispensed and debited. + [Tags] rule:debit-on-dispense + Given Alice has a balance of 100 EUR + When Alice withdraws 100 EUR + Then the ATM dispenses 100 EUR + And Alice's balance is 0 EUR +``` + +- The story lines appear at the top of the report and the log. `Metadata Feature` lists the feature in the suite metadata, and the `feature:` tag lets people run the feature alone. +- A feature that also has an outline becomes a directory. `assets/examples/resource-strategy/tests/cash_withdrawal/` shows the layout: `__init__.robot`, `withdrawal_rules.robot` and `remaining_balance.robot`. +- What an `__init__.robot` may contain, and what child suites inherit from it: see the `rf-language` skill. + +## Rule + +Robot Framework has no rule block. Put the rule in the test's `[Documentation]`, starting with `Rule:`, and tag the test `rule:`: +- The documentation keeps the rule next to its examples in the report. +- The tag runs one rule alone (`--include rule:no-overdraft`) and gives each rule a pass/fail row in the tag statistics. +- When every test in a file has the same rule, move the tag to `Test Tags` (Robocop TAG05 suggests it) and keep the `Rule:` line in each test. `assets/examples/resource-strategy/tests/daily_withdrawal_limit.robot` does this. + +## Background + +A Background is `Test Setup` with one `Given` sentence: `Test Setup Given the ATM holds 500 EUR`. +- Use it only for a starting state that two or more tests in the file share. Otherwise it stays in the test as the first `Given`. +- A setup is one keyword. When the shared state has several facts, name the whole state in one sentence (`Given a fully loaded ATM with a daily limit of 300 EUR`) and document what it sets up. Don't chain steps with `Run Keywords … AND …`. +- In a feature directory, `Test Setup` in `__init__.robot` applies to every test of the feature. Every child suite must import the steps resource that defines it. +- Never put business state in `Suite Setup`: the tests would share it. `Suite Setup` is for technical setup, such as starting the application. + +## Scenario and steps + +- The test name states the behaviour: "Withdrawal above the balance is refused", not "Test withdrawal 2". +- The body holds only prefixed steps, in the order Given, When, Then, with one `When`. `And` continues the previous phase; `But` adds a contrasting outcome (`But no fee is charged`). +- Robot Framework drops the prefix before it looks up the keyword, so one step keyword can serve as `Given` in one test and `Then` in another. Give it a sentence that reads well in both places, or define two sentences. + +## Scenario Outline and Examples + +An outline is a file of its own with `Test Template`, and each example row is a named test. The `*** Test Cases ***` header labels the columns, so the file reads like an Examples table. The runnable outline is `assets/examples/resource-strategy/tests/cash_withdrawal/remaining_balance.robot`. + +- The template keyword is the outline sentence. Its body holds the Given/When/Then steps, and it stays in the outline file. +- Name every example after what makes it different ("Withdrawal of the whole balance"). Don't put `${variables}` in example names; robotcode shows a `VariableNotReplaced` hint. +- The outline sentence must read correctly for every row. A row that needs other words ("on the due date" instead of "0 days late") becomes its own named test in the scenarios file, not an `IF` in the template keyword. +- Keep ordinary scenarios out of the outline file, because `Test Template` applies to every test in it. +- Template mechanics, row forms and status rules: see the `rf-language` skill. + +## Step arguments + +Gherkin data tables and doc strings become arguments after the step, on continuation lines when they are long: + +| Data | Step call | Keyword arguments | +|---|---|---| +| key and value (holder and balance) | `Given the following accounts exist Alice=100 Bob=500` | `[Arguments] &{balances}` | +| a list of values | `Then the menu offers Withdraw cash Check balance` | `[Arguments] @{entries}` | +| multi-line text (a receipt, a message) | `Then the receipt reads` with `... ` rows | `[Arguments] @{lines}`, joined in the step | + +The calls, with the steps resource below: + +```text +Given the following accounts exist Alice=100 Bob=500 +When Alice withdraws 20 EUR +Then the receipt reads +... Withdrawal: 20 EUR +... Balance: 80 EUR +``` + + +```robotframework +*** Settings *** +Documentation Step definitions for accounts and receipts. +Library AtmService + + +*** Keywords *** +The following accounts exist + [Documentation] Opens one account per ``holder=balance`` pair (balances in EUR). + [Arguments] &{balances} + FOR ${holder} ${balance} IN &{balances} + Open Account ${holder} ${balance} + END + +The receipt reads + [Documentation] The last receipt has exactly these ``lines``, in order. + [Arguments] @{lines} + ${expected} Catenate SEPARATOR=\n @{lines} + ${receipt} Get Last Receipt + Should Be Equal ${receipt} ${expected} +``` + +- `key=value` pairs suit items with one attribute each. The sentence says what the key and the value are, and `[Documentation]` gives the unit. +- When an item has several attributes, don't invent a table format. Write one step per fact, or name the starting state. +- Many rows of values for the same behaviour are an outline, not a table. +- For structured payloads, compare parsed fields (`The account summary contains holder=Alice balance=80`) instead of raw JSON text. + +## Constructs without a Robot Framework form + +| Gherkin | Write instead | +|---|---| +| `Or` steps | separate tests, or an outline | +| a step that calls another step | a technical keyword that both steps call | +| hooks with business meaning | a visible `Given` step or a Background | +| a header row on Examples | the `*** Test Cases ***` header of the outline file | +| feature description paragraphs | more lines in the suite `Documentation`, after the story | +| a step table with several columns | one step per fact, a named starting state, or an outline | +| `.feature` files | `.robot` suites built with this mapping; no Gherkin parser | + +## Translating pasted Gherkin + +Translate the meaning, not the exact words. Keep every rule and every example, apply the house rules, and list each wording change in the report so the author can confirm it. + +```gherkin +Feature: Late returns + Background: + Given the library charges 0.50 EUR per day late + + Rule: each late day costs a fee + Scenario Outline: Fee for a late return + When Alice returns a book days late + Then Alice pays EUR + Examples: + | days | fee | + | 0 | 0.00 | + | 1 | 0.50 | + | 3 | 1.50 | + | 5 | 2.50 | +``` + +The result is a feature directory, because the feature has an outline: + +```text +tests/late_returns/__init__.robot story, Metadata Feature, feature:late-returns, Test Setup (the Background) +tests/late_returns/return_fees.robot + Book returned on the due date costs nothing (row days=0) + Book returned one day late costs 0.50 EUR (row days=1) +tests/late_returns/late_fee.robot Test Template: Returning a book ${days} days late costs ${fee} EUR + Three days late (row days=3) + Five days late (row days=5) +``` + +Wording changes for the report: +- Row `0`: "0 days late" reads wrong, so it became the named test "Book returned on the due date costs nothing". +- Row `1`: "1 days late" reads wrong, so it became the named test "Book returned one day late costs 0.50 EUR". +- `When … Then …` became one template sentence, `Returning a book ${days} days late costs ${fee} EUR`, for the rows that fit it. +- Also list split conjunction steps ("has a card and 100 EUR"), vague outcomes made concrete ("it works"), and placeholder data replaced with realistic values. diff --git a/plugins/rf-core/skills/rf-bdd/references/review-checklist.md b/plugins/rf-core/skills/rf-bdd/references/review-checklist.md new file mode 100644 index 0000000..f24e5d9 --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/references/review-checklist.md @@ -0,0 +1,99 @@ +# Reviewing BDD Suites + +- [Gather evidence](#gather-evidence) +- [S1: correctness and analysis](#s1-correctness-and-analysis) +- [S2: layering and independence](#s2-layering-and-independence) +- [S3: language and style](#s3-language-and-style) +- [Findings format](#findings-format) + +A review answers two questions: can a business reader understand and trust these specifications, and will they stay consistent as the suite grows? Gather tool evidence first, then read every test and step against the checks below. In review mode, don't edit files unless the user asks for fixes. + +## Gather evidence + +| Command | Shows | +|---|---| +| `robotcode analyze code ` | `KeywordNotFound` (typos, undefined steps), `MultipleKeywords` (ambiguous steps), `VariableNotFound` (undeclared state), `PrivateKeyword` (tests calling helpers), `VariableNotReplaced` (`${var}` in templated test names) | +| `robotcode doc keywords ` | the vocabulary: synonyms, overlapping sentences, missing documentation | +| `robotcode discover tags` | tags outside the `feature:` and `rule:` scheme, two spellings of one tag | +| `robotcode discover tests --search ""` | where a behaviour is already specified | +| `robocop check --no-cache ` with the project's configuration | naming (NAME02, NAME18), missing documentation (DOC01 to DOC03), shared tags (TAG05), test-scope variables outside steps resources (VAR06) | +| `robotcode robot --dryrun` on the suite | steps that don't resolve, and a `Test Setup` from `__init__.robot` that a child suite can't find | + +## S1: correctness and analysis + +| ID | Problem | Detect | Fix | +|---|---|---|---| +| C1 | undefined, mistyped or ambiguous step | analyze: `KeywordNotFound`, `MultipleKeywords`; dry run | reuse the existing sentence; restrict values with types or patterns | +| C2 | trailing catch-all argument (`The order "${id}" is ${state}`) that swallows typos | read the step definitions | `${state:paid\|shipped\|cancelled}` | +| C3 | scenario state not declared | analyze: `VariableNotFound` on state | declare it in `*** Variables ***` of the steps resource | +| C4 | a declared list or dictionary changed in place, so state leaks into the next test | `Set To Dictionary`, `Append To List`, `Remove From …` on declared variables | `VAR &{NOTES} &{NOTES} ${k}=${v} scope=TEST` | +| C5 | a `Then` step that can't fail (only logs, or returns a boolean) | read the body; break the expected value and run | assert with `Should …` or raise `AssertionError` | +| C6 | a template for external data rows with embedded arguments | `Variable '${…}' not found` at run time; the dry run passes | a template keyword with normal `[Arguments]` | +| C7 | child suite of a feature directory without the steps import | `Setup failed: No keyword with name 'Given …'` | import the steps resource in every child suite | + +## S2: layering and independence + +| ID | Problem | Detect | Fix | +|---|---|---|---| +| L1 | library keywords, locators, URLs, HTTP verbs or SQL in a test body | any line that isn't a prefixed step or a template row; analyze `PrivateKeyword` | move the mechanics into a step; the test keeps the sentence | +| L2 | a UI script as steps: clicks, typing, navigation | steps name fields, buttons, pages, "click", "type", "open" | one declarative step per intent: `When Alice withdraws 20 EUR` | +| L3 | a test that depends on another test's state or order | no `Given` of its own; a `Then` reading what an earlier test created | give each test its own `Given` | +| L4 | business state in `Suite Setup`, hooks or listeners | read `Suite Setup` and `__init__.robot` | a visible `Given`, or `Test Setup Given …` | +| L5 | a Background (`Test Setup`) that only one test needs | count the tests that rely on it | inline it as that test's first `Given`, or split the file | +| L6 | repeated phases (`When … Then … When … Then …`) | read the prefix order | one test per behaviour | +| L7 | several behaviours in one test, or 10 or more steps | "and" in the test name; unrelated `Then`s | split; name the starting state in one sentence | +| L8 | a BDD prefix in a keyword name, or steps calling steps with prefixes | read keyword names and bodies | drop the prefix from the name; share logic through a technical keyword | +| L9 | values returned from steps into the test body (`${result} When …`) | read test bodies | keep the value in scenario state | + +## S3: language and style + +| ID | Problem | Detect | Fix | +|---|---|---|---| +| W1 | conjunction step with two facts or actions | "and", "then" or commas inside a step | split into `And` steps | +| W2 | vague outcome: `Then it works`, `the result is correct` | read the `Then` steps | name what the user or system shows | +| W3 | placeholder data: `foo`, `test`, `user1`, `123` | read arguments and rows | realistic values: `Alice`, `100 EUR` | +| W4 | synonym steps for one concept | `robotcode doc keywords` | keep one sentence, replace the others | +| W5 | unnamed template rows that need explaining, or `${variables}` in example names | analyze `VariableNotReplaced`; names like `Example 1` | one named test per example | +| W6 | missing feature documentation: no story, no `Metadata Feature`, no `feature:` tag | Robocop DOC03; read `*** Settings ***` | add the story, the metadata and the tag | +| W7 | missing rule documentation or `rule:` tag | Robocop DOC02; read the tests | add the `Rule:` sentence and the tag | +| W8 | missing step documentation | Robocop DOC01; `robotcode doc keywords` | one line on what the step establishes, does or checks | +| W9 | tags outside the vocabulary, two spellings, descriptions as tags | `robotcode discover tags` | `feature:` and `rule:` plus the project's tags | +| W10 | step casing that breaks the project's convention | read the steps; Robocop NAME02 and NAME18 with the project's configuration | the project's casing; sentence style when the project has none | +| W11 | first person, past tense or imperative steps (`I withdrew`, `Withdraw 20 EUR`) | read the steps | third person, present tense | +| W12 | test names that describe the procedure (`Test withdraw 2`) | read the names | name the behaviour | +| W13 | strings without quotes, or quotes around numbers | read the steps | `"Insufficient funds"`, but `20 EUR` | +| W14 | oversized outlines (20 or more rows) | count the rows | keep representative examples | +| W15 | imports that break the project's convention | read the imports in `*** Settings ***` | the project's style; by name with `python-path` when it has none | +| W16 | an outline sentence that doesn't fit every row (`1 days late`), or an `IF` in a template keyword | read the rows as sentences | a named test for the odd row | + +## Findings format + +Order findings by severity (S1, then S2, then S3), and within a severity by the number of tests affected. Write each finding on one line that starts with the check ID, then the location, then the problem, why it matters and the rewrite: + +```text +ID — file:line — problem. Why: one sentence. Rewrite: the replacement. +``` + +Given this test: + +```robotframework +# Wrong: a procedural test with technical keywords in its body. +*** Test Cases *** +Test withdraw 2 + Open Account user1 100 + Given user1 is logged in and has money + When user1 withdraws 20 + Then it works +``` + +the review reports: + +```text +L1 — tests/withdrawal.robot:12 — library keyword Open Account in a test body. Why: a business reader can't tell what is specified. Rewrite: Given Alice has a balance of 100 EUR +W1 — tests/withdrawal.robot:13 — conjunction step "is logged in and has money". Why: neither fact can be reused. Rewrite: Given Alice is signed in / And Alice has a balance of 100 EUR +W2 — tests/withdrawal.robot:15 — vague outcome "it works". Why: the test passes as long as nothing crashes. Rewrite: Then the ATM dispenses 20 EUR / And Alice's balance is 80 EUR +W3 — tests/withdrawal.robot:13 — placeholder data user1. Why: it says nothing about the example. Rewrite: Alice, with amounts in EUR +W12 — tests/withdrawal.robot:11 — procedural test name "Test withdraw 2". Why: the report doesn't say what behaviour failed. Rewrite: Account holder withdraws cash within their balance +``` + +End with a short summary: the count per severity, the three changes with the biggest effect, and the tool results (analysis errors, Robocop findings) before and, if fixes were applied, after. diff --git a/plugins/rf-core/skills/rf-bdd/references/robocop.md b/plugins/rf-core/skills/rf-bdd/references/robocop.md new file mode 100644 index 0000000..738c7e3 --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/references/robocop.md @@ -0,0 +1,62 @@ +# Robocop Settings for BDD Steps + +- [What each setting does](#what-each-setting-does) +- [Rules that stay on](#rules-that-stay-on) +- [Where the settings go](#where-the-settings-go) +- [Merging into an existing configuration](#merging-into-an-existing-configuration) +- [Running Robocop](#running-robocop) + +Robocop's defaults expect Title Case keyword names, so sentence-style steps get a naming finding on almost every line. `assets/robocop-bdd.toml` changes only what sentence-style steps need; all other rules stay on. Add it only when Step 0 chose sentence style. A project with Title Case steps keeps its configuration as it is. The settings were checked with Robocop 8.2.10 and 9.4.0. + +## What each setting does + +| Setting | Without it | With it | Versions | +|---|---|---|---| +| `wrong-case-in-keyword-name.convention=first_word_capitalized` (NAME02) | every step definition is flagged | a definition starts with a capital or an embedded argument; Title Case keywords still pass; a lower-case definition (`the ATM is broken`) is still flagged | 8.x and 9.x | +| `wrong-case-in-keyword-name.pattern='s` (NAME02) | `${holder}'s balance is …` is flagged, because `'s` is read as the first word | possessive steps pass | 8.x and 9.x | +| `wrong-case-in-keyword-call.convention=first_word_capitalized` (NAME18) | every call such as `Then the ATM dispenses 20 EUR` is flagged | the prefix provides the capital | 8.x and 9.x | +| `per-file-ignores = { "*_steps.resource" = ["VAR06"] }` | `VAR … scope=TEST` is flagged in steps resources | allowed there and only there | 8.x and 9.x | +| format: `extend-select = ["AlignTemplatedTestCases"]` | `robocop format` squeezes outline tables to single separators | outline rows align under the column labels | 8.x and 9.x | +| format: `AlignTemplatedTestCases.args_with_test=keep` (not in the asset) | 9.x moves each example's values to a line of their own | values stay on the line of the test name | 9.x only | +| format: `NormalizeSeparators.skip_keyword_call_pattern=^(Given\|When\|Then\|And\|But)\s` | `robocop format` collapses hand-aligned columns after a step | calls with a BDD prefix keep their alignment | 8.x and 9.x | + +- `args_with_test=keep` is left out of the asset, because Robocop 8.x doesn't know it: with that line, `robocop format` stops with `InvalidParameterError: AlignTemplatedTestCases: Failed to import`. 8.x keeps the values on the test line anyway. On 9.x, add the line to the format `configure` list. `robocop check` isn't affected. +- `per-file-ignores` needs steps resources named `_steps.resource`. For a steps resource with another name, put `# robocop: off=VAR06` on the first line of that file (works on 8.x and 9.x), or add its pattern to the table. + +## Rules that stay on + +| Rule | Why it helps a BDD suite | +|---|---| +| DOC01, DOC02, DOC03 (missing keyword, test and suite documentation) | step documentation is the glossary, test documentation holds the rule, suite documentation holds the story | +| TAG05 (all tests share a tag) | a rule tag that every test shares belongs in `Test Tags` | +| VAR01 (empty variable value) | declare state with `${None}`, `@{EMPTY}` or `&{EMPTY}` | +| ARG01 (unused argument) | a step that ignores its embedded argument is a sentence that lies | +| DEPR05 (`Set Variable` instead of `VAR`) | state uses `VAR` | +| MISC04 (inconsistent assignment sign) | on 9.x, `VAR` counts as "no `=`", so write assignments without `=`: `${balance} Get Balance ${holder}` | + +## Where the settings go + +- Robocop reads `robocop.toml`, `robot.toml` and `pyproject.toml`. In `robot.toml` and `pyproject.toml` the sections are `[tool.robocop.lint]` and `[tool.robocop.format]`; in `robocop.toml` they are `[lint]` and `[format]`. +- Prefer the file that exists. Most robotcode projects have a `robot.toml`, which accepts the `[tool.robocop.*]` sections next to robotcode's own settings. +- The nearest configuration file wins, and Robocop doesn't merge files. A `robot.toml`, `pyproject.toml` or `robocop.toml` in a subfolder that has its own Robocop section hides the root settings for every file below it. A file without a Robocop section is skipped. Put the BDD settings into the file the project's suites actually use, and keep one Robocop configuration per project. + +## Merging into an existing configuration + +Never overwrite a project's Robocop configuration. Propose the merge and explain it: +1. `configure`: append the three NAME02 and NAME18 lines. If the project already sets a `convention` for these rules, say that `first_word_capitalized` still accepts its Title Case keywords and only adds sentence-style steps. +2. `per-file-ignores`: add the `"*_steps.resource" = ["VAR06"]` entry to the existing table. +3. `select`: if the project enables only some rules, add the NAME02 and NAME18 lines only when those rules are in the list. +4. `extends`: if the project extends a shared configuration, change the project file, not the shared one. +5. Format: add the `[tool.robocop.format]` lines only when the project runs `robocop format`, and `args_with_test=keep` only on Robocop 9.x. + +## Running Robocop + +```bash +robocop check --no-cache tests resources # exit code 0: no findings +robocop check --no-cache --select NAME02 --select NAME18 tests # only the naming rules +robocop format --no-cache --check --diff tests # what the formatter would change +``` + +Repeat `--select` once per rule. A comma-separated value (`--select NAME02,NAME18`) selects nothing: Robocop 8.2.10 and 9.4.0 print "did not match with any rule name or id", then "No issues found.", and exit 0. + +Run Robocop from the project's environment; `robotcode[all]` installs it. diff --git a/plugins/rf-core/skills/rf-bdd/references/state.md b/plugins/rf-core/skills/rf-bdd/references/state.md new file mode 100644 index 0000000..78b313f --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/references/state.md @@ -0,0 +1,87 @@ +# Scenario State + +- [Choose the strategy](#choose-the-strategy) +- [Resource strategy](#resource-strategy) +- [The leak: never mutate a declared container](#the-leak-never-mutate-a-declared-container) +- [World-library strategy](#world-library-strategy) +- [What not to do](#what-not-to-do) + +A `When` step produces a result that a later `Then` step checks, so steps share state. Robot Framework has two clean places for it: variables declared in the steps resource, or a Python World library with test scope. Code blocks run in a copy of `assets/examples/resource-strategy/`. + +## Choose the strategy + +1. The project has step keywords in a Python library (`@keyword("…${arg}…")`): use the World-library strategy for that domain. +2. The project has `*_steps.resource` files: use the resource strategy. +3. A new project, or a new domain with no convention: use the resource strategy. +4. Don't mix the two strategies within one domain. + +## Resource strategy + +The steps resource declares every state variable in `*** Variables ***` with a neutral default, and steps assign it with `VAR … scope=TEST` (Robot Framework 7.0+). `assets/examples/resource-strategy/resources/cash_withdrawal_steps.resource` declares `${LAST_WITHDRAWAL} ${None}`; `${holder} withdraws ${amount: int} EUR` assigns it, and the `Then` steps read it. + +- Declare every state variable, with `${None}`, `@{EMPTY}` or `&{EMPTY}` as the default. Without the declaration, `robotcode analyze code` reports `VariableNotFound` on every step that reads it. Robocop VAR01 rejects an empty declaration. +- `scope=TEST` keeps the value for one scenario. The next test starts from the declared default. +- Upper-case names mark shared state; local variables stay lower case. +- The Robocop asset turns off VAR06 for `*_steps.resource` files only, because test-scope state is the design there. +- Often the best state is the system under test itself: `${holder}'s balance is …` asks the ATM. Keep variables for what the system doesn't keep for you, such as the result of the last action. + +## The leak: never mutate a declared container + +A declared list or dictionary is one object, shared by every test in the suite. Changing it in place with `Set To Dictionary`, `Append To List` or `Remove From Dictionary` changes the default, and the next test sees the leftovers. + +```robotframework +# Wrong: the next test in the suite still sees these notes. +*** Keywords *** +The ATM holds ${count: int} notes of ${value: int} EUR + [Documentation] Leaks into the next test. + Set To Dictionary ${NOTES} ${value} ${count} +``` + +Build a new value and assign it with `VAR … scope=TEST`: + + +```robotframework +*** Settings *** +Documentation Step definitions for loading notes into the ATM. + + +*** Variables *** +&{NOTES} &{EMPTY} + + +*** Keywords *** +The ATM holds ${count: int} notes of ${value: int} EUR + [Documentation] Adds ``count`` notes of ``value`` euros to the ATM's cassettes. + VAR &{NOTES} &{NOTES} ${value}=${count} scope=TEST + +The notes in the ATM add up to ${total: int} EUR + [Documentation] The value of all loaded notes is exactly ``total`` euros. + ${sum} Evaluate sum(int(value) * count for value, count in $NOTES.items()) + Should Be Equal ${sum} ${total} + ... msg=The notes add up to ${sum} EUR, expected ${total} EUR values=False +``` + +- `VAR &{NOTES} &{NOTES} ${value}=${count} scope=TEST` creates a new dictionary from the current one plus the new entry, so the declared default stays empty. Two tests in a row show it: the first loads 10 notes of 20 EUR and 4 of 50 EUR (400 EUR); the second loads 2 notes of 100 EUR and expects 200 EUR. With `Set To Dictionary` the second test sees 600 EUR; with `VAR` it sees 200 EUR. +- For a list, write `VAR @{ITEMS} @{ITEMS} ${new} scope=TEST`. + +## World-library strategy + +A Python library with `@library(scope="TEST")` keeps the state on the instance and implements the steps as `@keyword("…")` sentences. Robot Framework creates a new instance for every test, so nothing is declared and nothing leaks. The runnable project is `assets/examples/world-strategy/`, with the library in `lib/CashWithdrawalWorld.py`. + +- The `@keyword` name is the sentence; the method name isn't visible to readers. +- Type hints (`amount: int`) convert embedded arguments, like `${amount: int}` in a resource. +- Docstrings become the step documentation that `robotcode doc keywords` shows. +- Fail with `AssertionError` and a domain message; don't return `False`. +- Choose it when steps need real logic (parsing, calculation, retries), when the team prefers Python, or when the project already has one. The test files read the same under both strategies. +- Writing and checking the library itself: see the `rf-python-library` skill. + +## What not to do + +| Pattern | Problem | +|---|---| +| `VAR … scope=TEST` without a declaration | `VariableNotFound` from robotcode on every step that reads it | +| `Set Test Variable`, `Set Suite Variable`, `Set Global Variable` | the same analysis errors; suite and global scope leak between tests | +| `Set To Dictionary` or `Append To List` on a declared container | leaks into the next test | +| business state in `Suite Setup` | tests depend on run order | +| a `Then` that reads what a previous test created | tests are no longer independent | +| a value returned from a step into the test body (`${x} When …`) | test bodies hold only sentences; keep the value in scenario state | diff --git a/plugins/rf-core/skills/rf-bdd/references/step-design.md b/plugins/rf-core/skills/rf-bdd/references/step-design.md new file mode 100644 index 0000000..90fec8f --- /dev/null +++ b/plugins/rf-core/skills/rf-bdd/references/step-design.md @@ -0,0 +1,137 @@ +# Designing Step Keywords + +- [Sentence rules](#sentence-rules) +- [Naming](#naming) +- [Declarative, not procedural](#declarative-not-procedural) +- [Embedded arguments in steps](#embedded-arguments-in-steps) +- [Catch-all and ambiguous steps](#catch-all-and-ambiguous-steps) +- [Reusing the vocabulary](#reusing-the-vocabulary) +- [What goes inside a step](#what-goes-inside-a-step) + +A step keyword is one sentence of the domain language, and test cases are made of these sentences. This reference covers how to word, name and parameterise them, keep them unambiguous and reuse them. Code blocks run in a copy of `assets/examples/resource-strategy/`. + +## Sentence rules + +| Rule | Good | Bad | +|---|---|---| +| third person, present tense | `Alice withdraws 20 EUR` | `I withdrew 20 EUR`, `Withdraw 20 EUR` | +| subject and predicate | `The ATM dispenses 20 EUR` | `Dispensing of 20 EUR` | +| one idea per step | `Given Alice has a balance of 100 EUR` and `And Alice has already withdrawn 50 EUR today` | `Given Alice has 100 EUR and has withdrawn 50 EUR` | +| an observable outcome in `Then` | `Then the withdrawal is refused with "Insufficient funds"` | `Then it fails`, `Then the result is correct` | +| concrete, realistic data | `Alice`, `100 EUR`, `2026-08-31` | `user1`, `foo`, `123` | +| strings in double quotes | `is refused with "Insufficient funds"` | `is refused with Insufficient funds` | +| one word per concept | always "account holder", always "withdraws" | "customer" here, "user" there | +| grammatical for every value | `Alice has borrowed 3 books`, plus `Alice has borrowed one book` | `Alice has borrowed 1 books`, `${n} book(s)` | + +Write the sentence first and read it aloud inside its scenario. If it only makes sense to someone who knows the implementation, reword it. + +## Naming + +- Step 0 decides the casing. Follow the casing of the project's existing step keywords. A Title Case project (`The Cart Contains ${count} Items`) gets Title Case steps, and its Robocop configuration stays as it is. +- With no existing steps, the default is sentence style: only the first word capitalized, apart from names, acronyms and units (`The ATM dispenses ${amount: int} EUR`). A definition may start with an embedded argument (`${holder} has a balance of ${amount: int} EUR`); a BDD prefix before it needs Robot Framework 7.1. Merge `assets/robocop-bdd.toml` into the project's Robocop configuration so Robocop accepts this style. +- Calls put the prefix first and continue in the step's casing: `Then the ATM dispenses 20 EUR`. Name matching is case-insensitive. +- Never put `Given`, `When`, `Then`, `And` or `But` in a keyword name. A keyword defined as `Given the app is open` can't be called as `When the app is open`. +- Technical keywords below the steps keep Title Case (`Request Withdrawal`), so the layer shows in the name. +- Name step resources `_steps.resource`. The Robocop asset relies on this name. +- Every step keyword has `[Documentation]` (or a docstring in a Python library) that says what it establishes, does or checks. `robotcode doc keywords` shows the first line, so the steps resource becomes the domain glossary. + +## Declarative, not procedural + +Steps say what happens in the domain, not how a user operates the screen. The sentence should make sense without a computer. + +```robotframework +# Wrong: a UI script written as steps. +*** Test Cases *** +Alice withdraws cash + Given Alice opens the login page + When Alice types "alice@example.com" into the e-mail field + And Alice clicks "Withdraw" + And Alice types "20" into "#amount" + Then the element "div.result" contains "Please take your cash" +``` + +The declarative form is the test `Account holder withdraws cash within their balance` in `assets/examples/resource-strategy/tests/cash_withdrawal/withdrawal_rules.robot`: `Given Alice has a balance of 100 EUR`, `When Alice withdraws 20 EUR`, `Then the ATM dispenses 20 EUR`. + +- The clicks and fields still exist, inside the `Alice withdraws 20 EUR` step or the technical keywords it calls. +- Navigation belongs in a test only when navigation is the behaviour under test. +- A `Given` sets the starting state as directly as the system allows (an API call, a fixture) and states it as a fact: `Given Alice is signed in as an "Editor"`. + +## Embedded arguments in steps + +| Form | Use it for | Example | +|---|---|---| +| `"${name}"` | any string, including several words | `is refused with "${reason}"` | +| `${name: int}` | numbers, dates and other typed values (RF 7.3+) | `${amount: int} EUR` | +| `${name:regex}` | restricting what may match | `card "${number:\d{4} \d{4}}"` | +| `${name:a\|b}` | a closed set of words | `is ${outcome:dispensed\|refused}` | +| embedded plus `[Arguments]` | tables, lists and text after a sentence | `The receipt reads` with `@{lines}` | + +- Before Robot Framework 7.3, write `${amount:\d+}` and convert in the body: `${amount} Convert To Integer ${amount}`. +- Quote every string value, or restrict it with a pattern. Steps often put two values next to each other, and an unrestricted value takes the shortest match. +- An embedded argument can't be optional, and literal text in the name is matched exactly: `day(s)` matches only the characters `day(s)`. Write two sentences instead. +- Matching rules, custom patterns and conflicts between keywords: see the `rf-language` skill. + +## Catch-all and ambiguous steps + +When several step keywords match a call, Robot Framework picks the more specific one without telling you, and a dry run passes either way. The risk is a step that quietly matches the wrong sentence: + +```robotframework +# Wrong: a trailing catch-all. "is payed" and "is anything" both match. +*** Keywords *** +The order "${id}" is ${state} + [Documentation] Catch-all state check. + Log ${id} ${state} +``` + +A typo such as `Then the order "42" is payed` binds to it and fails with a confusing message, or passes. Restrict the trailing value instead, so robotcode reports `KeywordNotFound` for the typo before anything runs: + +A test then reads `Then the withdrawal is dispensed` or `Then the withdrawal is refused with "Insufficient funds"`, and both sentences resolve to one keyword each: + + +```robotframework +*** Settings *** +Documentation Step definitions for withdrawal outcomes. +Resource cash_withdrawal_steps.resource + + +*** Keywords *** +The withdrawal is ${outcome:dispensed|refused} + [Documentation] The last withdrawal ended with ``outcome``. + Withdrawal Outcome Should Be ${outcome} + +Withdrawal Outcome Should Be + [Documentation] Technical check shared by the outcome steps. + [Tags] robot:private + [Arguments] ${expected} + Should Be Equal ${LAST_WITHDRAWAL}[status] ${expected} + ... msg=Expected the withdrawal to be ${expected}, but it was ${LAST_WITHDRAWAL}[status] values=False +``` + +- No trailing catch-alls. A value at the end of a sentence matches anything; give it a type, an alternation or a pattern. +- Check a new sentence against the existing ones. `The withdrawal is ${outcome:dispensed|refused}` can't match `the withdrawal is refused with "…"`, because the whole name has to match. +- A stricter pattern alone doesn't make one keyword win over another; robotcode reports `MultipleKeywords`. Rename one of them. +- Only a real run shows which keyword and which values a step bound to. + +## Reusing the vocabulary + +Before writing a step: +1. List the existing sentences: `robotcode doc keywords resources/cash_withdrawal_steps.resource`, or add a filter such as `"balance"`. +2. If one has the right meaning, use it as written, even if you would word it differently. +3. If one is close but needs another value, generalise it with a typed argument instead of adding a near-copy. +4. Only then add a sentence to the project's existing steps resource or library for that domain. Create a new `_steps.resource` only for a new domain. + +Signs of drift: `The user is logged in` next to `The user has signed in`; `${holder} has ${amount} EUR` next to `${holder} has a balance of ${amount} EUR`; one concept called "customer", "client" and "account holder". + +## What goes inside a step + +- Steps call technical keywords: library keywords, the project's API client, or Title Case helpers. Tag helpers in a steps resource `[Tags] robot:private`; robotcode then warns (`PrivateKeyword`) when a test calls them. +- Steps don't call other steps with prefixes. Share logic through a technical keyword. +- A `Then` step asserts with a message in domain words, such as `msg=Expected the withdrawal to be dispensed, but it was refused`. +- A `Given` that can fail checks its own precondition (`Precondition failed: the earlier withdrawal of 200 EUR was refused`), so a broken setup doesn't surface later as a confusing assertion. +- Several lines of mechanics belong in a technical keyword. + +| Phase | Does | Avoid | +|---|---|---| +| `Given` | sets the starting state, as a fact | UI navigation; assertions on the system | +| `When` | one domain action; keeps its result as scenario state | several actions; assertions | +| `Then` | reads an observable outcome and asserts with a domain message | internal columns, private fields, log lines | diff --git a/plugins/rf-core/skills/rf-language/SKILL.md b/plugins/rf-core/skills/rf-language/SKILL.md index c8eec2c..c6063bc 100644 --- a/plugins/rf-core/skills/rf-language/SKILL.md +++ b/plugins/rf-core/skills/rf-language/SKILL.md @@ -23,7 +23,7 @@ Load this skill first, before opening the .robot and .resource files or answerin - selecting tests by tag on the robot command line; - `Multiple keywords with name '...'` and import scope between `__init__.robot` and child suites. -Not this skill: Python keyword libraries or listeners use `rf-python-library`; library keywords use `rf-browser` (plugin `rf-web`), `rf-requests` (plugin `rf-api`) or another library skill. Running, discovering, keyword lookup, result reading and debugging with the robotcode CLI are covered by the `robotcode` skill of the robotcode plugin. +Not this skill: Python keyword libraries or listeners use `rf-python-library`; library keywords use `rf-browser` (plugin `rf-web`), `rf-requests` (plugin `rf-api`) or another library skill. Writing or reviewing BDD specifications (features and rules, step vocabularies, pasted Gherkin) uses `rf-bdd` together with this skill. Running, discovering, keyword lookup, result reading and debugging with the robotcode CLI are covered by the `robotcode` skill of the robotcode plugin. ## Version gate @@ -247,7 +247,7 @@ The full table with minimum versions and robocop rule IDs is `references/migrati - wrong embedded-argument binding, including BDD steps matching an unintended keyword; - union-with-`str` conversion surprises (`${v: int | str}` keeps `"10"` a string), and typed arguments on RF < 7.3. These need a real run of at least one affected test before you report success. -5. Lint with `--target-version` set to the project's RF major version: `uv run robocop check --no-cache --target-version 7 -s "DEPR*" -s "ERR*" -s ORD02 -s "NAME*" -s VAR07 -s ARG03 tests resources` (repeat `-s`; a comma list is rejected), plus `-s KW06` for keyword name conflicts. Fix what it reports in the code you wrote. +5. Lint with `--target-version` set to the project's RF major version: `uv run robocop check --no-cache --target-version 7 -s "DEPR*" -s "ERR*" -s ORD02 -s "NAME*" -s VAR07 -s ARG03 tests resources` (repeat `-s`; a comma list selects nothing), plus `-s KW06` for keyword name conflicts. Fix what it reports in the code you wrote. 6. Check the changed files statically: `uv run robotcode analyze code `, and fix every error. 7. Run the affected tests for real by longname, `uv run robotcode robot -bl ""` (longnames from `robotcode discover tests`), or a whole directory, so that `__init__.robot` files apply. Don't pass a single `.robot` file. 8. Read the results: `uv run robotcode results show --failed`, then `uv run robotcode results log -bl ""`. @@ -277,7 +277,7 @@ If robotcode or Robocop is missing from the project environment, report steps 5 - An inline IF with no matching branch assigns `None`: `${v}= IF $n > 10 Set Variable big` leaves `${v}` as `None`; add an `ELSE` branch. - `Catenate` and `Set Variable If` are not flagged by robocop but have modern forms: `VAR ${s} a b separator=-` and an inline `IF`/`ELSE`. - A new file without a section header is not a suite ("contains no tests"); every new `.robot` or `.resource` file starts with `*** Settings ***`, `*** Test Cases ***` or `*** Keywords ***`. -- Robocop NAME18 flags keyword calls that are not Title Case, BDD steps included; write `Given The Calculator Is Cleared`, not `Given the calculator is cleared`. +- Robocop NAME02 and NAME18 check the casing of keyword names and calls, BDD steps included. Follow the casing the project's Robocop configuration and existing keywords use; with Robocop's defaults that is Title Case (`Given The Calculator Is Cleared`). For BDD specifications, `rf-bdd` sets the casing. ## When to read the references @@ -308,5 +308,6 @@ Examples in `assets/examples/` (run `robot --dryrun tests` from that folder): | API tests with RequestsLibrary | `rf-requests` (plugin `rf-api`) | | Install Robot Framework or a library, fix the environment | `rf-setup` | | Write or fix a Python keyword library or listener | `rf-python-library` | +| Write or review BDD (Given/When/Then) specifications and their step keywords | `rf-bdd` | For running, discovering, keyword documentation, result analysis and debugging with the robotcode CLI, use the `robotcode` skill of the robotcode plugin (`robotcode@robotframework-agent-plugins`); the commands above work without it. diff --git a/plugins/rf-core/skills/rf-language/references/embedded-arguments.md b/plugins/rf-core/skills/rf-language/references/embedded-arguments.md index d6f20f5..77d49a3 100644 --- a/plugins/rf-core/skills/rf-language/references/embedded-arguments.md +++ b/plugins/rf-core/skills/rf-language/references/embedded-arguments.md @@ -109,7 +109,7 @@ Pairs such as `${type} Framework` and `Robot ${action}` both match `Robot Framew ## BDD step definitions -Steps are called with `Given`, `When`, `Then`, `And` or `But`; Robot Framework removes the prefix and looks up the rest. Define step keywords without the prefix: +Steps are called with `Given`, `When`, `Then`, `And` or `But`; Robot Framework removes the prefix and looks up the rest. Define step keywords without the prefix (designing a step vocabulary for a specification: the `rf-bdd` skill): ```robotframework *** Test Cases *** diff --git a/plugins/rf-core/skills/rf-language/references/styles.md b/plugins/rf-core/skills/rf-language/references/styles.md index c5b756b..836f355 100644 --- a/plugins/rf-core/skills/rf-language/references/styles.md +++ b/plugins/rf-core/skills/rf-language/references/styles.md @@ -51,6 +51,8 @@ Prefer one test case per row: the report then shows which data row failed. Use m Scenarios that stakeholders read. Each step starts with `Given`, `When`, `Then`, `And` or `But`; Robot Framework drops the prefix before it looks the keyword up. Step keywords are therefore defined without a prefix, usually with embedded arguments (see the embedded-arguments reference for step definitions). +Writing or reviewing a BDD specification (features, rules, step vocabularies, scenario state, pasted Gherkin) is covered by the `rf-bdd` skill. + ```robotframework *** Settings *** Resource ../resources/calc.resource diff --git a/plugins/rf-core/tests/test_bdd_skill.py b/plugins/rf-core/tests/test_bdd_skill.py new file mode 100644 index 0000000..b830e01 --- /dev/null +++ b/plugins/rf-core/tests/test_bdd_skill.py @@ -0,0 +1,466 @@ +"""Structure, content and example checks for the rf-bdd skill (add-rf-bdd-skill, tasks 3.1-3.3). + +Layers: + +1. Structure (always): frontmatter, H2 order, size budget, the reference set and its + table, contents lists, no reference-to-reference links, no path into another + skill, version gate, Step 0 defaults, 11 house rules, gotchas, modes and the + companion rows. +2. Robot Framework (skipped without ``robot``): every ``robotframework`` code block + in SKILL.md and the references dry-runs against a copy of ``assets/examples``; + both example projects dry-run and run for real. +3. Robocop (the installed robocop, and Robocop 9.x through + ``uvx --from "robotframework-robocop>=9.1,<10"`` when available): the examples + are clean with ``assets/robocop-bdd.toml``, and every rule ID cited in + ``references/robocop.md`` exists. + +A code block whose first line starts with ``# Wrong`` is a counter-example and is not +run. An HTML comment ```` right above a block says where the block +goes in the example tree: a path starting with ``resource-strategy/``, +``world-strategy/`` or ``sut/`` is relative to ``assets/examples``, any other path is +relative to ``assets/examples/resource-strategy``. +""" + +from __future__ import annotations + +import functools +import importlib.util +import re +import shutil +import subprocess +import sys +import warnings +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[3] +SKILL = ROOT / "plugins" / "rf-core" / "skills" / "rf-bdd" +SKILL_MD = SKILL / "SKILL.md" +REFS = SKILL / "references" +EXAMPLES = SKILL / "assets" / "examples" +ROBOCOP_ASSET = SKILL / "assets" / "robocop-bdd.toml" + +HAS_ROBOT = importlib.util.find_spec("robot") is not None +needs_robot = pytest.mark.skipif(not HAS_ROBOT, reason="robotframework not installed") + +REQUIRED_SECTIONS = [ + "When to use", + "Version gate", + "Step 0: Match the project", + "House rules", + "Write a specification", + "Review a specification", + "Gherkin input", + "Gotchas", + "When to read the references", + "Companion Skills", +] +REFERENCES = { + "gherkin-mapping.md", + "step-design.md", + "state.md", + "data-driven.md", + "documentation-and-tags.md", + "robocop.md", + "review-checklist.md", +} +HARD_LIMIT, TARGET = 500, 300 +TOC_WINDOW, TOC_THRESHOLD = 15, 100 +# Version-gate rows: text that must appear in a row -> minimum RF version. +GATE = {"scope=TEST": "7.0", "BDD prefix": "7.1", "${amount: int}": "7.3", "[Metadata]": "7.5"} +HOUSE_RULE_TERMS = ("robot:private", "one `When`", "Test Template", "`IF`", "Metadata", "Robocop") +GOTCHA_TERMS = ("dry run", "`IF`", "scope=TEST", "__init__.robot", "--select", "DataDriver") +# Projects in assets/examples -> (python-path entries, number of tests). +PROJECTS = {"resource-strategy": (["resources", "../sut"], 8), "world-strategy": (["lib", "../sut"], 2)} +FILE_MARKER_RE = re.compile(r"\s*\n```(?:robotframework|robot)\n(.*?)^```", re.M | re.S) + + +def _text(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def _body(text: str) -> str: + return text.split("\n---\n", 1)[1] if text.startswith("---") else text + + +def _frontmatter(text: str) -> dict: + yaml = pytest.importorskip("yaml") + m = re.match(r"---\n(.*?)\n---\n", text, re.S) + assert m, "no frontmatter" + return yaml.safe_load(m.group(1)) + + +def _h2(text: str) -> list[str]: + out, fence = [], False + for line in text.splitlines(): + if line.lstrip().startswith("```"): + fence = not fence + elif not fence and line.startswith("## "): + out.append(line[3:].strip()) + return out + + +def _section(text: str, heading: str) -> str: + m = re.search(rf"^## {re.escape(heading)}[ \t]*\n(.*?)(?=^## |\Z)", text, re.M | re.S) + assert m, f"section {heading!r} missing" + return m.group(1) + + +def _rows(text: str) -> list[list[str]]: + rows = [] + for line in text.splitlines(): + s = line.strip() + if not s.startswith("|"): + continue + cells = [c.strip() for c in re.split(r"(? list[Path]: + return [SKILL_MD, *sorted(REFS.glob("*.md"))] + + +def _blocks(text: str) -> list[tuple[str | None, str]]: + """Every robotframework block with the file marker above it, if any.""" + marked = {m.start(2): m.group(1) for m in FILE_MARKER_RE.finditer(text)} + return [ + (marked.get(m.start(1)), m.group(1)) + for m in re.finditer(r"^```(?:robotframework|robot)\n(.*?)^```", text, re.M | re.S) + ] + + +def _is_counter_example(block: str) -> bool: + first = block.lstrip().splitlines()[0] if block.strip() else "" + return first.startswith("# Wrong") + + +def _list_items(section: str) -> list[str]: + return re.findall(r"^(?:- |\d+\. )(.+)$", section, re.M) + + +# --- 1. structure ---------------------------------------------------------------- + + +def test_frontmatter_and_layout() -> None: + fm = _frontmatter(_text(SKILL_MD)) + assert fm["name"] == "rf-bdd" == SKILL.name + assert fm["license"] == "Apache-2.0" + compat = fm["compatibility"] + for needle in ("Python 3.8", "robotframework>=7", "robotcode[all]>=2.8"): + assert needle in compat, needle + assert fm["metadata"].get("author") and "version" not in fm["metadata"] + assert "allowed-tools" not in fm + assert (SKILL / "references").is_dir() and (SKILL / "assets").is_dir() + assert not (SKILL / "scripts").exists() + assert not [p for p in SKILL.rglob("*") if p.is_symlink()] + + +def test_sections_ordered_and_budget() -> None: + text = _text(SKILL_MD) + heads = [h for h in _h2(_body(text)) if h in REQUIRED_SECTIONS] + assert heads == REQUIRED_SECTIONS + lines = len(text.splitlines()) + assert lines <= HARD_LIMIT, lines + if lines > TARGET: + warnings.warn(f"rf-bdd SKILL.md has {lines} lines (target {TARGET})", stacklevel=1) + + +def test_orientation_and_when_to_use() -> None: + body = _body(_text(SKILL_MD)) + head = body.split("## When to use", 1)[0].strip().splitlines() + assert len([line for line in head if line.strip()]) <= 5, head + assert re.fullmatch(r"Verified against Robot Framework \d+\.\d+(\.\d+)?\.?", head[-1].strip()), head[-1] + lines = body.lstrip("\n").splitlines() + idx = next(i for i, line in enumerate(lines) if line.strip() == "## When to use") + assert idx < 20, idx + block = _section(body, "When to use") + for needle in ("rf-language", "relies on `rf-language`", "`robotcode` skill of the robotcode plugin"): + assert needle in block, needle + + +def test_reference_set_matches_table() -> None: + assert {p.name for p in REFS.glob("*.md")} == REFERENCES + table = _section(_text(SKILL_MD), "When to read the references") + named = {m for cells in _rows(table) for m in re.findall(r"references/([a-z-]+\.md)", cells[0])} + assert named == REFERENCES + for cells in _rows(table)[1:]: + assert len(cells) >= 2 and cells[1], cells + + +def test_references_are_shallow() -> None: + for ref in sorted(REFS.glob("*.md")): + text = _text(ref) + assert not re.search(r"\]\((?!https?:)[^)]*\.md", text), f"{ref.name} links to another file" + assert "references/" not in text, f"{ref.name} names another reference by path" + lines = text.splitlines() + if len(lines) > TOC_THRESHOLD: + heads = [h for h in _h2(text)] + window = "\n".join(lines[:TOC_WINDOW]) + missing = [h for h in heads if h not in window] + assert not missing, f"{ref.name}: contents list lacks {missing}" + + +def test_no_path_into_another_skill() -> None: + for doc in [*_all_docs(), *EXAMPLES.rglob("*.*")]: + if doc.suffix not in {".md", ".robot", ".resource", ".py", ".toml"}: + continue + text = _text(doc) + assert not re.search(r"skills/rf-(?!bdd\b)|\.\./rf-[a-z-]+/", text), doc + assert "CLAUDE_SKILL_DIR" not in text or doc == SKILL_MD, doc + + +def test_version_gate() -> None: + gate = _section(_text(SKILL_MD), "Version gate") + rows = _rows(gate) + for feature, version in GATE.items(): + hit = [r for r in rows if feature in r[0]] + assert hit, feature + assert hit[0][1] == version, (feature, hit[0]) + assert r"${amount:\d+}" in gate and "Convert To Integer" in gate + + +def test_step_0_conventions_first() -> None: + step0 = _section(_text(SKILL_MD), "Step 0: Match the project") + for needle in ("Title Case", "python-path", "World library", "robotcode doc keywords", + "robotcode discover tags", "assets/robocop-bdd.toml", "first word capitalized", + "no convention", "rf-language"): + assert needle in step0, needle + + +def test_house_rules() -> None: + rules = _section(_text(SKILL_MD), "House rules") + numbered = re.findall(r"^(\d+)\. \*\*", rules, re.M) + assert numbered == [str(i) for i in range(1, 12)], numbered + for term in HOUSE_RULE_TERMS: + assert term in rules, term + assert rules.count("*Why:*") == 11 + + +def test_modes() -> None: + text = _text(SKILL_MD) + write = _section(text, "Write a specification") + for needle in ("robotcode analyze code", "--dryrun", 'robotcode robot -bl ""', "robocop check", + "Report"): + assert needle in write, needle + review = _section(text, "Review a specification") + assert "references/review-checklist.md" in review and "Don't edit files" in review + assert re.search(r"`ID — file:line — [^`]+`", review) + gherkin = _section(text, "Gherkin input") + for needle in (".feature", "parser", "every rule and every example", "1 days", "wording change"): + assert needle in gherkin, needle + + +def test_gotchas() -> None: + gotchas = _section(_text(SKILL_MD), "Gotchas") + items = _list_items(gotchas) + assert len(items) >= 11, len(items) + for term in GOTCHA_TERMS: + assert term in gotchas, term + + +def test_companion_skills() -> None: + text = _text(SKILL_MD) + assert _h2(_body(text))[-1] == "Companion Skills" + section = _section(text, "Companion Skills") + skills = {c for cells in _rows(section) for c in re.findall(r"`(rf-[a-z-]+)`", cells[-1])} + assert {"rf-language", "rf-setup", "rf-python-library"} <= skills + assert "`robotcode` skill of the robotcode plugin" in section + + +def test_documented_robocop_commands_disable_the_cache() -> None: + """A documented `robocop check` or `format` run must not leave .robocop_cache/ behind.""" + for doc in _all_docs(): + for line in _text(doc).splitlines(): + for cmd in re.findall(r"robocop (?:check|format)\b[^`\n]*", line): + if cmd.strip() in ("robocop check", "robocop format"): + continue # the bare tool name in prose, not a command to run + assert "--no-cache" in cmd, f"{doc.name}: {cmd}" + + +def test_examples_are_library_neutral() -> None: + forbidden = re.compile(r"^(Library|Resource)\s+(Browser|SeleniumLibrary|RequestsLibrary|REST|AppiumLibrary)\b", + re.M) + for path in EXAMPLES.rglob("*"): + if path.suffix in {".robot", ".resource"}: + assert not forbidden.search(_text(path)), path + for path in EXAMPLES.rglob("*.robot"): + assert "[Metadata]" not in _text(path), f"{path}: test-level [Metadata] needs RF 7.5" + + +# --- 2. Robot Framework layer --------------------------------------------------- + + +def _robot(args: list[str], cwd: Path) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-m", "robot", *args], capture_output=True, text=True, timeout=300, cwd=cwd, + ) + + +def _pythonpath(project: str) -> list[str]: + return [a for p in PROJECTS[project][0] for a in ("--pythonpath", p)] + + +@pytest.fixture() +def example_copy(tmp_path: Path) -> Path: + dest = tmp_path / "examples" + shutil.copytree(EXAMPLES, dest, ignore=shutil.ignore_patterns("__pycache__", ".robocop_cache", "results")) + return dest + + +def _place(marker: str | None, block: str, root: Path, idx: int) -> tuple[str, Path]: + """Write a block into the example copy; return (project, file or folder to dry-run).""" + if marker: + first = marker.split("/", 1)[0] + rel = marker if first in (*PROJECTS, "sut") else f"resource-strategy/{marker}" + project = rel.split("/", 1)[0] if rel.split("/", 1)[0] in PROJECTS else "resource-strategy" + path = root / rel + else: + project = "resource-strategy" + name = "resources" if "*** Test Cases ***" not in block and "*** Keywords ***" in block else "tests" + path = root / project / name / f"snippet_{idx}.{'resource' if name == 'resources' else 'robot'}" + if "*** Test Cases ***" not in block and "*** Keywords ***" not in block: + path = root / project / "tests" / f"snippet_{idx}" / "__init__.robot" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(block, encoding="utf-8") + if path.suffix == ".resource": + wrapper = root / project / "tests" / f"wrapper_{idx}.robot" + wrapper.write_text( + f"*** Settings ***\nResource {path.as_posix()}\n\n*** Test Cases ***\nImport only\n No Operation\n", + encoding="utf-8", + ) + return project, wrapper + if path.name == "__init__.robot": + (path.parent / "child.robot").write_text( + "*** Settings ***\nResource cash_withdrawal_steps.resource\n\n" + "*** Test Cases ***\nChild\n No Operation\n", + encoding="utf-8", + ) + return project, path.parent + return project, path + + +BLOCK_PARAMS = [ + pytest.param(marker, i, block, id=f"{doc.name}-{i}") + for doc in _all_docs() + for i, (marker, block) in enumerate(_blocks(_text(doc))) + if not _is_counter_example(block) +] + + +@needs_robot +@pytest.mark.parametrize(("marker", "idx", "block"), BLOCK_PARAMS) +def test_code_block_dry_runs(marker: str | None, idx: int, block: str, example_copy: Path) -> None: + project, target = _place(marker, block, example_copy, idx) + cwd = example_copy / project + res = _robot(["--dryrun", *_pythonpath(project), "--outputdir", str(example_copy / "out"), str(target)], cwd) + log = res.stdout + res.stderr + assert res.returncode == 0 and "[ ERROR ]" not in log, log + + +@needs_robot +@pytest.mark.parametrize("project", sorted(PROJECTS)) +def test_examples_dry_run_and_run(project: str, example_copy: Path) -> None: + from robot.api import ExecutionResult + + cwd = example_copy / project + out = example_copy / "out" + for extra in (["--dryrun"], []): + res = _robot([*extra, *_pythonpath(project), "--outputdir", str(out), "tests"], cwd) + log = res.stdout + res.stderr + assert res.returncode == 0 and "[ ERROR ]" not in log and "[ WARN ]" not in log, log + stats = ExecutionResult(str(out / "output.xml")).statistics.total + assert stats.passed == PROJECTS[project][1] and stats.failed == 0, log + + +@needs_robot +@pytest.mark.parametrize("project", sorted(PROJECTS)) +def test_examples_pass_robotcode_analyze(project: str, example_copy: Path) -> None: + robotcode = shutil.which("robotcode", path=str(Path(sys.executable).parent)) or shutil.which("robotcode") + if not robotcode: + pytest.skip("robotcode not installed") + res = subprocess.run([robotcode, "analyze", "code"], capture_output=True, text=True, timeout=300, + cwd=example_copy / project) + assert res.returncode == 0 and re.search(r"Errors: 0, Warnings: 0", res.stdout), res.stdout + res.stderr + + +# --- 3. Robocop layer ----------------------------------------------------------- + + +@functools.lru_cache(maxsize=1) +def _robocop_cmds() -> tuple[tuple[str, ...], ...]: + """The installed robocop and Robocop 9.x through uvx, those that answer --version.""" + candidates: list[tuple[str, ...]] = [] + local = shutil.which("robocop", path=str(Path(sys.executable).parent)) or shutil.which("robocop") + if local: + candidates.append((local,)) + uvx = shutil.which("uvx") + if uvx: + candidates.append((uvx, "--from", "robotframework-robocop>=9.1,<10", "robocop")) + found, majors = [], set() + for cmd in candidates: + try: + res = subprocess.run([*cmd, "--version"], capture_output=True, text=True, timeout=300) + except (OSError, subprocess.TimeoutExpired): + continue + m = re.search(r"(\d+)\.\d+", res.stdout + res.stderr) + if res.returncode == 0 and m and m.group(1) not in majors: + majors.add(m.group(1)) + found.append(cmd) + return tuple(found) + + +def _robocop_param(index: int) -> tuple[str, ...]: + cmds = _robocop_cmds() + if index >= len(cmds): + pytest.skip("robocop not available in this version") + return cmds[index] + + +@pytest.mark.parametrize("index", [0, 1], ids=["installed", "robocop-9"]) +@pytest.mark.parametrize("project", sorted(PROJECTS)) +def test_examples_are_clean_with_the_asset(index: int, project: str, example_copy: Path) -> None: + cmd = _robocop_param(index) + cwd = example_copy / project + # Lint with the asset only, not the example's own robot.toml. + (cwd / "robot.toml").unlink() + shutil.copy(ROBOCOP_ASSET, cwd / "pyproject.toml") + res = subprocess.run([*cmd, "check", "--no-cache", "."], capture_output=True, text=True, timeout=300, cwd=cwd) + found = re.findall(r"\b(NAME02|NAME18|VAR06)\b", res.stdout) + assert not found, res.stdout + res.stderr + + +@pytest.mark.parametrize("index", [0, 1], ids=["installed", "robocop-9"]) +def test_asset_format_settings_load(index: int, example_copy: Path) -> None: + """`robocop format` accepts the asset: a setting one version doesn't know stops it.""" + cmd = _robocop_param(index) + cwd = example_copy / "resource-strategy" + (cwd / "robot.toml").unlink() + shutil.copy(ROBOCOP_ASSET, cwd / "pyproject.toml") + res = subprocess.run([*cmd, "format", "--no-cache", "--check", "--diff", "tests"], + capture_output=True, text=True, timeout=300, cwd=cwd) + log = res.stdout + res.stderr + assert "Error" not in log and "Failed to import" not in log, log + + +def test_example_configs_carry_the_asset_settings() -> None: + asset = _text(ROBOCOP_ASSET) + settings = re.findall(r'^\s*["\']([^"\']+=[^"\']+)["\'],?$', asset, re.M) + assert settings + for project in PROJECTS: + config = _text(EXAMPLES / project / "robot.toml") + for setting in settings: + assert setting in config, (project, setting) + + +@pytest.mark.parametrize("index", [0, 1], ids=["installed", "robocop-9"]) +def test_cited_rule_ids_exist(index: int) -> None: + cmd = _robocop_param(index) + res = subprocess.run([*cmd, "list", "rules"], capture_output=True, text=True, timeout=300) + known = set(re.findall(r"\b([A-Z]{2,6}\d{2})\b", res.stdout)) + assert known, res.stdout + res.stderr + cited = set(re.findall(r"\b([A-Z]{2,6}\d{2})\b", _text(REFS / "robocop.md") + _text(SKILL_MD))) + assert cited <= known, sorted(cited - known) diff --git a/plugins/rf-core/tests/test_hook_scripts.py b/plugins/rf-core/tests/test_hook_scripts.py index 4ee25b3..99bc109 100644 --- a/plugins/rf-core/tests/test_hook_scripts.py +++ b/plugins/rf-core/tests/test_hook_scripts.py @@ -223,7 +223,7 @@ def test_injection_budget_and_routing(prompt: str) -> None: assert hso["hookEventName"] == "UserPromptSubmit" ctx = hso["additionalContext"] assert len(ctx) <= INJECTION_BUDGET, len(ctx) - for name in ("rf-language", "rf-python-library", "robotcode doc keyword", "robotcode results"): + for name in ("rf-language", "rf-bdd", "rf-python-library", "robotcode doc keyword", "robotcode results"): assert name in ctx, name assert ctx.rstrip().endswith("Write RF 7 syntax: RETURN, VAR, IF, Test Tags.") # No skill / subagent lists any more: the text routes by task. @@ -283,6 +283,20 @@ def test_injected_context_names_python_library_skill(prompt: str) -> None: assert "Python libraries/listeners -> rf-python-library" in ctx +@pytest.mark.parametrize( + "prompt", + [ + "use rf-bdd to turn these acceptance criteria into tests", + "Rewrite tests/checkout.robot as Given/When/Then scenarios", + ], +) +def test_injected_context_routes_bdd_specs(prompt: str) -> None: + """add-rf-bdd-skill: the rf-bdd id triggers, and BDD specs route to rf-bdd.""" + out, _err, rc = _run(INJECT_SCRIPT, {"prompt": prompt}) + assert rc == 0 + assert "Given/When/Then specs, Gherkin -> rf-bdd" in _expect_injection(out)["additionalContext"] + + def test_inject_fires_on_language_skill_id() -> None: out, _err, rc = _run(INJECT_SCRIPT, {"prompt": "Load rf-language for this suite"}) assert rc == 0 @@ -339,6 +353,9 @@ def test_injected_context_states_skill_boundaries() -> None: "What does RF stand for in your domain?", "run the unit tests and fix the RF amplifier model", "refactor this React component", + # Gherkin on its own is Cucumber, SpecFlow or pytest-bdd, not Robot Framework. + "write Cucumber step definitions in Java for: Given a cart, When I check out", + "Scenario: Given a logged-in user When they open the cart Then it is empty", # Markdown bold / emphasis is not an RF section header. "*** Important *** read this first", ], diff --git a/plugins/rf-core/tests/test_language_skill.py b/plugins/rf-core/tests/test_language_skill.py index df20947..c46f33e 100644 --- a/plugins/rf-core/tests/test_language_skill.py +++ b/plugins/rf-core/tests/test_language_skill.py @@ -121,6 +121,7 @@ "ERR17": "unsupported-setting-in-init-file", "NAME07": "not-capitalized-test-case-title", "NAME08": "section-variable-not-uppercase", + "NAME02": "wrong-case-in-keyword-name", "NAME18": "wrong-case-in-keyword-call", "ORD02": "keyword-section-out-of-order", "VAR05": "no-suite-variable", @@ -533,7 +534,7 @@ def test_agent_workflow_order_and_blind_spots() -> None: assert "skipped" in tail and "rf-setup" in tail and "robotcode[all]" in tail assert "--no-cache" in lint, "robocop leaves .robocop_cache/ in the project without --no-cache" assert lint.count(" -s ") >= 4 and "DEPR" in lint and "ORD02" in lint and "NAME" in lint - assert not re.search(r"-s\s+\S*,\S*", lint), "comma-separated robocop selection is rejected by robocop 9" + assert not re.search(r"-s\s+\S*,\S*", lint), "a comma-separated robocop selection selects nothing" assert "robotcode analyze code" in wf and "robotcode robot -bl" in wf and "robotcode results show --failed" in wf assert 'robotcode doc keyword' in joined[1] for retired in ("rf-libdoc", "rf-results", "rf-robotcode"): diff --git a/plugins/rf-core/variants/codex/agents/rf-test-architect.toml b/plugins/rf-core/variants/codex/agents/rf-test-architect.toml index 88dc5f7..990dac3 100644 --- a/plugins/rf-core/variants/codex/agents/rf-test-architect.toml +++ b/plugins/rf-core/variants/codex/agents/rf-test-architect.toml @@ -56,6 +56,7 @@ python-path belong to `rf-setup`. | Work | Skill | |------|-------| | Suites, test cases, `__init__.robot`, tags, templates, user keywords, resource and variable files | `rf-language` | +| BDD specifications: Given/When/Then suites, step vocabularies, translating Gherkin or acceptance criteria | `rf-bdd` | | Project keyword libraries and listeners in Python (`libraries/*.py`), checked with its library checker | `rf-python-library` | | Installing RF and libraries, project layout mechanics, `robot.toml`, python-path | `rf-setup` | | Web tests with Browser / SeleniumLibrary | `rf-browser` / `rf-selenium` (plugin `rf-web`) | diff --git a/plugins/rf-core/variants/opencode/agents/rf-test-architect.md b/plugins/rf-core/variants/opencode/agents/rf-test-architect.md index edb8640..dbe0fc0 100644 --- a/plugins/rf-core/variants/opencode/agents/rf-test-architect.md +++ b/plugins/rf-core/variants/opencode/agents/rf-test-architect.md @@ -61,6 +61,7 @@ python-path belong to `rf-setup`. | Work | Skill | |------|-------| | Suites, test cases, `__init__.robot`, tags, templates, user keywords, resource and variable files | `rf-language` | +| BDD specifications: Given/When/Then suites, step vocabularies, translating Gherkin or acceptance criteria | `rf-bdd` | | Project keyword libraries and listeners in Python (`libraries/*.py`), checked with its library checker | `rf-python-library` | | Installing RF and libraries, project layout mechanics, `robot.toml`, python-path | `rf-setup` | | Web tests with Browser / SeleniumLibrary | `rf-browser` / `rf-selenium` (plugin `rf-web`) | diff --git a/scripts/robotcode-first-allowlist.toml b/scripts/robotcode-first-allowlist.toml index 7d5a9de..a082416 100644 --- a/scripts/robotcode-first-allowlist.toml +++ b/scripts/robotcode-first-allowlist.toml @@ -39,4 +39,14 @@ file = "scripts/desktop-live-check.py" pattern = "parser-model" reason = "Screens model-written suites for disallowed libraries and keywords before they run on a maintainer's desktop; a safety screen, not keyword resolution." +[[allow]] +file = "src/rf_skill_eval/scoring/custom/bdd.py" +pattern = "parser-model" +reason = "BDD structure checks on the parsed model: test bodies hold only Given/When/Then calls or template rows, and suites carry Documentation and feature Metadata; robotcode has no such check." + +[[allow]] +file = "src/rf_skill_eval/scoring/custom/bdd.py" +pattern = "execution-result" +reason = "Collects the executed test and keyword names from the grader's own run to catch a wording that only appears after embedded arguments are filled in (\"1 days\"); robotcode results has no name search." + # --- Replaced by robotcode-first-tooling (tasks 2-5); remove with the code --- diff --git a/src/rf_skill_eval/cli.py b/src/rf_skill_eval/cli.py index c4baf82..afd8353 100644 --- a/src/rf_skill_eval/cli.py +++ b/src/rf_skill_eval/cli.py @@ -1418,7 +1418,8 @@ def preflight( fixture_skills: dict[str, set[str]] = {} for _p, t in index.values(): if t.fixture: - fixture_skills.setdefault(t.fixture, set()).add(t.skill) + # A variant directory (``sut-bdd-library/review``) belongs to its top-level fixture. + fixture_skills.setdefault(t.fixture.split("/", 1)[0], set()).add(t.skill) result = map_changed_paths( paths, dir_to_skill={d: n for n, d in shipped_skills(root, plugin).items()}, diff --git a/src/rf_skill_eval/scoring/custom/bdd.py b/src/rf_skill_eval/scoring/custom/bdd.py new file mode 100644 index 0000000..4e7314f --- /dev/null +++ b/src/rf_skill_eval/scoring/custom/bdd.py @@ -0,0 +1,405 @@ +"""Custom grader checks for the rf-bdd eval tasks (add-rf-bdd-skill D10). + +Referenced from task YAML as ``custom_python`` with +``func_ref: rf_skill_eval.scoring.custom.bdd:``. Every function +takes ``(run, params)`` and returns a tri-state :class:`Verdict`; a check that +cannot run (Robot Framework or robotcode missing in the grader environment) is +``skipped`` with a reason, never passed. + +* :func:`structure` — test bodies hold only BDD-prefixed step calls, or + template rows in templated tests (AST via ``robot.api``). +* :func:`living_docs` — every suite with tests has suite documentation and + ``Metadata Feature`` or a ``feature:`` tag (own or from ``__init__.robot``). +* :func:`native_only` — no ``.feature`` file and no parser configured. +* :func:`step_names_absent` — a real run whose executed step names never + match a regex (the outline row that reads "1 days"). +* :func:`review_findings` — no file changed against the fixture, and the + final message has at least N findings lines ``ID — file:line — …`` naming + seeded check IDs. + +The workspace fixtures are ``eval/fixtures/sut-bdd-library/``: a +``base`` project and the ``steps`` / ``review`` variants, which are the base +plus the files of that variant (the harness copies one fixture directory, so +an "overlay" is a variant directory selected by the task's ``fixture``). +""" + +from __future__ import annotations + +import hashlib +import re +from pathlib import Path +from typing import Any + +from ...domain.run import Run +from ...domain.verdict import Verdict +from ...grader.robot_runner import RobotcodeMissingError, run_robot +from ..session_based import _resolve_transcript + +_REPO = Path(__file__).resolve().parents[4] +FIXTURES_DIR = _REPO / "eval" / "fixtures" + +#: Settings a BDD test may carry besides its steps. +_ALLOWED_SETTINGS = ("Documentation", "Tags", "Setup", "Teardown", "Template", "Timeout") +#: Directories never part of the project's own files (agent tooling, caches, run output). +_IGNORED_DIRS = {"results", "__pycache__", "node_modules", "grader-out"} +_IGNORED_FILES = {"output.xml", "log.html", "report.html"} + +#: A findings line: ``L1 — tests/loans.robot:8 — …`` (also ``- **L1** – file:8-9 - …``, +#: ``S2 · L1 — …``, ``W3/W12 — …``). The ID part is captured, the location must name +#: a file with an extension and a line number. +_FINDING = re.compile( + r"""^\s*(?:[-*+]|\d+[.)]|\#{1,6})?\s* # list marker or heading + (?:\*\*|`)?\s*(?:S\d\s*[·.:]\s*)? # optional bold/code and severity + (?P[A-Z]\d{1,2}(?:\s*[/,+&]\s*[A-Z]\d{1,2})*) + \s*(?:\*\*|`)?\s* + (?:—|–|-{1,2})\s* # separator + (?:\*\*|`)?(?P[\w./\\-]+\.\w+:\d+(?:\s*[-–]\s*\d+)?)(?:\*\*|`)? + \s*(?:—|–|-{1,2})\s*\S # separator and some text + """, + re.VERBOSE, +) +_ID = re.compile(r"[A-Z]\d{1,2}") + + +def _name(params: dict[str, Any], default: str) -> str: + return str(params.get("name") or default) + + +def _hidden(rel: Path) -> bool: + return any(part.startswith(".") or part in _IGNORED_DIRS for part in rel.parts[:-1]) + + +def _robot_files(ws: Path, params: dict[str, Any]) -> list[Path]: + root = ws / str(params.get("path", "tests")) + if root.is_file(): + return [root] + if not root.is_dir(): + return [] + return sorted(p for p in root.rglob("*.robot") if p.is_file() and not _hidden(p.relative_to(ws))) + + +def _bdd_prefixes() -> set[str]: + """BDD prefixes of every built-in language, lowercased.""" + from robot.conf.languages import Language + + prefixes: set[str] = set() + for cls in Language.__subclasses__(): + try: + prefixes |= {p.lower() for p in cls().bdd_prefixes} + except Exception: # a broken language class must not break grading + continue + return prefixes or {"given", "when", "then", "and", "but"} + + +def _settings(model: Any) -> dict[str, Any]: + from robot.api.parsing import Documentation, Metadata, TestTags + + found: dict[str, Any] = {"documentation": False, "feature_metadata": False, "test_tags": []} + for section in model.sections: + if type(section).__name__ != "SettingSection": + continue + for node in section.body: + if isinstance(node, Documentation): + found["documentation"] = bool(node.value.strip()) + elif isinstance(node, Metadata) and node.name.strip().lower() == "feature" and node.value.strip(): + found["feature_metadata"] = True + elif isinstance(node, TestTags): + found["test_tags"] += list(node.values) + return found + + +def _inherited(path: Path, ws: Path) -> dict[str, Any]: + """Settings from the ``__init__.robot`` files between the workspace and ``path``.""" + from robot.api import get_model + + merged: dict[str, Any] = {"documentation": False, "feature_metadata": False, "test_tags": []} + for parent in path.parents: + if parent != ws and ws not in parent.parents: + break + init = parent / "__init__.robot" + if init.is_file() and init != path: + s = _settings(get_model(init)) + merged["documentation"] |= s["documentation"] + merged["feature_metadata"] |= s["feature_metadata"] + merged["test_tags"] += s["test_tags"] + return merged + + +def _tests(model: Any) -> list[Any]: + return [t for s in model.sections if type(s).__name__ == "TestCaseSection" + for t in s.body if hasattr(t, "name") and hasattr(t, "body")] + + +def _parse(files: list[Path]) -> list[tuple[Path, Any]]: + from robot.api import get_model + + return [(p, get_model(p)) for p in files if p.name != "__init__.robot"] + + +def _robot_missing(run: Run, name: str) -> Verdict | None: + try: + import robot.api # noqa: F401 + except ImportError: + return Verdict.skipped(run.id, name, "robotframework not importable in the grader environment") + return None + + +def _non_step_lines(test: Any, suite_template: bool, prefixes: set[str]) -> list[tuple[int, str]]: + """``(lineno, label)`` of each body line that is neither a BDD step nor a template row.""" + from robot.api.parsing import KeywordCall, Template, TemplateArguments + + templated = suite_template + for node in test.body: + if isinstance(node, Template): + templated = node.value not in (None, "", "NONE") + found: list[tuple[int, str]] = [] + for node in test.body: + kind = type(node).__name__ + if kind in ("EmptyLine", "Comment") or kind in _ALLOWED_SETTINGS: + continue + if isinstance(node, TemplateArguments) and templated: + continue + if isinstance(node, KeywordCall) and not node.assign: + words = (node.keyword or "").strip().lower() + if any(words.startswith(f"{p} ") for p in prefixes): + continue + found.append((node.lineno, node.keyword if isinstance(node, KeywordCall) else kind)) + return found + + +def structure(run: Run, params: dict[str, Any]) -> Verdict: + """Test bodies hold only ``Given/When/Then/And/But …`` calls or template rows. + + Params: ``path`` (workspace-relative file or directory, default ``tests``). + A call that assigns a variable, a ``VAR``, ``IF``/``FOR`` block or a + library keyword in a test body fails it; settings (``[Documentation]``, + ``[Tags]``, ``[Setup]``, ``[Teardown]``, ``[Template]``, ``[Timeout]``), + comments and empty lines are allowed. Fails when no test is found. + """ + name = _name(params, "bdd_structure") + if (missing := _robot_missing(run, name)) is not None: + return missing + from robot.api.parsing import TestTemplate + + ws = run.effective_workspace + prefixes = _bdd_prefixes() + problems: list[str] = [] + seen = 0 + for path, model in _parse(_robot_files(ws, params)): + suite_template = any(isinstance(n, TestTemplate) and n.value not in (None, "", "NONE") + for s in model.sections if type(s).__name__ == "SettingSection" for n in s.body) + rel = path.relative_to(ws) + for test in _tests(model): + seen += 1 + problems += [f"{rel}:{line} '{test.name}': {label}" + for line, label in _non_step_lines(test, suite_template, prefixes)] + if not seen: + return Verdict.of(run.id, name, False, f"no test cases under {params.get('path', 'tests')}") + return Verdict.of(run.id, name, not problems, + f"{seen} tests, {len(problems)} non-step lines" + (f": {problems[:6]}" if problems else "")) + + +def living_docs(run: Run, params: dict[str, Any]) -> Verdict: + """Every suite file with tests has documentation and a feature marker. + + The marker is ``Metadata Feature …`` or a ``feature:`` tag (``Test + Tags`` of the suite, or ``[Tags]`` of every test); both may come from an + ``__init__.robot`` above the file. Params: ``path`` (default ``tests``). + """ + name = _name(params, "bdd_living_docs") + if (missing := _robot_missing(run, name)) is not None: + return missing + from robot.api.parsing import Tags + + ws = run.effective_workspace + gaps: list[str] = [] + suites = 0 + for path, model in _parse(_robot_files(ws, params)): + tests = _tests(model) + if not tests: + continue + suites += 1 + own, parent = _settings(model), _inherited(path, ws) + rel = path.relative_to(ws) + if not (own["documentation"] or parent["documentation"]): + gaps.append(f"{rel}: no suite Documentation") + suite_tags = [t.lower() for t in own["test_tags"] + parent["test_tags"]] + tagged = any(t.startswith("feature:") for t in suite_tags) or all( + any(str(v).lower().startswith("feature:") for n in t.body if isinstance(n, Tags) for v in n.values) + for t in tests + ) + if not (own["feature_metadata"] or parent["feature_metadata"] or tagged): + gaps.append(f"{rel}: no `Metadata Feature` and no feature: tag") + if not suites: + return Verdict.of(run.id, name, False, f"no suites with tests under {params.get('path', 'tests')}") + return Verdict.of(run.id, name, not gaps, f"{suites} suites, gaps: {gaps[:6]}" if gaps else f"{suites} suites") + + +_PARSER_CONFIG = re.compile(r"(?im)^\s*\[(tool\.robot\.)?parsers\]|--parser\b|^\s*parsers\s*=|GherkinParser") + + +def native_only(run: Run, params: dict[str, Any]) -> Verdict: + """No ``.feature`` file in the project and no parser configured (``GherkinParser``).""" + name = _name(params, "native_only") + ws = run.effective_workspace + problems = [f"{p.relative_to(ws)}: .feature file" for p in sorted(ws.rglob("*.feature")) + if p.is_file() and not _hidden(p.relative_to(ws))] + for rel in ("robot.toml", "pyproject.toml", ".robot.toml"): + path = ws / rel + if path.is_file() and _PARSER_CONFIG.search(path.read_text(encoding="utf-8", errors="replace")): + problems.append(f"{rel}: configures a parser") + for path in sorted([*ws.rglob("*.robot"), *ws.rglob("*.resource")]): + if not _hidden(path.relative_to(ws)) and "GherkinParser" in path.read_text(encoding="utf-8", errors="replace"): + problems.append(f"{path.relative_to(ws)}: GherkinParser") + return Verdict.of(run.id, name, not problems, f"non-native: {problems[:6]}" if problems else "native .robot only") + + +def _executed_names(output_xml: Path) -> list[str]: + from robot.api import ExecutionResult, ResultVisitor + + names: list[str] = [] + + class _Collect(ResultVisitor): # type: ignore[misc] + def visit_test(self, test: Any) -> None: + names.append(test.name) + super().visit_test(test) + + def start_keyword(self, kw: Any) -> None: + names.append(str(kw.name or "")) + + ExecutionResult(str(output_xml)).visit(_Collect()) + return names + + +def step_names_absent(run: Run, params: dict[str, Any]) -> Verdict: + """Run ``path`` and fail when an executed test or keyword name matches ``regex``. + + Catches outline sentences that read wrongly for a row at run time (an + embedded ``${days} days`` called with ``1``), which the source files do + not show. Params: ``path`` (default ``tests``), ``regex``. The run itself + must pass with at least one test. + """ + name = _name(params, "step_names_absent") + if (missing := _robot_missing(run, name)) is not None: + return missing + ws = run.effective_workspace + target = ws / str(params.get("path", "tests")) + if not target.exists(): + return Verdict.of(run.id, name, False, f"missing {params.get('path', 'tests')}") + pattern = re.compile(str(params["regex"])) + outdir = run.artifacts_dir / "grader_bdd" / re.sub(r"[^A-Za-z0-9_.-]+", "_", name)[:60] + try: + result = run_robot([target], outdir, cwd=ws, timeout_seconds=int(params.get("timeout_seconds", 120))) + except RobotcodeMissingError: + return Verdict.skipped(run.id, name, "robotcode not installed") + if result.timed_out or result.output_xml is None: + return Verdict.of(run.id, name, False, "robot run timed out or wrote no output.xml") + if result.returncode != 0 or result.total_tests < 1: + return Verdict.of(run.id, name, False, f"run failed: exit={result.returncode} tests={result.total_tests}") + hits = sorted({n for n in _executed_names(result.output_xml) if pattern.search(n)}) + return Verdict.of(run.id, name, not hits, + f"regex={pattern.pattern!r} " + (f"matched executed names {hits[:4]}" if hits else "absent")) + + +# --- review ------------------------------------------------------------------------ + + +def _project_files(root: Path) -> dict[str, str]: + files: dict[str, str] = {} + for path in sorted(root.rglob("*")): + rel = path.relative_to(root) + if not path.is_file() or _hidden(rel) or (rel.name.startswith(".") and rel.name != ".gitignore"): + continue + if path.name in _IGNORED_FILES or path.suffix == ".pyc": + continue + files[rel.as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + return files + + +def changed_files(fixture_dir: Path, workspace: Path) -> list[str]: + """Files added, removed or modified in ``workspace`` relative to ``fixture_dir``. + + Hidden directories (``.claude``, ``.robotcode_cache``), ``results/``, caches + and Robot Framework output files are ignored. + """ + want, got = _project_files(fixture_dir), _project_files(workspace) + return sorted( + [f"added {p}" for p in got.keys() - want.keys()] + + [f"removed {p}" for p in want.keys() - got.keys()] + + [f"modified {p}" for p in want.keys() & got.keys() if want[p] != got[p]] + ) + + +def final_message(run: Run) -> str | None: + """The agent's final answer: the stream's ``result`` text, else its last text block.""" + import json + + path = _resolve_transcript(run) + if path is None: + return None + result: str | None = None + last_text: str | None = None + with path.open(encoding="utf-8", errors="replace") as fh: + for raw in fh: + raw = raw.strip() + if not raw: + continue + try: + entry = json.loads(raw) + except json.JSONDecodeError: + continue + if not isinstance(entry, dict): + continue + if entry.get("type") == "result" and isinstance(entry.get("result"), str): + result = entry["result"] + message = entry.get("message") + if isinstance(message, dict) and message.get("role") == "assistant": + content = message.get("content") + blocks = [{"type": "text", "text": content}] if isinstance(content, str) else content or [] + texts = [str(b.get("text", "")) for b in blocks if isinstance(b, dict) and b.get("type") == "text"] + if any(t.strip() for t in texts): + last_text = "\n".join(texts) + return result if result and result.strip() else last_text + + +def finding_lines(text: str) -> list[tuple[set[str], str]]: + """``(ids, line)`` for each line in the findings format ``ID — file:line — …``.""" + found: list[tuple[set[str], str]] = [] + for line in text.splitlines(): + match = _FINDING.match(line) + if match: + found.append((set(_ID.findall(match.group("ids"))), line.strip())) + return found + + +def review_findings(run: Run, params: dict[str, Any]) -> Verdict: + """Review mode: nothing edited, and at least ``min_found`` seeded issues reported. + + Params: ``fixture`` (fixture directory the workspace was copied from, e.g. + ``sut-bdd-library/review``), ``seeded`` (one entry per seeded issue; an + entry is a check ID or ``"W2|C5"`` when either ID names it), ``min_found`` + (default: all). Only lines in the findings format count; an issue is found + when a findings line names one of its IDs. Skipped without a transcript. + """ + name = _name(params, "review_findings") + fixture = FIXTURES_DIR / str(params["fixture"]) + if not fixture.is_dir(): + return Verdict.errored(run.id, name, f"fixture missing: {fixture}") + changes = changed_files(fixture, run.effective_workspace) + if changes: + return Verdict.of(run.id, name, False, f"review changed files: {changes[:6]}") + text = final_message(run) + if text is None: + return Verdict.skipped(run.id, name, "no session transcript (stdout.stream.jsonl missing)") + seeded = [{i.strip() for i in str(entry).split("|") if i.strip()} for entry in params.get("seeded") or ()] + if not seeded: + return Verdict.errored(run.id, name, "review_findings needs a non-empty 'seeded' list") + want = int(params.get("min_found", len(seeded))) + lines = finding_lines(text) + named = set().union(*(ids for ids, _ in lines)) if lines else set() + found = ["|".join(sorted(entry)) for entry in seeded if entry & named] + missed = ["|".join(sorted(entry)) for entry in seeded if not entry & named] + ok = len(found) >= want + return Verdict.of(run.id, name, ok, + f"no edits; {len(lines)} findings lines; seeded found {len(found)}/{len(seeded)} " + f"(need {want}): found={found} missed={missed}") diff --git a/tests/eval/golden/adv-bdd-gherkin-row-01/bad/resources/reservation_steps.resource b/tests/eval/golden/adv-bdd-gherkin-row-01/bad/resources/reservation_steps.resource new file mode 100644 index 0000000..59429ac --- /dev/null +++ b/tests/eval/golden/adv-bdd-gherkin-row-01/bad/resources/reservation_steps.resource @@ -0,0 +1,64 @@ +*** Settings *** +Documentation Step keywords for reservations and returns. +Library LibraryService + + +*** Variables *** +${DUE_DATE} 2026-12-03 +${LAST_LOAN} ${None} +${LAST_FINE} ${None} + + +*** Keywords *** +The only copy of "${title}" is on loan to ${member} + [Documentation] The library owns one copy of ``title``, and ``member`` has borrowed it. + Add Book ${title} 1 + Register Member ${member} + ${loan} Borrow ${member} ${title} + Should Be Equal ${loan}[status] lent + +${member} has reserved "${title}" + [Documentation] Registers ``member`` and puts them in the queue for ``title``. + Register Member ${member} + Reserve ${member} ${title} + +${member} returns "${title}" + [Documentation] ``member`` returns ``title`` today. + ${fine} Return Book ${member} ${title} + VAR ${LAST_FINE} ${fine} scope=TEST + +${member} returns "${title}" ${days: int} days after the due date + [Documentation] ``member`` returns ``title`` ``days`` days after its due date. + Return Late ${member} ${title} ${days} + +${member} returns "${title}" one day after the due date + [Documentation] ``member`` returns ``title`` the day after its due date. + Return Late ${member} ${title} 1 + +${member} can borrow "${title}" + [Documentation] ``member`` asks for ``title`` and gets it. + ${loan} Borrow ${member} ${title} + Should Be Equal ${loan}[status] lent msg=Loan refused: ${loan}[reason] values=False + +${member} cannot borrow "${title}" + [Documentation] ``member`` asks for ``title`` and is refused. + ${loan} Borrow ${member} ${title} + Should Be Equal ${loan}[status] refused + +${member} is charged a fine of ${fine} EUR + [Documentation] The last return cost ``member`` ``fine`` EUR. + Should Be Equal As Numbers ${LAST_FINE} ${fine} + +Returning "${title}" ${days: int} days late costs ${member} ${fine} EUR + [Documentation] Outline: ``member`` returns ``title`` ``days`` days late and pays ``fine`` EUR. + When ${member} returns "${title}" ${days} days after the due date + Then ${member} is charged a fine of ${fine} EUR + +Return Late + [Documentation] Returns ``title`` ``days`` days after the due date. + [Arguments] ${member} ${title} ${days} + [Tags] robot:private + ${day} Evaluate datetime.date.fromisoformat($DUE_DATE) + datetime.timedelta(days=${days}) + Set Today ${day} + ${fine} Return Book ${member} ${title} + VAR ${LAST_FINE} ${fine} scope=TEST diff --git a/tests/eval/golden/adv-bdd-gherkin-row-01/bad/tests/reservations/__init__.robot b/tests/eval/golden/adv-bdd-gherkin-row-01/bad/tests/reservations/__init__.robot new file mode 100644 index 0000000..36eb18d --- /dev/null +++ b/tests/eval/golden/adv-bdd-gherkin-row-01/bad/tests/reservations/__init__.robot @@ -0,0 +1,9 @@ +*** Settings *** +Documentation As a library member +... I want to reserve a book that is on loan +... So that I get it as soon as it is returned +Metadata Feature Reservations +Metadata Ticket LIB-214 +Resource reservation_steps.resource +Test Tags feature:reservations +Test Setup Given the only copy of "Dune" is on loan to Mia diff --git a/tests/eval/golden/adv-bdd-gherkin-row-01/bad/tests/reservations/late_returns.robot b/tests/eval/golden/adv-bdd-gherkin-row-01/bad/tests/reservations/late_returns.robot new file mode 100644 index 0000000..40f0bb3 --- /dev/null +++ b/tests/eval/golden/adv-bdd-gherkin-row-01/bad/tests/reservations/late_returns.robot @@ -0,0 +1,11 @@ +*** Settings *** +Documentation Rule: late returns are fined per day. +Resource reservation_steps.resource +Test Tags rule:fine-per-day +Test Template Returning "Dune" ${days} days late costs Mia ${fine} EUR + + +*** Test Cases *** DAYS FINE +One day late costs 0.50 EUR 1 0.50 +A week late costs 3.50 EUR 7 3.50 +A month late costs 15.00 EUR 30 15.00 diff --git a/tests/eval/golden/adv-bdd-gherkin-row-01/bad/tests/reservations/queue.robot b/tests/eval/golden/adv-bdd-gherkin-row-01/bad/tests/reservations/queue.robot new file mode 100644 index 0000000..c43a26b --- /dev/null +++ b/tests/eval/golden/adv-bdd-gherkin-row-01/bad/tests/reservations/queue.robot @@ -0,0 +1,13 @@ +*** Settings *** +Documentation Rule: reservations are served first come, first served. +Resource reservation_steps.resource +Test Tags rule:first-come-first-served + + +*** Test Cases *** +The returned copy is held for the first member in the queue + Given Noah has reserved "Dune" + And Lea has reserved "Dune" + When Mia returns "Dune" + Then Noah can borrow "Dune" + But Lea cannot borrow "Dune" diff --git a/tests/eval/golden/adv-bdd-gherkin-row-01/good/resources/reservation_steps.resource b/tests/eval/golden/adv-bdd-gherkin-row-01/good/resources/reservation_steps.resource new file mode 100644 index 0000000..59429ac --- /dev/null +++ b/tests/eval/golden/adv-bdd-gherkin-row-01/good/resources/reservation_steps.resource @@ -0,0 +1,64 @@ +*** Settings *** +Documentation Step keywords for reservations and returns. +Library LibraryService + + +*** Variables *** +${DUE_DATE} 2026-12-03 +${LAST_LOAN} ${None} +${LAST_FINE} ${None} + + +*** Keywords *** +The only copy of "${title}" is on loan to ${member} + [Documentation] The library owns one copy of ``title``, and ``member`` has borrowed it. + Add Book ${title} 1 + Register Member ${member} + ${loan} Borrow ${member} ${title} + Should Be Equal ${loan}[status] lent + +${member} has reserved "${title}" + [Documentation] Registers ``member`` and puts them in the queue for ``title``. + Register Member ${member} + Reserve ${member} ${title} + +${member} returns "${title}" + [Documentation] ``member`` returns ``title`` today. + ${fine} Return Book ${member} ${title} + VAR ${LAST_FINE} ${fine} scope=TEST + +${member} returns "${title}" ${days: int} days after the due date + [Documentation] ``member`` returns ``title`` ``days`` days after its due date. + Return Late ${member} ${title} ${days} + +${member} returns "${title}" one day after the due date + [Documentation] ``member`` returns ``title`` the day after its due date. + Return Late ${member} ${title} 1 + +${member} can borrow "${title}" + [Documentation] ``member`` asks for ``title`` and gets it. + ${loan} Borrow ${member} ${title} + Should Be Equal ${loan}[status] lent msg=Loan refused: ${loan}[reason] values=False + +${member} cannot borrow "${title}" + [Documentation] ``member`` asks for ``title`` and is refused. + ${loan} Borrow ${member} ${title} + Should Be Equal ${loan}[status] refused + +${member} is charged a fine of ${fine} EUR + [Documentation] The last return cost ``member`` ``fine`` EUR. + Should Be Equal As Numbers ${LAST_FINE} ${fine} + +Returning "${title}" ${days: int} days late costs ${member} ${fine} EUR + [Documentation] Outline: ``member`` returns ``title`` ``days`` days late and pays ``fine`` EUR. + When ${member} returns "${title}" ${days} days after the due date + Then ${member} is charged a fine of ${fine} EUR + +Return Late + [Documentation] Returns ``title`` ``days`` days after the due date. + [Arguments] ${member} ${title} ${days} + [Tags] robot:private + ${day} Evaluate datetime.date.fromisoformat($DUE_DATE) + datetime.timedelta(days=${days}) + Set Today ${day} + ${fine} Return Book ${member} ${title} + VAR ${LAST_FINE} ${fine} scope=TEST diff --git a/tests/eval/golden/adv-bdd-gherkin-row-01/good/tests/reservations/__init__.robot b/tests/eval/golden/adv-bdd-gherkin-row-01/good/tests/reservations/__init__.robot new file mode 100644 index 0000000..36eb18d --- /dev/null +++ b/tests/eval/golden/adv-bdd-gherkin-row-01/good/tests/reservations/__init__.robot @@ -0,0 +1,9 @@ +*** Settings *** +Documentation As a library member +... I want to reserve a book that is on loan +... So that I get it as soon as it is returned +Metadata Feature Reservations +Metadata Ticket LIB-214 +Resource reservation_steps.resource +Test Tags feature:reservations +Test Setup Given the only copy of "Dune" is on loan to Mia diff --git a/tests/eval/golden/adv-bdd-gherkin-row-01/good/tests/reservations/late_returns.robot b/tests/eval/golden/adv-bdd-gherkin-row-01/good/tests/reservations/late_returns.robot new file mode 100644 index 0000000..6def434 --- /dev/null +++ b/tests/eval/golden/adv-bdd-gherkin-row-01/good/tests/reservations/late_returns.robot @@ -0,0 +1,15 @@ +*** Settings *** +Documentation Rule: late returns are fined per day. +Resource reservation_steps.resource +Test Tags rule:fine-per-day +Test Template Returning "Dune" ${days} days late costs Mia ${fine} EUR + + +*** Test Cases *** DAYS FINE +A week late costs 3.50 EUR 7 3.50 +A month late costs 15.00 EUR 30 15.00 + +One day late costs 0.50 EUR + [Template] NONE + When Mia returns "Dune" one day after the due date + Then Mia is charged a fine of 0.50 EUR diff --git a/tests/eval/golden/adv-bdd-gherkin-row-01/good/tests/reservations/queue.robot b/tests/eval/golden/adv-bdd-gherkin-row-01/good/tests/reservations/queue.robot new file mode 100644 index 0000000..c43a26b --- /dev/null +++ b/tests/eval/golden/adv-bdd-gherkin-row-01/good/tests/reservations/queue.robot @@ -0,0 +1,13 @@ +*** Settings *** +Documentation Rule: reservations are served first come, first served. +Resource reservation_steps.resource +Test Tags rule:first-come-first-served + + +*** Test Cases *** +The returned copy is held for the first member in the queue + Given Noah has reserved "Dune" + And Lea has reserved "Dune" + When Mia returns "Dune" + Then Noah can borrow "Dune" + But Lea cannot borrow "Dune" diff --git a/tests/eval/golden/narrow-bdd-extend-vocabulary-01/bad/resources/borrowing_steps.resource b/tests/eval/golden/narrow-bdd-extend-vocabulary-01/bad/resources/borrowing_steps.resource new file mode 100644 index 0000000..1e5434c --- /dev/null +++ b/tests/eval/golden/narrow-bdd-extend-vocabulary-01/bad/resources/borrowing_steps.resource @@ -0,0 +1,57 @@ +*** Settings *** +Documentation Step definitions for borrowing books. +Library LibraryService + + +*** Variables *** +${LAST_LOAN} ${None} + + +*** Keywords *** +The library has ${copies: int} copies of "${title}" + [Documentation] Puts ``copies`` copies of ``title`` on the shelf. + Add Book ${title} ${copies} + +${member} is a ${membership:standard|premium} member + [Documentation] Registers ``member`` with a standard or premium membership. + Register Member ${member} ${membership} + +${member} borrows "${title}" + [Documentation] ``member`` asks to borrow ``title`` today. + ${result} Borrow ${member} ${title} + VAR ${LAST_LOAN} ${result} scope=TEST + +The loan is granted until ${due} + [Documentation] The last loan was granted and is due on ``due`` (YYYY-MM-DD). + Should Be Equal ${LAST_LOAN}[status] lent + ... msg=Expected the loan to be granted, but it was refused: ${LAST_LOAN}[reason] values=False + Should Be Equal As Strings ${LAST_LOAN}[due_date] ${due} + +The loan is refused with "${reason}" + [Documentation] The last loan was refused, and the library gave ``reason``. + Should Be Equal ${LAST_LOAN}[status] refused + Should Be Equal ${LAST_LOAN}[reason] ${reason} + +${member} has "${title}" on loan + [Documentation] ``title`` is among the books ``member`` currently has on loan. + ${loans} Get Loans ${member} + Should Contain ${loans} ${title} + +${membership:standard|premium} member ${member} already has ${count: int} books + Register Member ${member} ${membership} + FOR ${n} IN RANGE ${count} + Add Book Book ${n} 1 + Borrow ${member} Book ${n} + END + +Blocked member ${member} + Register Member ${member} + Block Member ${member} + +${member} tries to borrow another book + Add Book Another Book 1 + ${result} Borrow ${member} Another Book + VAR ${LAST_LOAN} ${result} scope=TEST + +The request is rejected because "${reason}" + Should Be Equal ${LAST_LOAN}[reason] ${reason} diff --git a/tests/eval/golden/narrow-bdd-extend-vocabulary-01/bad/tests/borrowing.robot b/tests/eval/golden/narrow-bdd-extend-vocabulary-01/bad/tests/borrowing.robot new file mode 100644 index 0000000..4ddf119 --- /dev/null +++ b/tests/eval/golden/narrow-bdd-extend-vocabulary-01/bad/tests/borrowing.robot @@ -0,0 +1,47 @@ +*** Settings *** +Documentation As a library member +... I want to borrow books +... So that I can read them at home +Metadata Feature Borrowing books +Resource borrowing_steps.resource +Test Tags feature:borrowing-books + + +*** Test Cases *** +Member borrows an available book + [Documentation] Rule: an available copy is lent for 21 days. + [Tags] rule:loan-period + Given the library has 2 copies of "Dune" + And Mia is a standard member + When Mia borrows "Dune" + Then the loan is granted until 2026-12-03 + And Mia has "Dune" on loan + +Book without an available copy cannot be borrowed + [Documentation] Rule: a book can only be lent while a copy is on the shelf. + [Tags] rule:copy-available + Given the library has 0 copies of "Dune" + And Mia is a standard member + When Mia borrows "Dune" + Then the loan is refused with "No copy available" + +Loan limit for standard members + [Documentation] Rule: standard members may have 3 books at a time. + [Tags] rule:loan-limit + Given standard member Mia already has 3 books + When Mia tries to borrow another book + Then the request is rejected because "Loan limit reached" + +Loan limit for premium members + [Documentation] Rule: premium members may have 10 books at a time. + [Tags] rule:loan-limit + Given premium member Noah already has 10 books + When Noah tries to borrow another book + Then the request is rejected because "Loan limit reached" + +Blocked members cannot borrow + [Documentation] Rule: blocked members cannot borrow at all. + [Tags] rule:blocked-member + Given blocked member Mia + When Mia tries to borrow another book + Then the request is rejected because "Member is blocked" diff --git a/tests/eval/golden/narrow-bdd-extend-vocabulary-01/good/resources/borrowing_steps.resource b/tests/eval/golden/narrow-bdd-extend-vocabulary-01/good/resources/borrowing_steps.resource new file mode 100644 index 0000000..3e1ec1e --- /dev/null +++ b/tests/eval/golden/narrow-bdd-extend-vocabulary-01/good/resources/borrowing_steps.resource @@ -0,0 +1,50 @@ +*** Settings *** +Documentation Step definitions for borrowing books. +Library LibraryService + + +*** Variables *** +${LAST_LOAN} ${None} + + +*** Keywords *** +The library has ${copies: int} copies of "${title}" + [Documentation] Puts ``copies`` copies of ``title`` on the shelf. + Add Book ${title} ${copies} + +${member} is a ${membership:standard|premium} member + [Documentation] Registers ``member`` with a standard or premium membership. + Register Member ${member} ${membership} + +${member} borrows "${title}" + [Documentation] ``member`` asks to borrow ``title`` today. + ${result} Borrow ${member} ${title} + VAR ${LAST_LOAN} ${result} scope=TEST + +The loan is granted until ${due} + [Documentation] The last loan was granted and is due on ``due`` (YYYY-MM-DD). + Should Be Equal ${LAST_LOAN}[status] lent + ... msg=Expected the loan to be granted, but it was refused: ${LAST_LOAN}[reason] values=False + Should Be Equal As Strings ${LAST_LOAN}[due_date] ${due} + +The loan is refused with "${reason}" + [Documentation] The last loan was refused, and the library gave ``reason``. + Should Be Equal ${LAST_LOAN}[status] refused + Should Be Equal ${LAST_LOAN}[reason] ${reason} + +${member} has "${title}" on loan + [Documentation] ``title`` is among the books ``member`` currently has on loan. + ${loans} Get Loans ${member} + Should Contain ${loans} ${title} + +${member} has ${count: int} books on loan + [Documentation] Lends ``count`` different books to ``member``. + FOR ${n} IN RANGE 1 ${count + 1} + Add Book Book ${n} 1 + ${loan} Borrow ${member} Book ${n} + Should Be Equal ${loan}[status] lent + END + +${member} is blocked + [Documentation] Blocks ``member``'s account. + Block Member ${member} diff --git a/tests/eval/golden/narrow-bdd-extend-vocabulary-01/good/tests/borrowing.robot b/tests/eval/golden/narrow-bdd-extend-vocabulary-01/good/tests/borrowing.robot new file mode 100644 index 0000000..63f7e29 --- /dev/null +++ b/tests/eval/golden/narrow-bdd-extend-vocabulary-01/good/tests/borrowing.robot @@ -0,0 +1,62 @@ +*** Settings *** +Documentation As a library member +... I want to borrow books +... So that I can read them at home +Metadata Feature Borrowing books +Resource borrowing_steps.resource +Test Tags feature:borrowing-books + + +*** Test Cases *** +Member borrows an available book + [Documentation] Rule: an available copy is lent for 21 days. + [Tags] rule:loan-period + Given the library has 2 copies of "Dune" + And Mia is a standard member + When Mia borrows "Dune" + Then the loan is granted until 2026-12-03 + And Mia has "Dune" on loan + +Book without an available copy cannot be borrowed + [Documentation] Rule: a book can only be lent while a copy is on the shelf. + [Tags] rule:copy-available + Given the library has 0 copies of "Dune" + And Mia is a standard member + When Mia borrows "Dune" + Then the loan is refused with "No copy available" + +Standard member with 3 books cannot borrow a fourth + [Documentation] Rule: standard members may have 3 books at a time. + [Tags] rule:loan-limit + Given the library has 2 copies of "Dune" + And Mia is a standard member + And Mia has 3 books on loan + When Mia borrows "Dune" + Then the loan is refused with "Loan limit reached" + +Premium member borrows beyond the standard limit + [Documentation] Rule: premium members may have 10 books at a time. + [Tags] rule:loan-limit + Given the library has 2 copies of "Dune" + And Noah is a premium member + And Noah has 9 books on loan + When Noah borrows "Dune" + Then the loan is granted until 2026-12-03 + +Premium member with 10 books cannot borrow an eleventh + [Documentation] Rule: premium members may have 10 books at a time. + [Tags] rule:loan-limit + Given the library has 2 copies of "Dune" + And Noah is a premium member + And Noah has 10 books on loan + When Noah borrows "Dune" + Then the loan is refused with "Loan limit reached" + +Blocked member cannot borrow + [Documentation] Rule: blocked members cannot borrow at all. + [Tags] rule:blocked-member + Given the library has 2 copies of "Dune" + And Mia is a standard member + And Mia is blocked + When Mia borrows "Dune" + Then the loan is refused with "Member is blocked" diff --git a/tests/eval/golden/narrow-bdd-greenfield-01/bad/tests/borrowing.robot b/tests/eval/golden/narrow-bdd-greenfield-01/bad/tests/borrowing.robot new file mode 100644 index 0000000..9ef7309 --- /dev/null +++ b/tests/eval/golden/narrow-bdd-greenfield-01/bad/tests/borrowing.robot @@ -0,0 +1,47 @@ +*** Settings *** +Library LibraryService + + +*** Test Cases *** +Borrow available book + Given the library has a copy of "Dune" + And Mia is a standard member + ${loan} Borrow Mia Dune + Then Should Be Equal ${loan}[status] lent + +No copy available + Register Member Mia + ${loan} Borrow Mia Dune + Should Be Equal ${loan}[reason] No copy available + +Loan limit standard + Register Member Mia + FOR ${n} IN RANGE 3 + Add Book Book ${n} + Borrow Mia Book ${n} + END + Add Book Dune + ${loan} Borrow Mia Dune + Should Be Equal ${loan}[reason] Loan limit reached + +Blocked member + Register Member Mia + Block Member Mia + Add Book Dune + ${loan} Borrow Mia Dune + Should Be Equal ${loan}[reason] Member is blocked + +Unpaid fines + Register Member Mia + Add Fine Mia 1 + Add Book Dune + ${loan} Borrow Mia Dune + Should Be Equal ${loan}[reason] Unpaid fines + + +*** Keywords *** +The library has a copy of "${title}" + Add Book ${title} 1 + +${member} is a standard member + Register Member ${member} diff --git a/tests/eval/golden/narrow-bdd-greenfield-01/good/resources/borrowing_steps.resource b/tests/eval/golden/narrow-bdd-greenfield-01/good/resources/borrowing_steps.resource new file mode 100644 index 0000000..f3d0bad --- /dev/null +++ b/tests/eval/golden/narrow-bdd-greenfield-01/good/resources/borrowing_steps.resource @@ -0,0 +1,53 @@ +*** Settings *** +Documentation Step keywords for borrowing books. +Library LibraryService + + +*** Variables *** +${LAST_LOAN} ${None} + + +*** Keywords *** +The library has ${copies: int} copies of "${title}" + [Documentation] Puts ``copies`` copies of ``title`` on the shelf. + Add Book ${title} ${copies} + +The library has a copy of "${title}" + [Documentation] Puts one copy of ``title`` on the shelf. + Add Book ${title} 1 + +${member} is a ${membership:standard|premium} member + [Documentation] Registers ``member`` with a standard or premium membership. + Register Member ${member} ${membership} + +${member} is blocked + [Documentation] Blocks ``member``'s account. + Block Member ${member} + +${member} has unpaid fines of ${amount} EUR + [Documentation] Adds an unpaid fine of ``amount`` EUR to ``member``'s account. + Add Fine ${member} ${amount} + +${member} has ${count: int} books on loan + [Documentation] Lends ``count`` different books to ``member``. + FOR ${n} IN RANGE 1 ${count + 1} + Add Book Book ${n} 1 + ${loan} Borrow ${member} Book ${n} + Should Be Equal ${loan}[status] lent + END + +${member} borrows "${title}" + [Documentation] ``member`` asks to borrow ``title`` today. + ${result} Borrow ${member} ${title} + VAR ${LAST_LOAN} ${result} scope=TEST + +The loan is granted until ${due} + [Documentation] The last loan was granted and is due on ``due`` (YYYY-MM-DD). + Should Be Equal ${LAST_LOAN}[status] lent + ... msg=Expected the loan to be granted, but it was refused: ${LAST_LOAN}[reason] values=False + Should Be Equal As Strings ${LAST_LOAN}[due_date] ${due} + +The loan is refused with "${reason}" + [Documentation] The last loan was refused, and the library gave ``reason``. + Should Be Equal ${LAST_LOAN}[status] refused + Should Be Equal ${LAST_LOAN}[reason] ${reason} diff --git a/tests/eval/golden/narrow-bdd-greenfield-01/good/tests/borrowing.robot b/tests/eval/golden/narrow-bdd-greenfield-01/good/tests/borrowing.robot new file mode 100644 index 0000000..2761f48 --- /dev/null +++ b/tests/eval/golden/narrow-bdd-greenfield-01/good/tests/borrowing.robot @@ -0,0 +1,70 @@ +*** Settings *** +Documentation As a library member +... I want to borrow books +... So that I can read them at home +Metadata Feature Borrowing books +Resource borrowing_steps.resource +Test Tags feature:borrowing-books + + +*** Test Cases *** +Member borrows an available book for 21 days + [Documentation] Rule: an available copy is lent for 21 days. + [Tags] rule:loan-period + Given the library has 2 copies of "Dune" + And Mia is a standard member + When Mia borrows "Dune" + Then the loan is granted until 2026-12-03 + +Book without an available copy cannot be borrowed + [Documentation] Rule: a book can only be lent while a copy is on the shelf. + [Tags] rule:copy-available + Given the library has 0 copies of "Dune" + And Mia is a standard member + When Mia borrows "Dune" + Then the loan is refused with "No copy available" + +Standard member borrows a third book + [Documentation] Rule: standard members may have 3 books at a time. + [Tags] rule:loan-limit + Given Mia is a standard member + And Mia has 2 books on loan + And the library has a copy of "Dune" + When Mia borrows "Dune" + Then the loan is granted until 2026-12-03 + +Standard member with 3 books cannot borrow a fourth + [Documentation] Rule: standard members may have 3 books at a time. + [Tags] rule:loan-limit + Given Mia is a standard member + And Mia has 3 books on loan + And the library has a copy of "Dune" + When Mia borrows "Dune" + Then the loan is refused with "Loan limit reached" + +Premium member with 10 books cannot borrow an eleventh + [Documentation] Rule: premium members may have 10 books at a time. + [Tags] rule:loan-limit + Given Noah is a premium member + And Noah has 10 books on loan + And the library has a copy of "Dune" + When Noah borrows "Dune" + Then the loan is refused with "Loan limit reached" + +Blocked member cannot borrow + [Documentation] Rule: blocked members cannot borrow. + [Tags] rule:blocked-member + Given Mia is a standard member + And Mia is blocked + And the library has a copy of "Dune" + When Mia borrows "Dune" + Then the loan is refused with "Member is blocked" + +Member with unpaid fines cannot borrow + [Documentation] Rule: members with unpaid fines cannot borrow. + [Tags] rule:unpaid-fines + Given Mia is a standard member + And Mia has unpaid fines of 2.50 EUR + And the library has a copy of "Dune" + When Mia borrows "Dune" + Then the loan is refused with "Unpaid fines" diff --git a/tests/eval/golden/narrow-bdd-outline-01/bad/resources/returning_steps.resource b/tests/eval/golden/narrow-bdd-outline-01/bad/resources/returning_steps.resource new file mode 100644 index 0000000..0c6c741 --- /dev/null +++ b/tests/eval/golden/narrow-bdd-outline-01/bad/resources/returning_steps.resource @@ -0,0 +1,14 @@ +*** Settings *** +Library LibraryService + + +*** Keywords *** +Return late + [Arguments] ${days} ${fine} + Register Member Mia + Add Book Dune 1 + Borrow Mia Dune + ${day} Evaluate datetime.date(2026, 12, 3) + datetime.timedelta(days=${days}) + Set Today ${day} + ${charged} Return Book Mia Dune + Should Be Equal As Numbers ${charged} ${fine} diff --git a/tests/eval/golden/narrow-bdd-outline-01/bad/tests/late_returns.robot b/tests/eval/golden/narrow-bdd-outline-01/bad/tests/late_returns.robot new file mode 100644 index 0000000..67bd397 --- /dev/null +++ b/tests/eval/golden/narrow-bdd-outline-01/bad/tests/late_returns.robot @@ -0,0 +1,13 @@ +*** Settings *** +Resource returning_steps.resource + + +*** Test Cases *** +Fine 0 + Return late 0 0 +Fine 1 + Return late 1 0.50 +Fine 7 + Return late 7 3.50 +Fine 30 + Return late 30 15.00 diff --git a/tests/eval/golden/narrow-bdd-outline-01/good/resources/returning_steps.resource b/tests/eval/golden/narrow-bdd-outline-01/good/resources/returning_steps.resource new file mode 100644 index 0000000..b876e48 --- /dev/null +++ b/tests/eval/golden/narrow-bdd-outline-01/good/resources/returning_steps.resource @@ -0,0 +1,58 @@ +*** Settings *** +Documentation Step keywords for returning books. +Library LibraryService + + +*** Variables *** +${DUE_DATE} 2026-12-03 +${LAST_FINE} ${None} + + +*** Keywords *** +${member} has borrowed "${title}" + [Documentation] Lends ``title`` to ``member`` today; the loan is due on ${DUE_DATE}. + Register Member ${member} + Add Book ${title} 1 + ${loan} Borrow ${member} ${title} + Should Be Equal As Strings ${loan}[due_date] ${DUE_DATE} + +${member} returns "${title}" on the due date + [Documentation] ``member`` returns ``title`` on its due date. + Return On Day ${member} ${title} ${DUE_DATE} + +${member} returns "${title}" one day late + [Documentation] ``member`` returns ``title`` the day after its due date. + Return Days Late ${member} ${title} 1 + +${member} returns "${title}" ${days: int} days late + [Documentation] ``member`` returns ``title`` ``days`` days after its due date. + Return Days Late ${member} ${title} ${days} + +${member} is charged a fine of ${fine} EUR + [Documentation] The last return cost ``fine`` EUR. + Should Be Equal As Numbers ${LAST_FINE} ${fine} + +${member} is charged no fine + [Documentation] The last return cost nothing. + Should Be Equal As Numbers ${LAST_FINE} 0 + +A late return of ${days: int} days costs ${fine} EUR + [Documentation] Outline: Mia returns "Dune" ``days`` days late and pays ``fine`` EUR. + Given Mia has borrowed "Dune" + When Mia returns "Dune" ${days} days late + Then Mia is charged a fine of ${fine} EUR + +Return Days Late + [Documentation] Returns ``title`` ``days`` days after the due date. + [Arguments] ${member} ${title} ${days} + [Tags] robot:private + ${day} Evaluate datetime.date.fromisoformat($DUE_DATE) + datetime.timedelta(days=${days}) + Return On Day ${member} ${title} ${day} + +Return On Day + [Documentation] Returns ``title`` on ``day``. + [Arguments] ${member} ${title} ${day} + [Tags] robot:private + Set Today ${day} + ${fine} Return Book ${member} ${title} + VAR ${LAST_FINE} ${fine} scope=TEST diff --git a/tests/eval/golden/narrow-bdd-outline-01/good/tests/late_returns.robot b/tests/eval/golden/narrow-bdd-outline-01/good/tests/late_returns.robot new file mode 100644 index 0000000..1784e16 --- /dev/null +++ b/tests/eval/golden/narrow-bdd-outline-01/good/tests/late_returns.robot @@ -0,0 +1,27 @@ +*** Settings *** +Documentation As the library +... I want late returns to be fined per day +... So that books come back on time +... +... Rule: a late return costs 0.50 EUR per day; a return on the due date is free. +Metadata Feature Late return fines +Resource returning_steps.resource +Test Tags feature:late-return-fines rule:fine-per-day +Test Template A late return of ${days} days costs ${fine} EUR + + +*** Test Cases *** DAYS FINE +A week late costs a week of fines 7 3.50 +A month late costs thirty days of fines 30 15.00 + +Return on the due date is free + [Template] NONE + Given Mia has borrowed "Dune" + When Mia returns "Dune" on the due date + Then Mia is charged no fine + +One day late costs one day of fines + [Template] NONE + Given Mia has borrowed "Dune" + When Mia returns "Dune" one day late + Then Mia is charged a fine of 0.50 EUR diff --git a/tests/eval/golden/narrow-bdd-review-01/bad/stdout.stream.jsonl b/tests/eval/golden/narrow-bdd-review-01/bad/stdout.stream.jsonl new file mode 100644 index 0000000..271d0a8 --- /dev/null +++ b/tests/eval/golden/narrow-bdd-review-01/bad/stdout.stream.jsonl @@ -0,0 +1,3 @@ +{"type": "system", "subtype": "init"} +{"type": "assistant", "message": {"role": "assistant", "content": [{"type": "text", "text": "## Review of tests/loans.robot and tests/loan_steps.resource\n\n### S2 (layering)\n- L1 \u2014 tests/loans.robot:8 \u2014 `Register Member` and `Add Book` are library calls in the test body; move them into Given steps such as `Given Mia is a standard member`.\n- L2 \u2014 tests/loans.robot:11 \u2014 \"opens the catalogue and selects book1 and presses borrow\" scripts the UI; write the intent: `When Mia borrows \"Dune\"`.\n- L6 \u2014 tests/loans.robot:20 \u2014 \"Check stuff\" has a second When/Then phase; split it into two tests.\n\n### S3 (language)\n- W1 \u2014 tests/loans.robot:10 \u2014 \"user1 is registered and the library has book1\" sets two facts in one step; split it into two Given/And steps.\n- W2 \u2014 tests/loans.robot:12 \u2014 \"Then everything is fine\" names nothing observable (and its keyword only logs, C5); state the loan outcome.\n- W3 \u2014 tests/loans.robot:8 \u2014 `user1` and `book1` are placeholders; use a member name and a real title.\n- W12 \u2014 tests/loans.robot:7 \u2014 \"Test loan 1\" and \"Check stuff\" describe nothing; name the behaviour.\n- W6 \u2014 tests/loans.robot:1 \u2014 no suite Documentation, no `Metadata Feature` and no feature: tag.\n\nSummary: 3 layering and 5 language findings; I applied the fixes to tests/loans.robot.\n"}]}} +{"type": "result", "subtype": "success", "is_error": false, "num_turns": 6, "result": "## Review of tests/loans.robot and tests/loan_steps.resource\n\n### S2 (layering)\n- L1 \u2014 tests/loans.robot:8 \u2014 `Register Member` and `Add Book` are library calls in the test body; move them into Given steps such as `Given Mia is a standard member`.\n- L2 \u2014 tests/loans.robot:11 \u2014 \"opens the catalogue and selects book1 and presses borrow\" scripts the UI; write the intent: `When Mia borrows \"Dune\"`.\n- L6 \u2014 tests/loans.robot:20 \u2014 \"Check stuff\" has a second When/Then phase; split it into two tests.\n\n### S3 (language)\n- W1 \u2014 tests/loans.robot:10 \u2014 \"user1 is registered and the library has book1\" sets two facts in one step; split it into two Given/And steps.\n- W2 \u2014 tests/loans.robot:12 \u2014 \"Then everything is fine\" names nothing observable (and its keyword only logs, C5); state the loan outcome.\n- W3 \u2014 tests/loans.robot:8 \u2014 `user1` and `book1` are placeholders; use a member name and a real title.\n- W12 \u2014 tests/loans.robot:7 \u2014 \"Test loan 1\" and \"Check stuff\" describe nothing; name the behaviour.\n- W6 \u2014 tests/loans.robot:1 \u2014 no suite Documentation, no `Metadata Feature` and no feature: tag.\n\nSummary: 3 layering and 5 language findings; I applied the fixes to tests/loans.robot.\n"} diff --git a/tests/eval/golden/narrow-bdd-review-01/bad/tests/loans.robot b/tests/eval/golden/narrow-bdd-review-01/bad/tests/loans.robot new file mode 100644 index 0000000..e54d87f --- /dev/null +++ b/tests/eval/golden/narrow-bdd-review-01/bad/tests/loans.robot @@ -0,0 +1,23 @@ +*** Settings *** +Documentation As a library member +... I want to borrow books +... So that I can read them at home +Metadata Feature Borrowing books +Resource loan_steps.resource +Test Tags feature:borrowing-books + + +*** Test Cases *** +Member borrows the last copy + Given Mia is a standard member + And the library has 1 copy of "Dune" + When Mia borrows "Dune" + Then the loan works + +Second member finds no copy + Given Mia is a standard member + And Noah is a standard member + And the library has 1 copy of "Dune" + And Mia borrows "Dune" + When Noah borrows "Dune" + Then the loan is refused with "No copy available" diff --git a/tests/eval/golden/narrow-bdd-review-01/good/stdout.stream.jsonl b/tests/eval/golden/narrow-bdd-review-01/good/stdout.stream.jsonl new file mode 100644 index 0000000..3463921 --- /dev/null +++ b/tests/eval/golden/narrow-bdd-review-01/good/stdout.stream.jsonl @@ -0,0 +1,3 @@ +{"type": "system", "subtype": "init"} +{"type": "assistant", "message": {"role": "assistant", "content": [{"type": "text", "text": "## Review of tests/loans.robot and tests/loan_steps.resource\n\n### S2 (layering)\n- L1 \u2014 tests/loans.robot:8 \u2014 `Register Member` and `Add Book` are library calls in the test body; move them into Given steps such as `Given Mia is a standard member`.\n- L2 \u2014 tests/loans.robot:11 \u2014 \"opens the catalogue and selects book1 and presses borrow\" scripts the UI; write the intent: `When Mia borrows \"Dune\"`.\n- L6 \u2014 tests/loans.robot:20 \u2014 \"Check stuff\" has a second When/Then phase; split it into two tests.\n\n### S3 (language)\n- W1 \u2014 tests/loans.robot:10 \u2014 \"user1 is registered and the library has book1\" sets two facts in one step; split it into two Given/And steps.\n- W2 \u2014 tests/loans.robot:12 \u2014 \"Then everything is fine\" names nothing observable (and its keyword only logs, C5); state the loan outcome.\n- W3 \u2014 tests/loans.robot:8 \u2014 `user1` and `book1` are placeholders; use a member name and a real title.\n- W12 \u2014 tests/loans.robot:7 \u2014 \"Test loan 1\" and \"Check stuff\" describe nothing; name the behaviour.\n- W6 \u2014 tests/loans.robot:1 \u2014 no suite Documentation, no `Metadata Feature` and no feature: tag.\n\nSummary: 3 layering and 5 language findings; no files were changed.\n"}]}} +{"type": "result", "subtype": "success", "is_error": false, "num_turns": 6, "result": "## Review of tests/loans.robot and tests/loan_steps.resource\n\n### S2 (layering)\n- L1 \u2014 tests/loans.robot:8 \u2014 `Register Member` and `Add Book` are library calls in the test body; move them into Given steps such as `Given Mia is a standard member`.\n- L2 \u2014 tests/loans.robot:11 \u2014 \"opens the catalogue and selects book1 and presses borrow\" scripts the UI; write the intent: `When Mia borrows \"Dune\"`.\n- L6 \u2014 tests/loans.robot:20 \u2014 \"Check stuff\" has a second When/Then phase; split it into two tests.\n\n### S3 (language)\n- W1 \u2014 tests/loans.robot:10 \u2014 \"user1 is registered and the library has book1\" sets two facts in one step; split it into two Given/And steps.\n- W2 \u2014 tests/loans.robot:12 \u2014 \"Then everything is fine\" names nothing observable (and its keyword only logs, C5); state the loan outcome.\n- W3 \u2014 tests/loans.robot:8 \u2014 `user1` and `book1` are placeholders; use a member name and a real title.\n- W12 \u2014 tests/loans.robot:7 \u2014 \"Test loan 1\" and \"Check stuff\" describe nothing; name the behaviour.\n- W6 \u2014 tests/loans.robot:1 \u2014 no suite Documentation, no `Metadata Feature` and no feature: tag.\n\nSummary: 3 layering and 5 language findings; no files were changed.\n"} diff --git a/tests/eval/test_arms.py b/tests/eval/test_arms.py index 3c27c9f..a861127 100644 --- a/tests/eval/test_arms.py +++ b/tests/eval/test_arms.py @@ -219,5 +219,5 @@ def test_partial_task_stages_only_its_plugins(tmp_path: Path, fake_claude: Path) repo_root=_REPO) run = runner.execute(_task(tier="partial", stage=["rf-core"]), Profile.for_arm("treatment", out / "cfg"), out) config = run.artifacts_dir / "claude_config" - assert sorted(p.name for p in (config / "skills").iterdir()) == ["rf-language", "rf-python-library", "rf-setup"] + assert sorted(p.name for p in (config / "skills").iterdir()) == ["rf-bdd", "rf-language", "rf-python-library", "rf-setup"] assert (config / "agents").is_dir() diff --git a/tests/eval/test_bdd_grader.py b/tests/eval/test_bdd_grader.py new file mode 100644 index 0000000..9bc7b86 --- /dev/null +++ b/tests/eval/test_bdd_grader.py @@ -0,0 +1,359 @@ +"""Custom grader checks for the rf-bdd tasks (add-rf-bdd-skill, task 5.2).""" + +from __future__ import annotations + +import json +import shutil +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from rf_skill_eval.domain.run import Run +from rf_skill_eval.scoring.custom import bdd +from rf_skill_eval.scoring.deterministic import check_custom_python + +_REPO = Path(__file__).resolve().parents[2] +_FIXTURE = _REPO / "eval" / "fixtures" / "sut-bdd-library" + +pytest.importorskip("robot") + + +def _run(tmp_path: Path, ws: Path) -> Run: + artifacts = tmp_path / "artifacts" + artifacts.mkdir(exist_ok=True) + now = datetime.now(UTC) + return Run(id="r", task_id="t", profile_name="treatment", started_at=now, + artifacts_dir=artifacts, workspace_dir=ws) + + +def _ws(tmp_path: Path, files: dict[str, str], variant: str = "base") -> Path: + ws = tmp_path / "ws" + shutil.copytree(_FIXTURE / variant, ws) + for rel, text in files.items(): + (ws / rel).parent.mkdir(parents=True, exist_ok=True) + (ws / rel).write_text(text, encoding="utf-8") + return ws + + +_STEPS = """\ +*** Settings *** +Documentation Steps. +Library LibraryService + + +*** Variables *** +${LAST_LOAN} ${None} + + +*** Keywords *** +The library has a copy of "${title}" + [Documentation] One copy on the shelf. + Add Book ${title} 1 + +${member} is a standard member + [Documentation] Registers a standard member. + Register Member ${member} + +${member} borrows "${title}" + [Documentation] Borrows today. + ${result} Borrow ${member} ${title} + VAR ${LAST_LOAN} ${result} scope=TEST + +The loan is granted + [Documentation] The last loan was granted. + Should Be Equal ${LAST_LOAN}[status] lent + +Borrowing "${title}" is ${status} + [Documentation] Outline: Mia borrows ``title``. + Given the library has a copy of "${title}" + And Mia is a standard member + When Mia borrows "${title}" + Then Should Be Equal ${LAST_LOAN}[status] ${status} +""" + +_SETTINGS = """\ +*** Settings *** +Documentation As a member I want to borrow books. +Metadata Feature Borrowing +Resource steps.resource +""" + +_GOOD = _SETTINGS + """ + +*** Test Cases *** +Member borrows an available book + [Documentation] Rule: an available copy is lent. + [Tags] rule:loan + # a comment is fine + Given the library has a copy of "Dune" + And Mia is a standard member + When Mia borrows "Dune" + Then the loan is granted +""" + +_TEMPLATED = _SETTINGS + """\ +Test Template Borrowing "${title}" is ${status} + + +*** Test Cases *** TITLE STATUS +Dune is lent Dune lent +Emma is lent Emma lent + +Spelled out + [Template] NONE + Given the library has a copy of "Dune" + And Mia is a standard member + When Mia borrows "Dune" + Then the loan is granted +""" + + +def _project(tmp_path: Path, suite: str, **extra: str) -> Path: + return _ws(tmp_path, {"resources/steps.resource": _STEPS, "tests/borrowing.robot": suite, **extra}) + + +# --- structure --------------------------------------------------------------------- + + +def test_structure_passes_for_steps_and_template_rows(tmp_path: Path) -> None: + ws = _project(tmp_path, _GOOD, **{"tests/outline.robot": _TEMPLATED}) + v = bdd.structure(_run(tmp_path, ws), {"path": "tests"}) + assert v.status == "passed", v.details + assert "4 tests" in v.details + + +@pytest.mark.parametrize( + ("line", "label"), + [ + (" Add Book Dune 1", "Add Book"), + (" ${loan} When Mia borrows \"Dune\"", "When Mia borrows"), + (" VAR ${x} 1", "Var"), + (" IF True Log x", "If"), + (" Givenish step", "Givenish step"), + ], +) +def test_structure_fails_on_non_step_lines(tmp_path: Path, line: str, label: str) -> None: + ws = _project(tmp_path, _GOOD + line + "\n") + v = bdd.structure(_run(tmp_path, ws), {"path": "tests"}) + assert v.status == "failed" + assert "tests/borrowing.robot:" in v.details and label in v.details + + +def test_structure_rejects_keyword_calls_in_a_test_that_turns_the_template_off(tmp_path: Path) -> None: + suite = _TEMPLATED.replace(' Given the library has a copy of "Dune"', " Add Book Dune 1") + ws = _project(tmp_path, suite) + v = bdd.structure(_run(tmp_path, ws), {"path": "tests"}) + assert v.status == "failed" and "Spelled out" in v.details + + +def test_structure_accepts_localised_prefixes(tmp_path: Path) -> None: + suite = "Language: German\n\n" + _GOOD.replace("Given the", "Angenommen the").replace("When Mia", "Wenn Mia") + ws = _project(tmp_path, suite) + v = bdd.structure(_run(tmp_path, ws), {"path": "tests"}) + assert v.status == "passed", v.details + + +def test_structure_fails_without_tests(tmp_path: Path) -> None: + ws = _ws(tmp_path, {}) + v = bdd.structure(_run(tmp_path, ws), {"path": "tests"}) + assert v.status == "failed" and "no test cases" in v.details + + +# --- living_docs ------------------------------------------------------------------- + + +def test_living_docs_passes_with_documentation_and_metadata(tmp_path: Path) -> None: + ws = _project(tmp_path, _GOOD) + assert bdd.living_docs(_run(tmp_path, ws), {}).status == "passed" + + +def test_living_docs_accepts_a_feature_tag_instead_of_metadata(tmp_path: Path) -> None: + suite = _GOOD.replace("Metadata Feature Borrowing", "Test Tags feature:borrowing") + ws = _project(tmp_path, suite) + v = bdd.living_docs(_run(tmp_path, ws), {}) + assert v.status == "passed", v.details + + +def test_living_docs_inherits_from_init(tmp_path: Path) -> None: + init = "*** Settings ***\nDocumentation As a member I want to borrow.\nMetadata Feature Borrowing\n" + suite = _GOOD.replace("Documentation As a member I want to borrow books.\n", "").replace( + "Metadata Feature Borrowing\n", "").replace("Resource steps.resource", + "Resource steps.resource") + ws = _ws(tmp_path, {"resources/steps.resource": _STEPS, "tests/borrowing/__init__.robot": init, + "tests/borrowing/scenarios.robot": suite}) + v = bdd.living_docs(_run(tmp_path, ws), {}) + assert v.status == "passed", v.details + + +@pytest.mark.parametrize( + ("drop", "gap"), + [ + ("Documentation As a member I want to borrow books.\n", "no suite Documentation"), + ("Metadata Feature Borrowing\n", "no `Metadata Feature`"), + ], +) +def test_living_docs_fails_on_missing_story_or_feature(tmp_path: Path, drop: str, gap: str) -> None: + ws = _project(tmp_path, _GOOD.replace(drop, "")) + v = bdd.living_docs(_run(tmp_path, ws), {}) + assert v.status == "failed" and gap in v.details + + +def test_living_docs_fails_without_suites(tmp_path: Path) -> None: + ws = _ws(tmp_path, {}) + assert bdd.living_docs(_run(tmp_path, ws), {}).status == "failed" + + +# --- native_only / step_names_absent ------------------------------------------------ + + +def test_native_only(tmp_path: Path) -> None: + ws = _project(tmp_path, _GOOD) + assert bdd.native_only(_run(tmp_path, ws), {}).status == "passed" + (ws / "features").mkdir() + (ws / "features" / "borrowing.feature").write_text("Feature: Borrowing\n") + v = bdd.native_only(_run(tmp_path, ws), {}) + assert v.status == "failed" and "borrowing.feature" in v.details + + +def test_native_only_flags_a_configured_parser(tmp_path: Path) -> None: + ws = _project(tmp_path, _GOOD) + (ws / "robot.toml").write_text((ws / "robot.toml").read_text() + 'parsers = { GherkinParser = [] }\n') + v = bdd.native_only(_run(tmp_path, ws), {}) + assert v.status == "failed" and "robot.toml" in v.details + + +def test_step_names_absent_sees_the_runtime_sentence(tmp_path: Path) -> None: + # The file says "${count} books"; only the run shows "1 books". + steps = _STEPS + '\n${member} has ${count: int} books\n [Documentation] x\n No Operation\n' + suite = _GOOD.replace(" Then the loan is granted", " And Mia has 1 books\n Then the loan is granted") + ws = _ws(tmp_path, {"resources/steps.resource": steps, "tests/borrowing.robot": suite}) + run = _run(tmp_path, ws) + v = bdd.step_names_absent(run, {"path": "tests", "regex": r"(? None: + ws = _project(tmp_path, _GOOD.replace("Then the loan is granted", "Then Fail broken")) + v = bdd.step_names_absent(_run(tmp_path, ws), {"path": "tests", "regex": "x"}) + assert v.status == "failed" and "run failed" in v.details + + +# --- review_findings --------------------------------------------------------------- + +_SEEDED = ["L1", "L2", "W1", "W2|C5", "W3", "L6", "W12|W6|W7"] + + +def _transcript(run: Run, text: str, *, result: bool = True) -> None: + events = [{"type": "assistant", "message": {"role": "assistant", "content": [{"type": "text", "text": text}]}}] + if result: + events.append({"type": "result", "subtype": "success", "result": text}) + (run.artifacts_dir / "stdout.stream.jsonl").write_text("\n".join(json.dumps(e) for e in events) + "\n") + + +_FINDINGS = """\ +Here is the review. + +- L1 — tests/loans.robot:8 — `Register Member` in the test body. +- **L2** – tests/loans.robot:11 – procedural UI step. +### S3 · W1 — tests/loans.robot:10 — conjunction step. +1. W3/W12 — tests/loans.robot:7-9 — placeholders and a procedural name. +""" + + +def _review(tmp_path: Path, text: str | None, **files: str) -> tuple[Run, dict[str, object]]: + ws = _ws(tmp_path, files, variant="review") + run = _run(tmp_path, ws) + if text is not None: + _transcript(run, text) + return run, {"fixture": "sut-bdd-library/review", "seeded": _SEEDED, "min_found": 4} + + +def test_review_findings_counts_findings_lines(tmp_path: Path) -> None: + run, params = _review(tmp_path, _FINDINGS) + v = bdd.review_findings(run, params) + assert v.status == "passed", v.details + # "W3/W12" names two seeded issues + assert "found 5/7" in v.details and "W12|W6|W7" in v.details and "missed=['C5|W2', 'L6']" in v.details + + +def test_review_findings_ignores_ids_outside_the_format(tmp_path: Path) -> None: + prose = ("L1: library calls in tests/loans.robot. W1 and W3 apply too, see line 10.\n" + "L2 — the UI step — rewrite it\n" + "- W2 — loans.robot — vague\n") + run, params = _review(tmp_path, prose) + v = bdd.review_findings(run, params) + assert v.status == "failed" + assert "0 findings lines" in v.details + + +def test_review_findings_needs_min_found(tmp_path: Path) -> None: + run, params = _review(tmp_path, _FINDINGS) + v = bdd.review_findings(run, {**params, "min_found": 6}) + assert v.status == "failed" and "need 6" in v.details + + +def test_review_findings_fails_on_any_edit(tmp_path: Path) -> None: + run, params = _review(tmp_path, _FINDINGS, **{"tests/loans.robot": "*** Test Cases ***\n"}) + v = bdd.review_findings(run, params) + assert v.status == "failed" and "modified tests/loans.robot" in v.details + run2, _ = _review(tmp_path / "2", _FINDINGS, **{"REVIEW.md": "notes"}) + v2 = bdd.review_findings(run2, params) + assert v2.status == "failed" and "added REVIEW.md" in v2.details + + +def test_review_findings_ignores_tool_output_and_agent_dirs(tmp_path: Path) -> None: + run, params = _review(tmp_path, _FINDINGS, **{ + "results/output.xml": "", ".claude/skills/x/SKILL.md": "x", + ".robotcode_cache/c.db": "x", "sut/__pycache__/LibraryService.cpython-312.pyc": "x", + }) + assert bdd.review_findings(run, params).status == "passed" + + +def test_review_findings_uses_the_last_text_without_a_result_line(tmp_path: Path) -> None: + run, params = _review(tmp_path, None) + _transcript(run, _FINDINGS, result=False) + assert bdd.review_findings(run, params).status == "passed" + + +def test_review_findings_skipped_without_transcript(tmp_path: Path) -> None: + run, params = _review(tmp_path, None) + assert bdd.review_findings(run, params).status == "skipped" + + +def test_finding_lines_parse_ids() -> None: + lines = bdd.finding_lines(_FINDINGS) + assert [sorted(ids) for ids, _ in lines] == [["L1"], ["L2"], ["W1"], ["W12", "W3"]] + + +def test_checks_resolve_through_custom_python(tmp_path: Path) -> None: + ws = _project(tmp_path, _GOOD) + for func in ("structure", "living_docs", "native_only"): + v = check_custom_python(_run(tmp_path, ws), func, + {"func_ref": f"rf_skill_eval.scoring.custom.bdd:{func}", "path": "tests"}) + assert v.status == "passed", (func, v.details) + + +# --- fixture ----------------------------------------------------------------------- + + +def test_fixture_variants_are_the_base_plus_their_files() -> None: + base = {p.relative_to(_FIXTURE / "base") for p in (_FIXTURE / "base").rglob("*") if p.is_file()} + shared = {p for p in base if p.name not in ("README.md", ".gitkeep")} + for variant in ("steps", "review"): + root = _FIXTURE / variant + for rel in shared: + assert (root / rel).read_bytes() == (_FIXTURE / "base" / rel).read_bytes(), (variant, rel) + assert not [p for p in (_FIXTURE / "base" / "tests").iterdir() if p.name != ".gitkeep"] + assert (_FIXTURE / "steps" / "resources" / "borrowing_steps.resource").is_file() + review = (_FIXTURE / "review" / "tests" / "loans.robot").read_text() + for seeded in ("Register Member user1", "presses borrow", "is registered and the library has", + "Then everything is fine", "Check stuff"): + assert seeded in review, seeded + + +def test_review_variant_is_unchanged_against_itself() -> None: + assert bdd.changed_files(_FIXTURE / "review", _FIXTURE / "review") == [] diff --git a/tests/eval/test_eval_content.py b/tests/eval/test_eval_content.py index 3f93210..0d226ef 100644 --- a/tests/eval/test_eval_content.py +++ b/tests/eval/test_eval_content.py @@ -149,12 +149,15 @@ def test_adversarial_tasks_are_tier_adversarial() -> None: ("sut-selenium", "tests/example.robot"), ("sut-api", "tests"), ("sut-browser", "tests/example.robot"), + ("sut-bdd-library/base", "smoke"), + ("sut-bdd-library/steps", "tests"), + ("sut-bdd-library/review", "tests"), ], ) def test_fixture_smoke_suite_passes(tmp_path: Path, fixture: str, target: str) -> None: from rf_skill_eval.scoring.deterministic import check_robot_pass - ws = tmp_path / fixture + ws = tmp_path / fixture.replace("/", "-") shutil.copytree(_FIXTURES / fixture, ws) now = datetime.now(UTC) run = Run( @@ -522,3 +525,83 @@ def test_legacy_syntax_bad_solution_fails_on_depr_checks(tmp_path: Path) -> None assert by_type["file_not_contains"].status == "failed" if shutil.which("robocop") is not None: assert by_type["lint_clean"].status == "failed" + + +# --- rf-bdd tasks (add-rf-bdd-skill, tasks 5.1-5.3) --------------------------------- + +BDD_NARROW = [ + "narrow-bdd-greenfield-01", + "narrow-bdd-outline-01", + "narrow-bdd-extend-vocabulary-01", + "narrow-bdd-review-01", +] +BDD_ADVERSARIAL = ["adv-bdd-gherkin-row-01"] +BDD_TASKS = BDD_NARROW + BDD_ADVERSARIAL + + +@pytest.mark.parametrize("task_id", BDD_TASKS) +def test_bdd_task_is_graded_for_rf_bdd(task_id: str) -> None: + task = _task(task_id) + assert task.skill == "rf-bdd" + assert task.tier == ("adversarial" if task_id.startswith("adv-") else "narrow") + assert task.fixture and task.fixture.split("/")[0] == "sut-bdd-library" + gating = task.gating_checks + assert gating + # No LLM judge and no transcript-only check gates a task (D10). + assert all(c.type not in ("tool_call_count", "tool_result_count", "tool_call_sequence") for c in gating) + refs = {str(c.params.get("func_ref", "")).rsplit(":", 1)[-1] for c in gating} + if task_id == "narrow-bdd-review-01": + assert refs == {"review_findings"} + else: + assert {"structure", "living_docs", "native_only"} <= refs + robot = [c for c in gating if c.type == "robot_pass"] + assert robot and robot[0].params.get("expected_tests") + + +@pytest.mark.parametrize("task_id", BDD_TASKS) +def test_bdd_gate_fails_on_unmodified_fixture(tmp_path: Path, task_id: str) -> None: + sc = _grade(tmp_path, task_id, None) + if task_id == "narrow-bdd-review-01": + # No transcript without a session: the review check can't decide. + assert sc.gate_result == "incomplete" + return + assert all(v.status in ("passed", "failed") for v in sc.gating_verdicts), [ + (v.check_name, v.status, v.reason) for v in sc.gating_verdicts + ] + assert sc.gate_result == "fail" + + +@pytest.mark.parametrize("task_id", BDD_TASKS) +def test_bdd_reference_solution_passes(tmp_path: Path, task_id: str) -> None: + sc = _grade(tmp_path, task_id, _GOLDEN / task_id / "good") + problems = [(v.check_name, v.status, v.details) for v in sc.verdicts if not v.passed] + assert sc.gate_result == "pass", problems + # The reference solutions also pass the reported (non-gating) checks. + assert not [p for p in problems if p[0] != "loaded_rf_bdd_skill"], problems + + +@pytest.mark.parametrize("task_id", BDD_TASKS) +def test_bdd_naive_solution_fails(tmp_path: Path, task_id: str) -> None: + sc = _grade(tmp_path, task_id, _GOLDEN / task_id / "bad") + assert sc.gate_result == "fail", [(v.check_name, v.status, v.details) for v in sc.verdicts] + + +def test_bdd_gherkin_row_kept_as_1_days_fails_only_on_the_sentence(tmp_path: Path) -> None: + sc = _grade(tmp_path, "adv-bdd-gherkin-row-01", _GOLDEN / "adv-bdd-gherkin-row-01" / "bad") + by_name = {v.check_name: v for v in sc.gating_verdicts} + assert by_name["no_step_reads_1_days"].status == "failed" + assert "1 days" in by_name["no_step_reads_1_days"].details + assert [n for n, v in by_name.items() if not v.passed] == ["no_step_reads_1_days"] + + +def test_bdd_review_that_edits_fails_although_the_findings_are_right(tmp_path: Path) -> None: + sc = _grade(tmp_path, "narrow-bdd-review-01", _GOLDEN / "narrow-bdd-review-01" / "bad") + review = next(v for v in sc.verdicts if v.check_name == "review_reports_seeded_issues_without_edits") + assert review.status == "failed" and "modified tests/loans.robot" in review.details + + +def test_bdd_naive_vocabulary_fails_on_reuse(tmp_path: Path) -> None: + sc = _grade(tmp_path, "narrow-bdd-extend-vocabulary-01", + _GOLDEN / "narrow-bdd-extend-vocabulary-01" / "bad") + failed = {v.check_name for v in sc.gating_verdicts if not v.passed} + assert failed == {"reuses_borrow_sentence", "reuses_refusal_sentence"}, failed diff --git a/tests/eval/test_trigger_harness.py b/tests/eval/test_trigger_harness.py index 5cd3a20..8474203 100644 --- a/tests/eval/test_trigger_harness.py +++ b/tests/eval/test_trigger_harness.py @@ -641,7 +641,7 @@ def behaviour(task, profile, replicate): # type: ignore[no-untyped-def] "--split", "validation", "--runs", "1", "--output", str(out), "--baseline", str(tmp_path / "none.json")]) assert res.exit_code == 0, res.output - assert staged_dirs == {frozenset({"rf-language", "rf-setup", "rf-python-library", "rf-browser", "rf-selenium"})} + assert staged_dirs == {frozenset({"rf-language", "rf-bdd", "rf-setup", "rf-python-library", "rf-browser", "rf-selenium"})} data = json.loads((out / "trigger-results.json").read_text()) assert data["staged"] == ["rf-core", "rf-web"] and "rf-requests" not in data["staged_skills"] partial = json.loads((out / "partial-install.json").read_text()) diff --git a/tests/eval/test_trigger_sets.py b/tests/eval/test_trigger_sets.py index af3d57c..49f2451 100644 --- a/tests/eval/test_trigger_sets.py +++ b/tests/eval/test_trigger_sets.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from pathlib import Path import pytest @@ -143,3 +144,21 @@ def test_readme_documents_discipline_and_is_linked() -> None: assert "Never edit validation queries" in readme assert "train" in readme and "validation" in readme assert "plugins/rf-core/evals/triggers/README.md" in (plugin_evals_dir(_REPO, "all") / "tasks" / "README.md").read_text() + + +def test_bdd_set_has_the_required_near_misses() -> None: + # add-rf-bdd-skill spec "Trigger query set for rf-bdd" + data = yaml.safe_load((_TRIGGERS / "rf-bdd.yaml").read_text()) + queries = data["queries"] + negatives = [q for q in queries if not q["should_trigger"]] + notes = {q.get("note", "") for q in negatives} + for sibling in ("sibling rf-language", "sibling rf-python-library", "sibling robotcode-plugin"): + assert sibling in notes, sibling + text = " | ".join(q["query"] for q in negatives) + for look_alike in ("Cucumber step definitions in Java", "pytest-bdd", "SpecFlow"): + assert look_alike in text, look_alike + for split in ("train", "validation"): + for polarity in (True, False): + assert any(q["split"] == split and q["should_trigger"] is polarity for q in queries) + skill_ids = re.compile(r"\b(rf-[a-z-]+|robotcode-plugin)\b|\brobotcode\b", re.IGNORECASE) + assert not [q["id"] for q in queries if skill_ids.search(q["query"])] diff --git a/tests/installer/test_plugin_selection.py b/tests/installer/test_plugin_selection.py index 550b8c6..5852d06 100644 --- a/tests/installer/test_plugin_selection.py +++ b/tests/installer/test_plugin_selection.py @@ -30,7 +30,7 @@ def test_plugins_flag_selects_plugins(two_plugin_bundle: Path, fake_home: Path, prefix = tmp_path / "root" assert main(["install", "--agent", "claude-code", "--prefix", str(prefix), "--yes", "--plugins", "rf-core,rf-web,rf-example"]) == 0 - assert _skills(prefix) == {"rf-language", "rf-setup", "rf-python-library", "rf-browser", "rf-selenium", + assert _skills(prefix) == {"rf-language", "rf-bdd", "rf-setup", "rf-python-library", "rf-browser", "rf-selenium", "rf-example"} @@ -42,7 +42,7 @@ def test_default_skips_the_bundle_and_installs_every_content_plugin( (path,) = list(tmp_path.rglob("installed.json")) files = [f for ins in json.loads(path.read_text())["installations"] for f in ins["files"]] assert {f["plugin"] for f in files} == {"rf-core", "rf-web", "rf-api", "rf-mobile", "rf-desktop"} - assert len(_skills(prefix)) == 9 + assert len(_skills(prefix)) == 10 def test_removed_bundle_is_an_unknown_plugin(two_plugin_bundle: Path, fake_home: Path, tmp_path: Path, diff --git a/tests/installer/test_reinstall_prune.py b/tests/installer/test_reinstall_prune.py index 07ac871..72f1c38 100644 --- a/tests/installer/test_reinstall_prune.py +++ b/tests/installer/test_reinstall_prune.py @@ -213,7 +213,7 @@ def test_upgrade_removes_managed_robotcode_overlap_skills_and_agent(fake_home: P assert not path.exists(), path for name in ("rf-libdoc", "rf-results", "rf-robotcode"): assert not (skills / name).exists(), name - assert len([d for d in skills.iterdir() if d.name.startswith("rf-")]) == 9 + assert len([d for d in skills.iterdir() if d.name.startswith("rf-")]) == 10 agents = sorted(p.stem for p in (fake_home / ".claude" / "agents").glob("rf-*.md")) assert agents == ["rf-debug-expert", "rf-migration-guide", "rf-test-architect"] assert "5 removed" in capsys.readouterr().out diff --git a/tests/installer/test_split_upgrade.py b/tests/installer/test_split_upgrade.py index 8a17f89..56272b0 100644 --- a/tests/installer/test_split_upgrade.py +++ b/tests/installer/test_split_upgrade.py @@ -67,7 +67,7 @@ def test_file_mode_upgrade(agent: str, old_bundle: Path, fake_home: Path, tmp_pa project.mkdir() _old_install(project, agent, old_bundle, monkeypatch) skills_before = _skills(project) - assert len(skills_before) == 9 + assert len(skills_before) == 10 assert _install(project, agent) == 0 root = project / ROOTS[agent] @@ -104,7 +104,7 @@ def test_narrowing_is_explicit(old_bundle: Path, fake_home: Path, tmp_path: Path assert _install(project, "claude-code") == 0 capsys.readouterr() assert _install(project, "claude-code", "--plugins", "rf-core,rf-web") == 0 - assert _skills(project) == {"rf-language", "rf-setup", "rf-python-library", "rf-browser", "rf-selenium"} + assert _skills(project) == {"rf-language", "rf-bdd", "rf-setup", "rf-python-library", "rf-browser", "rf-selenium"} assert set(_record(project)) == {"rf-core", "rf-web"} out = " ".join(capsys.readouterr().out.split()) assert "skills rf-appium, rf-platynui, rf-requests, rf-restinstance" in out diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 0400ddf..ad60a13 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -61,10 +61,10 @@ def test_libraries_live_in_the_plugins() -> None: "robotframework-requests", "RESTinstance", "robotframework-PlatynUI"} def test_every_skill_has_one_home() -> None: - """plugin-split: the nine skills appear once each, in the plugins the table names.""" + """plugin-split: the ten skills appear once each, in the plugins the table names.""" homes = {md.parent.name: md.parent.parent.parent.name for md in REPO.glob("plugins/*/skills/*/SKILL.md")} split = { - "rf-language": "rf-core", "rf-setup": "rf-core", "rf-python-library": "rf-core", + "rf-language": "rf-core", "rf-bdd": "rf-core", "rf-setup": "rf-core", "rf-python-library": "rf-core", "rf-browser": "rf-web", "rf-selenium": "rf-web", "rf-requests": "rf-api", "rf-restinstance": "rf-api", "rf-appium": "rf-mobile", "rf-platynui": "rf-desktop"} # Other plugins (a community plugin, the template in CI) may add skills. diff --git a/tests/test_skill_descriptions.py b/tests/test_skill_descriptions.py index 28730df..49e6216 100644 --- a/tests/test_skill_descriptions.py +++ b/tests/test_skill_descriptions.py @@ -114,6 +114,7 @@ "rf-setup": ("install", "pip", "uv", "venv", "environment"), "rf-language": (".robot", ".resource", "test case", "suite", "keyword"), "rf-python-library": ("Python",), + "rf-bdd": ("BDD", "Given/When/Then"), } # Trigger terms each skill's "## When to use" block must carry (they no longer @@ -143,6 +144,16 @@ "listener", "Literal", ), + "rf-bdd": ( + "BDD style", + "Given/When/Then", + "executable specification", + "acceptance criteria", + "Gherkin", + "Cucumber", + "scenario outline", + "Background", + ), } # Boundary (spec "Sibling skills state their boundary"): texts the @@ -156,8 +167,9 @@ "rf-appium": ("rf-platynui", "rf-browser", "rf-selenium"), "rf-platynui": ("rf-appium", "rf-browser", "rf-selenium"), "rf-language": ("Python keyword libraries", "rf-python-library", "rf-browser", "rf-requests", - "robotcode plugin"), + "robotcode plugin", "rf-bdd"), "rf-python-library": ("rf-language", ".resource", "library skill"), + "rf-bdd": ("rf-language", "robotcode plugin"), } # Default wording when no library is named (user decision 2026-09-27), in the block. @@ -180,6 +192,7 @@ "`rf-language`", ), "python-library": ("Write or fix a Python keyword library or listener", "`rf-python-library`"), + "bdd": ("Write or review BDD (Given/When/Then) specifications and their step keywords", "`rf-bdd`"), } REQUIRED_ROWS: dict[str, tuple[str, ...]] = { @@ -190,8 +203,9 @@ "rf-restinstance": ("requests", "setup", "language"), "rf-platynui": ("setup", "language"), "rf-setup": ("browser", "selenium", "appium", "requests", "restinstance", "platynui", "language"), - "rf-language": ("browser", "requests", "setup", "python-library"), + "rf-language": ("browser", "requests", "setup", "python-library", "bdd"), "rf-python-library": ("language", "setup"), + "rf-bdd": ("language", "setup", "python-library"), } THREE_COLUMN_HEADERS: dict[str, tuple[str, ...]] = {} @@ -799,7 +813,8 @@ def test_when_to_use_block_names_siblings_and_terms() -> None: def test_language_block_defers_cli_work_to_robotcode_plugin() -> None: block = ("## When to use\n\n- Test Template, Given/When/Then, __init__.robot, [Arguments], embedded, " "Force Tags, Multiple keywords with name.\n\nNot this skill: Python keyword libraries use " - "rf-python-library; library keywords use rf-browser or rf-requests. CLI work: the robotcode plugin.\n") + "rf-python-library; library keywords use rf-browser or rf-requests; BDD specifications use rf-bdd. " + "CLI work: the robotcode plugin.\n") assert when_to_use_problems("rf-language", _skill_md(block)) == [] assert any("robotcode plugin" in p for p in when_to_use_problems( "rf-language", _skill_md(block.replace("the robotcode plugin", "elsewhere")))) From bc4d96d34db8fcbbcf3de8c265e37029d6bafdd7 Mon Sep 17 00:00:00 2001 From: Many Kasiriha Date: Fri, 20 Nov 2026 15:29:42 +0100 Subject: [PATCH 2/2] chore(eval): GLM 5.3 Flash baselines for the rf-bdd tasks (run 44705577004) Adds only the rf-bdd entries (narrow, adversarial, triggers) from the candidate of run 44705577004: N=6, both arms, turn factor 1.5, $4.54 of OpenRouter spend. Existing entries are unchanged. The review grader counted the library_lending.egg-info/ that `uv run` builds as an edit, so every treatment run of narrow-bdd-review-01 failed. It now ignores *.egg-info; the stored treatment pass rate (5/6) is the fixed grader applied to the same six runs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FHEmzL3ErAhGQykKWDx2wP --- openspec/changes/add-rf-bdd-skill/tasks.md | 3 +- .../openrouter/glm-5.3-flash/adversarial.json | 42 +++++ .../openrouter/glm-5.3-flash/narrow.json | 168 ++++++++++++++++++ .../openrouter/glm-5.3-flash/triggers.json | 30 ++++ src/rf_skill_eval/scoring/custom/bdd.py | 6 +- tests/eval/test_bdd_grader.py | 1 + 6 files changed, 247 insertions(+), 3 deletions(-) diff --git a/openspec/changes/add-rf-bdd-skill/tasks.md b/openspec/changes/add-rf-bdd-skill/tasks.md index b3d1fec..4088a1d 100644 --- a/openspec/changes/add-rf-bdd-skill/tasks.md +++ b/openspec/changes/add-rf-bdd-skill/tasks.md @@ -31,7 +31,8 @@ - [x] 5.3 Write the five task YAMLs (`narrow-bdd-greenfield-01`, `narrow-bdd-outline-01`, `narrow-bdd-extend-vocabulary-01`, `narrow-bdd-review-01`, `adv-bdd-gherkin-row-01`), each with `skill: rf-bdd`, fixture `sut-bdd-library` and gating deterministic checks. Verify with `uv run rf-skill-eval doctor` and the task-loader tests. - [x] 5.4 Write `plugins/rf-core/evals/triggers/rf-bdd.yaml`: ≥ 8 should-trigger and ≥ 8 should-not-trigger queries, both polarities in train and validation, negatives for rf-language syntax, Python library, results, Cucumber/Java, pytest-bdd and SpecFlow, and no skill ids in the queries. Verify with `python -m pytest tests/eval/test_trigger_sets.py tests/test_skill_descriptions.py`. - [ ] 5.5 Tune rf-bdd's description on the train split (at most 5 iterations), then measure validation for rf-bdd and the rf-core siblings under the default budget. Verify the acceptance rule: rf-bdd recall ≥ 0.80 and should-not accuracy ≥ 0.90 (or rule (b) recorded), and no sibling below its 1.1 baseline. Run holdout and the 40000-character condition once each and report them. -- [ ] 5.6 Run the five tasks on the CI model, with and without rf-bdd installed. Store the baselines under `plugins/rf-core/evals/baselines/`, and verify that the eval gate passes with the new baselines. +- [x] 5.6 Run the five tasks on the CI model, with and without rf-bdd installed. Store the baselines under `plugins/rf-core/evals/baselines/`, and verify that the eval gate passes with the new baselines. + - Note: run 44705577004 (GLM 5.3 Flash, N=6, both arms, narrow and adversarial, $4.54). Only the rf-bdd entries were added to `baselines/openrouter/glm-5.3-flash/`. Treatment vs baseline arm: greenfield 6/6 vs 0/6, outline 6/6 vs 0/6, extend-vocabulary 6/6 vs 6/6, review 5/6 vs 0/6, gherkin-row 4/6 vs 0/6. The run's review grader counted the `library_lending.egg-info/` that `uv run` builds as an edit and failed every treatment run; the grader now ignores `*.egg-info`, and the review treatment pass rate (5/6) is the fixed grader applied to the same six transcripts and workspaces. Trigger baseline for rf-bdd from the same run (train 14/14, validation 7/8, false positive: SpecFlow in C#). rf-language's `lang-t14` (step keywords for Given/When/Then lines) now loads rf-bdd and rf-language only 1/3; the stored rf-language trigger baseline is unchanged. ## 6. Version, catalogue and docs diff --git a/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/adversarial.json b/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/adversarial.json index e966188..b0e1555 100644 --- a/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/adversarial.json +++ b/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/adversarial.json @@ -1,6 +1,48 @@ { "claude_code_version": "2.1.325 (Claude Code)", "entries": { + "adv-bdd-gherkin-row-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.012076, + "mean_duration_s": 103.93, + "mean_input_tokens": 345792.8, + "mean_output_tokens": 16608.2, + "mean_turns": 17.0, + "model": "z-ai/glm-5.3-flash", + "outcome_pass_rate": 0.0, + "pass_rate": 0.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-bdd", + "task_hash": "8aa7fa392de1dd2ef2c3f300f9b2a505081a9e8be9e1ef7be971c2a1298c0ff3", + "task_id": "adv-bdd-gherkin-row-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "adv-bdd-gherkin-row-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": false, + "incomplete": 0, + "mean_cost_usd": 0.054922, + "mean_duration_s": 340.92, + "mean_input_tokens": 1682189.3, + "mean_output_tokens": 41505.5, + "mean_turns": 46.67, + "model": "z-ai/glm-5.3-flash", + "outcome_pass_rate": 0.6667, + "pass_rate": 0.6667, + "provider": "openrouter", + "runs": 6, + "skill": "rf-bdd", + "task_hash": "8aa7fa392de1dd2ef2c3f300f9b2a505081a9e8be9e1ef7be971c2a1298c0ff3", + "task_id": "adv-bdd-gherkin-row-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, "adv-language-force-tags-01|baseline": { "arm": "baseline", "cost_source": "priced", diff --git a/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/narrow.json b/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/narrow.json index 958d7fc..7e4b968 100644 --- a/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/narrow.json +++ b/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/narrow.json @@ -1,6 +1,174 @@ { "claude_code_version": "2.1.325 (Claude Code)", "entries": { + "narrow-bdd-extend-vocabulary-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.012894, + "mean_duration_s": 128.86, + "mean_input_tokens": 390569.5, + "mean_output_tokens": 10806.5, + "mean_turns": 21.33, + "model": "z-ai/glm-5.3-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-bdd", + "task_hash": "ae7e267d68d486d877257532b1fd87d61454e288c9037b0e3b4f67d1daaa22bd", + "task_id": "narrow-bdd-extend-vocabulary-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-bdd-extend-vocabulary-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.021254, + "mean_duration_s": 121.43, + "mean_input_tokens": 652880.3, + "mean_output_tokens": 15447.0, + "mean_turns": 25.17, + "model": "z-ai/glm-5.3-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-bdd", + "task_hash": "ae7e267d68d486d877257532b1fd87d61454e288c9037b0e3b4f67d1daaa22bd", + "task_id": "narrow-bdd-extend-vocabulary-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-bdd-greenfield-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.020591, + "mean_duration_s": 124.55, + "mean_input_tokens": 627902.0, + "mean_output_tokens": 16739.3, + "mean_turns": 27.17, + "model": "z-ai/glm-5.3-flash", + "outcome_pass_rate": 0.0, + "pass_rate": 0.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-bdd", + "task_hash": "c5c073b27cb71122a4451bd52248005165816e025ba345fb73b66168827d14bb", + "task_id": "narrow-bdd-greenfield-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-bdd-greenfield-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.027216, + "mean_duration_s": 201.3, + "mean_input_tokens": 832894.3, + "mean_output_tokens": 21910.0, + "mean_turns": 31.0, + "model": "z-ai/glm-5.3-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-bdd", + "task_hash": "c5c073b27cb71122a4451bd52248005165816e025ba345fb73b66168827d14bb", + "task_id": "narrow-bdd-greenfield-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-bdd-outline-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.014835, + "mean_duration_s": 104.33, + "mean_input_tokens": 440259.0, + "mean_output_tokens": 15134.2, + "mean_turns": 20.17, + "model": "z-ai/glm-5.3-flash", + "outcome_pass_rate": 0.0, + "pass_rate": 0.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-bdd", + "task_hash": "9560feb6c2658c4763f9ae672b4e78d3269600914e5d9aa588a1a39babe892e0", + "task_id": "narrow-bdd-outline-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-bdd-outline-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.067143, + "mean_duration_s": 281.01, + "mean_input_tokens": 2122457.7, + "mean_output_tokens": 34666.0, + "mean_turns": 47.33, + "model": "z-ai/glm-5.3-flash", + "outcome_pass_rate": 1.0, + "pass_rate": 1.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-bdd", + "task_hash": "9560feb6c2658c4763f9ae672b4e78d3269600914e5d9aa588a1a39babe892e0", + "task_id": "narrow-bdd-outline-01", + "turn_limit_stops": 2, + "turns_factor": 1.5 + }, + "narrow-bdd-review-01|baseline": { + "arm": "baseline", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.00243, + "mean_duration_s": 31.18, + "mean_input_tokens": 66729.2, + "mean_output_tokens": 3115.7, + "mean_turns": 6.0, + "model": "z-ai/glm-5.3-flash", + "outcome_pass_rate": 0.0, + "pass_rate": 0.0, + "provider": "openrouter", + "runs": 6, + "skill": "rf-bdd", + "task_hash": "5c9fe14104dbb12c1d53b3b4fbc18784d47be2609a89e9b593421e9376061375", + "task_id": "narrow-bdd-review-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, + "narrow-bdd-review-01|treatment": { + "arm": "treatment", + "cost_source": "priced", + "gating": true, + "incomplete": 0, + "mean_cost_usd": 0.007814, + "mean_duration_s": 52.18, + "mean_input_tokens": 234331.8, + "mean_output_tokens": 6160.7, + "mean_turns": 15.5, + "model": "z-ai/glm-5.3-flash", + "outcome_pass_rate": 0.8333, + "pass_rate": 0.8333, + "provider": "openrouter", + "runs": 6, + "skill": "rf-bdd", + "task_hash": "5c9fe14104dbb12c1d53b3b4fbc18784d47be2609a89e9b593421e9376061375", + "task_id": "narrow-bdd-review-01", + "turn_limit_stops": 0, + "turns_factor": 1.5 + }, "narrow-language-bdd-01|baseline": { "arm": "baseline", "cost_source": "priced", diff --git a/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/triggers.json b/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/triggers.json index ac02f7a..75f1159 100644 --- a/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/triggers.json +++ b/plugins/rf-core/evals/baselines/openrouter/glm-5.3-flash/triggers.json @@ -5,6 +5,36 @@ "provider": "openrouter", "schema_version": 1, "skills": { + "rf-bdd": { + "train": { + "accuracy": 1.0, + "fn": 0, + "fp": 0, + "negative_loads": 2, + "negative_runs": 21, + "positive_loads": 21, + "positive_runs": 21, + "precision": 1.0, + "queries": 14, + "recall": 1.0, + "tn": 7, + "tp": 7 + }, + "validation": { + "accuracy": 0.875, + "fn": 0, + "fp": 1, + "negative_loads": 3, + "negative_runs": 12, + "positive_loads": 12, + "positive_runs": 12, + "precision": 0.8, + "queries": 8, + "recall": 1.0, + "tn": 3, + "tp": 4 + } + }, "rf-language": { "train": { "accuracy": 0.8, diff --git a/src/rf_skill_eval/scoring/custom/bdd.py b/src/rf_skill_eval/scoring/custom/bdd.py index 4e7314f..7dc4934 100644 --- a/src/rf_skill_eval/scoring/custom/bdd.py +++ b/src/rf_skill_eval/scoring/custom/bdd.py @@ -40,7 +40,8 @@ #: Settings a BDD test may carry besides its steps. _ALLOWED_SETTINGS = ("Documentation", "Tags", "Setup", "Teardown", "Template", "Timeout") -#: Directories never part of the project's own files (agent tooling, caches, run output). +#: Directories never part of the project's own files (agent tooling, caches, run output, +#: and the ``*.egg-info`` that ``uv run`` builds for the fixture project). _IGNORED_DIRS = {"results", "__pycache__", "node_modules", "grader-out"} _IGNORED_FILES = {"output.xml", "log.html", "report.html"} @@ -66,7 +67,8 @@ def _name(params: dict[str, Any], default: str) -> str: def _hidden(rel: Path) -> bool: - return any(part.startswith(".") or part in _IGNORED_DIRS for part in rel.parts[:-1]) + return any(part.startswith(".") or part in _IGNORED_DIRS or part.endswith(".egg-info") + for part in rel.parts[:-1]) def _robot_files(ws: Path, params: dict[str, Any]) -> list[Path]: diff --git a/tests/eval/test_bdd_grader.py b/tests/eval/test_bdd_grader.py index 9bc7b86..bcd413d 100644 --- a/tests/eval/test_bdd_grader.py +++ b/tests/eval/test_bdd_grader.py @@ -309,6 +309,7 @@ def test_review_findings_ignores_tool_output_and_agent_dirs(tmp_path: Path) -> N run, params = _review(tmp_path, _FINDINGS, **{ "results/output.xml": "", ".claude/skills/x/SKILL.md": "x", ".robotcode_cache/c.db": "x", "sut/__pycache__/LibraryService.cpython-312.pyc": "x", + "library_lending.egg-info/PKG-INFO": "x", }) assert bdd.review_findings(run, params).status == "passed"