diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-16-git-merge-plan-drops-non-conflicting.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-16-git-merge-plan-drops-non-conflicting.json new file mode 100644 index 0000000000..fc837ed196 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-16-git-merge-plan-drops-non-conflicting.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli", "date": "2026-11-16", "refs": ["ako/mxcli#2325"], "ce": [], "symptom": "`mxcli git merge p4 --plan` with \"take\": \"theirs\" on the conflicted Shop domain model committed a merge without Shop.OursOnly, an entity only ours had added and nobody conflicted on. The review listed it under Yours; nothing said the choice would drop it.", "file": "cmd/mxcli/cmd_git_merge.go wholeUnitDropReport / unitChanges / onlyIn; called in mergeRun's conflict stage and in applyPlan before the units are taken. Test: cmd/mxcli/gitmerge_whole_unit_drop_2325_test.go", "cause": "The plan resolves per unit by checking the chosen side's .mxunit out whole, so every change of the other side to that unit goes; the property report is per element and did not say the resolution is not.", "insight": "**The two mx diff files already hold the answer: a side's top-level change descriptions in a unit that the other side does not also list are exactly what taking the other side drops without a conflict.** Same description on both sides = the conflict itself (the choice), so it is not reported. The review prints both directions per unit before the plan is written; --plan prints only the chosen direction, before the commit. Element-level resolution (take per element) would need writing a merged unit, which the plan deliberately avoids (it checks a side's bytes out whole); warning is the honest scope. Live on 11.12.6 with real mx merge + mx diff: the review names Entity 'Shop.OursOnly' for theirs and 'Shop.TheirsOnly' for ours, and the theirs plan warns about OursOnly before 'Model merge committed'. Control: two sides changing only the same element report nothing."} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-16-traceability-hint-names-missing-flag.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-16-traceability-hint-names-missing-flag.json new file mode 100644 index 0000000000..a088cd1191 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-16-traceability-hint-names-missing-flag.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli", "date": "2026-11-16", "symptom": "`mxcli report traceability` on a brain with no plan tells you to run `mxcli brain promote --all`, which fails with 'unknown flag: --all'", "cause": "The hint was written from memory of a flag promote never had; nothing checked the commands an error message tells you to copy", "file": "`cmd/mxcli/cmd_report_kinds.go` (noRequirementsError)", "insight": "A copy-paste hint is code the user runs: test it against the cobra command tree (Find + Flags().Lookup) so a renamed or imagined flag fails the build, not the user. Found by following the hint during an end-to-end run", "refs": [], "test": "TestNoRequirementsHintUsesRealFlags"} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-17-fix-widgets-mismatched-mx.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-17-fix-widgets-mismatched-mx.json new file mode 100644 index 0000000000..7bb8893e52 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-17-fix-widgets-mismatched-mx.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli", "date": "2026-11-17", "refs": ["ako/mxcli#2321"], "ce": [], "symptom": "`mxcli fix widgets` on macOS with an 11.12.2 project failed: mx update-widgets 'Project is in different Mendix version - 11.12.2' (exit 4). The same project and command worked in a Linux container with --mx pointing at MxBuild's mx.", "file": "cmd/mxcli/cmd_fix.go runFixTool / resolveFixMx; cmd/mxcli/docker/check.go LocalMxForVersion. Test: cmd/mxcli/fix_mx_resolve_2321_test.go", "cause": "runFixTool resolved mx with docker.ResolveMxForVersion, which treats the project's version as a preference: an mx on PATH comes first, then the newest Studio Pro install or any cached mx. update-widgets and rename-design-properties only work on their own version, so any fallback is a guaranteed failure.", "insight": "**For a tool that edits the project in its own version's format, the version is a requirement, like `mxcli new` (ResolveMxForNewProject), never a preference.** resolveFixMx takes an explicit --mx as given, else an exact-version local mx, else (Linux) downloads that version, else (macOS/Windows, where the CDN's Linux mx cannot run) refuses naming the version and the --mx route. Not measured on macOS (none here): the fix is the resolution order, tested by injected lookups; on Linux an 11.12.6 project still resolves the cached 11.12.6 mx and fix widgets runs. The issue's other half — `replace with` writing a stale definition (CE0463) — did not reproduce with stock pluggable widgets (combo box, pop-up menu replaced inside a container on 11.12.6: no CE0463); it needs the tester's dropdown-menu widget."} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-17-parallel-writers-lose-updates.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-17-parallel-writers-lose-updates.json new file mode 100644 index 0000000000..b830918694 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-17-parallel-writers-lose-updates.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli", "date": "2026-11-17", "refs": ["ako/mxcli#2326"], "ce": [], "symptom": "Four `mxcli exec` processes creating one entity each on one project in parallel all failed ('disk I/O error (5898)', 'replaced on disk while unwritten changes were held', a temp-file rename error) and rolled back. Measured here (11.12.6 blank project, 8 parallel scripts of 2 entities + 1 association each, 3 rounds): every process exited 0 and the project held 10, 14 and 14 of the 16 entities — updates lost with success reported.", "file": "internal/projectlock/lock.go Acquire / BusyError; cmd/mxcli/project_write_lock.go programWritesModel / acquireProjectWriteLock (exit 75); wired in cmd/mxcli/cmd_exec.go and cmd/mxcli/oneliner.go. Test: internal/projectlock/lock_test.go", "cause": "A write run is many unit files plus the .mpr index, read at connect and written later, with a rollback that restores backups on failure; nothing serialised two processes, so each wrote over a project the other had changed, or rolled back over it.", "insight": "**The reported failure was the lucky case: the measured one is a silent lost update.** Four writers in this environment never collided; eight with an association each did, every round, and every process said it succeeded — so the control has to count what landed, not exit codes. The fix is a per-project O_EXCL lock file (.mxcli/write.lock, portable to Windows) held for the whole exec / -c run — not per write, since a failed run's rollback is itself a write — with a 2s heartbeat; a lock whose same-host pid is dead, or whose heartbeat stopped 30s ago, is taken over, so os.Exit paths that skip the deferred release cannot strand the project. The second writer waits up to 60s, then exits 75 (EX_TEMPFAIL) naming the holder. Programs of only show/describe/select/... take no lock. With the lock: 16/16 entities and 8/8 associations in all three rounds, mx check 0 errors. Not locked: the REPL session, and other writing commands (fix widgets, marketplace)."} diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-17-run-restart-refusal-names-no-flag.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-17-run-restart-refusal-names-no-flag.json new file mode 100644 index 0000000000..bd9913dba9 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-17-run-restart-refusal-names-no-flag.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli", "date": "2026-11-17", "refs": ["ako/mxcli#2313"], "ce": [], "symptom": "A run started with `--constant 'Encryption.EncryptionKey=@env:HELPDESK_ENC_KEY' --constant 'Helpdesk.OpenRouterKey=@env:OPENROUTER_KEY'` (and other arguments elided as '...') could not be restarted: `run restart` printed the generic '--db-password @env:' hint, which read as 'constant references are not replayed'.", "file": "cmd/mxcli/run_redact.go redactedArgNames; cmd/mxcli/cmd_run_lifecycle.go restartSecretRefusal. Test: cmd/mxcli/run_restart_redacted_names_2313_test.go", "cause": "The constant references are kept by redactArgs (since 21a9e0676) and replayed; with exactly the reported --constant arguments nothing is redacted and restart proceeds (measured through detachArgs + redactArgs). What restart refuses is a value stored redacted, and its message named none, so one literal flag among several references was indistinguishable from the references failing.", "insight": "**Reproduce the reported arguments through the same code path before believing the reported cause.** detachArgs + redactArgs on the issue's command line keep both @env: constants and redact nothing, so the refusal must have come from an elided literal (or a state file from an older binary). The fix is the message: it now lists each redacted flag by name ('--constant Helpdesk.OpenRouterKey', '--db-password', '--runtime-setting DatabasePassword') and says the reference values are replayed. Not booted live: the claim rests on the state-file arguments, which are what restart replays."} diff --git a/.claude/skills/fix-issue/findings/mdl-backend/2026-11-16-alter-navigation-insert-not-rerunnable.json b/.claude/skills/fix-issue/findings/mdl-backend/2026-11-16-alter-navigation-insert-not-rerunnable.json new file mode 100644 index 0000000000..bbbc10808e --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-backend/2026-11-16-alter-navigation-insert-not-rerunnable.json @@ -0,0 +1 @@ +{"area": "mdl/backend", "date": "2026-11-16", "refs": ["ako/mxcli#2312"], "ce": [], "symptom": "`alter navigation Responsive { insert { menu item 'Helpdesk chat' ( OnClick: show page Helpdesk.Chat ) } };` run twice left two 'Helpdesk chat' items, and the second run reported 'Altered navigation Responsive' instead of Unchanged.", "file": "mdl/backend/menumutator/mutator.go InsertMenuItems / withoutItemsAlreadyThere / sameMenuItem. Test: mdl/backend/menumutator/insert_rerun_2312_test.go", "cause": "InsertMenuItems spliced the built items in unconditionally; insert is the one menu operation whose second application is not a no-op.", "insight": "**An insert is re-runnable when 'already there' means 'already there as this statement builds it', compared without element ids and with the stored captions kept to the languages the statement writes.** Same caption and same content in the target list: dropped, so the patch write is elided and the run reports Unchanged. Same caption, different content (action, sub-items): refused naming `replace`, because inserting leaves two items of one caption and skipping drops the change. Comparing by caption alone would have silently ignored an edited action on re-run. Live on 11.12.6: run 1 Altered, run 2 Unchanged, one item; the edited OnClick is refused before anything is written; mx check 0 errors."} diff --git a/.claude/skills/fix-issue/findings/mdl-backend/2026-11-16-move-entity-lost-on-refused-target.json b/.claude/skills/fix-issue/findings/mdl-backend/2026-11-16-move-entity-lost-on-refused-target.json new file mode 100644 index 0000000000..82c66eb2bf --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-backend/2026-11-16-move-entity-lost-on-refused-target.json @@ -0,0 +1 @@ +{"area": "mdl/backend", "date": "2026-11-16", "refs": ["ako/mxcli#2323"], "ce": [], "symptom": "`exec --continue-on-error` of a MOVE ENTITY whose target write was refused reported the statement failed, and the entity was in neither module: removed from the source, not in the target. 12 chained moves on a generated project lost 9 entities and mx check then reported 53 'no longer exists' errors.", "file": "mdl/backend/modelsdk/association_move_write.go MoveEntity / refusedContents. Test: mdl/backend/modelsdk/issue2323_move_atomic_test.go", "cause": "MoveEntity persisted the source domain model, then built and persisted the target; a refusal of the target came after the source was on disk. The default exec rolls the whole script back, so only --continue-on-error kept the half-done move.", "insight": "**A statement that writes two units must decide both before writing either.** Both domain models are now built and encoded first and checked against the refusals the write path makes on content alone (canon.DuplicateElementIDError, canon.BareAttributeRefError); a target write that fails anyway restores the source's stored bytes with UpdateRawUnitPatch (no transplant onto the restore). The test provokes the refusal the way #2322 did — an element in the target with the moved entity's $ID — and without the fix it reports exactly the symptom ('entity Account is gone from Administration'). With #2322 fixed the reported chain no longer reaches the refusal, which is why the test plants one."} diff --git a/.claude/skills/fix-issue/findings/mdl-backend/2026-11-16-splice-edit-drops-other-translations.json b/.claude/skills/fix-issue/findings/mdl-backend/2026-11-16-splice-edit-drops-other-translations.json new file mode 100644 index 0000000000..1ef18c4859 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-backend/2026-11-16-splice-edit-drops-other-translations.json @@ -0,0 +1 @@ +{"area": "mdl/backend", "date": "2026-11-16", "refs": ["ako/mxcli#2327", "mendixlabs/mxcli#1870"], "ce": [], "symptom": "`create or modify microflow` that changed one show message's text reported `spliced: 1 edited in place`, and the message kept only en_US; its nl_NL translation was deleted. Measured on Administration.ChangePassword of the 11.12.6 blank project.", "file": "mdl/backend/mfmutator/splice_edit.go swapKeeping / carryTranslations. Test: mdl/backend/mfmutator/splice_edit_translations_2327_test.go", "cause": "The in-place edit serializes the activity from the statement (one language per text) and saves through UpdateRawUnitPatch, which passes ContentsOwnTranslations, so canon's translation carry never runs on it. Other unstatable properties of an edited activity already had their own carry (carryUnstated, #2169); translations had none.", "insight": "**A patch write opts out of every canon carry, so each thing a splice rebuilds from MDL needs its own.** The splice writes as a patch on purpose (a carry would undo a deliberate removal), but its in-place edit is a rebuild of one activity; it gets canon.CarryTranslations applied to that activity alone (stored activity vs replacement, paired by path), then unmarkRemoved on the result so the carried stored translation elements are not refused as removed. Control: an edit of the only language lands unchanged; without the carry the nl_NL assertion fails."} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-16-create-or-modify-insert-not-rerunnable.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-16-create-or-modify-insert-not-rerunnable.json new file mode 100644 index 0000000000..f307503d1a --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-16-create-or-modify-insert-not-rerunnable.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-16", "refs": ["ako/mxcli#2311"], "ce": [], "symptom": "A `create or modify microflow` that inserted statements succeeded ('spliced: 1 inserted'), and the second run of the same script was refused: 'the connectors of the DeleteObject at (1125, 200) are redrawn (its @anchor, @curve or @merge)' (MDL-V1-REBUILD). The refusal's own advice, drop + create in one script, rewrites the flow on every run. Measured on Administration.ChangePassword, 11.12.6 blank project.", "file": "mdl/executor/cmd_flow_modify.go patchDiff.before / statedCurves; mdl/executor/cmd_alter_flow.go flowVectors; mdl/backend/microflow_mutation.go MicroflowFragment.InCurve/OutCurve, FlowVectors; mdl/backend/mfmutator/splice.go spliceOnFlow; mdl/executor/flow_verdict.go deliberateRebuildAdvice. Tests: mdl/backend/mfmutator/splice_stated_curves_2311_test.go, mdl/executor/issue2311_insert_stated_curves_test.go", "cause": "The splice drew the connector into the inserted run and the one out of it with its own control vectors (sideVector, +/-15), where a create of the script draws them as its @curve states and straight ('0;0', which describe omits) where none is stated. The next diff compared the stated curve with the stored one and called the connector redrawn.", "insight": "**A splice that a re-run cannot recognise is not done: after any create-or-modify write, run the same script again and expect Unchanged.** That is how this was found: the first run of the reproduction succeeded, the second refused. The fix keeps the splice's own vectors for `alter` (which states no curves) and, for create or modify, passes the curve on the declared statement before the run (connector in) and on the run's last statement (connector out), only where both are plain statements. Live: run 1 spliced, runs 2 and 3 Unchanged, mx check 0 errors. The reporter's second refusal ('the stored statement \"Join\" has no activity to address') did not reproduce on flows mxcli wrote — describe prints their merges as @merge, and adding a handler before one spliced and re-ran Unchanged — so it needs their flow; the deliberate-rebuild advice now says to run drop + create once and keep the resulting create or modify, which is the reporter's own workaround."} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-16-grant-where-beside-open-rule-narrows-nothing.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-16-grant-where-beside-open-rule-narrows-nothing.json new file mode 100644 index 0000000000..91168f99af --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-16-grant-where-beside-open-rule-narrows-nothing.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-16", "refs": ["ako/mxcli#2320"], "ce": [], "symptom": "`grant read * on entity Shop.Customer to Shop.Clerk where [Name != '']` on a role that already had an unconstrained rule reported 'Granted access' and added a second rule; the role still read every row. exec, check and lint said nothing.", "file": "mdl/executor/cmd_security_write.go grantOverlapWarning; mdl/executor/validate_access_rule_overlap.go (ValidateAccessRuleOverlap no-project, CheckAccessRuleOverlap --references); wired in validate_program.go and cmd/mxcli/cmd_check.go. Test: mdl/executor/issue2320_access_rule_overlap_test.go", "cause": "A rule is keyed by role set plus constraint (#936), so a grant with a WHERE is a new rule, not a narrowing; access rules add up at runtime. That is correct model behaviour, but nothing said that the constraint limited nothing.", "insight": "**Report each overlap once, by where its other half comes from.** The check tracks per entity and role whether an unconstrained and a constrained rule are held and whether each came from the project or the script; the no-project pass reports script-vs-script overlaps and the --references pass only those involving a stored rule, so `check -p --references` does not print the same warning twice. `revoke all` clears the role (the documented way to narrow); a partial revoke changes rights, not which rules exist, so it clears nothing. exec prints the warning as a notice, so it still shows on a re-run that writes nothing (the overlap is still there). Live on 11.12.6: check --references and exec both warn; revoke all + grant where leaves one constrained rule and mx check reports 0 errors. Control: a constrained grant to a role with no unconstrained rule warns nothing."} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-16-declare-set-return-type-not-checked.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-16-declare-set-return-type-not-checked.json new file mode 100644 index 0000000000..de50e76a35 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-16-declare-set-return-type-not-checked.json @@ -0,0 +1 @@ +{"area": "mdl/other", "date": "2026-11-16", "refs": ["ako/mxcli#1258"], "ce": ["CE0117", "CE0844"], "symptom": "`declare $X Integer = 'abc';`, `set $X = 'abc';` and `return $Decimal` from a microflow that returns String all passed `mxcli check -p` and `exec`, then MxBuild reported CE0117 \"Error(s) in expression\" (Mendix CLI 2: CE0844 \"The expression is of type String but should be of type Integer/Long\").", "file": "mdl/exprcheck/target_kind.go checkTargetKind; Context.Target in mdl/exprcheck/interfaces.go; mdl/exprcheck/adapters/check.go checkExprAs / primitiveTarget / setTargetKind. Tests: mdl/exprcheck/target_kind_test.go TestTargetKindMatrix, mdl/exprcheck/adapters/target_kind_test.go", "cause": "The slot table declared DeclareStmt.InitialValue (DeclareType), MfSetStmt.Value (TargetVariable) and ReturnStmt.Value (MicroflowReturn) with a ResolveBy, and checkSlotKind skips every slot with a ResolveBy, so nothing compared the value with the declared type. Same gap #2096 closed for a call argument.", "insight": "**A ResolveBy entry in the slot table that nothing resolves is a rule that looks present and checks nothing; grep the ResolveBy values against what the adapter actually qualifies.** Measured before writing the rule: 216 microflows (3 positions x 6 primitive types x 12 value shapes) in one project, one `mx check -j` on 11.16.1, results keyed by the JSON document-name so one run answers the whole matrix. All three positions behave identically, every rejection is CE0117, and the accepted set equals callArgAccepted, so the predicate is reused and the matrix is the test. `empty` into a Boolean is also CE0117 and is deliberately left silent, as for a call argument. Overlap: MDL041 already reports a COMPUTED Decimal into Integer/Long by DECLARE/SET, so those are skipped here (a bare Decimal literal or variable, and any RETURN, are not MDL041's). False-positive sweep: a throwaway test parsed every .mdl in the repo and ran the adapter with no project, 1649 flows in 1267 scripts, 0 hits after the MDL041 skip (1 before: the intentional negative test bug-tests/f2-decimal-func-into-integer.fail.mdl). `check -p` is blind to this on a script that also has an unresolved reference, so a sweep through the CLI against a stand-in project reports zero for the wrong reason; run the adapter directly. Control: stubbing checkTargetKind to return nil fails TestTargetKindMatrix, the wiring test and the nanoflow test."} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-16-replace-is-not-an-expression-function.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-16-replace-is-not-an-expression-function.json new file mode 100644 index 0000000000..d3325b5804 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-16-replace-is-not-an-expression-function.json @@ -0,0 +1 @@ +{"area": "mdl/other", "date": "2026-11-16", "refs": ["mendixlabs/mxcli#2308", "ako/mxcli#1258"], "ce": ["CE0117", "CE7811"], "symptom": "`return replace($S, 'a', 'b');` passed `mxcli check -p` and `exec`, then MxBuild reported CE0117 \"Error(s) in expression\" (Mendix CLI 2: CE7811 \"Unknown function 'replace'\"). `mxcli syntax microflow.expressions.functions` listed `replace(String, String, String)`.", "file": "mdl/exprcheck/func_checker.go funcTable (the `replace` row, removed); mdl/exprcheck/unknown_funcs.go nearestFunc (sorted candidates). Test: mdl/exprcheck/unknown_funcs_test.go TestUnknownFunctionCalls; fixture mdl-examples/check-gap/fixtures/f09_badfunc.mdl on the check-gap branch", "cause": "funcTable is documented as every Mendix built-in and UnknownFunctionCalls (MDL044) trusts it, so one row for a function Mendix does not have makes the call valid to mxcli and the generated syntax docs. The string replacers are replaceAll and replaceFirst.", "insight": "**A wrong row in funcTable is a false negative in MDL044 and a false claim in the generated docs at once, and the fastest way to find one is to run a function name through an independent checker, not to read the table.** Found by the check-gap parity run (one broken script per miss, mxcli check -p vs mx check vs mendix check): only 1 of the 5 misses was a missing rule, this one was a wrong data row. Fixing it exposed a second defect: nearestFunc iterated the map, so replace (one edit from both replaceAll and replaceFirst) suggested one or the other at random per run; it now walks sorted names. Control: replaceAll and replaceFirst still pass; revert the row removal and TestUnknownFunctionCalls fails with the reported symptom (expected unknown func, got none)."} diff --git a/.claude/skills/fix-issue/findings/mdl-visitor/2026-11-16-visible-roles-and-expression-stored-as-text.json b/.claude/skills/fix-issue/findings/mdl-visitor/2026-11-16-visible-roles-and-expression-stored-as-text.json new file mode 100644 index 0000000000..e5cb453861 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-visitor/2026-11-16-visible-roles-and-expression-stored-as-text.json @@ -0,0 +1 @@ +{"area": "mdl/visitor", "date": "2026-11-16", "refs": ["ako/mxcli#2319"], "ce": [], "symptom": "`container c1 (Visible: roles (Shop.Admin) and $currentObject/Secret = 'x')` passed check and exec; the container's ConditionalVisibilitySettings stored Expression \"roles (Shop.Admin) and $currentObject/Secret = 'x'\" with ModuleRoles empty, and describe printed the input back.", "file": "mdl/visitor/visible_roles_split.go splitLeadingVisibleRoles (called in visitor_page_v3.go for `Visible: `); mdl/executor/validate_widgets.go exprCallsRoles / validateWidgetExpressionRoles (MDL-WIDGET54); mdl/executor/cmd_pages_builder_visible_when.go refuseRolesInExpressions (exec, also via childVisibility). Test: mdl/executor/issue2319_visible_roles_and_test.go", "cause": "The supported spelling is two `Visible:` properties (`Visible: roles (…)` has its own grammar rule). Combined with `and`, `roles (Shop.Admin)` parses as a function call inside an ordinary expression, so the visitor stored the whole text as VisibleIf.", "insight": "**When a grammar keyword can also parse as a function call, the combined form silently falls into the expression rule.** Rather than refuse the natural spelling, the leading `roles (…) and ` is split into ModuleRoles + Expression, which is exactly Studio Pro's 'visible for module roles' beside a condition (both apply). It declines on a top-level `or` in the rest, since `roles (A) and x or y` is not that. Any other `roles (` outside a string literal in DynamicClasses/Visible/Editable is MDL-WIDGET54 at check and refused at exec. Live on 11.12.6: ModuleRoles [Shop.Admin], Expression \"$currentObject/Secret = 'x'\", describe prints the two-property form, mx check 0 errors."} diff --git a/.claude/skills/fix-issue/findings/modelsdk/2026-11-16-transplant-gives-converted-assoc-neighbours-id.json b/.claude/skills/fix-issue/findings/modelsdk/2026-11-16-transplant-gives-converted-assoc-neighbours-id.json new file mode 100644 index 0000000000..b8237f3071 --- /dev/null +++ b/.claude/skills/fix-issue/findings/modelsdk/2026-11-16-transplant-gives-converted-assoc-neighbours-id.json @@ -0,0 +1 @@ +{"area": "modelsdk", "date": "2026-11-16", "refs": ["ako/mxcli#2322"], "ce": ["CE9208"], "symptom": "Moving Src.A1, Src.A2, Src.A3, Src.A4 (an association chain) to Dst in that order: after the second move mx check reported CE9208 (cross-module association Src.A3_A2 and association Dst.A2_A1 'map to the same database table'), and the third move was refused: 'element id(s) are used more than once' held by [Association CrossAssociation] and [DeleteBehavior DeleteBehavior]. Descending order worked.", "file": "modelsdk/canon/transplant.go pairer.keepsOwnID / namedElementIDs / pairDoc. Test: modelsdk/canon/transplant_keeps_stored_id_2322_test.go", "cause": "Moving A2 converts Src.A3_A2 into a cross-association that stays in Src, keeping its $ID. Reconcile's TransplantIDs then aligned Src's CrossAssociations list ([A2_A1] stored, [A3_A2] written) by position and mapped A3_A2's $ID onto A2_A1's; on an mxcli-created model GUID == $ID, so the substitution rewrote the GUID too. Two associations then shared one identity in two units, and the next move brought them into one.", "insight": "**An element that already carries a stored $ID under the same name is that element; the alignment must not hand it another.** The positional fallback exists so a renamed element keeps its id, but here the written element was not new: its own $ID was in the stored unit, on an Association it had been retyped from, in another list. The pairer now skips such an element and its whole subtree (the delete behaviour under it was the second duplicate). The rule is restricted to named elements: canon's own tests deliberately reuse id values across unnamed flow objects (round-trip, patch control), and those keep the old pairing. The GUID loss is the dangerous half: CLAUDE.md's GUID rule says a changed association GUID drops its table on deploy, and the storage-GUID guard did not see it because it pairs on $ID + Name and the names differed. Live, 11.12.6 blank project: all four ascending moves succeed, every association keeps its original $ID, mx check 0 errors."} diff --git a/.claude/skills/fix-issue/findings/modelsdk/2026-11-16-transplant-misses-reordered-flow-objects.json b/.claude/skills/fix-issue/findings/modelsdk/2026-11-16-transplant-misses-reordered-flow-objects.json new file mode 100644 index 0000000000..824275a86e --- /dev/null +++ b/.claude/skills/fix-issue/findings/modelsdk/2026-11-16-transplant-misses-reordered-flow-objects.json @@ -0,0 +1 @@ +{"area": "modelsdk", "date": "2026-11-16", "refs": ["ako/mxcli#2329", "mendixlabs/mxcli#1872"], "ce": [], "symptom": "`drop microflow` + `create microflow` of an unchanged describe lost a parameter's documentation on a flow Studio Pro authored, but not on a copy mxcli had created. Reproduced on Administration.ChangePassword in a 10.24.11 and an 11.12.6 blank project after setting the parameter's Documentation in the stored unit.", "file": "modelsdk/canon/transplant.go alignSlices / pairLeftovers / lcsPairsPlaced; modelsdk/canon/translations.go carryTranslationsByID, samePlaces; modelsdk/canon/identity.go Reconcile. Test: modelsdk/canon/transplant_unordered_2329_test.go", "cause": "Studio Pro stores a flow's objects in drawing order; mxcli writes the parameters and the start event first. TransplantIDs aligned the list on the longest common run of activities, so the parameter and start event fell in a gap with nothing (or another $Type) opposite and got fresh $IDs; every carry keyed on $ID (unstatedFlowDefaults' parameter Documentation) then missed them. The same reorder let two show messages cross: the alignment by order paired each with the other.", "insight": "**`Objects` is a set stored as a list, so an ordered alignment is the wrong tool for it, and the position is the key a describe preserves.** Two changes to the alignment: entries left unpaired (or held by a positional pair of different $Types, which pairs nothing) pair by matchKey when it is unique among the leftovers on both sides; and the LCS scores a same-place match (RelativeMiddlePoint) 12 against 2, so same-place pairs win over a longer run of guesses and an insertion that moved everything aligns as before. The translation carry then gained a pass after the transplant keyed on the paired Texts$Text $ID, applied only when the pairing is confirmed (same owner place, or every written language agrees), and the existing by-path pass is refused when two texts at one path are owned by objects at different places. Measured: drop + create of all 16 stock 10.24.11 microflows kept 612 of 821 element ids before and 722 after; 0 mx check errors; a describe → create or modify of all 16 is still 'already in sync'. A live check is what found the crossing: changing one message's text in the drop + create carried the OTHER message's nl_NL onto it until the place broke the tie. The reporter's translation loss itself (10.24.11, four languages) did not reproduce on the stock project; the ambiguous-source and changed-source cases are what the by-$ID pass covers."} diff --git a/.claude/skills/fix-issue/findings/sdk/2026-11-17-java-action-core-without-import.json b/.claude/skills/fix-issue/findings/sdk/2026-11-17-java-action-core-without-import.json new file mode 100644 index 0000000000..b9a17f0d62 --- /dev/null +++ b/.claude/skills/fix-issue/findings/sdk/2026-11-17-java-action-core-without-import.json @@ -0,0 +1 @@ +{"area": "sdk", "date": "2026-11-17", "refs": ["ako/mxcli#2310"], "ce": [], "symptom": "`create java action … as $$ … Core.getConfiguration() … $$` with no `import com.mendix.core.Core;` passed check, exec and docker check ('0 errors'); the build (test --local / run --local, mxbuild --target=deploy) failed: 'RunSparql.java:56: error: cannot find symbol … variable Core'.", "file": "sdk/javaactions/source.go withCoreImport / javaCodeOnly (called from GenerateSource). Test: sdk/javaactions/core_import_2310_test.go", "cause": "The generator writes the imports the generated scaffolding needs (IContext, UserAction, IMendixObject) and the ones the statement states; Core, which almost every body uses, needed an explicit import line. mx check does not compile Java, so nothing before the build saw it.", "insight": "**mx check is no evidence for a Java action body: only mxbuild --target=deploy compiles it.** Measured on 11.12.6: with the generated `import com.mendix.core.Core;` the deploy build succeeds; deleting that one line makes the same build fail with the reported 'cannot find symbol' (the control). The import is added only when `Core.` appears as code (comments and string literals blanked) and the list imports no `*.Core;` or `com.mendix.core.*;` of its own; mxbuild keeps imports it finds, so a regeneration does not undo it. describe then states the import explicitly, which re-executes to the same file."} diff --git a/.claude/skills/mendix/create-page/SKILL.md b/.claude/skills/mendix/create-page/SKILL.md index 32c940c358..e788d8bb98 100644 --- a/.claude/skills/mendix/create-page/SKILL.md +++ b/.claude/skills/mendix/create-page/SKILL.md @@ -15,12 +15,12 @@ description: "CREATE PAGE syntax reference — parameters, variables, layouts, a | 47-222 | Syntax | | 223-293 | Basic Examples | | 294-318 | Modifying Existing Pages | -| 319-431 | Conditional Visibility and Editability | -| 432-601 | Known Limitations | -| 602-610 | Tips | -| 611-637 | Related Commands | -| 638-662 | PLUGGABLEWIDGET Escape Hatch | -| 663-666 | See Also | +| 319-435 | Conditional Visibility and Editability | +| 436-605 | Known Limitations | +| 606-614 | Tips | +| 615-641 | Related Commands | +| 642-666 | PLUGGABLEWIDGET Escape Hatch | +| 667-670 | See Also | # CREATE PAGE - MDL Syntax Guide @@ -389,6 +389,10 @@ create page MyModule.Request_Review (title: 'Review', layout: Atlas_Core.Atlas_D }; ``` +`visible: roles (Mod.Role) and ` is the same two settings in one line +(both apply). `roles (…)` anywhere else in an expression is refused (MDL-WIDGET54): +Mendix expressions have no roles function. + > **`visible:`/`editable:` is a Mendix *expression*, not XPath** — a different > function set from a datasource `where [ ... ]` clause: diff --git a/.claude/skills/mendix/manage-security/SKILL.md b/.claude/skills/mendix/manage-security/SKILL.md index cc10d5e388..7d6f42fffe 100644 --- a/.claude/skills/mendix/manage-security/SKILL.md +++ b/.claude/skills/mendix/manage-security/SKILL.md @@ -13,11 +13,11 @@ description: "Configure Mendix security in MDL — module and user roles, access | 27-35 | When to Use This Skill | | 36-44 | Security Concepts | | 45-121 | The System-module ceiling — decide the role model around this FIRST | -| 122-568 | Syntax Reference | -| 569-583 | Starlark Lint Rule APIs | -| 584-645 | Common Workflow: Setting Up Module Security | -| 646-655 | Common Mistakes | -| 656-665 | Validation | +| 122-570 | Syntax Reference | +| 571-585 | Starlark Lint Rule APIs | +| 586-647 | Common Workflow: Setting Up Module Security | +| 648-657 | Common Mistakes | +| 658-667 | Validation | # Security Management Skill @@ -301,7 +301,9 @@ A rule is identified by its role set **and** its `where` constraint. Two grants with the same constraint update one rule; with different constraints they make two, which Mendix combines at runtime (a role's access is the union of every rule naming it). So adding a `where` to an existing grant creates a second rule — it -does not narrow the first. +does not narrow the first. To narrow, `revoke all on entity … from ;` first, +then grant with the `where`. `check` and exec warn **MDL-GRANT03** when a role +ends up with a constrained rule beside an unconstrained one. ```sql mdl 1; diff --git a/.claude/skills/mendix/report-map/SKILL.md b/.claude/skills/mendix/report-map/SKILL.md index 22ab43fdcc..f1c4462e30 100644 --- a/.claude/skills/mendix/report-map/SKILL.md +++ b/.claude/skills/mendix/report-map/SKILL.md @@ -10,10 +10,11 @@ description: "Produce and use the software map (mxcli report map): one interacti | lines | section | |---|---| -| 25-38 | Run it | -| 39-54 | Give each module a purpose | -| 55-72 | Reading it — and its traps | -| 73-76 | Related | +| 26-39 | Run it | +| 40-55 | Give each module a purpose | +| 56-76 | The app by feature | +| 77-94 | Reading it — and its traps | +| 95-98 | Related | # Software Map @@ -52,6 +53,27 @@ Better still, put the sentence in the model, so every tool (and Studio Pro) sees `alter module Rooster set documentation 'Builds and publishes the monthly roster.';` Lint rule QUAL002 reports app modules that have none. +## The app by feature + +With a project brain that has a plan (`docs/brain/plan/`), the map adds a +**Features** tab: one feature per slice, with its requirements and their status, +its milestone, and what it reaches — derived from the requirements' anchors, never +listed by hand, so there is no separate feature map to keep up to date: + +- **flows:** the anchored flows and the requirements' footprints (★ in the graph), + the flows behind buttons of anchored pages, and their call closure; +- **pages:** anchored, or opened by those flows; **entities:** anchored, or written + by them; +- **entry points:** reached flows started from *outside* the feature (a schedule, a + service, the menu, another page's button); **roles:** those that may open one of + its pages; **shares with:** other features reaching the same documents. + +**Reached by no feature** lists the app's pages, entry points and persistent +entities no anchor leads to: unspecified, or reached only through navigation the +plan does not anchor. A thin feature means thin anchors — anchor the page and the +microflow a requirement is built in (`brain check` flags requirements no test can +reach). Group slices into milestones with `mxcli brain milestone`. + ## Reading it — and its traps - **Module map:** screen links (layout, show-page, menu) are off by default, and so diff --git a/.claude/skills/mendix/report-traceability/SKILL.md b/.claude/skills/mendix/report-traceability/SKILL.md index 6e53692fef..1f958d3300 100644 --- a/.claude/skills/mendix/report-traceability/SKILL.md +++ b/.claude/skills/mendix/report-traceability/SKILL.md @@ -10,10 +10,10 @@ description: "Produce and read the PRD-to-app traceability report (mxcli report | lines | section | |---|---| -| 25-69 | Preconditions | -| 70-80 | Run it | -| 81-98 | Reading it | -| 99-102 | Related | +| 25-74 | Preconditions | +| 75-85 | Run it | +| 86-103 | Reading it | +| 104-107 | Related | # Traceability Report @@ -57,16 +57,21 @@ built and proven?". Nothing on it is ticked off by hand. a module question keeps them on the decision it becomes. `brain plan --json` reports each requirement's effective `status`, which this report shows. -4. **Milestones** map slices (and single requirements) to the spec's milestones in - `docs/brain/plan/traceability.json`: +4. **Milestones** live in the plan, on the slices they group. Put slices — or a + single requirement apart from its slice — in a milestone: - ```json - {"milestones": [ - {"name": "M1 — accounts", "slices": ["01-accounts"]}, - {"name": "M2 — swaps", "slices": ["02-swaps"], "requirements": ["a1b2c3"]} - ]} + ```bash + mxcli brain milestone "M1 — accounts" --slice 01-accounts -p app.mpr + mxcli brain milestone "M2 — swaps" --slice 02-swaps --requirement a1b2c3 -p app.mpr ``` + That writes `milestone: "M1 — accounts"` in the slice file's frontmatter (a + ``Milestone: `M2 — swaps` `` line on a requirement). `brain plan` shows the + column, and milestones are ordered by first appearance in slice-name order. + A project with the older `docs/brain/plan/traceability.json` moves it with + `mxcli brain milestone --import docs/brain/plan/traceability.json`; until then + the report still reads it, with a warning. + ## Run it ```bash diff --git a/.claude/skills/mendix/run-local/SKILL.md b/.claude/skills/mendix/run-local/SKILL.md index c511fe62f3..6a646dfad1 100644 --- a/.claude/skills/mendix/run-local/SKILL.md +++ b/.claude/skills/mendix/run-local/SKILL.md @@ -114,8 +114,8 @@ What each one does, precisely: orphaned `runtimelauncher` or mxbuild JVM is stopped too — including the leftovers of a run that was killed with `-9` (and its `--test-endpoint`, as on Windows, where the stop is always forced). Stopping when nothing runs is a no-op (exit 0). -- **`run restart`** reuses the arguments recorded in the state file; to change a flag, - `run stop` and `--detach` again. +- **`run restart`** reuses the recorded arguments (`@env:`/`@file:` secrets as references; + a literal one is refused, by flag name); to change a flag, `run stop` and `--detach` again. - **`.mxcli/run-local.json`** (ports, admin password — how `oql`, `log` and `constant set --apply` reach the app) is written when the app serves and removed only by the run diff --git a/.claude/skills/mendix/write-microflows/reference/pitfalls.md b/.claude/skills/mendix/write-microflows/reference/pitfalls.md index 8048e66401..14036f380e 100644 --- a/.claude/skills/mendix/write-microflows/reference/pitfalls.md +++ b/.claude/skills/mendix/write-microflows/reference/pitfalls.md @@ -477,6 +477,12 @@ has 3 paths". Those change where the flow's paths end or meet; rebuild the flow deliberately (drop the microflow and create it in the same script) or edit it in Studio Pro. Inside a loop body a decision is the loop's to replace (§11). +Make that rebuild a **one-time run**, not part of the script you re-run: a drop + create +rewrites the flow every time it runs. Run it once, then keep the flow's final +`create or modify` in the project script — the second run of that finds it unchanged. +A refusal that says there is no room for new statements has a re-runnable fix of its +own: leave out the `@position` of what comes after them, so it moves along. + ### 16. A Message Template Is a String, Not an Expression With `with (…)` The text of a `log` message, a `send email` subject or body, a `show message` and a diff --git a/CHANGELOG.md b/CHANGELOG.md index caf39c78eb..db1c4a63a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`fix widgets` and `fix design-properties` use the project's own Mendix version of `mx`** (ako/mxcli#2321) — they took the version as a preference, so on macOS an `mx` on PATH or another Studio Pro install ran, and `mx update-widgets` refused with "Project is in different Mendix version". They now take an `mx` of exactly the project's version. On Linux that version is downloaded when it is missing. On macOS and Windows, where no matching `mx` is installed, they say which version is needed and how to pass it with `--mx`. +- **Parallel `mxcli exec` runs on one project no longer lose each other's writes** (ako/mxcli#2326) — measured with 8 parallel scripts: every process reported success and 2 to 6 of the 16 entities were missing (the reporter saw all of them fail instead). A script, or a `-c`, that writes now holds the project's lock (`.mxcli/write.lock`) for its whole run. A second writer waits up to 60 seconds, then changes nothing and exits 75, naming the holder. A lock left by a crashed run is taken over. Read-only runs take no lock. With the lock, all 16 entities land in every round. +- **`run restart` names the secret it cannot replay** (ako/mxcli#2313) — a restart refused because of a literal secret printed only a generic `--db-password @env:` hint. The `--constant X=@env:VAR` references are recorded and replayed, so the hint read as if they were not. The refusal now lists each flag stored redacted, such as `--constant Helpdesk.OpenRouterKey` or `--db-password`, and says that the values given as references are replayed. +- **A Java action whose code uses `Core` compiles** (ako/mxcli#2310) — the generated `.java` file did not import `com.mendix.core.Core`, so a body calling `Core.getConfiguration()` passed check, exec and docker check (mx check does not compile Java) and failed the build with "cannot find symbol". The import is now added when the code uses `Core` and imports no `Core` of its own. A deploy build on 11.12.6 succeeds with it and fails without it. +- **A `create or modify microflow` that inserts statements can run again** (ako/mxcli#2311) — the insert drew the connectors at the new statements' ends with vectors of its own, not as the script's `@curve` states. The next run of the same script read them as redrawn and refused (MDL-V1-REBUILD). The connectors are now drawn as the script states them, and the second run reports Unchanged. A splice refusal's advice now says to make a deliberate drop + create a one-time run, then keep the flow's `create or modify`, because a drop + create rewrites the flow every time. +- **`alter navigation … insert` can run again** (ako/mxcli#2312) — re-running a script that inserts a menu item added it a second time and reported "Altered". An item the list already holds, as the statement builds it, is now already there, and the second run reports Unchanged. An item with the same caption but another action or other sub-items is refused, naming `replace`. The same applies to `alter menu`. +- **`Visible: roles (…) and ` stores the roles and the condition** (ako/mxcli#2319) — the combined form parsed as one expression, so the whole text, `roles (…)` included, was stored as the visibility expression and the role list stayed empty: an invalid expression, and the role restriction lost, while check and exec passed. The leading `roles (…) and` is now split into the module roles and the condition, as Studio Pro stores them. A `roles (…)` anywhere else in an expression is refused (MDL-WIDGET54). +- **`git merge` says what a plan's whole-unit choice drops** (ako/mxcli#2325) — a plan takes each conflicted unit whole from one side, so `"take": "theirs"` on a domain model also discarded an entity only your branch had added, without a word. The conflict review now lists, per unit, the changes each choice drops that did not conflict, and `--plan` repeats it for the side it takes before committing. +- **`move entity` of an association chain works in any order, and keeps each association's identity** (ako/mxcli#2322) — moving `Src.A1` then `Src.A2` gave the cross-association `Src.A3_A2` the `$ID` of `Dst.A2_A1`, and with it the GUID, so mx check reported CE9208 and the next move was refused for duplicate ids. A converted association now keeps its own `$ID`, and the four moves in ascending order build with 0 errors. +- **A `move entity` whose target is refused writes nothing** (ako/mxcli#2323) — the source domain model was written before the target, so under `exec --continue-on-error` a refused target left the entity in neither module. Both domain models are now checked before either is written, and the source is restored if the target write fails anyway. +- **`check` and exec warn when a `grant … where` narrows nothing** (ako/mxcli#2320) — a grant with a `where` writes a second rule beside the role's unconstrained one, and access rules add up, so the role still reached every row while exec reported success and check passed. Both now warn **MDL-GRANT03** and name the fix: `revoke all on entity … from ;` first, then grant with the `where`. The reverse also warns, an unconstrained grant beside a constrained rule. `mxcli syntax security entity-access` now says that grants add rules. +- **Editing a show message with `create or modify` keeps its other translations** (ako/mxcli#2327, mendixlabs/mxcli#1870) — the in-place edit of the activity wrote only the language MDL states, so the message's nl_NL text was deleted. The stored languages the statement does not write are now carried onto the edited activity, as Studio Pro keeps them when the default text is edited. +- **`drop` + `create` of a Studio Pro flow keeps its parameters' documentation** (ako/mxcli#2329, mendixlabs/mxcli#1872) — Studio Pro stores a flow's objects in drawing order and mxcli writes the parameters first, so the rebuilt parameter was not paired with the stored one and its documentation was lost. Flow objects now pair across a different order, preferring the element drawn at the same place. A drop + create of all 16 microflows in a stock 10.24.11 project keeps 722 of 821 element ids, up from 612. Translations are also carried by that pairing, and only where it is confirmed: same place, or the same text in every language the statement writes. - **A `drop` + `create` of a flow keeps a while loop's condition caption** (ako/mxcli#2298, mendixlabs/mxcli#1869) — the caption Studio Pro shows on a while loop (`WhileLoopCondition.Caption`) has no MDL spelling, and a rebuild wrote it empty although #2178 already kept the loop's other unstatable properties. It is now carried from the stored loop like them. - **An agent's System context entity is stored with its id** (ako/mxcli#2292) — `create agent … (Entity: System.User)` wrote `"entity":{"documentId":"",…}` because the System module is not in the project, while Studio Pro stores the System entity's `$ID` (a Studio Pro 11.16.1 save stores System.Language as `83d81a7b-4a84-416e-a64f-0ffa981c8408`). The ids are now taken from a table measured from built models: 42 System entities have a fixed `$ID`, the same in 10.24.4 and 11.16 builds. The 12 whose `$ID` mxbuild mints each time the model loads (System.Thumbnail, PrivateFileDocument, BackgroundJob, …) still have none to write and stay by name. - **`run --local`, `test --local` and `docker build` no longer hand MxBuild a JRE** (ako/mxcli#2186, mendixlabs/mxcli#1839) — on Windows with a JRE 21 and a JDK 21 side by side and `JAVA_HOME` unset, the JRE (first on `PATH`) was chosen, and the build failed with "Java compiler is not available". A JRE reports the same `java -version`; the JDK lookup now also requires `bin/javac`, so the JRE is skipped and the JDK found. A `JAVA_HOME` that points at a JRE is skipped too, and when no JDK is found the error says it is a runtime without a compiler. diff --git a/cmd/mxcli/brain/check.go b/cmd/mxcli/brain/check.go index 3c3df4b546..3af1308b93 100644 --- a/cmd/mxcli/brain/check.go +++ b/cmd/mxcli/brain/check.go @@ -151,6 +151,8 @@ type SliceProgress struct { // Unanchored is requirements with no anchor at all. They cannot be // measured, and are counted apart rather than silently called planned. Unanchored int `json:"unanchored"` + // Milestone is the milestone the slice's frontmatter names, if any. + Milestone string `json:"milestone,omitempty"` } // Total is every requirement in the slice. Open questions are excluded: they @@ -287,6 +289,7 @@ func Check(s *Store, r Resolver, shards []string) (Report, error) { rep.Anchors += progress.anchors rep.ResolvedN += progress.resolved rep.Unverifiable += progress.unverifiable + progress.Milestone = s.SliceMilestone(SliceOf(shard)) rep.Slices = append(rep.Slices, progress.SliceProgress) rep.Open = append(rep.Open, progress.open...) rep.Requirements = append(rep.Requirements, progress.requirements...) diff --git a/cmd/mxcli/brain/milestone.go b/cmd/mxcli/brain/milestone.go new file mode 100644 index 0000000000..f638d867f6 --- /dev/null +++ b/cmd/mxcli/brain/milestone.go @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: Apache-2.0 + +// milestone.go - which milestone of the specification a slice belongs to. +// +// Milestones lived in a separate docs/brain/plan/traceability.json, the one +// input the traceability report could not derive. It was a second file about +// the same slices, which nothing kept in step with them: rename a slice and the +// file named one that no longer existed. A milestone is now part of the slice: +// +// - a key in the slice file's frontmatter, which the store already carries +// across every rewrite — +// +// --- +// milestone: "M1 — accounts" +// --- +// +// - and, for a requirement placed apart from its slice, a line in its body: +// +// Milestone: `M2 — approvals` +// +// Milestones are ordered by where they first appear, walking the slices in name +// order — the same order `brain plan` sequences a roadmap in. +package brain + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" +) + +// Milestone is one milestone and what it holds. +type Milestone struct { + Name string `json:"name"` + Slices []string `json:"slices,omitempty"` + Requirements []string `json:"requirements,omitempty"` +} + +var ( + frontmatterMilestone = regexp.MustCompile(`(?m)^milestone:[ \t]*(.*?)[ \t]*$`) + bodyMilestone = regexp.MustCompile("(?im)^\\s*milestone:\\s*`([^`]+)`\\s*$") +) + +// milestoneOfFrontmatter reads the milestone key out of a frontmatter block. +func milestoneOfFrontmatter(fm string) string { + m := frontmatterMilestone.FindStringSubmatch(fm) + if m == nil { + return "" + } + v := m[1] + switch { + case strings.HasPrefix(v, `"`): + var s string + if json.Unmarshal([]byte(v), &s) == nil { + return strings.TrimSpace(s) + } + case strings.HasPrefix(v, "'") && strings.HasSuffix(v, "'") && len(v) >= 2: + return strings.TrimSpace(strings.ReplaceAll(v[1:len(v)-1], "''", "'")) + } + return strings.TrimSpace(v) +} + +// withFrontmatterMilestone sets (or, with name "", removes) the milestone key +// in a frontmatter block, leaving every other line as it was. A block that +// ends up empty is dropped. +func withFrontmatterMilestone(fm, name string) string { + line := "milestone: " + yamlQuote(name) + if fm == "" { + if name == "" { + return "" + } + return "---\n" + line + "\n---\n" + } + inner := strings.TrimSuffix(strings.TrimPrefix(fm, "---\n"), "---\n") + inner = strings.TrimSuffix(inner, "---") + if frontmatterMilestone.MatchString(inner) { + if name == "" { + inner = frontmatterMilestone.ReplaceAllString(inner, "") + inner = strings.ReplaceAll(inner, "\n\n", "\n") + inner = strings.TrimLeft(inner, "\n") + } else { + inner = frontmatterMilestone.ReplaceAllLiteralString(inner, line) + } + } else if name != "" { + if inner != "" && !strings.HasSuffix(inner, "\n") { + inner += "\n" + } + inner += line + "\n" + } + if strings.TrimSpace(inner) == "" { + return "" + } + if !strings.HasSuffix(inner, "\n") { + inner += "\n" + } + return "---\n" + inner + "---\n" +} + +// yamlQuote writes a JSON string, which is a valid YAML double-quoted scalar, +// without escaping & < > the way json.Marshal does for HTML. +func yamlQuote(s string) string { + var b strings.Builder + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + _ = enc.Encode(s) + return strings.TrimSuffix(b.String(), "\n") +} + +// RequirementMilestone is the milestone a requirement's own body places it in, +// or "". +func RequirementMilestone(e Entry) string { + if m := bodyMilestone.FindStringSubmatch(e.Body); m != nil { + return strings.TrimSpace(m[1]) + } + return "" +} + +// withRequirementMilestone sets (or with "" removes) a requirement's milestone +// line. +func withRequirementMilestone(e Entry, name string) Entry { + body := strings.TrimSpace(bodyMilestone.ReplaceAllString(e.Body, "")) + body = regexp.MustCompile(`\n{3,}`).ReplaceAllString(body, "\n\n") + if name != "" { + line := "Milestone: `" + name + "`" + if body == "" { + body = line + } else { + body += "\n\n" + line + } + } + e.Body = body + return e +} + +// SliceMilestone is the milestone a slice's frontmatter names, or "". +func (s *Store) SliceMilestone(slice string) string { + return milestoneOfFrontmatter(s.Frontmatter(PlanShard(slice))) +} + +// SetSliceMilestone puts a slice in a milestone; name "" takes it out. The +// slice must exist: a slice file is only written with requirements in it. +func (s *Store) SetSliceMilestone(slice, name string) error { + if strings.ContainsAny(name, "\n`") { + return fmt.Errorf("a milestone name is one line without backticks: %q", name) + } + shard := PlanShard(slice) + entries, _, err := s.LoadShard(shard) + if err != nil { + return err + } + if len(entries) == 0 { + return fmt.Errorf("no slice %q: capture a requirement in it first ('mxcli brain capture … --slice %s')", slice, slice) + } + return s.saveShardWith(shard, entries, withFrontmatterMilestone(s.Frontmatter(shard), strings.TrimSpace(name))) +} + +// SetRequirementMilestone places one requirement in a milestone apart from its +// slice; name "" puts it back with its slice. +func (s *Store) SetRequirementMilestone(id, name string) error { + if strings.ContainsAny(name, "\n`") { + return fmt.Errorf("a milestone name is one line without backticks: %q", name) + } + e, shard, err := s.Find(id) + if err != nil { + return err + } + if shard == "" { + return fmt.Errorf("no entry %s", id) + } + if e.EntryKind() != KindRequirement || e.Open { + return fmt.Errorf("%s is not a requirement: only requirements belong to a milestone", id) + } + return s.Replace(shard, withRequirementMilestone(e, strings.TrimSpace(name))) +} + +// Milestones reads every milestone from the plan, in order of first +// appearance. A requirement whose body names a milestone is listed under +// Requirements of that milestone and not counted with its slice's. Slices in +// no milestone are not listed; nil means the plan names no milestone at all. +func (s *Store) Milestones() ([]Milestone, error) { + slices, err := s.ListSlices() + if err != nil { + return nil, err + } + var out []Milestone + at := map[string]int{} + get := func(name string) *Milestone { + i, ok := at[name] + if !ok { + out = append(out, Milestone{Name: name}) + i = len(out) - 1 + at[name] = i + } + return &out[i] + } + for _, sh := range slices { + slice := SliceOf(sh) + if name := s.SliceMilestone(slice); name != "" { + m := get(name) + m.Slices = append(m.Slices, slice) + } + entries, _, err := s.LoadShard(sh) + if err != nil { + return nil, err + } + for _, e := range entries { + if e.Open { + continue + } + if name := RequirementMilestone(e); name != "" { + m := get(name) + m.Requirements = append(m.Requirements, e.ID) + } + } + } + return out, nil +} + +// ImportMilestones writes a traceability.json's milestones into the plan: each +// named slice gets the milestone in its frontmatter, each named requirement a +// milestone line. It returns what it could not place (a slice or requirement +// the plan does not have), and changes nothing when the file names a slice +// twice, which the new form cannot express. +func (s *Store) ImportMilestones(ms []Milestone) (skipped []string, err error) { + seen := map[string]string{} + for _, m := range ms { + for _, sl := range m.Slices { + if prev, ok := seen[sl]; ok && prev != m.Name { + return nil, fmt.Errorf("slice %q is in two milestones (%q and %q); a slice belongs to one", sl, prev, m.Name) + } + seen[sl] = m.Name + } + } + for _, m := range ms { + for _, sl := range m.Slices { + if err := s.SetSliceMilestone(sl, m.Name); err != nil { + skipped = append(skipped, fmt.Sprintf("milestone %q: %v", m.Name, err)) + } + } + for _, id := range m.Requirements { + if err := s.SetRequirementMilestone(id, m.Name); err != nil { + skipped = append(skipped, fmt.Sprintf("milestone %q: %v", m.Name, err)) + } + } + } + return skipped, nil +} diff --git a/cmd/mxcli/brain/milestone_test.go b/cmd/mxcli/brain/milestone_test.go new file mode 100644 index 0000000000..f0a00e8306 --- /dev/null +++ b/cmd/mxcli/brain/milestone_test.go @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: Apache-2.0 + +package brain + +import ( + "os" + "reflect" + "strings" + "testing" +) + +func milestoneStore(t *testing.T) (*Store, map[string]Entry) { + t.Helper() + s := NewStore(t.TempDir()) + if _, err := s.Init(); err != nil { + t.Fatal(err) + } + reqs := map[string]Entry{} + for _, r := range []struct{ key, text, slice, anchor string }{ + {"login", "Users can log in", "01-accounts", "@Accounts.ACT_Login"}, + {"reset", "Users can reset a password", "01-accounts", "@Accounts.ACT_Reset"}, + {"offer", "Colleagues can offer a shift", "02-swaps", "@Swaps.ACT_Offer"}, + {"take", "Colleagues can take a shift", "02-swaps", "@Swaps.ACT_Take"}, + {"report", "Managers see swap totals", "03-reports", "@Swaps.Report"}, + } { + e := mustRequirement(t, r.text, r.slice, r.anchor) + if err := s.Promote(e, e.Shard()); err != nil { + t.Fatal(err) + } + reqs[r.key] = e + } + return s, reqs +} + +// A milestone is part of its slice, so it is written +// into the slice file, survives every rewrite of it, and comes back in the +// order the slices are sequenced. +func TestMilestonesLiveOnTheirSlices(t *testing.T) { + s, reqs := milestoneStore(t) + if ms, err := s.Milestones(); err != nil || ms != nil { + t.Fatalf("a plan without milestones has none: %v %v", ms, err) + } + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(s.SetSliceMilestone("02-swaps", "M2 — swaps & trades")) + must(s.SetSliceMilestone("01-accounts", "M1 — accounts")) + must(s.SetSliceMilestone("03-reports", "M2 — swaps & trades")) + // One requirement of slice 02 waits for the next milestone. + must(s.SetRequirementMilestone(reqs["take"].ID, "M3 — later")) + + b, _ := os.ReadFile(s.ShardPath(PlanShard("02-swaps"))) + if !strings.HasPrefix(string(b), "---\nmilestone: \"M2 — swaps & trades\"\n---\n") { + t.Errorf("the slice file carries its milestone in frontmatter, unescaped:\n%s", b) + } + // A later write to the slice keeps it. + extra := mustRequirement(t, "Colleagues can withdraw an offer", "02-swaps", "@Swaps.ACT_Withdraw") + must(s.Promote(extra, extra.Shard())) + + ms, err := s.Milestones() + must(err) + want := []Milestone{ + {Name: "M1 — accounts", Slices: []string{"01-accounts"}}, + {Name: "M2 — swaps & trades", Slices: []string{"02-swaps", "03-reports"}}, + {Name: "M3 — later", Requirements: []string{reqs["take"].ID}}, + } + if !reflect.DeepEqual(ms, want) { + t.Errorf("milestones:\n got %+v\nwant %+v", ms, want) + } + + // Clearing: the slice key goes, and other frontmatter stays. + must(s.SetRequirementMilestone(reqs["take"].ID, "")) + e, _, _ := s.Find(reqs["take"].ID) + if strings.Contains(e.Body, "Milestone") { + t.Errorf("cleared requirement still names a milestone: %q", e.Body) + } + path := s.ShardPath(PlanShard("01-accounts")) + b, _ = os.ReadFile(path) + must(os.WriteFile(path, []byte(strings.Replace(string(b), "---\n", "---\ntags: [auth]\n", 1)), 0644)) + must(s.SetSliceMilestone("01-accounts", "")) + b, _ = os.ReadFile(path) + if !strings.HasPrefix(string(b), "---\ntags: [auth]\n---\n") || strings.Contains(string(b), "milestone") { + t.Errorf("clearing must remove only the milestone key:\n%s", b) + } + must(s.SetSliceMilestone("02-swaps", "")) + b, _ = os.ReadFile(s.ShardPath(PlanShard("02-swaps"))) + if strings.HasPrefix(string(b), "---") { + t.Errorf("a frontmatter block with nothing left in it is dropped:\n%s", b) + } +} + +func TestMilestoneRefusals(t *testing.T) { + s, _ := milestoneStore(t) + if err := s.SetSliceMilestone("09-nothing", "M9"); err == nil { + t.Error("a slice with no requirements cannot be put in a milestone") + } + d := mustEntry(t, "Swaps go through a manager", "@Swaps.ACT_Offer") + if err := s.Promote(d, d.Shard()); err != nil { + t.Fatal(err) + } + if err := s.SetRequirementMilestone(d.ID, "M1"); err == nil { + t.Error("a decision is not in a milestone") + } + if err := s.SetSliceMilestone("01-accounts", "M`1"); err == nil { + t.Error("a backtick would break the requirement line form") + } +} + +// The old traceability.json moves into the plan; a slice in two milestones is +// refused rather than half-imported, and what cannot be placed is reported. +func TestImportMilestones(t *testing.T) { + s, reqs := milestoneStore(t) + skipped, err := s.ImportMilestones([]Milestone{ + {Name: "M1", Slices: []string{"01-accounts", "07-gone"}}, + {Name: "M2", Slices: []string{"02-swaps"}, Requirements: []string{reqs["report"].ID, "ffffff"}}, + }) + if err != nil { + t.Fatal(err) + } + if len(skipped) != 2 { + t.Errorf("want the missing slice and the unknown id reported, got %v", skipped) + } + ms, _ := s.Milestones() + want := []Milestone{{Name: "M1", Slices: []string{"01-accounts"}}, {Name: "M2", Slices: []string{"02-swaps"}, Requirements: []string{reqs["report"].ID}}} + if !reflect.DeepEqual(ms, want) { + t.Errorf("imported:\n got %+v\nwant %+v", ms, want) + } + + s2, _ := milestoneStore(t) + if _, err := s2.ImportMilestones([]Milestone{{Name: "A", Slices: []string{"01-accounts"}}, {Name: "B", Slices: []string{"01-accounts"}}}); err == nil { + t.Error("a slice in two milestones must be refused") + } + if ms, _ := s2.Milestones(); ms != nil { + t.Errorf("a refused import changes nothing: %+v", ms) + } +} + +func TestFrontmatterMilestoneQuoting(t *testing.T) { + for fm, want := range map[string]string{ + "---\nmilestone: M1\n---\n": "M1", + "---\nmilestone: \"M1 — a: b\"\n---\n": "M1 — a: b", + "---\nmilestone: 'it''s M2'\n---\n": "it's M2", + "---\ntitle: x\n---\n": "", + "---\nmilestone: \"R&D \\u0026 more\"\n---\n": "R&D & more", + } { + if got := milestoneOfFrontmatter(fm); got != want { + t.Errorf("%q: got %q, want %q", fm, got, want) + } + } +} diff --git a/cmd/mxcli/cmd_brain.go b/cmd/mxcli/cmd_brain.go index be1ec75b85..28a04ac49a 100644 --- a/cmd/mxcli/cmd_brain.go +++ b/cmd/mxcli/cmd_brain.go @@ -778,13 +778,25 @@ func printBrainPlan(slices []brain.SliceProgress) { } } var built, total int - fmt.Printf("%-*s %8s %8s %12s\n", width, "SLICE", "BUILT", "PLANNED", "UNANCHORED") + withMilestones := false + for _, sl := range slices { + withMilestones = withMilestones || sl.Milestone != "" + } + header := fmt.Sprintf("%-*s %8s %8s %12s", width, "SLICE", "BUILT", "PLANNED", "UNANCHORED") + if withMilestones { + header += " MILESTONE" + } + fmt.Println(header) for _, sl := range slices { un := "" if sl.Unanchored > 0 { un = fmt.Sprint(sl.Unanchored) } - fmt.Printf("%-*s %8d %8d %12s\n", width, sl.Slice, sl.Built, sl.Planned, un) + line := fmt.Sprintf("%-*s %8d %8d %12s", width, sl.Slice, sl.Built, sl.Planned, un) + if withMilestones { + line += " " + sl.Milestone + } + fmt.Println(strings.TrimRight(line, " ")) built += sl.Built total += sl.Total() } @@ -1329,7 +1341,7 @@ func brainLocatedCommands() []*cobra.Command { brainInitCmd, brainCaptureCmd, brainStagedCmd, brainPromoteCmd, brainDropCmd, brainShowCmd, brainCheckCmd, brainPlanCmd, brainResolveCmd, brainBriefCmd, brainEditCmd, brainMoveCmd, - brainListCmd, + brainListCmd, brainMilestoneCmd, } } diff --git a/cmd/mxcli/cmd_brain_milestone.go b/cmd/mxcli/cmd_brain_milestone.go new file mode 100644 index 0000000000..83943b72a6 --- /dev/null +++ b/cmd/mxcli/cmd_brain_milestone.go @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/mendixlabs/mxcli/cmd/mxcli/brain" + "github.com/spf13/cobra" +) + +var brainMilestoneCmd = &cobra.Command{ + Use: "milestone [] [--slice ]… [--requirement ]…", + Short: "Group slices into the specification's milestones", + Long: `Put slices — or single requirements — in a milestone of the specification. + +A milestone is part of the slice it groups: the slice file's frontmatter gets +'milestone: ""', and a requirement placed apart from its slice gets a +"Milestone: ` + "``" + `" line. 'brain plan' shows them, and +'report traceability' and 'report map' group by them. + +Milestones are ordered by where they first appear, walking the slices in name +order (01-…, 02-…). + +With no name, lists the milestones. --clear takes the slices or requirements +out of their milestone. --import reads the traceability.json the traceability +report used to take, writes it into the plan, and removes the file.`, + Example: ` mxcli brain milestone "M1 — accounts" --slice 01-accounts --slice 02-profiles + mxcli brain milestone "M2 — approvals" --requirement a1b2c3 + mxcli brain milestone --clear --slice 02-profiles + mxcli brain milestone --import docs/brain/plan/traceability.json + mxcli brain milestone`, + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + store := brain.NewStore(brainProjectDir(cmd)) + if !store.Exists() { + brainFatal(fmt.Errorf("no brain here; 'mxcli brain init' creates one")) + } + slices, _ := cmd.Flags().GetStringArray("slice") + reqs, _ := cmd.Flags().GetStringArray("requirement") + clear, _ := cmd.Flags().GetBool("clear") + imp, _ := cmd.Flags().GetString("import") + + switch { + case imp != "": + if len(args) > 0 || len(slices) > 0 || len(reqs) > 0 || clear { + brainFatal(fmt.Errorf("--import takes no name, --slice, --requirement or --clear")) + } + runMilestoneImport(store, imp) + return + case len(args) == 0 && !clear: + if len(slices) > 0 || len(reqs) > 0 { + brainFatal(fmt.Errorf("name the milestone, or pass --clear")) + } + ms, err := store.Milestones() + if err != nil { + brainFatal(err) + } + if globalJSONFlag { + brainJSON(map[string]any{"milestones": ms}) + return + } + if len(ms) == 0 { + fmt.Println("No milestones. Group slices with:") + fmt.Println(" mxcli brain milestone \"M1 — \" --slice 01-") + return + } + for _, m := range ms { + fmt.Println(m.Name) + for _, sl := range m.Slices { + fmt.Printf(" slice %s\n", sl) + } + for _, id := range m.Requirements { + fmt.Printf(" requirement %s\n", id) + } + } + return + } + if len(slices) == 0 && len(reqs) == 0 { + brainFatal(fmt.Errorf("say what goes in it: --slice or --requirement ")) + } + name := "" + if !clear { + name = args[0] + } else if len(args) > 0 { + brainFatal(fmt.Errorf("--clear takes no milestone name")) + } + for _, sl := range slices { + if err := store.SetSliceMilestone(sl, name); err != nil { + brainFatal(err) + } + reportMilestoneChange("slice "+sl, name) + } + for _, id := range reqs { + if err := store.SetRequirementMilestone(id, name); err != nil { + brainFatal(err) + } + reportMilestoneChange("requirement "+id, name) + } + }, +} + +func reportMilestoneChange(what, name string) { + if globalJSONFlag { + return + } + if name == "" { + fmt.Printf("Took %s out of its milestone\n", what) + return + } + fmt.Printf("Put %s in milestone %q\n", what, name) +} + +func runMilestoneImport(store *brain.Store, path string) { + b, err := os.ReadFile(path) + if err != nil { + brainFatal(err) + } + var file struct { + Milestones []brain.Milestone `json:"milestones"` + } + if err := json.Unmarshal(b, &file); err != nil { + brainFatal(fmt.Errorf("%s: %w", path, err)) + } + skipped, err := store.ImportMilestones(file.Milestones) + if err != nil { + brainFatal(err) + } + removed := false + if len(skipped) == 0 { + removed = os.Remove(path) == nil + } + if globalJSONFlag { + brainJSON(map[string]any{"imported": len(file.Milestones), "skipped": skipped, "removed": removed}) + return + } + fmt.Printf("Imported %d milestone(s) from %s into the plan\n", len(file.Milestones), filepath.Base(path)) + for _, s := range skipped { + fmt.Printf(" not placed: %s\n", s) + } + if removed { + fmt.Printf("Removed %s: the slices carry their milestones now\n", path) + } else if len(skipped) > 0 { + fmt.Printf("Kept %s, since not everything in it was placed; delete it once the rest is settled\n", path) + } +} + +func init() { + brainMilestoneCmd.Flags().StringArray("slice", nil, "A slice to put in the milestone; repeatable") + brainMilestoneCmd.Flags().StringArray("requirement", nil, "A requirement placed in the milestone apart from its slice; repeatable") + brainMilestoneCmd.Flags().Bool("clear", false, "Take the slices or requirements out of their milestone") + brainMilestoneCmd.Flags().String("import", "", "Write a traceability.json's milestones into the plan, then remove the file") + brainCmd.AddCommand(brainMilestoneCmd) +} diff --git a/cmd/mxcli/cmd_check.go b/cmd/mxcli/cmd_check.go index b50efcfa79..7178d16642 100644 --- a/cmd/mxcli/cmd_check.go +++ b/cmd/mxcli/cmd_check.go @@ -558,6 +558,10 @@ func runCheckFileAgainst(cmd *cobra.Command, filePath, against string) int { // A consistency rule between two documents, not a reference, so the // reference check above cannot see it. projectViolations = append(projectViolations, exec.CheckEntityAccessCalls(prog)...) + // MDL-GRANT03: a grant whose where clause narrows nothing because the + // role already holds an unconstrained rule on the entity in the + // project, or the reverse (ako/mxcli#2320). + projectViolations = append(projectViolations, exec.CheckAccessRuleOverlap(prog)...) // MDL-I18N01 (MxBuild CE4899): a script that changes the default // language leaves every required caption written in the old one empty // in the new one (ako/mxcli#944). diff --git a/cmd/mxcli/cmd_exec.go b/cmd/mxcli/cmd_exec.go index af7e356f9d..4f4c731905 100644 --- a/cmd/mxcli/cmd_exec.go +++ b/cmd/mxcli/cmd_exec.go @@ -62,6 +62,12 @@ mprcontents/) is restored; generated .java/.js source is not. With Pass "-" as the file to read the script from standard input, so MDL can be piped or written inline as a heredoc without a temporary file. +Writers of one project run one at a time: a script that writes (and a -c that +does) holds the project's lock, .mxcli/write.lock, for its whole run, and a +second one waits up to 60 seconds for it. Past that it changes nothing and +exits 75 ("try again"), naming the process that holds the lock. Scripts that +only read take no lock. + Example: mxcli exec setup.mdl mxcli exec -p app.mpr script.mdl @@ -94,6 +100,20 @@ Example: os.Exit(1) } + // Writers of one project run one at a time: four in parallel all + // failed, each rolling back over the others (ako/mxcli#2326). A script + // that only reads takes no lock. Deferred before the executor's Close, + // so the lock is held until the last deferred write has landed. + if projectPath != "" { + if early, _ := visitor.Build(string(content)); early == nil || programWritesModel(early) { + release, code := acquireProjectWriteLock(projectPath, "exec "+filePath, os.Stderr) + if code != 0 { + os.Exit(code) + } + defer release() + } + } + // A watching dev loop is told when this script starts writing and when // the last write has landed, so it builds at once instead of waiting // for the model to go quiet (ako/mxcli#2151). Deferred first, so it diff --git a/cmd/mxcli/cmd_fix.go b/cmd/mxcli/cmd_fix.go index a993203fe1..f9031eb900 100644 --- a/cmd/mxcli/cmd_fix.go +++ b/cmd/mxcli/cmd_fix.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "runtime" modelsdk "github.com/mendixlabs/mxcli" "github.com/mendixlabs/mxcli/cmd/mxcli/docker" @@ -111,9 +112,16 @@ func runFixTool(cmd *cobra.Command, subcommand, what string) error { mendixVer, _ := reader.GetMendixVersion() _ = reader.Disconnect() - mxPath, err := docker.ResolveMxForVersion(mxOverride, mendixVer) - if err != nil { - return fmt.Errorf("locate mx for Mendix %s: %w\nhint: run 'mxcli setup mxbuild -p %s'", mendixVer, err, mprPath) + var mxPath string + if mxOverride != "" || mendixVer == "" { + mxPath, err = docker.ResolveMxForVersion(mxOverride, mendixVer) + if err != nil { + return fmt.Errorf("locate mx for Mendix %s: %w\nhint: run 'mxcli setup mxbuild -p %s'", mendixVer, err, mprPath) + } + } else if mxPath, err = resolveFixMx("", mendixVer, runtime.GOOS, docker.LocalMxForVersion, func(v string) (string, error) { + return docker.ResolveMxForNewProject(v, out) + }); err != nil { + return err } fmt.Fprintf(out, "Updating %s in %s (Mendix %s)...\n", what, mprPath, mendixVer) @@ -159,3 +167,31 @@ func reportFix(out io.Writer, what string, res *docker.HarvestResult) { } fmt.Fprintln(out, "\n Next: 'mxcli docker check -p '") } + +// resolveFixMx finds the mx a fix tool runs. `mx update-widgets` and `mx +// rename-design-properties` work on a project of their own version only and +// refuse any other ("Project is in different Mendix version", exit 4), so the +// version is a requirement here, not the preference docker.ResolveMxForVersion +// takes it as: on macOS that fell back to an mx on PATH or another Studio Pro +// and fix widgets failed (ako/mxcli#2321). An explicit override is used as +// given; otherwise an exact-version local mx, then — on Linux, where the CDN's +// mx runs — a download of that version; on macOS and Windows, a refusal that +// names the version and how to provide it. +func resolveFixMx(override, version, goos string, exact func(string) string, download func(string) (string, error)) (string, error) { + if override != "" { + return override, nil + } + if p := exact(version); p != "" { + return p, nil + } + if goos == "linux" && download != nil { + p, err := download(version) + if err != nil { + return "", fmt.Errorf("download mx %s, which this project's version needs: %w", version, err) + } + return p, nil + } + return "", fmt.Errorf("this project is Mendix %s, and no mx %s is installed: the fix tools run only on their own "+ + "version (another refuses with \"Project is in different Mendix version\"). Install Studio Pro %s, or pass "+ + "--mx with the mx of an MxBuild %s (in a Linux container, the one MxBuild ships)", version, version, version, version) +} diff --git a/cmd/mxcli/cmd_git_merge.go b/cmd/mxcli/cmd_git_merge.go index ec0751d4b7..3fb321e957 100644 --- a/cmd/mxcli/cmd_git_merge.go +++ b/cmd/mxcli/cmd_git_merge.go @@ -67,6 +67,11 @@ the engine chokes on the conflicted state. Its result is copied back: those units (a stale or hand-widened plan is refused). Each planned unit is taken whole from its side, its conflict marker cleared, and the .mpr index repaired. + Whole means the other side's changes to that unit go + too, not only the conflicting one: the review lists, + per unit, what each choice drops that did not conflict + (an entity only one side added), and --plan repeats it + for the side it takes before committing. Modified-on-both units only — added/deleted/moved conflicts go to Studio Pro. --ours / --theirs take one side of every conflicted unit @@ -320,6 +325,7 @@ func mergeRun(mprPath, ref, message, strategy, plan string, push bool, runners m // A failed side is reported, not dropped: skipping the report made // "the diff crashed" read as "no details" (ako/mxcli#1981). fmt.Fprintln(out, propertyReport(diffSide{oursJSON, d1}, diffSide{theirsJSON, d2}, manual)) + fmt.Fprint(out, wholeUnitDropReport(diffSide{oursJSON, d1}, diffSide{theirsJSON, d2}, manual, nil)) } // The abort below resets the index; what the human resolved and staged // for the non-model files is recorded first, and --plan restores it @@ -966,6 +972,17 @@ func applyPlan(root string, repo *gitNoteRepo, mprAbs, mprPath, ref, planPath, m } } + // Say what the whole-unit choices drop before they are applied: changes + // of the other side to the same unit that did not conflict (ako/mxcli#2325). + if runners.diff != nil { + base := filepath.Join(revs.baseDir, filepath.FromSlash(repo.MPRPath)) + oursJSON := filepath.Join(revs.tmp, "ours-diff.json") + theirsJSON := filepath.Join(revs.tmp, "theirs-diff.json") + d1 := runners.diff(base, filepath.Join(revs.oursDir, filepath.FromSlash(repo.MPRPath)), oursJSON) + d2 := runners.diff(base, filepath.Join(revs.theirsDir, filepath.FromSlash(repo.MPRPath)), theirsJSON) + fmt.Fprint(out, wholeUnitDropReport(diffSide{oursJSON, d1}, diffSide{theirsJSON, d2}, open, take)) + } + // The engine's result first, then each planned unit whole from its // side's commit, its marker cleared. The hashes are rewritten from the // files — files are the truth — and the gate refuses (and aborts) a @@ -1056,3 +1073,102 @@ func realMxDiff(base, mine, outJSON string) error { } return nil } + +// --- what a whole-unit choice drops (ako/mxcli#2325) ------------------------- + +// unitChanges reads one side's mx diff into its top-level changes per unit: +// the elements the side added, changed or removed ("Entity 'Shop.OursOnly'", +// "Attribute 'Shop.Customer.Name'"). ok is false for a side whose diff failed +// or cannot be read. +func unitChanges(side diffSide) (changes map[string][]string, types map[string]string, ok bool) { + if side.err != nil { + return nil, nil, false + } + raw, err := os.ReadFile(side.path) + if err != nil { + return nil, nil, false + } + var d mxDiffFile + if err := json.Unmarshal(bytes.TrimPrefix(raw, []byte("\xef\xbb\xbf")), &d); err != nil { + return nil, nil, false + } + changes, types = map[string][]string{}, map[string]string{} + for _, u := range d.UnitDifferences { + types[u.ID] = u.Type + for _, c := range u.ChangeDetails { + desc := c.ChangeDescription + if desc == "" { + desc = c.Property + } + if desc != "" { + changes[u.ID] = append(changes[u.ID], desc) + } + } + } + return changes, types, true +} + +// onlyIn returns the entries of a that b does not have, in a's order. +func onlyIn(a, b []string) []string { + in := map[string]bool{} + for _, x := range b { + in[x] = true + } + var out []string + for _, x := range a { + if !in[x] { + out = append(out, x) + } + } + return out +} + +// wholeUnitDropReport says, per conflicted unit, what each "take" drops that +// does not conflict: a plan takes a unit whole, from one side's commit, so +// every change of the other side to that unit goes — not only the conflicting +// element. The property report lists both sides' changes; nothing said that +// "theirs" also discards an entity only ours added (ako/mxcli#2325). take, when +// non-nil, limits the report to the side each unit's plan chose. Empty when +// neither side's diff can be read or nothing would be dropped. +func wholeUnitDropReport(ours, theirs diffSide, unitIDs []string, take map[string]string) string { + oc, ot, ok1 := unitChanges(ours) + tc, tt, ok2 := unitChanges(theirs) + if !ok1 || !ok2 { + return "" + } + var b strings.Builder + for _, id := range unitIDs { + typ := ot[id] + if typ == "" { + typ = tt[id] + } + label := shortCommit(id) + if typ != "" { + label += " (" + typ + ")" + } + dropsOurs := onlyIn(oc[id], tc[id]) + dropsTheirs := onlyIn(tc[id], oc[id]) + if (take == nil || take[id] == "theirs") && len(dropsOurs) > 0 { + fmt.Fprintf(&b, " %s — \"take\": \"theirs\" also drops these changes of yours, which do not conflict:\n", label) + for _, c := range dropsOurs { + fmt.Fprintf(&b, " %s\n", c) + } + } + if (take == nil || take[id] == "ours") && len(dropsTheirs) > 0 { + fmt.Fprintf(&b, " %s — \"take\": \"ours\" also drops these changes of theirs, which do not conflict:\n", label) + for _, c := range dropsTheirs { + fmt.Fprintf(&b, " %s\n", c) + } + } + } + if b.Len() == 0 { + return "" + } + head := "A plan takes each unit whole, from one side:\n" + if take != nil { + head = "Warning: the plan takes each unit whole, so it drops changes that did not conflict:\n" + } + return head + b.String() + + " To keep both sides' changes, resolve the unit in Studio Pro (Version Control › Merge Changes Here), " + + "or re-apply the dropped changes after the merge.\n" +} diff --git a/cmd/mxcli/cmd_report_kinds.go b/cmd/mxcli/cmd_report_kinds.go index 69714dc8b4..5b7a574f21 100644 --- a/cmd/mxcli/cmd_report_kinds.go +++ b/cmd/mxcli/cmd_report_kinds.go @@ -501,9 +501,11 @@ and three gap lists: not fully built as asked, built but untested, and open questions that affect a requirement. It needs requirements in docs/brain/plan/ (see 'mxcli brain capture --slice'). -Milestones come from docs/brain/plan/traceability.json, or --milestones: - {"milestones": [{"name": "M1", "slices": ["01-accounts"], "requirements": ["a1b2c3"]}]} -Without it, each slice is a milestone. Evidence flags are the tests report's. +Milestones come from the plan ('mxcli brain milestone "M1" --slice 01-accounts'); +without any, each slice is a milestone. A milestones file (--milestones, or the +older docs/brain/plan/traceability.json) is still read, with a warning; +'mxcli brain milestone --import ' moves it into the plan. Evidence flags +are the tests report's. Annotation keys: req:.`, example: ` mxcli report traceability -p app.mpr -o traceability.html @@ -511,7 +513,7 @@ Annotation keys: req:.`, flags: func(cmd *cobra.Command) { scopeFlags(cmd) evidenceFlags(cmd, true) - cmd.Flags().String("milestones", "", "Milestones file (default: docs/brain/plan/traceability.json)") + cmd.Flags().String("milestones", "", "Milestones file, instead of the milestones in the plan (deprecated: mxcli brain milestone --import moves it into the plan)") }, collect: func(cmd *cobra.Command, projectPath string) (*projectreport.Envelope, error) { loc, err := resolveBrainLocation(projectPath, "") @@ -524,10 +526,7 @@ Annotation keys: req:.`, return nil, err } if len(sliceShards) == 0 { - return nil, fmt.Errorf("no requirements: %s has no plan slices.\n"+ - "Capture the specification's requirements first, one per line of the PRD, anchored where they will be built:\n"+ - " mxcli brain capture \" (p.3)\" --slice 01- -a @Module.Element -p %s\n"+ - " mxcli brain promote --all -p %s", filepath.Join(loc.Dir, brain.StoreDir, "plan"), projectPath, projectPath) + return nil, noRequirementsError(filepath.Join(loc.Dir, brain.StoreDir, "plan"), projectPath) } var warnings []string in := projectreport.TraceabilityInput{} @@ -586,17 +585,12 @@ Annotation keys: req:.`, warnings = append(warnings, "Malformed brain entry left out: "+m) } in.Entries = list.Entries - msPath, _ := cmd.Flags().GetString("milestones") - if msPath == "" { - if def := filepath.Join(loc.Dir, brain.StoreDir, "plan", "traceability.json"); reportFileExists(def) { - msPath = def - } - } - if msPath != "" { - if in.Milestones, err = projectreport.ReadMilestones(msPath); err != nil { - return nil, err - } + ms, msWarnings, err := traceabilityMilestones(cmd, store, loc.Dir) + if err != nil { + return nil, err } + in.Milestones = ms + warnings = append(warnings, msWarnings...) data, err := projectreport.CollectTraceability(q, in) if err != nil { return nil, err @@ -611,6 +605,48 @@ Annotation keys: req:.`, render: projectreport.RenderTraceability, }) +// noRequirementsError is what the traceability report says on a brain with no +// plan: how to capture requirements and commit them. +func noRequirementsError(planDir, projectPath string) error { + return fmt.Errorf("no requirements: %s has no plan slices.\n"+ + "Capture the specification's requirements first, one per line of the PRD, anchored where they will be built:\n"+ + " mxcli brain capture \" (p.3)\" --slice 01- -a @Module.Element -p %s\n"+ + " mxcli brain staged -p %s\n"+ + " mxcli brain promote ... -p %s", planDir, projectPath, projectPath, projectPath) +} + +// traceabilityMilestones reads the milestones from the plan, where the slices +// carry them. A --milestones file still works, and so does the old +// docs/brain/plan/traceability.json when the plan names no milestone, each with +// a warning that says how to move it into the plan. +func traceabilityMilestones(cmd *cobra.Command, store *brain.Store, dir string) (*projectreport.Milestones, []string, error) { + if path, _ := cmd.Flags().GetString("milestones"); path != "" { + ms, err := projectreport.ReadMilestones(path) + return ms, []string{"Milestones were read from " + path + ". The plan can carry them instead: mxcli brain milestone --import " + path}, err + } + planned, err := store.Milestones() + if err != nil { + return nil, nil, err + } + old := filepath.Join(dir, brain.StoreDir, "plan", "traceability.json") + if len(planned) == 0 { + if !reportFileExists(old) { + return nil, nil, nil + } + ms, err := projectreport.ReadMilestones(old) + return ms, []string{"Milestones were read from " + old + ", which is deprecated. Move them into the plan: mxcli brain milestone --import " + old}, err + } + var warnings []string + if reportFileExists(old) { + warnings = append(warnings, "Ignored "+old+": the plan names milestones, which take precedence. Delete the file.") + } + out := &projectreport.Milestones{} + for _, m := range planned { + out.Milestones = append(out.Milestones, projectreport.MilestoneSpec{Name: m.Name, Slices: m.Slices, Requirements: m.Requirements}) + } + return out, warnings, nil +} + func reportFileExists(p string) bool { fi, err := os.Stat(p) return err == nil && !fi.IsDir() @@ -631,6 +667,10 @@ describes what is there and how it fits together: Landscape user roles and where each starts; the systems around the app; what runs without a user + Features with a project brain plan: one feature per slice, with + its requirements, milestone and what its anchors reach + (flows, pages, entities, entry points, roles), and what + no feature reaches Module map app modules as a dependency graph, domain use apart from screen links, the hub and weak links behind toggles Inside a module data model (other modules' entities as ghosts, the flows @@ -670,13 +710,61 @@ modules that have none one sentence each in --annotations: if err != nil { return nil, err } + trace, sliceMilestones, warnings, err := mapFeatureTrace(cmd, projectPath, exec) + if err != nil { + return nil, err + } + projectreport.AddFeatures(data, trace, sliceMilestones) return projectreport.NewEnvelope("map", reportProjectName(projectPath), reportNow(), version, data, - projectreport.MapMethod, projectreport.MapLimits, nil) + projectreport.MapMethod, projectreport.MapLimits, warnings) }, render: projectreport.RenderMap, renderHTML: projectreport.RenderMapHTML, }) +// mapFeatureTrace reads the project brain's plan, when there is one, as the +// traceability report does — the same statuses, footprints and milestones — so +// the map can show the app by feature. No brain, or no slices, is no feature +// view, not an error. +func mapFeatureTrace(cmd *cobra.Command, projectPath string, exec *executor.Executor) (*projectreport.TraceabilityData, map[string]string, []string, error) { + loc, err := resolveBrainLocation(projectPath, "") + if err != nil { + return nil, nil, nil, nil + } + store := brain.NewStore(loc.Dir) + if !store.Exists() { + return nil, nil, nil, nil + } + sliceShards, err := store.ListSlices() + if err != nil || len(sliceShards) == 0 { + return nil, nil, nil, err + } + rep, err := brain.Check(store, &catalogResolver{cat: exec.Catalog(), be: exec.Backend()}, sliceShards) + if err != nil { + return nil, nil, nil, err + } + list, err := store.List(brain.ListFilter{}) + if err != nil { + return nil, nil, nil, err + } + ms, warnings, err := traceabilityMilestones(cmd, store, loc.Dir) + if err != nil { + return nil, nil, nil, err + } + sliceMilestones := map[string]string{} + if ms != nil { + for _, m := range ms.Milestones { + for _, sl := range m.Slices { + sliceMilestones[sl] = m.Name + } + } + } + trace, err := projectreport.CollectTraceability(exec.Catalog(), projectreport.TraceabilityInput{ + Requirements: rep.Requirements, Entries: list.Entries, Milestones: ms, + }) + return trace, sliceMilestones, warnings, err +} + func init() { reportCmd.AddCommand(reportMapCmd) } diff --git a/cmd/mxcli/cmd_report_milestones_test.go b/cmd/mxcli/cmd_report_milestones_test.go new file mode 100644 index 0000000000..18a7bf916c --- /dev/null +++ b/cmd/mxcli/cmd_report_milestones_test.go @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/mendixlabs/mxcli/cmd/mxcli/brain" + "github.com/spf13/cobra" +) + +// The traceability report reads milestones from the plan. The old +// traceability.json is read only when the plan names none, and either way the +// report says how to move it into the plan. +func TestTraceabilityMilestonesComeFromThePlan(t *testing.T) { + dir := t.TempDir() + store := brain.NewStore(dir) + if _, err := store.Init(); err != nil { + t.Fatal(err) + } + r, err := brain.NewRequirement("Users can log in", []string{"@Accounts.ACT_Login"}, "01-accounts", time.Now()) + if err != nil { + t.Fatal(err) + } + if err := store.Promote(r, r.Shard()); err != nil { + t.Fatal(err) + } + cmd := &cobra.Command{} + cmd.Flags().String("milestones", "", "") + read := func() (string, []string) { + t.Helper() + ms, warnings, err := traceabilityMilestones(cmd, store, dir) + if err != nil { + t.Fatal(err) + } + if ms == nil { + return "", warnings + } + return ms.Milestones[0].Name, warnings + } + if name, w := read(); name != "" || len(w) > 0 { + t.Fatalf("no milestones anywhere: got %q %v", name, w) + } + old := filepath.Join(dir, "docs", "brain", "plan", "traceability.json") + if err := os.WriteFile(old, []byte(`{"milestones":[{"name":"From file","slices":["01-accounts"]}]}`), 0644); err != nil { + t.Fatal(err) + } + if name, w := read(); name != "From file" || len(w) != 1 || !strings.Contains(w[0], "brain milestone --import") { + t.Errorf("old file only: got %q %v", name, w) + } + if err := store.SetSliceMilestone("01-accounts", "From plan"); err != nil { + t.Fatal(err) + } + if name, w := read(); name != "From plan" || len(w) != 1 || !strings.Contains(w[0], "Ignored") { + t.Errorf("plan and old file: the plan wins and the file is called out: got %q %v", name, w) + } + _ = cmd.Flags().Set("milestones", old) + if name, _ := read(); name != "From file" { + t.Errorf("an explicit --milestones file still wins: got %q", name) + } +} + +// The hint on an empty plan is a set of commands to copy. Every flag it names +// must exist on the brain command it names: it once said `brain promote --all`, +// which promote does not have. +func TestNoRequirementsHintUsesRealFlags(t *testing.T) { + msg := noRequirementsError("docs/brain/plan", "app.mpr").Error() + checked := 0 + for _, line := range strings.Split(msg, "\n") { + fields := strings.Fields(line) + if len(fields) < 3 || fields[0] != "mxcli" || fields[1] != "brain" { + continue + } + sub, _, err := brainCmd.Find([]string{fields[2]}) + if err != nil || sub == brainCmd { + t.Errorf("%q: no brain command %q", line, fields[2]) + continue + } + for _, f := range fields[3:] { + if !strings.HasPrefix(f, "-") { + continue + } + name := strings.TrimLeft(f, "-") + var ok bool + if strings.HasPrefix(f, "--") { + ok = sub.Flags().Lookup(name) != nil || sub.InheritedFlags().Lookup(name) != nil + } else { + ok = sub.Flags().ShorthandLookup(name) != nil || sub.InheritedFlags().ShorthandLookup(name) != nil + } + if !ok { + t.Errorf("%q: brain %s has no flag %s", line, fields[2], f) + } + } + checked++ + } + if checked < 2 { + t.Fatalf("expected the hint to name brain commands:\n%s", msg) + } +} diff --git a/cmd/mxcli/cmd_run_lifecycle.go b/cmd/mxcli/cmd_run_lifecycle.go index 866ef4db3d..2a294dbc85 100644 --- a/cmd/mxcli/cmd_run_lifecycle.go +++ b/cmd/mxcli/cmd_run_lifecycle.go @@ -181,7 +181,7 @@ shell.`, // restart would hand the app the placeholder instead of the value. Say so // before stopping anything. if hasRedactedArgs(st.Args) { - fmt.Fprint(os.Stderr, restartSecretRefusal(projectPath)) + fmt.Fprint(os.Stderr, restartSecretRefusal(projectPath, redactedArgNames(st.Args))) os.Exit(runExitFailed) } if code := runStop(projectPath, os.Stdout); code != runExitOK { @@ -218,13 +218,21 @@ func lifecycleProject(cmd *cobra.Command) string { // database password never reads (only --ensure-db's psql does): a run started // that way connected as the default password. A reference is what restart can // resolve again. -func restartSecretRefusal(projectPath string) string { +// +// literal names the flags whose value was stored redacted (redactedArgNames): +// a reference — @env:VAR, @file:path — is stored and replayed, so the +// refusal says which values were not one (ako/mxcli#2313). +func restartSecretRefusal(projectPath string, literal []string) string { + which := "a --constant value, --db-password, ..." + if len(literal) > 0 { + which = strings.Join(literal, ", ") + } return fmt.Sprintf("cannot restart: this run was started with a secret written literally on the command line "+ - "(a --constant value, --db-password, ...), and it is not stored.\n"+ - " Stop it and start it again with the secret as a reference, which is stored and resolved again on restart:\n"+ + "(%s), and a literal secret is not stored. The values given as @env:VAR or @file:path are stored and replayed.\n"+ + " Stop it and start it again with each of those as a reference too:\n"+ " mxcli run stop -p %s\n"+ - " mxcli run --local --detach -p %s --db-password @env:MX_DB_PASSWORD ... (or @file:path)\n", - projectPath, projectPath) + " mxcli run --local --detach -p %s --constant Module.Key=@env:VAR --db-password @env:MX_DB_PASSWORD ... (or @file:path)\n", + which, projectPath, projectPath) } // --- detach ------------------------------------------------------------ diff --git a/cmd/mxcli/docker/check.go b/cmd/mxcli/docker/check.go index d5c45f546f..d7386e6cd3 100644 --- a/cmd/mxcli/docker/check.go +++ b/cmd/mxcli/docker/check.go @@ -532,3 +532,7 @@ func CachedMxPath(version string) string { func AnyCachedMxPath() string { return anyCachedBinaryPath(mxBinaryNames()) } + +// LocalMxForVersion is localMxForVersion: an installed or cached mx of exactly +// version, or "". +func LocalMxForVersion(version string) string { return localMxForVersion(version) } diff --git a/cmd/mxcli/fix_mx_resolve_2321_test.go b/cmd/mxcli/fix_mx_resolve_2321_test.go new file mode 100644 index 0000000000..a4453beea2 --- /dev/null +++ b/cmd/mxcli/fix_mx_resolve_2321_test.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "strings" + "testing" +) + +// ako/mxcli#2321: `mxcli fix widgets` on macOS ran an mx of another version +// than the project's — ResolveMxForVersion treats the version as a preference +// and falls back to an mx on PATH or the newest install — and `mx +// update-widgets` refused: "Project is in different Mendix version - 11.12.2" +// (exit 4). update-widgets only works on its own version, so fix needs that +// exact mx, or says which one it needs. + +func TestResolveFixMx_ExactLocalVersion_Issue2321(t *testing.T) { + exact := func(v string) string { + if v == "11.12.2" { + return "/Applications/Studio Pro 11.12.2.app/modeler/mx" + } + return "" + } + got, err := resolveFixMx("", "11.12.2", "darwin", exact, nil) + if err != nil || got != "/Applications/Studio Pro 11.12.2.app/modeler/mx" { + t.Errorf("got %q, %v; want the exact-version mx", got, err) + } +} + +// No exact mx on macOS: a downloaded Linux mx cannot run there, so the refusal +// names the version and the ways to provide it — not a mismatched mx. +func TestResolveFixMx_MacWithoutExactVersionSaysWhich_Issue2321(t *testing.T) { + none := func(string) string { return "" } + download := func(string) (string, error) { t.Fatal("must not download a Linux mx on macOS"); return "", nil } + _, err := resolveFixMx("", "11.12.2", "darwin", none, download) + if err == nil { + t.Fatal("resolved an mx although none of 11.12.2 exists") + } + for _, want := range []string{"11.12.2", "--mx", "different Mendix version"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q lacks %q", err, want) + } + } +} + +// On Linux the exact version is downloaded, as `mxcli new` does. +func TestResolveFixMx_LinuxDownloadsTheExactVersion_Issue2321(t *testing.T) { + none := func(string) string { return "" } + var asked string + download := func(v string) (string, error) { asked = v; return "/cache/" + v + "/mx", nil } + got, err := resolveFixMx("", "11.12.2", "linux", none, download) + if err != nil || asked != "11.12.2" || got != "/cache/11.12.2/mx" { + t.Errorf("got %q, %v (downloaded %q); want the 11.12.2 download", got, err, asked) + } + failing := func(string) (string, error) { return "", errors.New("offline") } + if _, err := resolveFixMx("", "11.12.2", "linux", none, failing); err == nil || !strings.Contains(err.Error(), "11.12.2") { + t.Errorf("failed download: %v", err) + } +} + +// CONTROL: an explicit --mx is used as given. +func TestResolveFixMx_ExplicitOverride_Issue2321(t *testing.T) { + got, err := resolveFixMx("/opt/mxbuild/modeler/mx", "11.12.2", "darwin", func(string) string { return "" }, nil) + if err != nil || got != "/opt/mxbuild/modeler/mx" { + t.Errorf("got %q, %v", got, err) + } +} diff --git a/cmd/mxcli/gitmerge_whole_unit_drop_2325_test.go b/cmd/mxcli/gitmerge_whole_unit_drop_2325_test.go new file mode 100644 index 0000000000..1d38b59de0 --- /dev/null +++ b/cmd/mxcli/gitmerge_whole_unit_drop_2325_test.go @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +// ako/mxcli#2325: `git merge --plan` resolves per unit, so "take": "theirs" on +// a domain model replaced the whole unit: Shop.OursOnly, which no one else +// touched, was gone after the merge. The property report listed it under Yours +// and nothing said the choice would drop it. + +const oursDiff2325 = `{"unitDifferences":[{"id":"` + tDM + `","type":"DomainModels$DomainModel","change":"Modified","changeDetails":[` + + `{"change":"Added","changeDescription":"Entity 'Shop.OursOnly'"},` + + `{"change":"Modified","changeDescription":"Attribute 'Shop.Customer.Name'","changeDetails":[{"property":"New type > Length","baseValue":"200","mineValue":"150"}]}]}]}` + +const theirsDiff2325 = `{"unitDifferences":[{"id":"` + tDM + `","type":"DomainModels$DomainModel","change":"Modified","changeDetails":[` + + `{"change":"Added","changeDescription":"Entity 'Shop.TheirsOnly'"},` + + `{"change":"Modified","changeDescription":"Attribute 'Shop.Customer.Name'","changeDetails":[{"property":"New type > Length","baseValue":"200","mineValue":"300"}]}]}]}` + +func diffSides2325(t *testing.T) (diffSide, diffSide) { + t.Helper() + dir := t.TempDir() + o, th := filepath.Join(dir, "ours.json"), filepath.Join(dir, "theirs.json") + if err := os.WriteFile(o, []byte(oursDiff2325), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(th, []byte(theirsDiff2325), 0o644); err != nil { + t.Fatal(err) + } + return diffSide{path: o}, diffSide{path: th} +} + +func TestWholeUnitDropReport_NamesTheNonConflictingChanges_Issue2325(t *testing.T) { + ours, theirs := diffSides2325(t) + got := wholeUnitDropReport(ours, theirs, []string{tDM}, nil) + for _, want := range []string{ + `"take": "theirs" also drops these changes of yours`, "Entity 'Shop.OursOnly'", + `"take": "ours" also drops these changes of theirs`, "Entity 'Shop.TheirsOnly'", + } { + if !strings.Contains(got, want) { + t.Errorf("report lacks %q:\n%s", want, got) + } + } + // The conflicting attribute is the choice itself, not a loss to warn of. + if strings.Contains(got, "Shop.Customer.Name") { + t.Errorf("report lists the conflicting attribute as dropped:\n%s", got) + } +} + +// With a plan, only the side each unit's choice drops is reported. +func TestWholeUnitDropReport_PlanNamesWhatItsChoiceDrops_Issue2325(t *testing.T) { + ours, theirs := diffSides2325(t) + got := wholeUnitDropReport(ours, theirs, []string{tDM}, map[string]string{tDM: "theirs"}) + if !strings.Contains(got, "Warning") || !strings.Contains(got, "Entity 'Shop.OursOnly'") { + t.Errorf("plan warning lacks the dropped entity:\n%s", got) + } + if strings.Contains(got, "TheirsOnly") { + t.Errorf("plan warning names what the choice keeps:\n%s", got) + } +} + +// CONTROL: when the two sides changed only the same element, the choice drops +// nothing else and nothing is reported. +func TestWholeUnitDropReport_NothingElseChangedSaysNothing_Issue2325(t *testing.T) { + dir := t.TempDir() + one := `{"unitDifferences":[{"id":"` + tDM + `","type":"DomainModels$DomainModel","changeDetails":[{"changeDescription":"Attribute 'Shop.Customer.Name'"}]}]}` + o, th := filepath.Join(dir, "o.json"), filepath.Join(dir, "t.json") + _ = os.WriteFile(o, []byte(one), 0o644) + _ = os.WriteFile(th, []byte(one), 0o644) + if got := wholeUnitDropReport(diffSide{path: o}, diffSide{path: th}, []string{tDM}, nil); got != "" { + t.Errorf("report = %q, want none", got) + } +} + +// End to end through the conflict stage: the report printed before a plan is +// written says what each choice drops. +func TestGitMerge_ConflictReportSaysWhatATakeDrops_Issue2325(t *testing.T) { + dir, mprPath := modelConflictRepo(t) + doctorEnv(t, dir) + mx := mxRunners{ + merge: func(base, mine, theirs string, shas [3]string) (int, string, error) { + return 2, `[{"ConflictResolutionDetails":"ManualResolutionRequired","UnitId":"` + tDM + `"}]`, nil + }, + diff: func(base, mine, outJSON string) error { + body := oursDiff2325 + if strings.Contains(mine, "theirs") { + body = theirsDiff2325 + } + return os.WriteFile(outJSON, []byte(body), 0o644) + }, + } + var out bytes.Buffer + if err := mergeRun(mprPath, "side", "", "", "", false, mx, &out); err == nil { + t.Fatal("a manual conflict must end with the decision error") + } + if !strings.Contains(out.String(), "Entity 'Shop.OursOnly'") || !strings.Contains(out.String(), "also drops") { + t.Errorf("conflict report does not say what taking theirs drops:\n%s", out.String()) + } + gitInE(t, dir, "merge", "--abort") +} + +// And when the plan is applied: the warning names what that plan's choice +// drops, before the merge is committed. +func TestGitMerge_PlanWarnsWhatItDrops_Issue2325(t *testing.T) { + dir, mprPath := twoUnitConflictRepo(t) + doctorEnv(t, dir) + result := baseUnits(t) + result[tDM] = tUnit{tMod, unitDoc(t, "DomainModels$DomainModel", "", mainSideKey)} + result[tMf] = tUnit{tMod, unitDoc(t, "Microflows$Microflow", "ACT_Save", mainSideKey)} + planPath := writePlanFor(t, dir, map[string]string{tDM: "theirs", tMf: "ours"}) + mx := mxRunners{ + merge: engineDecides(t, result, []string{tDM, tMf}, nil), + diff: func(base, mine, outJSON string) error { + body := oursDiff2325 + if strings.Contains(mine, "theirs") { + body = theirsDiff2325 + } + return os.WriteFile(outJSON, []byte(body), 0o644) + }, + } + var out bytes.Buffer + if err := mergeRun(mprPath, "side", "", "", planPath, false, mx, &out); err != nil { + t.Fatalf("mergeRun --plan: %v\n%s", err, out.String()) + } + got := out.String() + if !strings.Contains(got, "Warning: the plan takes each unit whole") || !strings.Contains(got, "Entity 'Shop.OursOnly'") { + t.Errorf("plan run does not warn what taking theirs drops:\n%s", got) + } + if i, j := strings.Index(got, "Warning: the plan"), strings.Index(got, "committed"); j >= 0 && i > j { + t.Errorf("the warning comes after the commit:\n%s", got) + } + assertPlanMergeCommitted(t, dir, mprPath) +} diff --git a/cmd/mxcli/oneliner.go b/cmd/mxcli/oneliner.go index 19ff0be486..8dc3a370c8 100644 --- a/cmd/mxcli/oneliner.go +++ b/cmd/mxcli/oneliner.go @@ -38,6 +38,18 @@ func runCommandLine(cmd *cobra.Command, commands, projectPath string, continueOn return 1 } + // As in `exec`: one writer of a project at a time (ako/mxcli#2326); + // statements that only read take no lock. Released after exec.Close. + if projectPath != "" { + if early, _ := visitor.Build(commands); early == nil || programWritesModel(early) { + release, code := acquireProjectWriteLock(projectPath, "-c", errOut) + if code != 0 { + return code + } + defer release() + } + } + // As in `exec`: a watching dev loop builds as soon as these statements // are done (ako/mxcli#2151). Deferred first, so it runs after exec.Close. defer docker.BeginModelWrite(projectPath)() diff --git a/cmd/mxcli/project_write_lock.go b/cmd/mxcli/project_write_lock.go new file mode 100644 index 0000000000..6e497d03d9 --- /dev/null +++ b/cmd/mxcli/project_write_lock.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "fmt" + "io" + "reflect" + "strings" + + "github.com/mendixlabs/mxcli/internal/projectlock" + "github.com/mendixlabs/mxcli/mdl/ast" +) + +// exitProjectBusy is the exit code of a run that gave up waiting for another +// mxcli process to finish writing the project: EX_TEMPFAIL, "try again +// later", so a script or an agent can tell it from a failed statement +// (ako/mxcli#2326). +const exitProjectBusy = 75 + +// readOnlyStatementPrefixes are the statement types that never write the +// model: a program of only these runs without the project's write lock, so +// agents reading one project in parallel do not queue behind each other. +var readOnlyStatementPrefixes = []string{ + "Show", "Describe", "Select", "Search", "Help", "Exit", "Status", "Lint", + "Connect", "Disconnect", "Set", "Refresh", "SQLConnect", "SQLDisconnect", + "SQLConnections", "SQLQuery", "SQLShow", "SQLDescribe", +} + +// programWritesModel reports whether prog has a statement that may write the +// project. Unknown statement types count as writes: an extra wait is a cost, +// a missing lock is the failure this guards against. +func programWritesModel(prog *ast.Program) bool { + if prog == nil { + return false + } + for _, s := range prog.Statements { + if s == nil { + continue + } + name := reflect.TypeOf(s).String() + name = name[strings.LastIndexByte(name, '.')+1:] + readOnly := false + for _, p := range readOnlyStatementPrefixes { + if strings.HasPrefix(name, p) { + readOnly = true + break + } + } + if !readOnly { + return true + } + } + return false +} + +// acquireProjectWriteLock takes the project's write lock for a run that +// writes it, waiting for another mxcli process that holds it. It returns the +// release, or the exit code to stop with (and has said why on errOut). +func acquireProjectWriteLock(projectPath, purpose string, errOut io.Writer) (release func(), exitCode int) { + release, err := projectlock.Acquire(projectPath, projectlock.Options{Purpose: purpose}) + if err == nil { + return release, 0 + } + var busy *projectlock.BusyError + if errors.As(err, &busy) { + fmt.Fprintf(errOut, "Error: %v\nNothing was written.\n", err) + return nil, exitProjectBusy + } + fmt.Fprintf(errOut, "Error: take the project's write lock: %v\n", err) + return nil, 1 +} diff --git a/cmd/mxcli/projectreport/features.go b/cmd/mxcli/projectreport/features.go new file mode 100644 index 0000000000..8e63d2ad90 --- /dev/null +++ b/cmd/mxcli/projectreport/features.go @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: Apache-2.0 + +package projectreport + +import ( + "fmt" + "sort" + "strings" + + "github.com/mendixlabs/mxcli/cmd/mxcli/brain" +) + +// The feature view of the software map: the app organised by what it does for +// its users rather than by module. A feature is a plan slice of the project +// brain; what belongs to it is DERIVED from its requirements' anchors and their +// footprints, never listed by hand, so there is no feature map to maintain and +// nothing to go stale. What no feature reaches is listed as unowned: the part +// of the model nobody specified. +// +// Reach, from a feature's anchors: +// +// - flows: the anchored flows, the requirement footprints (which already +// include the flows anchored pages, services and entities lead to), the +// flows behind buttons of anchored pages, and the call closure of all of +// those; +// - pages: the anchored pages and the pages those flows open; +// - entities: the anchored entities (and the entity of an anchored attribute +// or association) and the entities those flows write. +// +// Reach stops there. Following the pages a feature opens on into their own +// buttons would walk the navigation and give every feature the whole app. + +// MapFeature is one feature: a slice of the plan and what it reaches. +type MapFeature struct { + Name string `json:"name"` + Milestone string `json:"milestone,omitempty"` + Requirements []MapFeatureReq `json:"requirements"` + Status map[string]int `json:"status"` + Modules []string `json:"modules"` + Pages []string `json:"pages"` + // Flows is every flow in the feature's reach; Core is the ones its + // requirements are anchored on or whose footprint they are. + Flows []string `json:"flows"` + Core []string `json:"core"` + Entities []string `json:"entities"` + Other []string `json:"other,omitempty"` + EntryPoints []MapFeatureEntry `json:"entryPoints"` + Roles []string `json:"roles"` + // SharesWith names the other features that reach some of the same pages, + // flows or entities. + SharesWith []string `json:"sharesWith,omitempty"` +} + +// MapFeatureReq is a requirement of a feature. +type MapFeatureReq struct { + ID string `json:"id"` + Title string `json:"title"` + Status string `json:"status"` + // Milestone is set when the requirement is in a different milestone from + // its feature. + Milestone string `json:"milestone,omitempty"` +} + +// MapFeatureEntry is a way into a feature: a flow in its reach that something +// outside the feature starts — not a button on one of its own pages. +type MapFeatureEntry struct { + Flow string `json:"flow"` + Kind string `json:"kind"` + Source string `json:"source"` +} + +// MapUnowned is the app no feature reaches. +type MapUnowned struct { + Pages []string `json:"pages"` + EntryPoints []string `json:"entryPoints"` + Entities []string `json:"entities"` +} + +// AddFeatures derives the feature view from the traceability data of the +// project brain and adds it to the map. sliceMilestones is each slice's own +// milestone (brain.Store.SliceMilestone). With no requirements it does nothing. +func AddFeatures(d *MapData, t *TraceabilityData, sliceMilestones map[string]string) { + if d == nil || t == nil { + return + } + flows := map[string]MapFlow{} + for _, f := range d.Flows { + flows[f.Name] = f + } + pages := map[string]MapPage{} + for _, p := range d.Pages { + pages[p.Name] = p + } + entities := map[string]MapEntity{} + writtenBy := map[string][]string{} // flow -> entities it writes + for _, e := range d.Entities { + entities[e.Name] = e + for _, f := range e.WrittenBy { + writtenBy[f] = append(writtenBy[f], e.Name) + } + } + + type acc struct { + f *MapFeature + pages, flows, ents, other map[string]bool + reachFlows, reachPages, mods map[string]bool + } + var order []string + bySlice := map[string]*acc{} + reqMilestone := map[string]string{} + for _, m := range t.Milestones { + for _, r := range m.Requirements { + if m.Name != r.Slice && m.Name != notInMilestone { + reqMilestone[r.ID] = m.Name + } + } + } + for _, m := range t.Milestones { + for _, r := range m.Requirements { + a := bySlice[r.Slice] + if a == nil { + a = &acc{f: &MapFeature{Name: r.Slice, Status: map[string]int{}}, + pages: map[string]bool{}, flows: map[string]bool{}, ents: map[string]bool{}, other: map[string]bool{}, + reachFlows: map[string]bool{}, reachPages: map[string]bool{}, mods: map[string]bool{}} + bySlice[r.Slice] = a + order = append(order, r.Slice) + } + a.f.Requirements = append(a.f.Requirements, MapFeatureReq{ID: r.ID, Title: r.Title, Status: r.Status, Milestone: reqMilestone[r.ID]}) + a.f.Status[r.Status]++ + for _, an := range r.Anchors { + if an.State != brain.Resolved { + continue + } + name := strings.TrimPrefix(an.Anchor, "@") + switch strings.ToLower(strings.ReplaceAll(an.Kind, " ", "_")) { + case "page", "snippet": + a.pages[name] = true + case "microflow", "nanoflow": + a.flows[name] = true + case "entity": + a.ents[name] = true + case "attribute", "association": + if i := strings.LastIndexByte(name, '.'); i > 0 { + a.ents[name[:i]] = true + } + default: + a.other[name] = true + } + } + for _, fp := range r.Footprint { + a.flows[fp] = true + } + } + } + + // Reach. + owner := map[string]map[string]bool{} // element -> features reaching it + own := func(el, feature string) { + if owner[el] == nil { + owner[el] = map[string]bool{} + } + owner[el][feature] = true + } + for _, slice := range order { + a := bySlice[slice] + var seeds []string + for f := range a.flows { + seeds = append(seeds, f) + } + for p := range a.pages { + a.reachPages[p] = true + seeds = append(seeds, pages[p].Flows...) + } + sort.Strings(seeds) + for len(seeds) > 0 { + n := seeds[0] + seeds = seeds[1:] + if a.reachFlows[n] { + continue + } + f, ok := flows[n] + if !ok { + continue // outside the app (Marketplace, System) or not a flow + } + a.reachFlows[n] = true + seeds = append(seeds, f.Calls...) + for _, p := range f.Pages { + if _, ok := pages[p]; ok { + a.reachPages[p] = true + } + } + for _, e := range writtenBy[n] { + a.ents[e] = true + } + } + for p := range a.reachPages { + a.pages[p] = true + } + // Only what is in the app is shown: an anchor on a Marketplace page is + // a requirement on it, not a part of this app. + for p := range a.pages { + if _, ok := pages[p]; !ok { + delete(a.pages, p) + } + } + for f := range a.flows { + if _, ok := flows[f]; !ok { + delete(a.flows, f) + } + } + for e := range a.ents { + if _, ok := entities[e]; !ok { + delete(a.ents, e) + } + } + for p := range a.pages { + own("page:"+p, slice) + a.mods[moduleOf(p)] = true + } + for f := range a.reachFlows { + own("flow:"+f, slice) + a.mods[moduleOf(f)] = true + } + for e := range a.ents { + own("entity:"+e, slice) + a.mods[moduleOf(e)] = true + } + f := a.f + f.Pages, f.Flows, f.Core = sortedKeys(a.pages), sortedKeys(a.reachFlows), sortedKeys(a.flows) + f.Entities, f.Other = sortedKeys(a.ents), sortedKeys(a.other) + f.Modules = sortedKeys(a.mods) + for _, n := range sortedKeys(a.reachFlows) { + for _, tr := range flows[n].Triggers { + // A button on one of the feature's own pages is how the + // feature works inside, not a way into it. + if a.pages[tr.Source] && (tr.Kind == "page button" || tr.Kind == "page data source") { + continue + } + f.EntryPoints = append(f.EntryPoints, MapFeatureEntry{Flow: n, Kind: tr.Kind, Source: tr.Source}) + } + } + sort.SliceStable(f.EntryPoints, func(i, j int) bool { return kindOrder(f.EntryPoints[i].Kind) < kindOrder(f.EntryPoints[j].Kind) }) + // The user roles that may open one of its pages. + for _, r := range d.Roles { + mr := setOf(r.ModuleRoles) + page: + for p := range a.pages { + for _, x := range pages[p].Roles { + if mr[x] { + f.Roles = append(f.Roles, r.Name) + break page + } + } + } + } + f.Milestone = sliceMilestones[slice] + for i := range f.Requirements { + if f.Requirements[i].Milestone == f.Milestone { + f.Requirements[i].Milestone = "" + } + } + } + for _, slice := range order { + a := bySlice[slice] + shares := map[string]bool{} + for _, fs := range owner { + if !fs[slice] { + continue + } + for o := range fs { + if o != slice { + shares[o] = true + } + } + } + a.f.SharesWith = sortedKeys(shares) + d.Features = append(d.Features, *a.f) + } + + // Unowned: the app no feature reaches. + if len(d.Features) == 0 { + return + } + u := &MapUnowned{} + for _, p := range d.Pages { + if owner["page:"+p.Name] == nil { + u.Pages = append(u.Pages, p.Name) + } + } + for _, f := range d.Flows { + if len(f.Triggers) > 0 && owner["flow:"+f.Name] == nil { + u.EntryPoints = append(u.EntryPoints, f.Name) + } + } + for _, e := range d.Entities { + if e.Persistent && owner["entity:"+e.Name] == nil { + u.Entities = append(u.Entities, e.Name) + } + } + d.Unowned = u +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// featureStatus is "3 built, 1 partial" for a feature's requirements. +func featureStatus(st map[string]int) string { + var parts []string + for _, s := range []string{StatusBuilt, StatusPartial, StatusChanged, StatusNotBuilt, StatusNotTraceable} { + if n := st[s]; n > 0 { + parts = append(parts, fmt.Sprintf("%d %s", n, s)) + } + } + return strings.Join(parts, ", ") +} diff --git a/cmd/mxcli/projectreport/features_test.go b/cmd/mxcli/projectreport/features_test.go new file mode 100644 index 0000000000..23aaafda41 --- /dev/null +++ b/cmd/mxcli/projectreport/features_test.go @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: Apache-2.0 + +package projectreport + +import ( + "bytes" + "reflect" + "testing" + + "github.com/mendixlabs/mxcli/cmd/mxcli/brain" +) + +func resolved(anchor, kind string) brain.AnchorResult { + return brain.AnchorResult{Anchor: anchor, State: brain.Resolved, Kind: kind} +} + +// The feature view is derived from the plan: a slice reaches what its +// requirements' anchors lead to, and what no slice reaches is listed apart. +func TestFeatureView(t *testing.T) { + f := mapFixture(t) + d, err := CollectMap(f.cat, ScopeOptions{}, nil) + if err != nil { + t.Fatal(err) + } + orders := TraceRequirement{ID: "r1", Slice: "01-orders", Title: "Place orders", Status: StatusBuilt, + Anchors: []brain.AnchorResult{resolved("@Sales.Order_Overview", "page")}} + invoice := TraceRequirement{ID: "r2", Slice: "02-billing", Title: "Invoice nightly", Status: StatusBuilt, + Anchors: []brain.AnchorResult{resolved("@Billing.ACT_Invoice", "microflow")}, Footprint: []string{"Billing.ACT_Invoice"}} + later := TraceRequirement{ID: "r3", Slice: "02-billing", Title: "Credit notes", Status: StatusNotBuilt, + Anchors: []brain.AnchorResult{{Anchor: "@Billing.ACT_Credit", State: brain.NotFound}}} + trace := &TraceabilityData{Milestones: []TraceMilestone{ + {Name: "M1", Requirements: []TraceRequirement{orders}}, + {Name: "M2", Requirements: []TraceRequirement{invoice}}, + {Name: "M3", Requirements: []TraceRequirement{later}}, + }} + AddFeatures(d, trace, map[string]string{"01-orders": "M1", "02-billing": "M2"}) + if len(d.Features) != 2 { + t.Fatalf("one feature per slice: %+v", d.Features) + } + o, b := d.Features[0], d.Features[1] + + // 01-orders: the anchored page, the flows behind its buttons and their + // closure, the page one of them opens, the entity one writes. + if !reflect.DeepEqual(o.Pages, []string{"Admin.Purge", "Sales.Order_Overview"}) { + t.Errorf("orders pages = %v (Order_Edit is opened by the page itself, which reach does not follow)", o.Pages) + } + if !reflect.DeepEqual(o.Flows, []string{"Admin.ACT_Purge", "Admin.SUB_Audit", "Core.SUB_Log", "Sales.ACT_Place", "Sales.SUB_Price"}) { + t.Errorf("orders flows = %v", o.Flows) + } + if !reflect.DeepEqual(o.Entities, []string{"Sales.Order"}) || len(o.EntryPoints) != 0 { + t.Errorf("orders entities %v, entry points %v (its own page's buttons are not a way in)", o.Entities, o.EntryPoints) + } + if !reflect.DeepEqual(o.Roles, []string{"Admin", "User"}) && !reflect.DeepEqual(o.Roles, []string{"User", "Admin"}) { + t.Errorf("orders roles = %v", o.Roles) + } + + // 02-billing: the anchored flow is core, the schedule is its way in, it + // shares the logging helper with orders, and the unbuilt requirement keeps + // its own milestone. + if !reflect.DeepEqual(b.Core, []string{"Billing.ACT_Invoice"}) || b.Milestone != "M2" { + t.Errorf("billing core %v milestone %q", b.Core, b.Milestone) + } + if len(b.EntryPoints) != 1 || b.EntryPoints[0].Kind != "schedule" || b.EntryPoints[0].Flow != "Billing.ACT_Invoice" { + t.Errorf("billing entry points = %+v", b.EntryPoints) + } + if !reflect.DeepEqual(b.SharesWith, []string{"01-orders"}) { + t.Errorf("billing shares with %v", b.SharesWith) + } + if b.Requirements[0].Milestone != "" || b.Requirements[1].Milestone != "M3" { + t.Errorf("only a requirement placed apart names its milestone: %+v", b.Requirements) + } + if b.Status[StatusBuilt] != 1 || b.Status[StatusNotBuilt] != 1 { + t.Errorf("billing status %v", b.Status) + } + + // What no feature reaches. + u := d.Unowned + if !reflect.DeepEqual(u.Pages, []string{"Sales.Order_Edit", "Sales.Task_Approve"}) { + t.Errorf("unowned pages = %v", u.Pages) + } + for _, want := range []string{"Core.SSO_Provision", "Billing.Worker", "Sales.REST_GetOrder"} { + if !containsStr(u.EntryPoints, want) { + t.Errorf("unowned entry points %v lack %s", u.EntryPoints, want) + } + } + if containsStr(u.EntryPoints, "Billing.ACT_Invoice") || !reflect.DeepEqual(u.Entities, []string{"Core.Account"}) { + t.Errorf("unowned entry points %v, entities %v", u.EntryPoints, u.Entities) + } + + // Rendered: the markdown section and the HTML tab are there. + env, _ := NewEnvelope("map", "Shop", "now", "t", d, MapMethod, MapLimits, nil) + doc, err := RenderMap(env, nil) + if err != nil { + t.Fatal(err) + } + var md bytes.Buffer + _ = RenderMarkdown(&md, doc) + mustContain(t, md.String(), "## Features", "| 02-billing | M2 | 2 (1 built, 1 not built) |", "schedule `Billing.ACT_Invoice` (Billing.SE_Nightly)", "Reached by no feature") + var h bytes.Buffer + if err := RenderMapHTML(&h, env, nil); err != nil { + t.Fatal(err) + } + mustContain(t, h.String(), `"features":[`, "['features','Features']") +} + +// Without a plan the map is unchanged: no feature tab, no unowned list. +func TestNoPlanNoFeatureView(t *testing.T) { + f := mapFixture(t) + d, err := CollectMap(f.cat, ScopeOptions{}, nil) + if err != nil { + t.Fatal(err) + } + AddFeatures(d, nil, nil) + AddFeatures(d, &TraceabilityData{}, nil) + if d.Features != nil || d.Unowned != nil { + t.Errorf("no requirements, no feature view: %+v %+v", d.Features, d.Unowned) + } +} diff --git a/cmd/mxcli/projectreport/softwaremap.go b/cmd/mxcli/projectreport/softwaremap.go index 21289de4b7..581f4d292f 100644 --- a/cmd/mxcli/projectreport/softwaremap.go +++ b/cmd/mxcli/projectreport/softwaremap.go @@ -44,6 +44,10 @@ type MapData struct { Prefixes []MapPrefix `json:"prefixes"` SharedFlows []MapShared `json:"sharedFlows"` Index []MapIndexDoc `json:"index"` + // Features is the app by plan slice, when the project has a brain with + // requirements (AddFeatures); Unowned is what no feature reaches. + Features []MapFeature `json:"features,omitempty"` + Unowned *MapUnowned `json:"unowned,omitempty"` } // MapModule is one app module. diff --git a/cmd/mxcli/projectreport/softwaremap_page.go b/cmd/mxcli/projectreport/softwaremap_page.go index 41df3b15dd..1a3d952978 100644 --- a/cmd/mxcli/projectreport/softwaremap_page.go +++ b/cmd/mxcli/projectreport/softwaremap_page.go @@ -104,6 +104,49 @@ V.landscape=function(){ h+=table(['Started by','Source','Flow'],(D.background||[]).map(function(t){return [esc(t.kind),esc(t.source),link(t.flow)];})); return h; }; +V.features=function(arg){ + function arr(x){return x||[];} + var fs=arr(D.features).map(function(f){['requirements','modules','pages','flows','core','entities','other','entryPoints','roles','sharesWith'].forEach(function(k){f[k]=arr(f[k]);});f.status=f.status||{};return f;}); + var u=D.unowned||{}; u={pages:arr(u.pages),entryPoints:arr(u.entryPoints),entities:arr(u.entities)}; + var h='
Feature
'; + var f=fs.filter(function(x){return x.name===arg;})[0]; + function status(st){return ['built','partial','changed','not built','not traceable'].filter(function(k){return st[k];}).map(function(k){return ''+st[k]+' '+esc(k)+'';}).join('');} + if(!f){ + h+=h2('Features','Each feature is a slice of the project brain\'s plan; what belongs to it is derived from its requirements\' anchors. Pick one to see its pages, flows and data.'); + h+='
'+fs.map(function(f){return '

'+esc(f.name)+''+(f.milestone?' '+esc(f.milestone)+'':'')+'

'+ + '
'+status(f.status)+'
'+f.requirements.length+' requirement(s) · '+f.pages.length+' page(s) · '+f.flows.length+' flow(s) · '+f.entities.length+' entit'+(f.entities.length===1?'y':'ies')+'
'+ + '
'+esc(f.modules.join(', '))+'
'+(f.sharesWith&&f.sharesWith.length?'
shares with '+esc(f.sharesWith.join(', '))+'
':'')+'
';}).join('')+'
'; + h+=h2('Reached by no feature','The part of the app no requirement leads to: specified nowhere, or reached only through navigation the plan does not anchor.'); + h+=table(['Kind','Count','Documents'],[['pages',u.pages.length,links(u.pages)],['entry points',u.entryPoints.length,links(u.entryPoints)],['persistent entities',u.entities.length,links(u.entities)]].map(function(r){return [esc(r[0]),r[1],r[2]];})); + return h; + } + h+='

'+(f.milestone?'Milestone '+esc(f.milestone)+'. ':'')+'Modules: '+esc(f.modules.join(', ')||'none')+'.'+(f.roles.length?' Roles that may open its pages: '+esc(f.roles.join(', '))+'.':'')+ + (f.sharesWith&&f.sharesWith.length?' Shares pages, flows or data with '+f.sharesWith.map(function(s){return ''+esc(s)+'';}).join(', ')+'.':'')+'

'; + h+=h2('Requirements'); + h+=table(['Id','Requirement','Status','Milestone'],f.requirements.map(function(r){return [''+esc(r.id)+'',esc(r.title),esc(r.status),esc(r.milestone||'')];})); + var keep={},nodes=[],edges=[]; + f.pages.forEach(function(p){keep[p]=1;nodes.push({id:p,label:p,fill:'var(--warnbg)'});}); + var core={}; f.core.forEach(function(n){core[n]=1;}); + var shown=f.core.concat(f.flows.filter(function(n){return !core[n];})).slice(0,60); + shown.forEach(function(n){keep[n]=1;var fl=flows[n];nodes.push({id:n,label:(core[n]?'★ ':'')+n,fill:fl&&fl.nanoflow?'var(--infobg)':'var(--okbg)'});}); + f.entities.forEach(function(e){keep[e]=1;nodes.push({id:e,label:e,fill:'var(--card)',dashed:1});}); + f.flows.forEach(function(n){var fl=flows[n]; if(!fl||!keep[n])return; + (fl.calls||[]).forEach(function(c){if(keep[c])edges.push({from:n,to:c,title:'calls'});}); + (fl.pages||[]).forEach(function(p){if(keep[p])edges.push({from:n,to:p,nav:1,title:'opens'});});}); + f.pages.forEach(function(p){var pg=pages[p]; (pg&&pg.flows||[]).forEach(function(n){if(keep[n])edges.push({from:p,to:n,title:'button runs'});});}); + f.entities.forEach(function(e){(ents[e]&&ents[e].writtenBy||[]).forEach(function(n){if(keep[n])edges.push({from:n,to:e,title:'writes'});});}); + h+=h2('Pages, flows and data','Amber: pages; green: microflows; blue: nanoflows; dashed: entities. Arrows: a button runs a flow, a flow calls, opens or writes. ★ marks the '+f.core.length+' flow(s) the requirements are anchored on or whose footprint they are; the rest is their reach'+(f.flows.length>60?' (60 of '+f.flows.length+' shown)':'')+'.'); + h+=nodes.length?layout(nodes,edges):'

Nothing in the app is anchored yet.

'; + h+=h2('Entry points','Flows in the feature\'s reach that something outside it starts: a schedule, a service, the menu, or a button on a page that is not the feature\'s own.'); + h+=table(['Started by','Flow','From'],(f.entryPoints||[]).map(function(e){return [esc(e.kind),link(e.flow),esc(e.source)];})); + if((f.other||[]).length)h+=h2('Other anchors','Anchors that are not pages, flows or entities: services, workflows, roles, enumerations.')+'

'+links(f.other)+'

'; + return h; +}; +V.features.after=function(root){ + root.querySelector('#ft-sel').addEventListener('change',function(e){go('features',e.target.value);}); + root.querySelectorAll('a[data-feature]').forEach(function(a){a.addEventListener('click',function(){go('features',a.getAttribute('data-feature'));});}); + root.querySelectorAll('.node').forEach(function(g){g.addEventListener('click',function(){openDoc(g.getAttribute('data-id'));});}); +}; V.modules=function(){ var h=h2('Module map','Arrows point from a module to what it uses, thicker for more references. Screen links (dashed) and the hub are hidden unless switched on.'); h+='
'+(D.hub?'':'')+ @@ -265,7 +308,7 @@ function showDoc(name){ function openDoc(name){ if(cur!=='index') go('index',''); showDoc(name); } // ---- shell -var tabs=[['landscape','Landscape'],['modules','Module map'],['module','Inside a module'],['pageflows','Page flows'],['data','Data across modules'],['conventions','Conventions'],['apis','APIs & background'],['index','Index'],['about','How this was made']]; +var tabs=[['landscape','Landscape']].concat((D.features||[]).length?[['features','Features']]:[]).concat([['modules','Module map'],['module','Inside a module'],['pageflows','Page flows'],['data','Data across modules'],['conventions','Conventions'],['apis','APIs & background'],['index','Index'],['about','How this was made']]); V.about=function(){ return h2('How this was measured')+'
    '+P.method.map(function(m){return '
  • '+esc(m)+'
  • ';}).join('')+'
'+ h2('What this does not cover')+'
    '+P.limits.map(function(m){return '
  • '+esc(m)+'
  • ';}).join('')+'
'; diff --git a/cmd/mxcli/projectreport/softwaremap_render.go b/cmd/mxcli/projectreport/softwaremap_render.go index d1815bbf94..1f3dae2d01 100644 --- a/cmd/mxcli/projectreport/softwaremap_render.go +++ b/cmd/mxcli/projectreport/softwaremap_render.go @@ -17,6 +17,7 @@ var MapMethod = []string{ "Page flows per role start at the role's home pages and the menu, then follow show-page actions, page buttons and the flows behind them, through snippets and layouts, limited to pages the role may view and flows it may execute.", "Index: attribute and enumeration-value references are folded into their entity or enumeration.", "Module purposes come from the module's domain-model documentation, or from `--annotations` (`module:` notes) when the model has none.", + "Features (when the project has a brain with a plan) are its slices. A feature reaches its anchored flows and the requirements' footprints (as `report traceability` computes them), the flows behind buttons of its anchored pages, and the call closure of all of those; the pages anchored or opened by those flows; the entities anchored or written by them. Entry points are the reached flows something outside the feature starts; roles are the user roles that may open one of its pages. Milestones come from the plan.", } var MapLimits = []string{ @@ -24,6 +25,7 @@ var MapLimits = []string{ "A page the role may open but no click path reaches is listed, not judged: workflow task pages and links from notifications are reached that way on purpose.", "Page flows follow the model, not the running app: a button hidden by a condition still counts as a path.", "Clients of published services and the systems behind consumed ones are outside the model; the map names the services, not who calls them.", + "A feature is only as complete as its anchors: a requirement anchored on one microflow of a larger flow shows that flow's closure, not the pages that lead to it. \"Reached by no feature\" is what no anchor leads to — unspecified, or reached only through navigation the plan does not anchor.", } // applyPurposes copies annotation notes keyed module: onto modules that @@ -79,6 +81,10 @@ func RenderMap(env *Envelope, ann *Annotations) (Doc, error) { } doc.Sections = append(doc.Sections, Section{ID: "landscape", Title: "Landscape", Blocks: []Block{rt, st, bt}}) + if len(d.Features) > 0 { + doc.Sections = append(doc.Sections, featureSection(d)) + } + // Modules. mt := Table{Columns: []string{"Module", "Purpose", "Entities", "Microflows", "Pages", "Entry points"}} for _, m := range d.Modules { @@ -153,6 +159,35 @@ func RenderMap(env *Envelope, ann *Annotations) (Doc, error) { return Frame(env, ann, doc), nil } +// featureSection is the app by feature: one row per plan slice, and what no +// feature reaches. +func featureSection(d MapData) Section { + ft := Table{Columns: []string{"Feature", "Milestone", "Requirements", "Modules", "Pages", "Flows", "Entities", "Entry points", "Roles that may open it", "Shares with"}} + for _, f := range d.Features { + var eps []string + for _, e := range f.EntryPoints { + eps = append(eps, e.Kind+" `"+e.Flow+"` ("+e.Source+")") + } + ft.Rows = append(ft.Rows, []string{f.Name, f.Milestone, fmt.Sprintf("%d (%s)", len(f.Requirements), featureStatus(f.Status)), + strings.Join(f.Modules, ", "), strings.Join(codeList(f.Pages), ", "), fmt.Sprintf("%d (%d anchored)", len(f.Flows), len(f.Core)), + strings.Join(codeList(f.Entities), ", "), strings.Join(eps, ", "), strings.Join(f.Roles, ", "), strings.Join(f.SharesWith, ", ")}) + } + blocks := []Block{ft} + if u := d.Unowned; u != nil { + ut := Table{Caption: "Reached by no feature", Columns: []string{"Kind", "Count", "Documents"}} + for _, r := range []struct { + kind string + xs []string + }{{"pages", u.Pages}, {"entry points", u.EntryPoints}, {"persistent entities", u.Entities}} { + ut.Rows = append(ut.Rows, []string{r.kind, fmt.Sprint(len(r.xs)), strings.Join(codeList(r.xs), ", ")}) + } + blocks = append(blocks, ut) + } + return Section{ID: "features", Title: "Features", + Intro: "Each feature is a slice of the project brain's plan. What belongs to it is derived from its requirements' anchors: the anchored flows and their footprints with their call closure, the pages anchored or opened by those flows, and the entities anchored or written by them. What no feature reaches is the model nobody specified.", + Blocks: blocks} +} + func hubNote(hub string) string { if hub == "" { return "" diff --git a/cmd/mxcli/projectreport/traceability.go b/cmd/mxcli/projectreport/traceability.go index 3fe9c524b8..326a1a5e5e 100644 --- a/cmd/mxcli/projectreport/traceability.go +++ b/cmd/mxcli/projectreport/traceability.go @@ -14,9 +14,9 @@ import ( ) // Milestones groups plan slices — and single requirements placed apart from -// their slice — into the specification's milestones. It is the one input the -// traceability report cannot derive: the brain knows slices, the PRD knows -// milestones. +// their slice — into the specification's milestones. The plan carries them +// (brain.Store.Milestones); this file form is what --milestones and the older +// docs/brain/plan/traceability.json still read. // // {"milestones": [ // {"name": "M1 — accounts", "slices": ["01-accounts"]}, @@ -369,7 +369,7 @@ func CollectTraceability(q Querier, in TraceabilityInput) (*TraceabilityData, er d.Milestones = append(d.Milestones, tm) } var rest TraceMilestone - rest.Name = "Not in any milestone" + rest.Name = notInMilestone for _, sl := range sliceOrder { for _, id := range bySlice[sl] { if !placed[id] { @@ -397,6 +397,9 @@ func CollectTraceability(q Querier, in TraceabilityInput) (*TraceabilityData, er return d, nil } +// notInMilestone names the group of requirements no milestone holds. +const notInMilestone = "Not in any milestone" + // isFootprintKind is brain's list, so `brain check` and this report agree on // which anchors a test can ever reach (ako/mxcli#2303). func isFootprintKind(kind string) bool { return brain.ProvableKind(kind) } @@ -444,7 +447,7 @@ var TraceabilityMethod = []string{ "Footprint: the microflows that implement a requirement — an anchored microflow plus its same-module callees; microflows a page, workflow, scheduled event or published service anchor calls; microflows that write an anchored entity or retrieve an anchored view entity; microflows that use an anchored attribute or association. Helpers with more than three callers are left out.", "A microflow test proves a requirement when what it calls, with their call closure, touches the footprint. A journey proves it when a footprint microflow appears in its spans, or its log names an anchored page.", "Decisions bear on a requirement when they share an anchor (module anchors excluded) or cite its id; open questions also when filed on its slice.", - "Milestones come from `docs/brain/plan/traceability.json` (or `--milestones`); without one, each slice is a milestone.", + "Milestones come from the plan: a slice's frontmatter (`milestone:`), or a requirement's ``Milestone: `` `` line (`mxcli brain milestone`); without any, each slice is a milestone. They are ordered by first appearance, walking the slices in name order.", } var TraceabilityLimits = []string{ diff --git a/cmd/mxcli/run_redact.go b/cmd/mxcli/run_redact.go index 0bea5dabe2..6a25689922 100644 --- a/cmd/mxcli/run_redact.go +++ b/cmd/mxcli/run_redact.go @@ -137,3 +137,32 @@ func hasRedactedArgs(args []string) bool { } return false } + +// redactedArgNames names each value of args that redactArgs replaced, as the +// flag a user would change: "--db-password", "--constant Mod.Key", +// "--runtime-setting DatabasePassword". run restart's refusal lists them +// (ako/mxcli#2313): a run with one literal secret among several @env: +// references read as "references are not replayed". +func redactedArgNames(args []string) []string { + var out []string + for i := 0; i < len(args); i++ { + a := args[i] + flag, val, inline := a, "", false + if k := strings.Index(a, "="); strings.HasPrefix(a, "--") && k > 0 { + flag, val, inline = a[:k], a[k+1:], true + } + if !inline && i+1 < len(args) && (secretRunFlags[flag] || flag == "--constant" || flag == "--runtime-setting") { + i++ + val = args[i] + } + switch { + case secretRunFlags[flag] && val == redactedValue: + out = append(out, flag) + case flag == "--constant" || flag == "--runtime-setting": + if name, v, ok := strings.Cut(val, "="); ok && v == redactedValue { + out = append(out, flag+" "+name) + } + } + } + return out +} diff --git a/cmd/mxcli/run_restart_redacted_names_2313_test.go b/cmd/mxcli/run_restart_redacted_names_2313_test.go new file mode 100644 index 0000000000..4fd6b76b12 --- /dev/null +++ b/cmd/mxcli/run_restart_redacted_names_2313_test.go @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "reflect" + "strings" + "testing" +) + +// ako/mxcli#2313: `run restart` refused a run started with --constant +// X=@env:VAR and printed only the generic hint ("--db-password @env:…"). The +// @env: references themselves are stored and replayed; what restart refuses is +// a value stored redacted, and the refusal did not say which one — so a run +// with one literal flag among several references read as "references do not +// work". It now names each. +func TestRedactedArgNames_NamesEachLiteralSecret_Issue2313(t *testing.T) { + stored, _ := redactArgs([]string{"run", "--local", + "--constant", "Encryption.EncryptionKey=@env:HELPDESK_ENC_KEY", + "--constant", "Helpdesk.OpenRouterKey=sk-literal", + "--db-password", "hunter2", + "--runtime-setting=DatabasePassword=x", + "-p", "/abs/App.mpr"}) + got := redactedArgNames(stored) + want := []string{"--constant Helpdesk.OpenRouterKey", "--db-password", "--runtime-setting DatabasePassword"} + if !reflect.DeepEqual(got, want) { + t.Errorf("redactedArgNames = %q, want %q", got, want) + } + msg := restartSecretRefusal("/abs/App.mpr", got) + for _, w := range want { + if !strings.Contains(msg, w) { + t.Errorf("refusal does not name %s:\n%s", w, msg) + } + } + if strings.Contains(msg, "EncryptionKey") { + t.Errorf("refusal names a constant that is a reference, which restart replays:\n%s", msg) + } +} + +// CONTROL: a run whose secrets are all references is restartable — nothing is +// redacted, so nothing is named and restart is not refused. +func TestRedactedArgNames_ReferencesOnly_Issue2313(t *testing.T) { + stored, changed := redactArgs([]string{"run", "--local", + "--constant", "Encryption.EncryptionKey=@env:HELPDESK_ENC_KEY", + "--constant=Helpdesk.OpenRouterKey=@file:/run/secrets/key", + "--db-password", "@env:MX_DB_PASSWORD"}) + if changed || hasRedactedArgs(stored) || len(redactedArgNames(stored)) != 0 { + t.Errorf("references were redacted: %q", stored) + } +} diff --git a/cmd/mxcli/run_secrets_test.go b/cmd/mxcli/run_secrets_test.go index a91696f300..4bd814135d 100644 --- a/cmd/mxcli/run_secrets_test.go +++ b/cmd/mxcli/run_secrets_test.go @@ -113,7 +113,7 @@ func TestDetachArgs_KeepsAFlagValueThatLooksLikeAFlag(t *testing.T) { // The refusal names a way that works: the runtime's database password does not // read PGPASSWORD, so suggesting it gave a run with the default password. func TestRestartSecretRefusal_SuggestsAReference(t *testing.T) { - msg := restartSecretRefusal("/abs/app.mpr") + msg := restartSecretRefusal("/abs/app.mpr", nil) if strings.Contains(msg, "PGPASSWORD") { t.Errorf("the refusal still suggests PGPASSWORD:\n%s", msg) } diff --git a/cmd/mxcli/syntax/features_integration.go b/cmd/mxcli/syntax/features_integration.go index 203b3cbc9a..0dfaaf0873 100644 --- a/cmd/mxcli/syntax/features_integration.go +++ b/cmd/mxcli/syntax/features_integration.go @@ -704,7 +704,7 @@ DESCRIBE DATABASE CONNECTION Ops.Erp;`, "exposed as", "returns", "generics", "drop java action", "extra code", "helper method", "import", }, - Syntax: "CREATE [OR MODIFY] JAVA ACTION Module.Name[] [FOLDER 'path'](\n [/** description */] Param: Type [NOT NULL],\n EntityType: ENTITY NOT NULL,\n Obj: pEntity\n) RETURNS ReturnType [AS $ReturnName]\n[EXPOSED AS 'Label' IN 'Category'\n [ICON 'icon.png'] [ICON DARK 'icon-dark.png']\n [IMAGE 'image.png'] [IMAGE DARK 'image-dark.png']]\n[NOT EXPOSED]\nAS $$\n// Java code — AS $$ ... $$ is mandatory, cannot be omitted\n$$\n[EXTRA CODE $$\n// helper methods, constants, nested classes beside executeAction()\n$$];\n\nOR MODIFY: updates signature/body in-place, preserves UUID.\n\nEXPOSED AS puts the action in Studio Pro's toolbox. The icon is a 64x64 PNG\nand the image a 256x192 PNG, read from disk relative to the .mdl file's own directory.\nAn OMITTED clause preserves what is stored — including bitmaps set in Studio\nPro — so removing an entry is NOT EXPOSED, and clearing one bitmap is\nDROP ICON / DROP IMAGE [DARK].\n\nA parameter's description is a /** ... */ doc comment before it, as on an\nentity attribute; AS $ReturnName names the return value (the default name of\nthe variable a microflow call stores it in), as on a microflow. Leaving either\nout keeps what is stored; a new action gets no description and ReturnValueName.\n\nThe .java file is written as mxbuild writes it, so a build leaves it unchanged:\nthe action's own /** ... */ (before CREATE) is the class javadoc, and a rewrite\nwithout one keeps the stored documentation.\n\nThe file has two more sections a regeneration keeps, and the statement states\nboth: an import line in the body goes to the import list, and EXTRA CODE $$ ... $$\nis the EXTRA CODE section (helpers the body calls). An OMITTED EXTRA CODE keeps\nthe stored section; EXTRA CODE $$ $$ clears it. DESCRIBE prints the imports a\ndeveloper added and the section, so its output rebuilds the same file\n(ako/mxcli#1702).\n\nA $$ … $$ literal ends at the first $$ it contains, so a body or EXTRA CODE\nholding $$ is written as a string literal instead (AS '…', apostrophes doubled);\nDESCRIBE does this itself (ako/mxcli#1003).\n\nType parameters: an ENTITY parameter declares T. A type parameter no\nENTITY parameter names is declared in the header, Module.Name; an\nunqualified type that is neither is refused (MDL-JA01), since it would be\nwritten as an entity with no module (CE1613).\n\nA bare Module.Name type is an entity or an enumeration, whichever the project\n(or an earlier statement of the script) has; ENUM Module.Name says enumeration\nexplicitly. A name that is neither is refused by check --references and exec,\nsince it would be written as an entity that does not exist (CE1613).\n\nRETURNS takes Boolean, Integer, Long, Decimal, String, DateTime, an enumeration,\nan entity, a list, a type parameter or Void; leaving it out returns Boolean.\nAny other type is refused (MDL-JA03) — StringTemplate(...) is a parameter\ntype only.", + Syntax: "CREATE [OR MODIFY] JAVA ACTION Module.Name[] [FOLDER 'path'](\n [/** description */] Param: Type [NOT NULL],\n EntityType: ENTITY NOT NULL,\n Obj: pEntity\n) RETURNS ReturnType [AS $ReturnName]\n[EXPOSED AS 'Label' IN 'Category'\n [ICON 'icon.png'] [ICON DARK 'icon-dark.png']\n [IMAGE 'image.png'] [IMAGE DARK 'image-dark.png']]\n[NOT EXPOSED]\nAS $$\n// Java code — AS $$ ... $$ is mandatory, cannot be omitted\n$$\n[EXTRA CODE $$\n// helper methods, constants, nested classes beside executeAction()\n$$];\n\nOR MODIFY: updates signature/body in-place, preserves UUID.\n\nEXPOSED AS puts the action in Studio Pro's toolbox. The icon is a 64x64 PNG\nand the image a 256x192 PNG, read from disk relative to the .mdl file's own directory.\nAn OMITTED clause preserves what is stored — including bitmaps set in Studio\nPro — so removing an entry is NOT EXPOSED, and clearing one bitmap is\nDROP ICON / DROP IMAGE [DARK].\n\nA parameter's description is a /** ... */ doc comment before it, as on an\nentity attribute; AS $ReturnName names the return value (the default name of\nthe variable a microflow call stores it in), as on a microflow. Leaving either\nout keeps what is stored; a new action gets no description and ReturnValueName.\n\nThe .java file is written as mxbuild writes it, so a build leaves it unchanged:\nthe action's own /** ... */ (before CREATE) is the class javadoc, and a rewrite\nwithout one keeps the stored documentation.\n\nThe file has two more sections a regeneration keeps, and the statement states\nboth: an import line in the body goes to the import list, and EXTRA CODE $$ ... $$\nis the EXTRA CODE section (helpers the body calls). An OMITTED EXTRA CODE keeps\nthe stored section; EXTRA CODE $$ $$ clears it. DESCRIBE prints the imports a\ndeveloper added and the section, so its output rebuilds the same file\n(ako/mxcli#1702). Code that uses Core (Core.getConfiguration(), Core.getLogger(...))\ngets import com.mendix.core.Core; added when the list has no Core of its own;\nother classes need their import line (ako/mxcli#2310).\n\nA $$ … $$ literal ends at the first $$ it contains, so a body or EXTRA CODE\nholding $$ is written as a string literal instead (AS '…', apostrophes doubled);\nDESCRIBE does this itself (ako/mxcli#1003).\n\nType parameters: an ENTITY parameter declares T. A type parameter no\nENTITY parameter names is declared in the header, Module.Name; an\nunqualified type that is neither is refused (MDL-JA01), since it would be\nwritten as an entity with no module (CE1613).\n\nA bare Module.Name type is an entity or an enumeration, whichever the project\n(or an earlier statement of the script) has; ENUM Module.Name says enumeration\nexplicitly. A name that is neither is refused by check --references and exec,\nsince it would be written as an entity that does not exist (CE1613).\n\nRETURNS takes Boolean, Integer, Long, Decimal, String, DateTime, an enumeration,\nan entity, a list, a type parameter or Void; leaving it out returns Boolean.\nAny other type is refused (MDL-JA03) — StringTemplate(...) is a parameter\ntype only.", Example: "mdl 1;\nCREATE JAVA ACTION Utils.FormatCurrency(\n /** The amount to format. */\n Amount: Decimal NOT NULL\n) RETURNS String AS $FormattedAmount\nEXPOSED AS 'Format Currency' IN 'Formatting'\n ICON 'assets/currency-64.png'\n IMAGE 'assets/currency-256.png'\nAS $$\nreturn String.format(\"%.2f\", Amount);\n$$;\n\n-- Generic entity validator with type parameter\nCREATE JAVA ACTION Utils.IsValid(\n EntityType: ENTITY NOT NULL,\n Obj: pEntity NOT NULL\n) RETURNS Boolean\nAS $$\nreturn Obj != null;\n$$;\n\n-- Type parameter with no entity selector: the argument's entity binds it\nCREATE JAVA ACTION Utils.Describe(\n Obj: T NOT NULL\n) RETURNS String\nAS $$\nreturn Obj.toString();\n$$;\n\n-- A helper in the EXTRA CODE section, and the import it needs\nCREATE JAVA ACTION Utils.Twice(N: Integer NOT NULL) RETURNS Integer\nAS $$\nimport java.util.Objects;\nreturn twice(Objects.requireNonNull(N));\n$$\nEXTRA CODE $$\nprivate static long twice(long n) {\n\treturn n * 2L;\n}\n$$;\n\n-- Idempotent update (preserves UUID)\nCREATE OR MODIFY JAVA ACTION Utils.FormatCurrency(\n Amount: Decimal NOT NULL,\n Decimals: Integer NOT NULL\n) RETURNS String\nAS $$\nreturn String.format(\"%.\" + Decimals + \"f\", Amount);\n$$;", SeeAlso: []string{"java-action", "javascript-action"}, }) diff --git a/cmd/mxcli/syntax/features_misc.go b/cmd/mxcli/syntax/features_misc.go index eadb27289b..6816ac7ca3 100644 --- a/cmd/mxcli/syntax/features_misc.go +++ b/cmd/mxcli/syntax/features_misc.go @@ -657,6 +657,11 @@ LIST NAVIGATION MENU prints every item's shortest address. -- carries the item itself, sub-items included. Fragment items are written -- exactly as in CREATE NAVIGATION, so DESCRIBE output pastes in. -- +-- INSERT is re-runnable: an item the target list already holds, as the +-- statement builds it, is already there and is not added again (a second +-- run reports Unchanged). An item of that caption built differently (another +-- action, other sub-items) is refused -- REPLACE changes it. +-- -- mxcli never guesses: an address that matches nothing or several items is -- refused, listing the address of each candidate. Positional addresses do -- not exist. diff --git a/cmd/mxcli/syntax/features_security.go b/cmd/mxcli/syntax/features_security.go index 6510b3891f..72b4894a1f 100644 --- a/cmd/mxcli/syntax/features_security.go +++ b/cmd/mxcli/syntax/features_security.go @@ -47,6 +47,13 @@ func init() { "MDL-DEPR082 and is rewritten the same way.\n\n" + "A module role is always Module.Role. A bare role name parses but is\n" + "refused (MDL-GRANT02) — mxcli cannot tell which module it belongs to.\n\n" + + "Grants add rules:\n" + + " A rule is keyed by its roles and its WHERE constraint, and a role's\n" + + " access is the union of every rule naming it. A GRANT with a WHERE\n" + + " beside the role's unconstrained rule narrows nothing; check and exec\n" + + " warn MDL-GRANT03. To narrow, revoke first, then grant:\n" + + " REVOKE ALL ON ENTITY Shop.Customer FROM Shop.Clerk;\n" + + " GRANT READ * ON ENTITY Shop.Customer TO Shop.Clerk WHERE [Name != ''];\n\n" + "Members added later:\n" + " A rule also carries a default for members added AFTER it was written,\n" + " derived from the grant: WRITE * gives ReadWrite, READ * gives ReadOnly,\n" + diff --git a/docs-site/src/tools/project-brain.md b/docs-site/src/tools/project-brain.md index 94886837fe..39619239b3 100644 --- a/docs-site/src/tools/project-brain.md +++ b/docs-site/src/tools/project-brain.md @@ -207,6 +207,28 @@ implement it. Misfiling is not checked for slices: a slice spans modules by design. +### Milestones + +A specification's milestones group its slices. The milestone is part of the +slice, not a separate file: + +```bash +mxcli brain milestone "M1 — accounts" --slice 01-accounts --slice 02-profiles -p app.mpr +mxcli brain milestone "M2 — approvals" --requirement a1b2c3 -p app.mpr +mxcli brain milestone -p app.mpr # list them +``` + +That writes `milestone: "M1 — accounts"` in the slice file's frontmatter, and a +``Milestone: `M2 — approvals` `` line on a requirement placed apart from its +slice. `brain plan` adds a MILESTONE column, `brain plan --json` a `milestone` +per slice, and `report traceability` groups by them. Milestones are ordered by +where they first appear, walking the slices in name order. `--clear` takes +slices or requirements out again. + +A project that kept milestones in `docs/brain/plan/traceability.json` moves them +with `mxcli brain milestone --import docs/brain/plan/traceability.json`, which +removes the file once everything in it is placed. + ### Before there is a project The brain does not need a model to exist: capture requirements from a @@ -503,6 +525,7 @@ progress from 65/65 to 63/65 and the number was the only symptom. | `brain capture "" --open [-a @Anchor]…` | Queues an **open question**; anchors not checked | | `brain resolve ""` | Answers it, turning it into a decision in place | | `brain plan [--slice ]` | Each slice's requirements counted against the model | +| `brain milestone [""] [--slice ]… [--requirement ]… [--clear] [--import ]` | Groups slices (or single requirements) into milestones; with no name, lists them | | `brain brief --slice \| --module ` | The reading pack: exactly the shards that work needs | | `brain check [--changed]` | Anchors resolve, entries filed correctly, plus slice progress | | `brain list [--slice ] [--kind ] [--open]` | The committed entries; with `--json`, as data | diff --git a/docs-site/src/tools/project-reports.md b/docs-site/src/tools/project-reports.md index 742377ed13..2c109cdc99 100644 --- a/docs-site/src/tools/project-reports.md +++ b/docs-site/src/tools/project-reports.md @@ -130,9 +130,12 @@ mxcli report traceability -p app.mpr --junit results.xml --journeys journeys.jso is brain's: `brain plan --json` reports the same `status` per requirement, and `brain check` fails on a verdict naming an unknown id, one on a requirement, and a requirement that is both changed and partly met. -Milestones are read from `docs/brain/plan/traceability.json` -(`{"milestones":[{"name":"M1","slices":["01-accounts"]}]}`); `brain plan --json` -now lists each requirement with each anchor's resolution. +Milestones are part of the plan: `mxcli brain milestone "M1" --slice 01-accounts` +writes `milestone: "M1"` in the slice's frontmatter, and `--requirement ` +places one requirement apart from its slice. The older +`docs/brain/plan/traceability.json` is still read, with a warning, until +`mxcli brain milestone --import` moves it into the plan. `brain plan --json` lists +each requirement with each anchor's resolution, and each slice's milestone. ## map @@ -152,6 +155,14 @@ mxcli report map -p app.mpr -o map.html mxcli report map -p app.mpr --format json -o map.json # the same data for another viewer ``` +With a project brain that has a plan, the map adds a **Features** tab: one feature +per slice, with its requirements, milestone and status, and what its anchors reach +— flows (the anchored ones, their footprints and call closure), the pages and +entities those lead to, entry points started from outside the feature, and the +roles that may open its pages. **Reached by no feature** lists the pages, entry +points and persistent entities no requirement leads to. Features are derived from +the plan, so there is no feature map to maintain alongside it. + The page is self-contained and draws one diagram at a time. Module purposes come from domain-model documentation, or from `module:` notes in `--annotations`. diff --git a/docs/01-project/MDL_QUICK_REFERENCE.md b/docs/01-project/MDL_QUICK_REFERENCE.md index 5384096fad..bb298d4abf 100644 --- a/docs/01-project/MDL_QUICK_REFERENCE.md +++ b/docs/01-project/MDL_QUICK_REFERENCE.md @@ -713,7 +713,7 @@ Nested folders use `/` separator: `'Parent/Child/Grandchild'`. Missing folders a | Revoke nanoflow access | `revoke execute on nanoflow Mod.NF from Mod.Role, ...;` | | | Grant page access | `grant view on page Mod.Page to Mod.Role, ...;` | | | Revoke page access | `revoke view on page Mod.Page from Mod.Role, ...;` | | -| Grant entity access | `grant create, delete, read *, write * on entity Mod.Entity to Mod.Role;` / `grant read * on entity Mod.Entity to Mod.Role where [Status = 'Open'];` | The XPath is in brackets, quotes written once; the old `grant Mod.Role on Mod.Entity (…) where '…'` warns MDL-DEPR030 (`fmt --upgrade` rewrites it). Additive — merges with existing. A module role must be qualified: a bare `Role` parses but is refused (MDL-GRANT02). Inherited members are named like the entity's own (`read *` covers them); an unknown name is an error. Entities extending `System.User` are the exception — their platform members must not be granted | +| Grant entity access | `grant create, delete, read *, write * on entity Mod.Entity to Mod.Role;` / `grant read * on entity Mod.Entity to Mod.Role where [Status = 'Open'];` | The XPath is in brackets, quotes written once; the old `grant Mod.Role on Mod.Entity (…) where '…'` warns MDL-DEPR030 (`fmt --upgrade` rewrites it). Additive — merges with existing; a `where` makes a second rule beside the role's unconstrained one and narrows nothing (MDL-GRANT03) — revoke first, then grant. A module role must be qualified: a bare `Role` parses but is refused (MDL-GRANT02). Inherited members are named like the entity's own (`read *` covers them); an unknown name is an error. Entities extending `System.User` are the exception — their platform members must not be granted | | Access for members added later | — | A rule's default for new members is derived from the grant: `write *` → ReadWrite, `read *` → ReadOnly, member lists alone → **None**. So an attribute added later is granted None on a member-listed rule — clean build, blank field. `alter entity … add attribute` warns and prints the widening grant. The rule's *default* decides this, not how narrow its member list is | | Revoke entity access | `revoke all on entity Mod.Entity from Mod.Role;` | Full revoke — removes entire rule | | Revoke entity access (partial) | `revoke read (attr) on entity Mod.Entity from Mod.Role;` | Partial — downgrades specific rights | diff --git a/docs/11-proposals/PROPOSAL_check_diagnostics_catalog.md b/docs/11-proposals/PROPOSAL_check_diagnostics_catalog.md index 05a87de61a..e0a99f6882 100644 --- a/docs/11-proposals/PROPOSAL_check_diagnostics_catalog.md +++ b/docs/11-proposals/PROPOSAL_check_diagnostics_catalog.md @@ -64,11 +64,58 @@ Integration/REST/OData 129, Security 29, Workflows 15, Navigation 7. `CE`/`CW` codes** across ~24 `validate_*.go` files and ~30 lint rules — proof the approach works; the catalog just turns "port as we hit them" into an ordered backlog of ~1240 left. -**There is no cheaper Mendix-native shortcut.** `mx check` (consistency only, no build) -measured **~22 s** — *slower* than an incremental build — because both cold-start the .NET -model engine + load the full `.mpr`. So mxcli's Go-native `check` is the **only** path under -the ~18–60 s floor. That is the entire justification for porting rules into Go rather than -shelling out to `mx check`. +**Is there a cheaper Mendix-native shortcut?** Not established. `mx check` (consistency +only, no build) was measured **once, at ~22 s** (project size and warm/cold state not +recorded), *slower* than an incremental build, because both cold-start the .NET model +engine and load the full `.mpr`. A later measurement on one small project (mxcli v0.32.0, +Mendix 11.16.1, ako/mxcli#1258) gave `mxcli docker check` 7.6 s and Mendix CLI 2's +`mendix check` (an in-process checker that emits CE codes such as CE0109/CE0844) 1.2 s. +Two single-project numbers do not settle it, but they show 22 s is not a constant. +`mendix check` also reported false errors on some real apps (tester report), so it is a +candidate *pre-filter* for the harvest loop, not a replacement oracle: any finding must be +confirmed against MxBuild before a rule is ported. + +The justification for a Go-native `check` therefore rests on more than speed: it needs no +.NET runtime or mxbuild download, runs in the pre-write path before any model exists on +disk, and can be wired into `exec`. A faster Mendix-native checker would change the cost of +the *harvest loop*, not the case for porting. + +## Spike results: Mendix CLI 2 `mendix check` as a fast oracle (2026-11-16) + +Harness: `mdl-examples/check-gap/run-parity.sh` (fixtures beside it). Mendix 11.16.1 +MxBuild, Mendix CLI 2 (11.16 build), Linux x64, one clean project from `mxcli new` +(0 errors on both checkers), single runs, no cold/warm split. + +| fixture | `mxcli check -p` | `mx check` (MxBuild) | `mendix check` (CLI 2) | +|---|---|---|---| +| duplicate variable | rejected | CE0111 | CE0111 | +| wrong parameter name | rejected | CE1613 | CE0115, CE7710 | +| undefined variable | **passed** | CE0109 | CE0109, CE7823 | +| `DECLARE $X Integer = 'abc'` | **passed** | CE0117 | CE0844 | +| branch with no return | **passed** | CE0109 | CE0109, CE7823 | +| Decimal returned from String microflow | **passed** | CE0117 | CE0844 | +| unknown function `replace` | **passed** | CE0117 | CE7811 | +| commit of non-persistable entity (control) | passed | none | none | + +- **Detection.** Both Mendix checkers flag all 7 broken fixtures and agree on the control. + `mxcli check -p` misses 5 of 7, all Tier B (expression and flow typing). +- **Speed.** `mendix check` 1.3-1.7 s, `mx check` 6.3-8.5 s on the clean project (one 18.3 s outlier on the first fixture of a later run, so run-to-run spread is wide; 4.0 s on + a 28-error project); `mxcli docker check` 7.5 s. None is near the 22 s above. +- **More specific.** Where `mx check` reports the generic CE0117 "Error(s) in + expression", `mendix check` names the cause (CE0844 type mismatch, CE7811 unknown + function). That is the better porting target. +- **Codes differ for the same finding.** On a 28-error project (`testdata/testapp-views`), + all 28 "... no longer exists" findings are CE1613 in `mx check` (11.14.0 and 11.16.1 + alike) and CE7710 in `mendix check`. The harness must compare messages, not only codes. +- **Unconfirmed extras.** On that project `mendix check` reports 34 errors to `mx + check`'s 28: 3x CE0136 (retrieve without entity), 2x CE0535 (grid column weights), + 1x CE6083 (design property unsupported by theme). MxBuild 11.16.1 reports none of + them, so they stay unconfirmed. The CE0136 ones sit on retrieves whose entity no + longer exists, which `mx check` already reports as CE1613, so they look like + cascades rather than independent findings. No extras on the clean project. +- **Conclusion.** `mendix check` works as a first-pass oracle: run it, then confirm any + code it reports against `mx check` before porting. It is a proprietary binary, so the + harness takes its path as an option and nothing is committed. ## Extraction methodology @@ -128,11 +175,14 @@ button, or a URL."* 2. **Port by tier then frequency.** Tier A first (cheapest, highest hit-rate), Tier B behind the type-inference work, Tier C case-by-case, Tier D never. Route `CE`→`check` (hard), `CW`→`lint` (advisory) — severity is free from the prefix. -3. **Harvest-against-the-oracle loop.** Use `mx check` (~22 s, the full 1318-code truth) as - an **offline oracle** over a corpus of deliberately-broken projects; diff its findings +3. **Harvest-against-the-oracle loop.** Use `mx check` (the full 1318-code truth; ~22 s in + one measurement, see above) as an **offline oracle** over a corpus of deliberately-broken projects; diff its findings against `mxcli check`; the misses are the ranked backlog; port; re-run to confirm parity. This is the build tool for the porting effort, and reuses the "run mxbuild over broken projects and harvest output" fallback from the very first spike as a *deliberate* tool. + Open spike: run Mendix CLI 2's `mendix check` over the same fixtures as a fast + pre-filter and record its agreement with `mx check` per code (false-positive and + false-negative rate) before trusting it for anything. 4. `PROPOSAL_check_mxbuild_gap_heuristics.md` is the first tactical instance; this proposal is the strategic program it fits into. @@ -176,6 +226,9 @@ button, or a URL."* per-rule trigger predicate lives in Mendix's `.cs` (the `LOCATION` breadcrumb) and must be reverse-engineered per rule. The catalog gives the *target + message*, not the predicate — sizing must account for that. -4. **Overlap with `PROPOSAL_check_mxbuild_gap_heuristics.md`.** Fold that proposal's 6 cases +4. **Fast oracle.** The spike above supports `mendix check` as a first-pass oracle with + `mx check` confirming. Still open: the 6 unconfirmed extras on a broken project, a + corpus larger than 8 fixtures, and cold/warm timings. +5. **Overlap with `PROPOSAL_check_mxbuild_gap_heuristics.md`.** Fold that proposal's 6 cases in as the first harvested batch, or keep it separate and cross-link? (Recommend: keep it as the tactical pilot; this proposal owns the methodology + backlog.) diff --git a/docs/11-proposals/PROPOSAL_verify_gate.md b/docs/11-proposals/PROPOSAL_verify_gate.md new file mode 100644 index 0000000000..9e0eefcefb --- /dev/null +++ b/docs/11-proposals/PROPOSAL_verify_gate.md @@ -0,0 +1,223 @@ +--- +title: mxcli verify — one gate that reports only what a change made worse +status: draft +date: 2026-11-16 +related: + - PROPOSAL_check_mxbuild_gap_heuristics.md + - PROPOSAL_check_diagnostics_catalog.md + - PROPOSAL_agent_loop_efficiency.md +--- + +# Proposal: `mxcli verify` — a staged gate with baseline comparison + +**Status:** Draft +**Date:** 2026-11-16 +**Issue:** ako/mxcli#2333 +**Relates to:** ako/mxcli#1313 (`mx check` baseline), #1312 (`diff --mx-check`), +#1258 (`check --full`), #2308 (`replace()` passes `check`, fails `mx check`) + +## Problem + +An agent or CI job that changes a project must know which of six commands to run, +in which order, and how to separate its own new findings from pre-existing ones. +Only `lint` can compare against a baseline (`--baseline`); every other stage has +its own output format and no notion of "new". + +Measured in #2333: `mxcli check -p` said "Check passed!" for five of eight scripts +that an independent checker rejected (undefined variable CE0109, assignment type +mismatch CE0844, branch without return CE0109, Decimal returned as String CE0844, +`replace()` CE7811). `lint` added nothing for those. `mxcli docker check` caught +the undefined variable in 7.6 s. So the fast checks cannot be the whole gate; the +gate must include `mx check`. + +**Relation to the gap programme.** These five misses are intake for +`PROPOSAL_check_mxbuild_gap_heuristics.md` (a standing programme that adds a static +heuristic each time `check` passes what `mx check` rejects) and +`PROPOSAL_check_diagnostics_catalog.md`. `verify` fixes none of them. It exists so +that the gap, whatever its size at any moment, cannot pass unnoticed: it makes the +~8 s authoritative stage part of the loop. Closing the gap stays with `check`. + +## Proposed behaviour + +``` +mxcli verify -p app.mpr [--base | --baseline ] + [--stages check,lint,mxcheck,build,test,smoke] + [--fail-fast] [--format text|json|sarif] + [--update-baseline] +``` + +| Stage | Reuses | Default | Cost | +|-------|--------|---------|------| +| `check` | project-wide reference check (`check -p … --references`) | on | ~0.2 s | +| `lint` | `linter` + `Baseline` | on | ~0.2–0.9 s | +| `mxcheck` | `docker.Check` | on | ~8 s | +| `build` | `run --local` build/boot | opt-in | 30–60 s | +| `test` | `test --local` | opt-in | ~24 s | +| `smoke` | `playwright check --role X` | opt-in | ~24 s | + +- Stages run in cost order. **Default: run every selected stage** and report all + new findings in one result. The consumer is an agent, and stopping at the first + failing stage costs one round trip per stage (fix, rerun, next stage fails), + each re-reading the whole conversation (the cost model in + `PROPOSAL_agent_loop_efficiency.md`). Spending ~8 s of `mxcheck` once saves two + reruns. `--fail-fast` stops at the first stage with new findings, for CI. + Opt-in stages (`build`, `test`, `smoke`) are still skipped unless selected, and + are skipped when an earlier stage failed, since they need a model that builds. +- Findings are classified **new / resolved / pre-existing** against the baseline, + keyed by `(code, document, element path)` — never by message text or line + number, which drift between runs (the problem #1313 describes). +- **Exit code:** 0 when nothing is new; 2 when the gate could not run (no `mx`, + no runtime for a selected stage), so "gate failed" and "gate could not run" are + never confused. When stages fail, the code is a **bitmask** (1 check, 2 lint, + 4 mxcheck, 8 build, 16 test, 32 smoke) offset so it cannot collide with 0 or 2 + — exact encoding to be fixed in phase 2 — because with run-all as the default + several stages can fail at once. The per-stage detail is always in the output; + the code is only a summary. **Relation to existing codes:** `check` exits with a + variable code (`cmd_check.go`) and `lint` exits 1. `verify`'s table is its own + contract and must say so in `--help`; an agent running `check` then `verify` + will see different codes for the same finding. +- **Output:** `--format text|json|sarif`, matching `check` and `lint` (the global + `--json` is the existing alias via `resolveFormat`), not a new flag. JSON is one + object `{stages:[{name,status,durationMs,new:[...],resolved:, + preExisting:int}], exit}`, so an agent reads one result instead of six formats. + `resolved` is typed honestly per stage: a list where the key identifies the + element, a count where it cannot (see Finding identity). +- **Machine-output discipline.** `verify` must register in `machineReadableFormats` + (`output_stream.go`), send progress through `progressSink(format)` so stdout + carries only the payload, and get a row in `json_output_purity_test.go` + (modelled on `TestCheckStructuredOutputIsOneDocumentOnStdout`). `lint --format + json` once printed "Connected to:" ahead of the JSON; a new command must not + repeat that. +- **SARIF.** `check` and `lint` already emit SARIF, which has `partialFingerprints` + and `baselineState: new | unchanged | absent` — the same three states as this + proposal. `--format sarif` is therefore supported rather than a seventh bespoke + format being the only option. The proposal still defines the JSON shape because + SARIF is verbose and awkward for an agent to read in one call, and because it + has no place for per-stage durations or "could not run"; the JSON is a + projection of the same internal findings and the SARIF writer is its second + rendering. + +### Baseline sources + +1. `--baseline ` — saved findings (cheap; ships first). `--update-baseline` + records the current state. +2. `--base ` — run the same stages on a scratch export of that ref, then + compare. Roughly doubles `mxcheck` cost; results are cacheable per commit. + +## Existing pieces and gaps (checked against `main` at `ecdd9b7c`) + +- `mdl/linter/baseline.go` has `WriteBaseline`/`LoadBaseline`/`Compare`, but it is + **not reusable as is** against the `Finding` below: + - its key prefers `DocumentID` (`baselineKey`: `rule \0 id \0 ` when an ID is + present), so a verify finding needs an `ID` field to reproduce it; + - `Compare` pairs multiplicities on identical **message text** within a key group + (pass 1), then treats an unmatched one as a reworded form (pass 2), so + "message is display only" cannot hold for the lint stage unless `Compare` is + wrapped or replaced; + - `BaselineComparison` is `{New []Violation, Unchanged int, Fixed int}`, and the + file stores messages per key, not whole violations: lint's `resolved` is a + **count**, not a list. + The lint stage therefore either keeps lint's own file and `Compare` and adapts + the result, or `Finding` grows an `ID` and the matching rules are restated. + Phase 2 decides; it is not "reuse as is". +- `docker.Check(opts) error` (`cmd/mxcli/docker/check.go`) runs `mx` on a temporary + copy and **writes free text** to `opts.Stdout`; it returns only an error. + (`appErrorCount` is an unexported helper.) **There are no structured `mx check` + findings to baseline — this is the central gap.** The `mxCheckCmd` and + `resolveMxForCheck` test seams are worth keeping. +- `check` and `lint` take `--format text|json|sarif` (`main.go`). A project-wide + reference-check entry point for the `check` stage needs a thin adapter. +- `test --local` (`testRunCmd`) and `playwright check` have their own reporting; + each needs an adapter to the common shape. +- `diff-local --ref` resolves git refs for mxunit diffs; exporting an MPR at a ref + (v1 needs an extraction step, v2 is `mprcontents/`) is new. +- **Name:** the only existing `verify` is `playwright verify ` (runs + Playwright test scripts). `mxcli verify` next to `mxcli playwright verify` is the + ordinary noun-under-a-namespace case, not a collision of two top-level commands. + +## Finding identity: for which codes can the key be built at all? + +The key `(code, document, element path)` is only as good as `mx check`'s text. In +`recheckCE0463` (`docker/check.go`): *"mx check's text output does not name a +CE0463's page, so two same-named widgets on different pages print the same line; +the lines are compared as counts."* For CE0463 the element identity is **not in +the output**. Phase 1 must therefore classify each code the parser sees: + +1. **Located** — the line names document and element (e.g. CE0109 on a microflow + activity). Key = `(code, document, path)`; `resolved` is a list. +2. **Document-only** — names the document but not the element. Key = + `(code, document)` with multiplicity; matching is by count per key. +3. **Unlocated** — names neither (CE0463). Fallback: **count-based matching per + distinct line**, exactly as `recheckCE0463` does today; `resolved` is a count + delta, never a list, and `--format json` says so. + +The CE0463 stale/persisting labelling from the re-check must survive unchanged as +a label on class 3 findings. The first question for phase 1 is the classification +table (code → class), built from real `mx check` output; stability under rename +or move (formerly the only question) applies only to class 1. + +## Design + +New package `cmd/mxcli/verify/` (orchestrator, no business logic): + +```go +type Finding struct { Stage, Code, Document, DocumentID, Path, Message string } +type Stage interface { + Name() string + Run(ctx context.Context, p Project) ([]Finding, error) +} +``` + +- `Finding.Key()` is **per class** (see Finding identity): `Stage + Code + Document + Path` for located findings, down to count-per-line for unlocated ones. The lint stage's key additionally uses `ID` when present, as `baselineKey` does. Message is display only except where a stage's existing matcher (lint `Compare`) uses it, and then that is stated. +- Shared **`mx check` output parser** (`docker.ParseCheckOutput`) producing + `[]Finding` from `mx` lines (`ERROR [CE0109] "…" at Microflow 'M.F', Activity …`). + Also unblocks #1313 and #1312; lands as its own PR. +- Baseline file is versioned JSON, sorted for stable diffs, one section per stage + (lint reuses its existing format inside it or keeps its own file — see questions). +- Infrastructure failure (no `mx`, no Docker/runtime for opt-in stages) is a + distinct status `error`, never `passed`. + +## Out of scope + +- Fixing findings; running stages in parallel (cost order + early stop is the point); + replacing `docker check`, `lint`, `test` or `playwright check`. +- `check --full` (#1258): `mxcheck` stage calls `docker check` until it exists, + then may be swapped or made optional. + +## Phasing + +1. **Structured `mx check` findings** — parser + `docker.CheckFindings()`, tests + against recorded `mx` output fixtures. Standalone PR; unblocks #1313. +2. **`mxcli verify`** with `check`, `lint`, `mxcheck`; `--baseline`, + `--update-baseline`, `--format`, exit-code table, `--fail-fast`. +3. **`--base `** via scratch export. +4. **Opt-in `build`, `test`, `smoke` stages.** + +## Testing + +- Unit: finding key normalisation per class (located / document-only / unlocated) (same finding, different line/message → same + key); classification new/resolved/pre-existing; default run-all vs `--fail-fast`. +- Parser: golden `mx check` outputs (CE0109, CE0844, CE7811, CE0463 labelled). +- Control (per CLAUDE.md): a "nothing new" test paired with one that injects a + finding and must fail. +- Output purity row in `json_output_purity_test.go`; exit-code table incl. several failing stages at once and "could not run". +- Executor-level orchestration with stub stages (same seam style as + `mxCheckCmd`); integration (`-tags integration`) for real `mx`. +- Runtime stages follow `.claude/skills/verify-in-runtime.md`. + +## Open questions + +1. **Name.** `mxcli verify` alongside `mxcli playwright verify`: acceptable, or + prefer `gate`/`validate`? (Much milder than first drafted: one existing + `verify`, under a namespace.) +2. **One baseline file or one per stage?** One file is simpler for agents; + per-stage keeps `lint --baseline` files valid unchanged. +3. **Code classification** (phase 1): which `mx check` codes are located, + document-only, or unlocated, and is count-based matching acceptable for the + unlocated ones? +4. **`--base` for MPR v1** — extract via `git show` of the single `.mpr`, or require v2? +5. **Should `check` stage fail on warnings,** or only errors? +6. **Exit-code encoding** — bitmask offsets, and whether to align with `check`'s + and `lint`'s existing codes. +7. **Default stage set** — with run-all as the default, `mxcheck` costs ~8 s once + per iteration; is on-by-default accepted? diff --git a/internal/projectlock/lock.go b/internal/projectlock/lock.go new file mode 100644 index 0000000000..eb9e5657de --- /dev/null +++ b/internal/projectlock/lock.go @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package projectlock serialises the mxcli processes that write one project. +// +// Four `mxcli exec` processes writing one project at once all failed +// (ako/mxcli#2326): each held the .mpr open, wrote units, and found the +// others' writes under it — "disk I/O error (5898)", "replaced on disk while +// unwritten changes were held", a temp-file rename that lost its race — and +// each rolled back, so none of the work landed and no message said to retry. +// A write run is not a single SQLite transaction that SQLite could serialise: +// it is a run of unit files and an index, and a failing run restores its +// backups, which would put back files a parallel run had just written. +// +// So a run that writes holds the project's lock from start to finish, and a +// second one waits for it. The lock is a file created with O_EXCL — the one +// primitive that means the same on Linux, macOS and Windows — under the +// project's .mxcli folder. It names its holder, so a refusal can say who, and +// is kept fresh by a heartbeat, so a lock left by a crashed run (killed -9, a +// container gone) is taken over instead of locking the project for good: at +// once when its holder is a process on this host that no longer exists, else +// once the heartbeat has stopped for staleAfter. +package projectlock + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/mendixlabs/mxcli/internal/procalive" +) + +const ( + // heartbeat is how often a holder touches its lock. + heartbeat = 2 * time.Second + // staleAfter is how long a lock may go untouched before it is taken over + // although its holder cannot be checked (another host, a container). + staleAfter = 30 * time.Second + poll = 50 * time.Millisecond + // DefaultWait is how long a writer waits for another to finish. + DefaultWait = 60 * time.Second +) + +// Options configure Acquire. +type Options struct { + // Wait bounds the wait for a live holder; 0 is DefaultWait, negative is + // no wait. + Wait time.Duration + // Purpose says what the run does ("exec script.mdl"), for the message a + // waiting writer gets. + Purpose string +} + +// BusyError is the refusal when another process held the lock for the whole +// wait. +type BusyError struct { + Project string + Holder string + Waited time.Duration + Lock string +} + +func (e *BusyError) Error() string { + return fmt.Sprintf("another mxcli process is writing %s (%s), and it still was after %s: "+ + "retry when it has finished — writes to one project run one at a time. "+ + "If no mxcli process is running, delete %s", + filepath.Base(e.Project), e.Holder, e.Waited.Round(time.Second), e.Lock) +} + +// Path is the lock file of the project at mprPath. +func Path(mprPath string) string { + return filepath.Join(filepath.Dir(mprPath), ".mxcli", "write.lock") +} + +// Acquire blocks until this process holds the project's write lock, and +// returns the function that releases it. Release is safe to call more than +// once, and never removes a lock another process has taken since. +func Acquire(mprPath string, opts Options) (release func(), err error) { + path := Path(mprPath) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, err + } + wait := opts.Wait + if wait == 0 { + wait = DefaultWait + } + token := newToken() + host, _ := os.Hostname() + content := fmt.Sprintf("pid=%d\nhost=%s\npurpose=%s\nsince=%s\ntoken=%s\n", + os.Getpid(), host, oneLine(opts.Purpose), time.Now().UTC().Format(time.RFC3339), token) + + start := time.Now() + for { + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err == nil { + _, werr := f.WriteString(content) + cerr := f.Close() + if werr != nil || cerr != nil { + _ = os.Remove(path) + return nil, fmt.Errorf("write the project lock %s: %v", path, firstErr(werr, cerr)) + } + return holdLock(path, token), nil + } + if !os.IsExist(err) { + return nil, err + } + holder, abandoned := inspect(path, host) + if abandoned { + _ = os.Remove(path) // a race to take it over is settled by O_EXCL + continue + } + if wait < 0 || time.Since(start) >= wait { + return nil, &BusyError{Project: mprPath, Holder: holder, Waited: time.Since(start), Lock: path} + } + time.Sleep(poll) + } +} + +// holdLock starts the heartbeat and returns the release. +func holdLock(path, token string) func() { + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + t := time.NewTicker(heartbeat) + defer t.Stop() + for { + select { + case <-stop: + return + case now := <-t.C: + if owns(path, token) { + _ = os.Chtimes(path, now, now) + } + } + } + }() + released := false + return func() { + if released { + return + } + released = true + close(stop) + <-done + if owns(path, token) { + _ = os.Remove(path) + } + } +} + +// inspect describes the lock's holder and reports whether the lock is +// abandoned: its holder is a process on this host that is gone, or its +// heartbeat stopped staleAfter ago. +func inspect(path, host string) (holder string, abandoned bool) { + raw, err := os.ReadFile(path) + if err != nil { + // Released between our create and this read: try again. + return "", os.IsNotExist(err) + } + fields := map[string]string{} + for _, line := range strings.Split(string(raw), "\n") { + if k, v, ok := strings.Cut(line, "="); ok { + fields[k] = v + } + } + holder = "pid " + fields["pid"] + if fields["host"] != "" && fields["host"] != host { + holder += " on " + fields["host"] + } + if p := fields["purpose"]; p != "" { + holder += ", " + p + } + if fields["host"] == host { + if pid, err := strconv.Atoi(fields["pid"]); err == nil && !procalive.Alive(pid) { + return holder, true + } + } + if st, err := os.Stat(path); err == nil && time.Since(st.ModTime()) > staleAfter { + return holder, true + } + return holder, false +} + +// owns reports whether the lock file is still the one this holder wrote. +func owns(path, token string) bool { + raw, err := os.ReadFile(path) + return err == nil && strings.Contains(string(raw), "token="+token+"\n") +} + +func newToken() string { + b := make([]byte, 8) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") } + +func firstErr(errs ...error) error { + for _, e := range errs { + if e != nil { + return e + } + } + return nil +} diff --git a/internal/projectlock/lock_test.go b/internal/projectlock/lock_test.go new file mode 100644 index 0000000000..f0a970961e --- /dev/null +++ b/internal/projectlock/lock_test.go @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: Apache-2.0 + +package projectlock + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// ako/mxcli#2326: four `mxcli exec` processes writing one project at once +// all failed, with "disk I/O error (5898)", "replaced on disk while unwritten +// changes were held" and a temp-file rename error, and none said to retry. A +// writer now holds the project's lock for its run; a second one waits, and +// past its wait fails naming the holder. + +func project(t *testing.T) string { + t.Helper() + dir := t.TempDir() + mpr := filepath.Join(dir, "App.mpr") + if err := os.WriteFile(mpr, nil, 0o644); err != nil { + t.Fatal(err) + } + return mpr +} + +func TestAcquire_SecondWriterWaitsThenFailsNamingTheHolder_Issue2326(t *testing.T) { + mpr := project(t) + release, err := Acquire(mpr, Options{Wait: time.Second, Purpose: "exec a.mdl"}) + if err != nil { + t.Fatal(err) + } + defer release() + start := time.Now() + _, err = Acquire(mpr, Options{Wait: 300 * time.Millisecond, Purpose: "exec b.mdl"}) + var busy *BusyError + if !errors.As(err, &busy) { + t.Fatalf("second Acquire = %v, want a BusyError", err) + } + if time.Since(start) < 250*time.Millisecond { + t.Errorf("did not wait before failing") + } + for _, want := range []string{"another mxcli process", "exec a.mdl", "retry"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q lacks %q", err, want) + } + } +} + +func TestAcquire_WaitingWriterGetsTheLockOnRelease_Issue2326(t *testing.T) { + mpr := project(t) + release, err := Acquire(mpr, Options{Wait: time.Second}) + if err != nil { + t.Fatal(err) + } + go func() { + time.Sleep(150 * time.Millisecond) + release() + }() + r2, err := Acquire(mpr, Options{Wait: 3 * time.Second}) + if err != nil { + t.Fatalf("waiting writer did not get the lock after release: %v", err) + } + r2() +} + +// A lock left by a process that is gone (crashed, killed -9) does not block +// the project: one whose holder is not alive on this host is taken over. +func TestAcquire_TakesOverADeadHoldersLock_Issue2326(t *testing.T) { + mpr := project(t) + path := Path(mpr) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + host, _ := os.Hostname() + stale := "pid=999999999\nhost=" + host + "\npurpose=exec old.mdl\n" + if err := os.WriteFile(path, []byte(stale), 0o600); err != nil { + t.Fatal(err) + } + release, err := Acquire(mpr, Options{Wait: 200 * time.Millisecond}) + if err != nil { + t.Fatalf("a dead holder's lock blocked the project: %v", err) + } + release() + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Errorf("release left the lock file behind") + } +} + +// CONTROL: release is idempotent and does not remove a lock someone else has +// taken since. +func TestRelease_DoesNotRemoveAnotherHoldersLock_Issue2326(t *testing.T) { + mpr := project(t) + release, err := Acquire(mpr, Options{Wait: time.Second}) + if err != nil { + t.Fatal(err) + } + release() + r2, err := Acquire(mpr, Options{Wait: time.Second}) + if err != nil { + t.Fatal(err) + } + release() // the first holder's release again + if _, err := os.Stat(Path(mpr)); err != nil { + t.Errorf("a second release removed the new holder's lock: %v", err) + } + r2() +} diff --git a/mdl-examples/check-gap/README.md b/mdl-examples/check-gap/README.md new file mode 100644 index 0000000000..e6cb017a6e --- /dev/null +++ b/mdl-examples/check-gap/README.md @@ -0,0 +1,22 @@ +# check-gap: broken-script fixtures for the check-vs-Mendix parity harness + +Each `fixtures/fNN_*.mdl` is a small script that `mxcli exec --no-check` writes +happily but a Mendix checker rejects (or, for `f10`, accepts: a neutral control). +`fixtures/00_base.mdl` creates the module the fixtures build on and is prepended +to each. See `docs/11-proposals/PROPOSAL_check_diagnostics_catalog.md` +(harvest-against-the-oracle loop). + +`run-parity.sh` writes every fixture into a copy of a clean project and prints +the error codes reported by `mxcli check -p` (pre-write), MxBuild `mx check`, +and optionally Mendix CLI 2 `mendix check` (a proprietary binary that is never +committed; pass its path). Gated: needs mxbuild, so it is not part of `make test`. + + mxcli setup mxbuild --version 11.16.1 + mxcli new Spike --version 11.16.1 --skip-build --skip-init --theme none \ + --layout none --output-dir /tmp/clean + mdl-examples/check-gap/run-parity.sh --project /tmp/clean/Spike.mpr \ + --mx ~/.mxcli/mxbuild/11.16.1/modeler/mx [--mendix /path/to/mendix] + +A fixture that `mxcli check -p` passes while `mx check` reports a code is a +backlog item. Codes can differ between checkers for the same finding (CE1613 in +mx is CE7710 in `mendix check`), so compare messages when a code differs. diff --git a/mdl-examples/check-gap/fixtures/00_base.mdl b/mdl-examples/check-gap/fixtures/00_base.mdl new file mode 100644 index 0000000000..b83a4cfa64 --- /dev/null +++ b/mdl-examples/check-gap/fixtures/00_base.mdl @@ -0,0 +1,4 @@ +mdl 1; +create module Sp; +create persistent entity Sp.Item ( Name: String(100), Amount: Decimal, Qty: Integer ); +create microflow Sp.Helper ($Text: String) returns String as $R begin return $Text; end; diff --git a/mdl-examples/check-gap/fixtures/f01_dupvar.mdl b/mdl-examples/check-gap/fixtures/f01_dupvar.mdl new file mode 100644 index 0000000000..75f5486b96 --- /dev/null +++ b/mdl-examples/check-gap/fixtures/f01_dupvar.mdl @@ -0,0 +1,2 @@ +mdl 1; +create microflow Sp.F01 () returns Integer as $R begin declare $X Integer = 1; declare $X Integer = 2; return $X; end; diff --git a/mdl-examples/check-gap/fixtures/f02_badparam.mdl b/mdl-examples/check-gap/fixtures/f02_badparam.mdl new file mode 100644 index 0000000000..471c65f25c --- /dev/null +++ b/mdl-examples/check-gap/fixtures/f02_badparam.mdl @@ -0,0 +1,2 @@ +mdl 1; +create microflow Sp.F02 () returns String as $R begin $S = call microflow Sp.Helper(Wrong = 'a'); return $S; end; diff --git a/mdl-examples/check-gap/fixtures/f05_undef.mdl b/mdl-examples/check-gap/fixtures/f05_undef.mdl new file mode 100644 index 0000000000..6498aa0133 --- /dev/null +++ b/mdl-examples/check-gap/fixtures/f05_undef.mdl @@ -0,0 +1,2 @@ +mdl 1; +create microflow Sp.F05 () returns Integer as $R begin return $Nope + 1; end; diff --git a/mdl-examples/check-gap/fixtures/f06_assigntype.mdl b/mdl-examples/check-gap/fixtures/f06_assigntype.mdl new file mode 100644 index 0000000000..edc452fb4d --- /dev/null +++ b/mdl-examples/check-gap/fixtures/f06_assigntype.mdl @@ -0,0 +1,2 @@ +mdl 1; +create microflow Sp.F06 () returns Integer as $R begin declare $X Integer = 'abc'; return $X; end; diff --git a/mdl-examples/check-gap/fixtures/f07_noreturn.mdl b/mdl-examples/check-gap/fixtures/f07_noreturn.mdl new file mode 100644 index 0000000000..01c17b2eb9 --- /dev/null +++ b/mdl-examples/check-gap/fixtures/f07_noreturn.mdl @@ -0,0 +1,2 @@ +mdl 1; +create microflow Sp.F07 ($B: Boolean) returns Boolean as $R begin if $B then return true; end if; end; diff --git a/mdl-examples/check-gap/fixtures/f08_rettype.mdl b/mdl-examples/check-gap/fixtures/f08_rettype.mdl new file mode 100644 index 0000000000..922b61ccc5 --- /dev/null +++ b/mdl-examples/check-gap/fixtures/f08_rettype.mdl @@ -0,0 +1,2 @@ +mdl 1; +create microflow Sp.F08 ($I: Sp.Item) returns String as $R begin return $I/Amount; end; diff --git a/mdl-examples/check-gap/fixtures/f09_badfunc.mdl b/mdl-examples/check-gap/fixtures/f09_badfunc.mdl new file mode 100644 index 0000000000..3739af8f3c --- /dev/null +++ b/mdl-examples/check-gap/fixtures/f09_badfunc.mdl @@ -0,0 +1,2 @@ +mdl 1; +create microflow Sp.F09 ($S: String) returns String as $R begin return replace($S, 'a', 'b'); end; diff --git a/mdl-examples/check-gap/fixtures/f10_commit_nonpers.mdl b/mdl-examples/check-gap/fixtures/f10_commit_nonpers.mdl new file mode 100644 index 0000000000..ee666e45ba --- /dev/null +++ b/mdl-examples/check-gap/fixtures/f10_commit_nonpers.mdl @@ -0,0 +1,3 @@ +mdl 1; +create non-persistent entity Sp.Np ( A: String(10) ); +create microflow Sp.F10 ($N: Sp.Np) returns Boolean as $R begin commit $N; return true; end; diff --git a/mdl-examples/check-gap/run-parity.sh b/mdl-examples/check-gap/run-parity.sh new file mode 100755 index 0000000000..36bbcd331a --- /dev/null +++ b/mdl-examples/check-gap/run-parity.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Parity harness: mxcli check -p vs mx check (vs mendix check) over broken fixtures. +set -uo pipefail +PROJECT=""; MX=""; MENDIX=""; MXCLI="${MXCLI:-./bin/mxcli}" +while [ $# -gt 0 ]; do case "$1" in + --project) PROJECT="$2"; shift 2;; --mx) MX="$2"; shift 2;; + --mendix) MENDIX="$2"; shift 2;; --mxcli) MXCLI="$2"; shift 2;; + *) echo "unknown arg $1" >&2; exit 2;; esac; done +[ -f "$PROJECT" ] && [ -x "$MX" ] || { echo "usage: $0 --project clean.mpr --mx .../modeler/mx [--mendix path]" >&2; exit 2; } +HERE="$(cd "$(dirname "$0")" && pwd)"; PDIR="$(dirname "$PROJECT")"; PNAME="$(basename "$PROJECT")" +LIB=$(ls /usr/lib/$(uname -m)-linux-gnu/libfreetype.so.6 2>/dev/null || true) +[ -n "$LIB" ] && export LD_PRELOAD="$LIB" +codes() { grep -o '\[C[EW][0-9]*\]' | sort -u | tr -d '[]' | tr '\n' ' '; } +TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT +printf '%-22s | %-14s | %-16s | %-16s | %s\n' fixture "check -p" "mx check" "mendix check" "secs mx/mendix" +for f in "$HERE"/fixtures/f*.mdl; do + n=$(basename "$f" .mdl); W="$TMP/$n"; cp -r "$PDIR" "$W" + { cat "$HERE/fixtures/00_base.mdl"; grep -v '^mdl 1;$' "$f"; } > "$W/script.mdl" # one header only + pre=$("$MXCLI" check "$W/script.mdl" -p "$PROJECT" 2>&1); case "$pre" in *"Check passed"*) pre=passed;; *) pre=REJECTED;; esac + "$MXCLI" exec "$W/script.mdl" -p "$W/$PNAME" --no-check >/dev/null 2>&1 || { printf '%-22s | exec refused\n' "$n"; continue; } + t0=$(date +%s.%N); b=$(cd "$W" && "$MX" check "$PNAME" 2>&1); t1=$(date +%s.%N) + a="-"; t2=$t1; if [ -n "$MENDIX" ]; then a=$(cd "$W" && "$MENDIX" check "$PNAME" 2>&1 | codes); t2=$(date +%s.%N); fi + printf '%-22s | %-14s | %-16s | %-16s | %.1f/%.1f\n' "$n" "$pre" "$(echo "$b"|codes)" "$a" "$(echo "$t1-$t0"|bc)" "$(echo "$t2-$t1"|bc)" +done diff --git a/mdl/ast/ast_alter_flow.go b/mdl/ast/ast_alter_flow.go index b583837c8d..4b4875df9c 100644 --- a/mdl/ast/ast_alter_flow.go +++ b/mdl/ast/ast_alter_flow.go @@ -82,4 +82,12 @@ type AlterFlowOperation struct { // in place, never replaced (mendixlabs/mxcli#1517). Set only by // `create or modify`. SetNotes bool + // StatesCurves, on an insert, says InCurve and OutCurve are the curves the + // script states for the connector into the inserted statements (on the + // statement before them) and the one out of them (on their last), nil + // meaning a straight line, as describe prints none for one. Set only by + // `create or modify`, whose declared flow is drawn whole; an `alter` + // insert states no curves (ako/mxcli#2311). + StatesCurves bool + InCurve, OutCurve *FlowCurve } diff --git a/mdl/backend/menumutator/insert_rerun_2312_test.go b/mdl/backend/menumutator/insert_rerun_2312_test.go new file mode 100644 index 0000000000..f4726a2190 --- /dev/null +++ b/mdl/backend/menumutator/insert_rerun_2312_test.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 + +package menumutator + +import ( + "strings" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/types" +) + +// ako/mxcli#2312: `alter navigation … { insert { menu item 'Helpdesk chat' (…) } }` +// run twice put the item in the menu twice, and the second run reported +// "Altered navigation" — a project script that is not re-runnable. An item the +// list already holds, as the statement builds it, is now already there. + +func menuDocWith(items ...bson.D) bson.D { + arr := bson.A{int32(3)} + for _, it := range items { + arr = append(arr, it) + } + return bson.D{{Key: "$Type", Value: "Menus$MenuDocument"}, {Key: "ItemCollection", Value: bson.D{ + {Key: "$Type", Value: "Menus$MenuItemCollection"}, {Key: "Items", Value: arr}}}} +} + +func TestInsertMenuItems_RerunInsertsNothing_Issue2312(t *testing.T) { + m, err := NewMenuDocument(menuDocWith(item("Home")), "u", &fakeDeps{}) + if err != nil { + t.Fatal(err) + } + spec := []types.NavMenuItemSpec{{Caption: "Helpdesk chat"}} + for run := 0; run < 2; run++ { + if err := m.InsertMenuItems(nil, len(m.MenuTree()), spec); err != nil { + t.Fatalf("run %d: %v", run+1, err) + } + } + if tree := m.MenuTree(); len(tree) != 2 || tree[1].Caption != "Helpdesk chat" { + t.Errorf("tree after two runs = %+v, want Home and one Helpdesk chat", captions(tree)) + } +} + +// actionDeps builds an item whose action differs from the stored one's. +type actionDeps struct{ fakeDeps } + +func (actionDeps) BuildMenuItem(spec types.NavMenuItemSpec) (bson.D, error) { + it := item(spec.Caption) + for i, e := range it { + if e.Key == "Action" { + it[i].Value = bson.D{{Key: "$Type", Value: "Forms$FormAction"}, {Key: "FormSettings", Value: "Helpdesk.Chat"}} + } + } + return it, nil +} + +// An item of the same caption that the statement builds differently is not a +// re-run: inserting it would leave two items of one caption, and skipping it +// would drop the statement's change. It is refused, naming replace. +func TestInsertMenuItems_SameCaptionDifferentItemRefused_Issue2312(t *testing.T) { + m, err := NewMenuDocument(menuDocWith(item("Helpdesk chat")), "u", &actionDeps{}) + if err != nil { + t.Fatal(err) + } + err = m.InsertMenuItems(nil, 1, []types.NavMenuItemSpec{{Caption: "Helpdesk chat"}}) + if err == nil || !strings.Contains(err.Error(), "replace") { + t.Errorf("err = %v, want a refusal naming replace", err) + } + if tree := m.MenuTree(); len(tree) != 1 { + t.Errorf("tree = %v, want it unchanged", captions(m.MenuTree())) + } +} + +// CONTROL: an item whose caption is not in the list is inserted, and the same +// caption in another list (a sub-menu) is not this list's. +func TestInsertMenuItems_NewCaptionInserted_Issue2312(t *testing.T) { + m, err := NewMenuDocument(menuDocWith(item("Home", item("Helpdesk chat"))), "u", &fakeDeps{}) + if err != nil { + t.Fatal(err) + } + if err := m.InsertMenuItems(nil, 1, []types.NavMenuItemSpec{{Caption: "Helpdesk chat"}}); err != nil { + t.Fatal(err) + } + if tree := m.MenuTree(); len(tree) != 2 { + t.Errorf("tree = %v, want the top-level item added", captions(tree)) + } +} + +func captions(tree []*backend.MenuTreeNode) []string { + var out []string + for _, n := range tree { + out = append(out, n.Caption) + } + return out +} diff --git a/mdl/backend/menumutator/mutator.go b/mdl/backend/menumutator/mutator.go index 14f433f3d5..25a9212ceb 100644 --- a/mdl/backend/menumutator/mutator.go +++ b/mdl/backend/menumutator/mutator.go @@ -267,10 +267,113 @@ func (m *Mutator) InsertMenuItems(parent backend.MenuItemPath, index int, specs if index < 0 || index > len(items) { return fmt.Errorf("menu position %d is outside a list of %d items", index, len(items)) } + built, err = withoutItemsAlreadyThere(items, built) + if err != nil { + return err + } + if len(built) == 0 { + return nil + } out := append(append(append([]bson.D(nil), items[:index]...), built...), items[index:]...) return setItems(holder, out) } +// withoutItemsAlreadyThere drops from built each item the list already holds +// as the statement builds it, so a script that inserts a menu item can run +// again without adding it twice (ako/mxcli#2312). An item of the same caption +// that is built differently — another action, other sub-items — is refused: +// inserting it would leave two items of one caption, and skipping it would +// drop the statement's change. +func withoutItemsAlreadyThere(items, built []bson.D) ([]bson.D, error) { + var out []bson.D + for _, b := range built { + caption := captionOf(b) + var same *bson.D + for i := range items { + if captionOf(items[i]) == caption { + same = &items[i] + break + } + } + if same == nil { + out = append(out, b) + continue + } + if sameMenuItem(*same, b) { + continue + } + return nil, fmt.Errorf("the list already has an item %q, with another action or other sub-items — "+ + "`replace '%s' with { … }` changes it; insert would leave two items of one caption", caption, caption) + } + return out, nil +} + +// sameMenuItem reports whether stored is the item built would write: equal +// once element ids are ignored and stored's captions are kept to the languages +// built writes (a translation added in Studio Pro is not a difference). +func sameMenuItem(stored, built bson.D) bool { + langs := map[string]bool{} + collectLanguages(built, langs) + a, err1 := bson.Marshal(comparableItem(stored, langs)) + b, err2 := bson.Marshal(comparableItem(built, langs)) + if err1 != nil || err2 != nil { + return false + } + return bytes.Equal(a, b) +} + +func collectLanguages(v any, into map[string]bool) { + switch n := v.(type) { + case bson.D: + if bsonnav.DGetString(n, "$Type") == "Texts$Translation" { + into[bsonnav.DGetString(n, "LanguageCode")] = true + } + for _, e := range n { + collectLanguages(e.Value, into) + } + case bson.A: + for _, e := range n { + collectLanguages(e, into) + } + } +} + +// comparableItem is a copy of v without $IDs, with its keys sorted, and with +// only the translations in langs. +func comparableItem(v any, langs map[string]bool) any { + switch n := v.(type) { + case bson.D: + out := bson.D{} + for _, e := range n { + if e.Key == "$ID" { + continue + } + out = append(out, bson.E{Key: e.Key, Value: comparableItem(e.Value, langs)}) + } + sortKeys(out) + return out + case bson.A: + out := bson.A{} + for _, e := range n { + if d, ok := e.(bson.D); ok && bsonnav.DGetString(d, "$Type") == "Texts$Translation" && + !langs[bsonnav.DGetString(d, "LanguageCode")] { + continue + } + out = append(out, comparableItem(e, langs)) + } + return out + } + return v +} + +func sortKeys(d bson.D) { + for i := 1; i < len(d); i++ { + for j := i; j > 0 && d[j-1].Key > d[j].Key; j-- { + d[j-1], d[j] = d[j], d[j-1] + } + } +} + func (m *Mutator) build(specs []types.NavMenuItemSpec) ([]bson.D, error) { var out []bson.D for _, s := range specs { diff --git a/mdl/backend/mfmutator/splice.go b/mdl/backend/mfmutator/splice.go index e43e7e5c49..6a5b011218 100644 --- a/mdl/backend/mfmutator/splice.go +++ b/mdl/backend/mfmutator/splice.go @@ -366,15 +366,23 @@ func (m *Mutator) spliceOnFlow(g *graph, f flowRef, frag *backend.MicroflowFragm setPointer(f.doc, "DestinationPointer", frag.Entry) setInt(f.doc, "DestinationConnectionIndex", entrySide) setVector(f.doc, "DestinationControlVector", sideVector(entrySide)) + if c := frag.InCurve; c != nil { + setVector(f.doc, "OriginControlVector", c.Origin) + setVector(f.doc, "DestinationControlVector", c.Destination) + } // And a new flow carries on from the fragment to Y. + outOrigin, outDest := sideVector(exitSide), destVec + if c := frag.OutCurve; c != nil { + outOrigin, outDest = c.Origin, c.Destination + } if err := m.addFlow(µflows.SequenceFlow{ BaseElement: model.BaseElement{ID: model.ID(types.GenerateID())}, OriginID: frag.Exit, DestinationID: model.ID(y.id), OriginConnectionIndex: exitSide, DestinationConnectionIndex: destIdx, - OriginControlVector: sideVector(exitSide), - DestinationControlVector: destVec, + OriginControlVector: outOrigin, + DestinationControlVector: outDest, CaseValue: frag.ExitCase, }); err != nil { return err diff --git a/mdl/backend/mfmutator/splice_edit.go b/mdl/backend/mfmutator/splice_edit.go index dfecc2a6a0..9153a9b17d 100644 --- a/mdl/backend/mfmutator/splice_edit.go +++ b/mdl/backend/mfmutator/splice_edit.go @@ -10,6 +10,7 @@ import ( "github.com/mendixlabs/mxcli/mdl/backend" "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/modelsdk/canon" ) // # Editing an activity in place (ako/mxcli#1761) @@ -69,10 +70,15 @@ func sameKind(d, stored bson.D) bool { // under it that pair up (carryPairs), put on at Bytes. func (m *Mutator) swapKeeping(g *graph, x *node, oc, d bson.D, id model.ID) error { same := sameKind(d, x.doc) + if same { + d = carryTranslations(d, x.doc) + } if err := m.swapInPlace(g, x, oc, d, id); err != nil { return err } if same { + // The carried translations are the stored elements, still in the unit. + m.unmarkRemoved(d) m.carryUnstated(d, x.doc) carryPairs(d, x.doc, m.carried) m.edited[x.id] = true @@ -80,6 +86,35 @@ func (m *Mutator) swapKeeping(g *graph, x *node, oc, d bson.D, id model.ID) erro return nil } +// carryTranslations returns d, the serialized replacement of stored, with the +// languages of stored's texts that d does not write put back +// (ako/mxcli#2327, mendixlabs/mxcli#1870). The statement writes one language +// per text, and the splice saves as a patch, which canon does not carry +// translations onto — so an edited show message kept only that language. The +// carry is canon.CarryTranslations, the one a rebuild's write gets, applied to +// the activity: it pairs texts by place, never overwrites a language the +// statement wrote, and keeps a carried translation's stored $ID. On anything +// it cannot read, d is returned as it is. +func carryTranslations(d, stored bson.D) bson.D { + nb, err := bson.Marshal(d) + if err != nil { + return d + } + ob, err := bson.Marshal(stored) + if err != nil { + return d + } + out := canon.CarryTranslations(nb, ob) + if bytes.Equal(out, nb) { + return d + } + var carried bson.D + if bson.Unmarshal(out, &carried) != nil { + return d + } + return carried +} + // unstated are the properties, by element $Type, that MDL has no spelling for: // describe does not print them, so no statement states them, and the writer puts // its default where they go. On an edit the stored value is the only one there diff --git a/mdl/backend/mfmutator/splice_edit_translations_2327_test.go b/mdl/backend/mfmutator/splice_edit_translations_2327_test.go new file mode 100644 index 0000000000..69a3f4d43c --- /dev/null +++ b/mdl/backend/mfmutator/splice_edit_translations_2327_test.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mfmutator + +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// ako/mxcli#2327 (mendixlabs/mxcli#1870): `create or modify` of a microflow +// whose show message text changed edits the activity in place, and the edit +// is written from the statement — one language. The splice saves as a patch, +// which canon does not carry translations onto, so the message kept only the +// language MDL wrote: its nl_NL text was deleted. Studio Pro keeps the other +// languages when the default text is edited, stale as they then are. + +func showMessage(act, text string, langs ...[2]string) bson.D { + var items []bson.D + for i, l := range langs { + id := "" + if act != "" { + id = act + "t" + string(rune('0'+i)) + } + items = append(items, el(id, "Texts$Translation", kv("LanguageCode", l[0]), kv("Text", l[1]))) + } + textID, tplID := "", "" + if act != "" { + textID, tplID = act+"x", act+"p" + } + return el(act, "Microflows$ShowMessageAction", + kv("Blocking", true), + kv("ErrorHandlingType", "Rollback"), + kv("Template", el(tplID, "Microflows$TextTemplate", + kv("Parameters", list(2)), + kv("Text", el(textID, "Texts$Text", kv("Items", list(3, items...)))))), + kv("Type", "Information")) +} + +func messageTranslations(a bson.D) map[string]string { + out := map[string]string{} + text := dDoc(dDoc(dDoc(a, "Action"), "Template"), "Text") + for _, it := range arrayElements(dGet(text, "Items")) { + if d, ok := it.(bson.D); ok { + out[dString(d, "LanguageCode")] = dString(d, "Text") + } + } + return out +} + +func TestSplice_EditKeepsOtherTranslations(t *testing.T) { + a := editA(t, + showMessage("act", "", [2]string{"en_US", "The password has been updated."}, + [2]string{"nl_NL", "Het wachtwoord is gewijzigd."}), + showMessage("", "", [2]string{"en_US", "The password has been updated!"}), + "") + got := messageTranslations(a) + if got["en_US"] != "The password has been updated!" { + t.Errorf("en_US = %q, want the edited text", got["en_US"]) + } + if got["nl_NL"] != "Het wachtwoord is gewijzigd." { + t.Errorf("nl_NL = %q, want the stored translation kept (got %v)", got["nl_NL"], got) + } +} + +// Control: the language the statement writes is the statement's, so an edit of +// it lands even when the stored text had only that language. +func TestSplice_EditOfTheOnlyLanguageLands(t *testing.T) { + a := editA(t, + showMessage("act", "", [2]string{"en_US", "Saved."}), + showMessage("", "", [2]string{"en_US", "Saved!"}), + "") + got := messageTranslations(a) + if len(got) != 1 || got["en_US"] != "Saved!" { + t.Errorf("translations = %v, want only en_US Saved!", got) + } +} diff --git a/mdl/backend/mfmutator/splice_stated_curves_2311_test.go b/mdl/backend/mfmutator/splice_stated_curves_2311_test.go new file mode 100644 index 0000000000..d58d099f18 --- /dev/null +++ b/mdl/backend/mfmutator/splice_stated_curves_2311_test.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mfmutator + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/model" +) + +// ako/mxcli#2311: `create or modify` inserted statements and drew the two +// connectors at their ends with the splice's own control vectors (±15), +// where a create of the same script draws them as its @curve states — and +// straight ("0;0") where it states none. The next run of the same script read +// the difference as a connector the script redraws and refused it +// (MDL-V1-REBUILD): a script that had just succeeded could not run again. +// The fragment now carries the curves the script states for both connectors. + +func vectors(t *testing.T, m *Mutator, origin, dest string) (string, string) { + t.Helper() + for _, f := range m.graph().flows { + if f.origin == origin && f.dest == dest { + _, o := flowEnd(f.doc, "Origin") + _, d := flowEnd(f.doc, "Destination") + return o, d + } + } + t.Fatalf("no flow %s -> %s", origin, dest) + return "", "" +} + +func TestSplice_InsertDrawsTheStatedCurves_Issue2311(t *testing.T) { + m, _ := newMutator(t, unit(line(), lineFlows())) + frag := oneActivity() + frag.InCurve = &backend.FlowVectors{Origin: "30;0", Destination: "-30;0"} + frag.OutCurve = &backend.FlowVectors{Origin: "0;0", Destination: "0;0"} + if err := m.InsertAfter(model.ID(uid("a")), frag); err != nil { + t.Fatal(err) + } + if o, d := vectors(t, m, uid("a"), string(frag.Entry)); o != "30;0" || d != "-30;0" { + t.Errorf("connector into the fragment = %s / %s, want the stated 30;0 / -30;0", o, d) + } + if o, d := vectors(t, m, string(frag.Exit), uid("b")); o != "0;0" || d != "0;0" { + t.Errorf("connector out of the fragment = %s / %s, want the stated straight line", o, d) + } +} + +// CONTROL: an `alter` insert states no curves, and keeps the splice's own. +func TestSplice_InsertWithoutStatedCurvesKeepsItsOwn_Issue2311(t *testing.T) { + m, _ := newMutator(t, unit(line(), lineFlows())) + frag := oneActivity() + if err := m.InsertAfter(model.ID(uid("a")), frag); err != nil { + t.Fatal(err) + } + if _, d := vectors(t, m, uid("a"), string(frag.Entry)); d != sideVector(sideLeft) { + t.Errorf("unstated connector into the fragment ends with %s, want the splice's %s", d, sideVector(sideLeft)) + } +} diff --git a/mdl/backend/microflow_mutation.go b/mdl/backend/microflow_mutation.go index 9ae785682f..4f3bbe8c64 100644 --- a/mdl/backend/microflow_mutation.go +++ b/mdl/backend/microflow_mutation.go @@ -35,6 +35,19 @@ type MicroflowFragment struct { // where they are when the fragment is translated into place, and take no // part in the room it needs (mendixlabs/mxcli#1598, case 1). Pinned map[model.ID]bool + // InCurve and OutCurve are the control vectors the script states for the + // connector into the fragment and the one out of it, when it states them + // (`create or modify`, whose declared flow is drawn whole): the splice + // draws those two connectors with them, so the flow it leaves is the flow + // the script describes and the next run of it finds nothing to change + // (ako/mxcli#2311). nil keeps the splice's own vectors (an `alter`). + InCurve, OutCurve *FlowVectors +} + +// FlowVectors are a sequence flow's two bezier control vectors as stored, +// "x;y"; "0;0" is a straight end. +type FlowVectors struct { + Origin, Destination string } // MicroflowMutator splices into one stored microflow or nanoflow (ADR-0012 diff --git a/mdl/backend/modelsdk/association_move_write.go b/mdl/backend/modelsdk/association_move_write.go index 63a4ce8bd2..40d5e381b5 100644 --- a/mdl/backend/modelsdk/association_move_write.go +++ b/mdl/backend/modelsdk/association_move_write.go @@ -11,6 +11,7 @@ import ( "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/modelsdk/canon" "github.com/mendixlabs/mxcli/modelsdk/codec" "github.com/mendixlabs/mxcli/modelsdk/element" genDm "github.com/mendixlabs/mxcli/modelsdk/gen/domainmodels" @@ -425,10 +426,6 @@ func (b *Backend) MoveEntity(entity *domainmodel.Entity, sourceDMID, targetDMID } } - if err := b.persistDM(sourceDMID, sourceDM); err != nil { - return nil, fmt.Errorf("MoveEntity: persist source: %w", err) - } - // Add the (rebuilt) entity to the target DM, carrying the identities the // rebuild has no business re-minting — the same two carries UpdateEntity does, // which MoveEntity never got (#657 for the entity, #1119 for its children). @@ -450,8 +447,41 @@ func (b *Backend) MoveEntity(entity *domainmodel.Entity, sourceDMID, targetDMID // and the access rules' captions (ako/mxcli#801). carryStoredEntity(ge, orig, entity) targetDM.AddEntities(ge) - if err := b.persistDM(targetDMID, targetDM); err != nil { - return nil, fmt.Errorf("MoveEntity: persist target: %w", err) + + // The move writes two units, and a failure between them lost the entity: + // removed from the source, refused by the target, in neither module + // (ako/mxcli#2323). A script that stops rolls the source back with the rest + // of the run, but `exec --continue-on-error` goes on to the next statement + // with the source written. So both units are checked against the refusals + // the write path makes on content before either is written, and a target + // write that fails anyway puts the source back as it was. + sourceContents, err := (&codec.Encoder{}).Encode(sourceDM) + if err != nil { + return nil, fmt.Errorf("MoveEntity: encode source: %w", err) + } + targetContents, err := (&codec.Encoder{}).Encode(targetDM) + if err != nil { + return nil, fmt.Errorf("MoveEntity: encode target: %w", err) + } + if err := refusedContents(sourceDMID, sourceContents); err != nil { + return nil, fmt.Errorf("MoveEntity: source: %w; nothing was written", err) + } + if err := refusedContents(targetDMID, targetContents); err != nil { + return nil, fmt.Errorf("MoveEntity: target: %w; nothing was written", err) + } + storedSource, err := b.reader.GetRawUnitBytes(string(sourceDMID)) + if err != nil { + return nil, fmt.Errorf("MoveEntity: read source: %w", err) + } + if err := b.writer.UpdateRawUnit(string(sourceDMID), sourceContents); err != nil { + return nil, fmt.Errorf("MoveEntity: persist source: persist domain model %s: %w", sourceDMID, err) + } + if err := b.writer.UpdateRawUnit(string(targetDMID), targetContents); err != nil { + if rerr := b.writer.UpdateRawUnitPatch(string(sourceDMID), storedSource); rerr != nil { + return nil, fmt.Errorf("MoveEntity: persist target: persist domain model %s: %w; "+ + "restoring the source domain model also failed, so the entity is in neither module: %v", targetDMID, err, rerr) + } + return nil, fmt.Errorf("MoveEntity: persist target: persist domain model %s: %w; the source domain model was restored", targetDMID, err) } // A cross-association in any OTHER module that names the moved entity is // re-pointed; nothing else in the move rewrites that module's unit (#628). @@ -460,3 +490,13 @@ func (b *Backend) MoveEntity(entity *domainmodel.Entity, sourceDMID, targetDMID } return converted, nil } + +// refusedContents is the refusal the write path makes on a unit's content +// alone, before anything is stored (Writer.updateUnit): two elements sharing an +// $ID, or a bare attribute reference. +func refusedContents(unitID model.ID, contents []byte) error { + if err := canon.DuplicateElementIDError(string(unitID), contents); err != nil { + return err + } + return canon.BareAttributeRefError(string(unitID), contents) +} diff --git a/mdl/backend/modelsdk/issue2323_move_atomic_test.go b/mdl/backend/modelsdk/issue2323_move_atomic_test.go new file mode 100644 index 0000000000..8c0db9102d --- /dev/null +++ b/mdl/backend/modelsdk/issue2323_move_atomic_test.go @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "strings" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/mdl/bsonutil" + "github.com/mendixlabs/mxcli/model" +) + +// TestIssue2323_RefusedTargetWritesNothing guards ako/mxcli#2323: MoveEntity +// writes two units, the source without the entity and the target with it. When +// the target write was refused, the source had already been written, so the +// entity was in neither module. A script that stops is rolled back as a whole, +// but `exec --continue-on-error` goes on to the next statement — and on a +// generated project lost 9 of 12 moved entities. +// +// The refusal is provoked the way #2322 provoked it: the target unit already +// holds an element with the moved entity's $ID, so the target with the entity +// added holds that $ID twice. +func TestIssue2323_RefusedTargetWritesNothing(t *testing.T) { + proj := copyFixture(t) + b := New() + if err := b.Connect(proj); err != nil { + t.Fatalf("connect: %v", err) + } + srcID, srcMod, _, _, childID, _ := associationSubject(t, b) + dstID, dstMod := otherDomainModel(t, b, srcID) + if dstID == "" { + t.Skip("fixture has no second domain model to move into") + } + ent := entityByID(t, b, srcID, childID) + plantElementWithID(t, b, dstID, childID) + + _, err := b.MoveEntity(ent, srcID, dstID, srcMod, dstMod) + if err == nil { + t.Fatal("MoveEntity succeeded; the planted duplicate should have refused the target") + } + if !strings.Contains(err.Error(), "nothing was written") { + t.Errorf("error does not say nothing was written: %v", err) + } + if err := b.Disconnect(); err != nil { + t.Fatalf("disconnect: %v", err) + } + + b2 := New() + if err := b2.Connect(proj); err != nil { + t.Fatalf("reconnect: %v", err) + } + t.Cleanup(func() { _ = b2.Disconnect() }) + dm, err := b2.loadDomainModelGen(srcID) + if err != nil { + t.Fatal(err) + } + for _, e := range dm.EntitiesItems() { + if string(e.ID()) == string(childID) { + return + } + } + t.Errorf("entity %s is gone from %s after a move whose target was refused", ent.Name, srcMod) +} + +// plantElementWithID adds to the domain model dmID an annotation carrying id +// as its $ID. +func plantElementWithID(t *testing.T, b *Backend, dmID, id model.ID) { + t.Helper() + raw, err := b.reader.GetRawUnitBytes(string(dmID)) + if err != nil { + t.Fatal(err) + } + var d bson.D + if err := bson.Unmarshal(raw, &d); err != nil { + t.Fatal(err) + } + ann := bson.D{ + {Key: "$ID", Value: bsonutil.IDToBsonBinary(string(id))}, + {Key: "$Type", Value: "DomainModels$Annotation"}, + {Key: "Caption", Value: "planted"}, + {Key: "Location", Value: "0;0"}, + {Key: "Width", Value: int32(100)}, + } + planted := false + for i, e := range d { + if e.Key == "Annotations" { + list, _ := e.Value.(bson.A) + if len(list) == 0 { + list = bson.A{int32(3)} + } + d[i].Value = append(list, ann) + planted = true + } + } + if !planted { + t.Fatal("domain model has no Annotations list") + } + out, err := bson.Marshal(d) + if err != nil { + t.Fatal(err) + } + if err := b.writer.UpdateRawUnit(string(dmID), out); err != nil { + t.Fatalf("plant: %v", err) + } +} diff --git a/mdl/executor/cmd_alter_flow.go b/mdl/executor/cmd_alter_flow.go index 3e2068655f..799e6703b7 100644 --- a/mdl/executor/cmd_alter_flow.go +++ b/mdl/executor/cmd_alter_flow.go @@ -222,6 +222,11 @@ func (a *alterFlowContext) applyTo(ctx *ExecContext, mut backend.MicroflowMutato return fail(inLoop("the fragment holds " + what)) } } + // The connectors at the fragment's ends, as the script draws them + // (ako/mxcli#2311). + if op.StatesCurves { + frag.InCurve, frag.OutCurve = flowVectors(op.InCurve), flowVectors(op.OutCurve) + } // A stated @position on the fragment is where it goes (ako/mxcli#818). frag.Placed = firstStatementPlaced(op.Body) if !frag.Placed && laterStatementPlaced(op.Body) { @@ -1658,3 +1663,18 @@ func (a *alterFlowContext) checkDeclaredUnclashed(frag *backend.MicroflowFragmen } return nil } + +// flowVectors is a stated @curve as the two control vectors a flow stores; an +// end it leaves out, and a statement with no @curve, is straight ("0;0"). +func flowVectors(c *ast.FlowCurve) *backend.FlowVectors { + end := func(p *ast.Position) string { + if p == nil { + return "0;0" + } + return fmt.Sprintf("%d;%d", p.X, p.Y) + } + if c == nil { + return &backend.FlowVectors{Origin: "0;0", Destination: "0;0"} + } + return &backend.FlowVectors{Origin: end(c.From), Destination: end(c.To)} +} diff --git a/mdl/executor/cmd_flow_modify.go b/mdl/executor/cmd_flow_modify.go index 51313524fb..c6ddc1ce1b 100644 --- a/mdl/executor/cmd_flow_modify.go +++ b/mdl/executor/cmd_flow_modify.go @@ -889,6 +889,10 @@ func withImplicitEnd(declared, stored []ast.MicroflowStatement, returnVar string // patchDiff accumulates the operations of one diff. type patchDiff struct { + // before is the declared statement before the run gap is diffing, nil at + // the start of a list: the statement whose @curve draws the connector + // into an inserted run (ako/mxcli#2311). + before ast.MicroflowStatement loc *storedLocator ops []*ast.AlterFlowOperation targets []mfmutator.Candidate @@ -914,6 +918,10 @@ func (pd *patchDiff) statements(declared, stored []ast.MicroflowStatement) error if p < len(pairs) { dEnd, sEnd = pairs[p][0], pairs[p][1] } + pd.before = nil + if di > 0 { + pd.before = declared[di-1] + } if err := pd.gap(declared[di:dEnd], stored, si, sEnd); err != nil { return err } @@ -974,6 +982,7 @@ func (pd *patchDiff) gap(ins []ast.MicroflowStatement, stored []ast.MicroflowSta return err } pd.add(op, c, ins) + pd.statedCurves(pd.ops[len(pd.ops)-1], ins) return nil } if len(ins) == 1 && len(del) == 1 { @@ -2151,3 +2160,23 @@ func patchSummary(ops []*ast.AlterFlowOperation, edited map[*ast.AlterFlowOperat } return strings.Join(out, "; ") } + +// statedCurves gives an insert the curves the script states for the two +// connectors the splice draws: into the run, on the declared statement before +// it, and out of it, on the run's last statement (ako/mxcli#2311). Only where +// both ends are plain statements — an activity's @curve is its one outgoing +// connector; a decision's or a loop's says nothing about the one at hand. +// The splice drew those connectors with vectors of its own, and the next run +// of the same script read the difference as a redrawn connector and refused. +func (pd *patchDiff) statedCurves(op *ast.AlterFlowOperation, ins []ast.MicroflowStatement) { + if len(ins) == 0 || pd.before == nil || notPlainInBody(pd.before) != "" || notPlainInBody(ins[len(ins)-1]) != "" { + return + } + op.StatesCurves = true + if ann := statementAnnotations(pd.before); ann != nil { + op.InCurve = ann.Curve + } + if ann := statementAnnotations(ins[len(ins)-1]); ann != nil { + op.OutCurve = ann.Curve + } +} diff --git a/mdl/executor/cmd_pages_builder_v3_layout.go b/mdl/executor/cmd_pages_builder_v3_layout.go index 9d869fba0d..d36ba81570 100644 --- a/mdl/executor/cmd_pages_builder_v3_layout.go +++ b/mdl/executor/cmd_pages_builder_v3_layout.go @@ -546,6 +546,9 @@ func (pb *pageBuilder) buildControlBarV3(w *ast.WidgetV3) (*pages.Container, err // layout-grid row (#1461) — in either form: an expression, or `Attr in (…)` — // plus any `Visible: roles (…)` (#1460). nil when there is none. func (pb *pageBuilder) childVisibility(w *ast.WidgetV3) (*pages.ConditionalVisibilitySettings, error) { + if err := refuseRolesInVisibility(w); err != nil { + return nil, err + } if vw, ok := w.Properties["VisibleWhen"].(*ast.VisibleWhenV3); ok && vw != nil { cvs, err := pb.visibleWhenSettings(w, vw) if err != nil { diff --git a/mdl/executor/cmd_pages_builder_visible_when.go b/mdl/executor/cmd_pages_builder_visible_when.go index 0aeb85183d..1ec48a2d0a 100644 --- a/mdl/executor/cmd_pages_builder_visible_when.go +++ b/mdl/executor/cmd_pages_builder_visible_when.go @@ -208,6 +208,9 @@ func (pb *pageBuilder) attributeConditions(w *ast.WidgetV3, vw *ast.VisibleWhenV // condition on the same element (TestApp WorkflowCommons), so they never // replace it. Runs after applyConditionalSettings and applyVisibleWhen. func applyVisibleRoles(widget pages.Widget, w *ast.WidgetV3) error { + if err := refuseRolesInExpressions(w); err != nil { + return err + } if len(widgetVisibleRoles(w)) == 0 { return nil } @@ -314,3 +317,31 @@ func (pb *pageBuilder) getEnumerations() ([]*model.Enumeration, error) { } return enums, nil } + +// refuseRolesInExpressions is MDL-WIDGET54 at exec: a `roles (…)` call left in +// a widget expression would be stored as invalid expression text with the +// role restriction lost (ako/mxcli#2319). +func refuseRolesInExpressions(w *ast.WidgetV3) error { + // The keys of expressionWidgetProps, each read by literal: the builder's + // property reads are checked that way (widget_native_props_source_test.go). + for _, p := range []struct{ key, expr string }{ + {"DynamicClasses", w.GetStringProp("DynamicClasses")}, + {"VisibleIf", w.GetStringProp("VisibleIf")}, + {"EditableIf", w.GetStringProp("EditableIf")}, + } { + if exprCallsRoles(p.expr) { + return mdlerrors.NewValidation(rolesInExpressionMessage(w.Type, w.Name, p.key)) + } + } + return nil +} + +// refuseRolesInVisibility is refuseRolesInExpressions for an element that has +// a visibility and nothing else (a tab page, a layout-grid row): only its +// `Visible:` is read. +func refuseRolesInVisibility(w *ast.WidgetV3) error { + if exprCallsRoles(w.GetStringProp("VisibleIf")) { + return mdlerrors.NewValidation(rolesInExpressionMessage(w.Type, w.Name, "VisibleIf")) + } + return nil +} diff --git a/mdl/executor/cmd_security_write.go b/mdl/executor/cmd_security_write.go index 59b6548046..51d0c180f8 100644 --- a/mdl/executor/cmd_security_write.go +++ b/mdl/executor/cmd_security_write.go @@ -659,9 +659,47 @@ func execGrantEntityAccess(ctx *ExecContext, s *ast.GrantEntityAccessStmt) error text: text, unchanged: fmt.Sprintf("entity access: %s.%s (%s)", s.Entity.Module, s.Entity.Name, strings.Join(roleNames, ", ")), }) + // Printed on a re-run too: an unchanged write leaves the overlap in place. + if w := grantOverlapWarning(entityQN, entity.AccessRules, roleNames, xpathConstraint); w != "" { + ctx.reportAccessRule(heldReport{notice: true, text: w}) + } return nil } +// grantOverlapWarning is MDL-GRANT03 at exec (ako/mxcli#2320): the roles of a +// grant that, beside the rule it writes, hold a rule on the entity of the other +// kind — one with no constraint beside a constrained grant, or a constrained one +// beside an unconstrained grant. Access rules add up, so the constraint limits +// nothing. rules are the entity's rules as they were before the grant. +func grantOverlapWarning(entityQN string, rules []*domainmodel.AccessRule, roleNames []string, xpath string) string { + constrained := strings.TrimSpace(xpath) != "" + want := map[string]bool{} + for _, r := range roleNames { + want[r] = true + } + hit := map[string]bool{} + for _, rule := range rules { + if (strings.TrimSpace(rule.XPathConstraint) != "") == constrained { + continue + } + for _, role := range rule.ModuleRoleNames { + if want[role] { + hit[role] = true + } + } + } + if len(hit) == 0 { + return "" + } + roles := make([]string, 0, len(hit)) + for r := range hit { + roles = append(roles, r) + } + v := accessRuleOverlapViolation(0, entityQN, roles, constrained) + msg := strings.TrimPrefix(v.Message, "statement 0: ") + return fmt.Sprintf("Warning (%s): %s.\n %s\n", v.RuleID, msg, v.Suggestion) +} + // execRevokeEntityAccess handles REVOKE roles ON Module.Entity [(rights...)]. // nothingToRevoke is a revoke's report when the roles already lack the rights: // the idempotent re-run of a script, worded as the state it found rather than diff --git a/mdl/executor/flow_verdict.go b/mdl/executor/flow_verdict.go index 50df94e58f..ca2db5a41f 100644 --- a/mdl/executor/flow_verdict.go +++ b/mdl/executor/flow_verdict.go @@ -84,7 +84,9 @@ func flowRefusal(d *flowDecl, why error) error { // rebuild the author means: drop the flow and create it in the same script. func deliberateRebuildAdvice(kind string) string { return fmt.Sprintf("to rebuild the flow deliberately, drop the %s and create it in the same script "+ - "(the drop's module-role grants carry to a create in the same script or session, not to one in a later run)", kind) + "(the drop's module-role grants carry to a create in the same script or session, not to one in a later run) — "+ + "once, in a run of its own: a drop + create rewrites the flow on every run, so the script you keep should "+ + "then hold the `create or modify` of the result, which a second run finds unchanged", kind) } // replaceBodyAdvice is what `create or modify` and `alter` both say about a diff --git a/mdl/executor/issue2311_insert_stated_curves_test.go b/mdl/executor/issue2311_insert_stated_curves_test.go new file mode 100644 index 0000000000..89e5695b29 --- /dev/null +++ b/mdl/executor/issue2311_insert_stated_curves_test.go @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +// ako/mxcli#2311: a `create or modify` that inserted statements drew the +// connectors at their ends with the splice's own vectors, and the next run of +// the same script refused the flow as "redrawn". The insert now carries the +// curves the script states: the one on the statement before the run, for the +// connector into it, and the one on the run's last statement. +func TestStatedCurves_InsertCarriesTheScriptsCurves_Issue2311(t *testing.T) { + before := &ast.LogStmt{Annotations: &ast.ActivityAnnotations{ + Curve: &ast.FlowCurve{From: &ast.Position{X: 30}, To: &ast.Position{X: -30}}}} + last := &ast.LogStmt{} + pd := &patchDiff{before: before} + op := &ast.AlterFlowOperation{Op: ast.AlterFlowInsertAfter} + pd.statedCurves(op, []ast.MicroflowStatement{&ast.LogStmt{}, last}) + if !op.StatesCurves || op.InCurve != before.Annotations.Curve || op.OutCurve != nil { + t.Fatalf("op = %+v, want the predecessor's curve in and none (straight) out", op) + } + in, out := flowVectors(op.InCurve), flowVectors(op.OutCurve) + if in.Origin != "30;0" || in.Destination != "-30;0" || out.Origin != "0;0" || out.Destination != "0;0" { + t.Errorf("vectors in %+v out %+v", in, out) + } +} + +// CONTROL: a run at the start of a list (no statement before it) or after a +// decision states nothing about the connector into it, and keeps the splice's. +func TestStatedCurves_NotAfterADecision_Issue2311(t *testing.T) { + for _, before := range []ast.MicroflowStatement{nil, &ast.IfStmt{}} { + pd := &patchDiff{before: before} + op := &ast.AlterFlowOperation{Op: ast.AlterFlowInsertBefore} + pd.statedCurves(op, []ast.MicroflowStatement{&ast.LogStmt{}}) + if op.StatesCurves { + t.Errorf("before %T: curves stated, want the splice's own", before) + } + } +} diff --git a/mdl/executor/issue2319_visible_roles_and_test.go b/mdl/executor/issue2319_visible_roles_and_test.go new file mode 100644 index 0000000000..d0fb959364 --- /dev/null +++ b/mdl/executor/issue2319_visible_roles_and_test.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "reflect" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +// ako/mxcli#2319: `Visible: roles (Shop.Admin) and $currentObject/Secret = +// 'x'` passed check and exec, and the whole text — `roles (…)` is not Mendix +// expression syntax — was stored as the visibility expression, with +// ModuleRoles empty. The role restriction was lost and the expression invalid. +// +// Studio Pro keeps "visible for module roles" and the condition on one +// settings element, both applying; that is what the leading `roles (…) and` +// means, so it is stored that way. A `roles (…)` anywhere else in an +// expression has no such reading and is refused. + +func visibleWidget(t *testing.T, visible string) *ast.WidgetV3 { + t.Helper() + prog, errs := visitor.Build("create page Shop.P (Title: 'x', Layout: A.L) {\n container c1 (Visible: " + visible + ") { }\n}\n") + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + return prog.Statements[0].(*ast.CreatePageStmtV3).Widgets[0] +} + +func TestVisibleRolesAndExpression_SplitIntoRolesAndCondition_Issue2319(t *testing.T) { + w := visibleWidget(t, "roles (Shop.Admin, Shop.User) and $currentObject/Secret = 'x'") + if roles, _ := w.Properties["VisibleRoles"].([]string); !reflect.DeepEqual(roles, []string{"Shop.Admin", "Shop.User"}) { + t.Errorf("VisibleRoles = %#v, want [Shop.Admin Shop.User]", w.Properties["VisibleRoles"]) + } + if got := w.GetStringProp("VisibleIf"); got != "$currentObject/Secret = 'x'" { + t.Errorf("VisibleIf = %q, want the condition alone", got) + } +} + +// CONTROL: a plain expression stays an expression, roles untouched. +func TestVisibleExpressionWithoutRoles_Unchanged_Issue2319(t *testing.T) { + w := visibleWidget(t, "$currentObject/Secret = 'roles (x) and y'") + if _, has := w.Properties["VisibleRoles"]; has { + t.Errorf("VisibleRoles set from a string literal: %#v", w.Properties["VisibleRoles"]) + } + if got := w.GetStringProp("VisibleIf"); got != "$currentObject/Secret = 'roles (x) and y'" { + t.Errorf("VisibleIf = %q", got) + } +} + +func TestVisibleRolesElsewhereInExpression_Refused_Issue2319(t *testing.T) { + for _, visible := range []string{ + "$currentObject/Secret = 'x' and roles (Shop.Admin)", + "roles (Shop.Admin) or $currentObject/Secret = 'x'", + } { + w := visibleWidget(t, visible) + vs := validateWidgetExpressionRoles(w, "page Shop.P") + if len(vs) != 1 || vs[0].RuleID != "MDL-WIDGET54" || !strings.Contains(vs[0].Message, "Visible: roles (") { + t.Errorf("%s: violations = %v, want one MDL-WIDGET54 naming the supported spelling", visible, vs) + } + } + // CONTROL: the split form and a plain expression are clean. + for _, visible := range []string{"roles (Shop.Admin) and $currentObject/Secret = 'x'", "$currentObject/Secret = 'x'"} { + if vs := validateWidgetExpressionRoles(visibleWidget(t, visible), "page Shop.P"); len(vs) != 0 { + t.Errorf("%s: unexpected %v", visible, vs) + } + } +} + +// exec refuses it too, for a script run past check (--no-check). +func TestVisibleRolesElsewhereInExpression_ExecRefuses_Issue2319(t *testing.T) { + w := visibleWidget(t, "$currentObject/Secret = 'x' and roles (Shop.Admin)") + if err := refuseRolesInExpressions(w); err == nil || !strings.Contains(err.Error(), "roles (") { + t.Errorf("exec guard = %v, want a refusal", err) + } + if err := refuseRolesInExpressions(visibleWidget(t, "roles (Shop.Admin) and $currentObject/Secret = 'x'")); err != nil { + t.Errorf("split form refused: %v", err) + } +} diff --git a/mdl/executor/issue2320_access_rule_overlap_test.go b/mdl/executor/issue2320_access_rule_overlap_test.go new file mode 100644 index 0000000000..ab3ae6b4f9 --- /dev/null +++ b/mdl/executor/issue2320_access_rule_overlap_test.go @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" + "github.com/mendixlabs/mxcli/sdk/security" +) + +// ako/mxcli#2320: a GRANT with a WHERE meant to narrow a role's access wrote a +// constrained rule beside the role's unconstrained one. Access rules add up, so +// the role still read every row — and exec, check and lint said nothing. + +func overlapFixture(t *testing.T, rules []*domainmodel.AccessRule) (*ExecContext, func() string) { + t.Helper() + mod := mkModule("Shop") + h := mkHierarchy(mod) + cust := &domainmodel.Entity{ + BaseElement: model.BaseElement{ID: model.ID("e-cust")}, + Name: "Customer", + Attributes: []*domainmodel.Attribute{{Name: "Name", Type: &domainmodel.StringAttributeType{Length: 200}}}, + AccessRules: rules, + } + dm := &domainmodel.DomainModel{BaseElement: model.BaseElement{ID: "dm-shop"}, ContainerID: mod.ID, + Entities: []*domainmodel.Entity{cust}} + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + GetModuleByNameFunc: func(string) (*model.Module, error) { return mod, nil }, + ListDomainModelsFunc: func() ([]*domainmodel.DomainModel, error) { return []*domainmodel.DomainModel{dm}, nil }, + GetDomainModelFunc: func(model.ID) (*domainmodel.DomainModel, error) { return dm, nil }, + GetModuleSecurityFunc: func(model.ID) (*security.ModuleSecurity, error) { + return &security.ModuleSecurity{ModuleRoles: []*security.ModuleRole{{Name: "Clerk"}, {Name: "Manager"}}}, nil + }, + AddEntityAccessRuleFunc: func(backend.EntityAccessRuleParams) error { return nil }, + ReconcileMemberAccessesFunc: func(model.ID, string) (int, error) { return 0, nil }, + } + ctx, buf := newMockCtx(t, withBackend(mb), withHierarchy(h)) + return ctx, buf.String +} + +func openClerkRule() []*domainmodel.AccessRule { + return []*domainmodel.AccessRule{{ModuleRoleNames: []string{"Shop.Clerk"}, AllowCreate: true, + DefaultMemberAccessRights: "ReadWrite"}} +} + +func constrainedGrant(role string) *ast.GrantEntityAccessStmt { + return &ast.GrantEntityAccessStmt{ + Roles: []ast.QualifiedName{{Module: "Shop", Name: role}}, + Entity: ast.QualifiedName{Module: "Shop", Name: "Customer"}, + Rights: []ast.EntityAccessRight{{Type: ast.EntityAccessReadAll}}, + XPathConstraint: "[Name != '']", + } +} + +func TestGrantWhere_WarnsBesideAnUnconstrainedRule_Issue2320(t *testing.T) { + ctx, out := overlapFixture(t, openClerkRule()) + if err := execGrantEntityAccess(ctx, constrainedGrant("Clerk")); err != nil { + t.Fatal(err) + } + got := out() + if !strings.Contains(got, "MDL-GRANT03") || !strings.Contains(got, "narrows nothing") { + t.Fatalf("no overlap warning in:\n%s", got) + } + if !strings.Contains(got, "revoke all on entity Shop.Customer from Shop.Clerk;") { + t.Errorf("warning does not name the revoke that narrows it:\n%s", got) + } +} + +// CONTROL: a constrained grant to a role with no unconstrained rule is the +// ordinary row-level rule and warns nothing. +func TestGrantWhere_NoWarningWithoutAnOpenRule_Issue2320(t *testing.T) { + ctx, out := overlapFixture(t, openClerkRule()) + if err := execGrantEntityAccess(ctx, constrainedGrant("Manager")); err != nil { + t.Fatal(err) + } + if got := out(); strings.Contains(got, "MDL-GRANT03") { + t.Errorf("unexpected warning for a role with no other rule:\n%s", got) + } +} + +func TestGrantOpen_WarnsBesideAConstrainedRule_Issue2320(t *testing.T) { + ctx, out := overlapFixture(t, []*domainmodel.AccessRule{{ModuleRoleNames: []string{"Shop.Clerk"}, + XPathConstraint: "[Name != '']", DefaultMemberAccessRights: "ReadOnly"}}) + g := constrainedGrant("Clerk") + g.XPathConstraint = "" + if err := execGrantEntityAccess(ctx, g); err != nil { + t.Fatal(err) + } + if got := out(); !strings.Contains(got, "MDL-GRANT03") || !strings.Contains(got, "restricts nothing") { + t.Errorf("no overlap warning in:\n%s", got) + } +} + +func TestCheck_ScriptOverlap_Issue2320(t *testing.T) { + both := parseMDL(t, `mdl 1; +grant read * on entity Shop.Customer to Shop.Clerk; +grant read * on entity Shop.Customer to Shop.Clerk where [Name != ''];`) + if v := ValidateAccessRuleOverlap(both); len(v) != 1 || v[0].RuleID != "MDL-GRANT03" { + t.Fatalf("violations = %v, want one MDL-GRANT03", v) + } + // CONTROL: revoking the rule first is the way to narrow, and is clean. + revoked := parseMDL(t, `mdl 1; +grant read * on entity Shop.Customer to Shop.Clerk; +revoke all on entity Shop.Customer from Shop.Clerk; +grant read * on entity Shop.Customer to Shop.Clerk where [Name != ''];`) + if v := ValidateAccessRuleOverlap(revoked); len(v) != 0 { + t.Errorf("violations = %v, want none after the revoke", v) + } +} + +// With a project, the stored unconstrained rule is what the grant overlaps — +// and the no-project pass does not report it, so it is said once. +func TestCheck_StoredOverlap_Issue2320(t *testing.T) { + prog := parseMDL(t, `mdl 1; +grant read * on entity Shop.Customer to Shop.Clerk where [Name != ''];`) + stored := func(entity string) (map[string]bool, map[string]bool, bool) { + if entity != "Shop.Customer" { + return nil, nil, false + } + return map[string]bool{"Shop.Clerk": true}, map[string]bool{}, true + } + if v := accessRuleOverlaps(prog, stored); len(v) != 1 { + t.Fatalf("violations = %v, want one", v) + } + if v := ValidateAccessRuleOverlap(prog); len(v) != 0 { + t.Errorf("no-project pass reported a stored overlap: %v", v) + } + revoked := parseMDL(t, `mdl 1; +revoke all on entity Shop.Customer from Shop.Clerk; +grant read * on entity Shop.Customer to Shop.Clerk where [Name != ''];`) + if v := accessRuleOverlaps(revoked, stored); len(v) != 0 { + t.Errorf("violations = %v, want none after the revoke", v) + } +} diff --git a/mdl/executor/validate_access_rule_overlap.go b/mdl/executor/validate_access_rule_overlap.go new file mode 100644 index 0000000000..33b8ba5fa7 --- /dev/null +++ b/mdl/executor/validate_access_rule_overlap.go @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// accessRuleOverlapRule is MDL-GRANT03 (ako/mxcli#2320): a role holds a rule +// with no XPath constraint on an entity and another one with a constraint. +// +// Access rules add up. A GRANT with a WHERE writes a rule of its own beside the +// role's existing rules — a rule is keyed by its roles and its constraint +// (#936) — so a constrained grant meant to narrow an unconstrained one leaves +// the unconstrained rule in place, and the role still reads every row. exec +// reported success, check passed and lint said nothing; only a diff showed it. +// The same holds the other way round: an unconstrained grant makes the role's +// constrained rules restrict nothing. +// +// It is a warning, not an error: a rule with a constraint beside one without is +// a valid model, and a role may hold both on purpose (a constraint granting +// write on its own rows beside a read on every row). The defect was saying +// nothing. +const accessRuleOverlapRule = "MDL-GRANT03" + +// accessRuleOrigin says where a role's rule on an entity comes from, so the +// check without a project reports what the script does to itself and the check +// with one only what it does to the project's rules — each finding once. +type accessRuleOrigin int + +const ( + ruleNone accessRuleOrigin = iota + ruleStored + ruleScript +) + +// roleRules is what one role holds on one entity: an unconstrained rule and a +// constrained one, each by where it came from. +type roleRules struct { + open, constrained accessRuleOrigin +} + +// storedAccessRules answers, for an entity, the roles holding a rule with no +// constraint and those holding one with a constraint; ok is false where the +// project cannot say (no project, or no such entity). +type storedAccessRules func(entity string) (open, constrained map[string]bool, ok bool) + +// ValidateAccessRuleOverlap reports MDL-GRANT03 for the overlaps the script +// makes with itself. It needs no project. +func ValidateAccessRuleOverlap(prog *ast.Program) []linter.Violation { + return accessRuleOverlaps(prog, nil) +} + +// CheckAccessRuleOverlap reports MDL-GRANT03 for the overlaps the script makes +// with the rules the project already holds. +func (e *Executor) CheckAccessRuleOverlap(prog *ast.Program) []linter.Violation { + if e == nil { + return nil + } + ctx := e.newExecContext(context.Background()) + if !ctx.Connected() { + return nil + } + return accessRuleOverlaps(prog, func(entity string) (map[string]bool, map[string]bool, bool) { + e := lookupEntityForAccess(ctx, entity) + if e == nil { + return nil, nil, false + } + open, constrained := map[string]bool{}, map[string]bool{} + for _, r := range e.AccessRules { + for _, role := range r.ModuleRoleNames { + if strings.TrimSpace(r.XPathConstraint) == "" { + open[role] = true + } else { + constrained[role] = true + } + } + } + return open, constrained, true + }) +} + +// accessRuleOverlaps walks the program in order, tracking per entity and role +// which rules are held and where they came from. stored is nil for the check +// without a project, which then reports script-made overlaps only; with it, +// only the overlaps a stored rule takes part in are reported. +func accessRuleOverlaps(prog *ast.Program, stored storedAccessRules) []linter.Violation { + if prog == nil { + return nil + } + state := map[string]map[string]*roleRules{} + entityState := func(entity string) map[string]*roleRules { + if m, ok := state[entity]; ok { + return m + } + m := map[string]*roleRules{} + if stored != nil { + if open, constrained, ok := stored(entity); ok { + for role := range open { + rr := m[role] + if rr == nil { + rr = &roleRules{} + m[role] = rr + } + rr.open = ruleStored + } + for role := range constrained { + rr := m[role] + if rr == nil { + rr = &roleRules{} + m[role] = rr + } + rr.constrained = ruleStored + } + } + } + state[entity] = m + return m + } + report := func(a, b accessRuleOrigin) bool { + if stored == nil { + return a == ruleScript && b == ruleScript + } + return a == ruleStored || b == ruleStored + } + + var out []linter.Violation + for i, stmt := range prog.Statements { + switch s := stmt.(type) { + case *ast.GrantEntityAccessStmt: + entity := s.Entity.String() + m := entityState(entity) + constrained := strings.TrimSpace(s.XPathConstraint) != "" + var hit []string + for _, r := range s.Roles { + role := r.String() + rr := m[role] + if rr == nil { + rr = &roleRules{} + m[role] = rr + } + if constrained { + if rr.open != ruleNone && report(rr.open, ruleScript) { + hit = append(hit, role) + } + rr.constrained = ruleScript + } else { + if rr.constrained != ruleNone && report(rr.constrained, ruleScript) { + hit = append(hit, role) + } + rr.open = ruleScript + } + } + if len(hit) > 0 { + out = append(out, accessRuleOverlapViolation(i+1, entity, hit, constrained)) + } + case *ast.RevokeEntityAccessStmt: + if len(s.Rights) > 0 { + continue // a partial revoke changes rights, not which rules there are + } + m := entityState(s.Entity.String()) + for _, r := range s.Roles { + delete(m, r.String()) + } + case *ast.DropEntityStmt: + state[s.Name.String()] = map[string]*roleRules{} + } + } + return out +} + +func accessRuleOverlapViolation(stmt int, entity string, roles []string, constrained bool) linter.Violation { + sort.Strings(roles) + who := strings.Join(roles, ", ") + if constrained { + return linter.Violation{ + RuleID: accessRuleOverlapRule, + Severity: linter.SeverityWarning, + Message: fmt.Sprintf("statement %d: %s already has an access rule on %s with no constraint; access rules add up, "+ + "so this grant's where clause narrows nothing — the role still reaches every %s", stmt, who, entity, entity), + Suggestion: fmt.Sprintf("To narrow the access, remove the unconstrained rule first: "+ + "`revoke all on entity %s from %s;` before this grant.", entity, who), + } + } + return linter.Violation{ + RuleID: accessRuleOverlapRule, + Severity: linter.SeverityWarning, + Message: fmt.Sprintf("statement %d: %s also has an access rule on %s with a constraint; access rules add up, "+ + "so with this unconstrained grant that constraint restricts nothing", stmt, who, entity), + Suggestion: fmt.Sprintf("If the constraint is meant to limit the role, grant with the where clause instead, "+ + "after `revoke all on entity %s from %s;`.", entity, who), + } +} + +// lookupEntityForAccess is the stored entity Module.Name, or nil where the +// project has no such module or entity. +func lookupEntityForAccess(ctx *ExecContext, qn string) *domainmodel.Entity { + mod, name, ok := strings.Cut(qn, ".") + if !ok { + return nil + } + module, err := findModule(ctx, mod) + if err != nil || module == nil { + return nil + } + dm, err := ctx.Backend.GetDomainModel(module.ID) + if err != nil || dm == nil { + return nil + } + return dm.FindEntityByName(name) +} diff --git a/mdl/executor/validate_program.go b/mdl/executor/validate_program.go index 22b635ff4e..fc22662bc6 100644 --- a/mdl/executor/validate_program.go +++ b/mdl/executor/validate_program.go @@ -348,6 +348,12 @@ func ValidateProgram(prog *ast.Program, projectPath string) []linter.Violation { // under --references, where it would only fire with -p (#836). violations = append(violations, ValidateGrantRoles(prog)...) + // MDL-GRANT03: a role given a constrained and an unconstrained rule on one + // entity in this script — rules add up, so the constraint limits nothing + // (ako/mxcli#2320). The overlap with the project's rules is reported under + // --references (CheckAccessRuleOverlap). + violations = append(violations, ValidateAccessRuleOverlap(prog)...) + // Warn on an AI model or knowledge base Provider no Studio Pro document has // been seen to hold. mxbuild never reads it, so a typo builds and only // shows in Studio Pro (ako/mxcli#2023). diff --git a/mdl/executor/validate_widgets.go b/mdl/executor/validate_widgets.go index f72f627bac..82d3885063 100644 --- a/mdl/executor/validate_widgets.go +++ b/mdl/executor/validate_widgets.go @@ -1665,6 +1665,7 @@ func validateStaticWidgetOf(w *ast.WidgetV3, def *WidgetDefinition, locationPref // over, because a data binding on the SAME widget can traverse the same // association legitimately. (findings #4) out = append(out, validateWidgetExpressionAssociations(w, locationPrefix)...) + out = append(out, validateWidgetExpressionRoles(w, locationPrefix)...) // A contentparams/captionparams value is a DATA BINDING (an attribute path), // not a client-side expression: mxcli stores an unquoted value as an attribute @@ -2407,3 +2408,56 @@ func sortedPropertyKeys(w *ast.WidgetV3) []string { sort.Strings(out) return out } + +// rolesCallRe finds `roles (` in an expression, checked outside string +// literals by exprCallsRoles. +var rolesCallRe = regexp.MustCompile(`(?i)\broles\s*\(`) + +// exprCallsRoles reports whether expr calls `roles (…)` outside a string +// literal. Mendix has no such function: it is MDL's "visible for module +// roles", which only the leading `roles (…) and ` form, split by +// the parser, can carry (ako/mxcli#2319). +func exprCallsRoles(expr string) bool { + var b strings.Builder + inStr := false + for i := 0; i < len(expr); i++ { + c := expr[i] + if c == '\'' { + inStr = !inStr + b.WriteByte(' ') + continue + } + if inStr { + b.WriteByte(' ') + continue + } + b.WriteByte(c) + } + return rolesCallRe.MatchString(b.String()) +} + +// rolesInExpressionMessage is MDL-WIDGET54's text, shared by check and exec. +func rolesInExpressionMessage(where, widget, prop string) string { + return fmt.Sprintf("%s: widget `%s` property `%s` calls `roles (…)` inside an expression. Mendix expressions have no roles "+ + "function: it would be stored as invalid expression text and the role restriction lost. Write the roles as their own "+ + "property beside the condition — `Visible: roles (Mod.Role), Visible: ` — or lead with them: "+ + "`Visible: roles (Mod.Role) and ` (both apply)", where, widget, prop) +} + +// validateWidgetExpressionRoles refuses a `roles (…)` call left inside a widget +// expression property (MDL-WIDGET54, ako/mxcli#2319). +func validateWidgetExpressionRoles(w *ast.WidgetV3, locationPrefix string) []linter.Violation { + var out []linter.Violation + for _, key := range expressionWidgetProps { + expr, ok := lookupWidgetProp(w, key) + if !ok || !exprCallsRoles(expr) { + continue + } + out = append(out, linter.Violation{ + RuleID: "MDL-WIDGET54", + Severity: linter.SeverityError, + Message: rolesInExpressionMessage(locationPrefix, w.Name, key), + }) + } + return out +} diff --git a/mdl/exprcheck/adapters/check.go b/mdl/exprcheck/adapters/check.go index 5174db1a13..934ddd8b07 100644 --- a/mdl/exprcheck/adapters/check.go +++ b/mdl/exprcheck/adapters/check.go @@ -22,6 +22,9 @@ type CheckAdapter struct { // entities and kinds are set for the duration of one flow's walk. entities exprcheck.EntityScope kinds exprcheck.Scope + // returnKind is the primitive type the flow under walk returns, or + // KindUnknown; a RETURN is checked against it. + returnKind exprcheck.TypeKind } // Option configures a CheckAdapter. @@ -71,6 +74,7 @@ func (c *CheckAdapter) CheckMicroflow(stmt *ast.CreateMicroflowStmt) *Result { if stmt == nil { return r } + c.returnKind = primitiveTarget(returnDataType(stmt.ReturnType)) c.walkFlow(stmt.Body, stmt.Parameters, stmt.Name.String(), r) return r } @@ -87,6 +91,7 @@ func (c *CheckAdapter) CheckNanoflow(stmt *ast.CreateNanoflowStmt) *Result { if stmt == nil { return r } + c.returnKind = primitiveTarget(returnDataType(stmt.ReturnType)) c.walkFlow(stmt.Body, stmt.Parameters, stmt.Name.String(), r) return r } @@ -104,7 +109,7 @@ func (c *CheckAdapter) walkFlow(body []ast.MicroflowStatement, params []ast.Micr scope := buildFlowScope(body, params, c.assoc) c.entities = entityScope{vars: scope.entities, assoc: c.assoc} c.kinds = kindScope(scope.kinds) - defer func() { c.entities, c.kinds = nil, nil }() + defer func() { c.entities, c.kinds, c.returnKind = nil, nil, exprcheck.KindUnknown }() c.walkBodyWithScope(body, mf, scope.entities, r) } @@ -123,11 +128,11 @@ func (c *CheckAdapter) walkBodyWithScope(body []ast.MicroflowStatement, mf strin case *ast.ListOperationStmt: c.checkListOperationCondition(n, mf, scope, r) case *ast.ReturnStmt: - c.checkExpr(n.Value, "ReturnStmt.Value", mf, r) + c.checkExprAs(n.Value, "ReturnStmt.Value", mf, c.returnKind, r) case *ast.DeclareStmt: - c.checkExpr(n.InitialValue, "DeclareStmt.InitialValue", mf, r) + c.checkExprAs(n.InitialValue, "DeclareStmt.InitialValue", mf, primitiveTarget(&n.Type), r) case *ast.MfSetStmt: - c.checkExpr(n.Value, "MfSetStmt.Value", mf, r) + c.checkExprAs(n.Value, "MfSetStmt.Value", mf, c.setTargetKind(n.Target), r) case *ast.LogStmt: c.checkExpr(n.Message, "LogStmt.Message", mf, r) // The template parameters were not walked at all, so a non-String @@ -220,6 +225,13 @@ func (c *CheckAdapter) checkListOperationCondition(n *ast.ListOperationStmt, mf } func (c *CheckAdapter) checkExpr(expr ast.Expression, slot, mf string, r *Result) { + c.checkExprAs(expr, slot, mf, exprcheck.KindUnknown, r) +} + +// checkExprAs is checkExpr for a position whose type is declared elsewhere — a +// DECLARE, a SET, a RETURN. target is the primitive kind the expression must +// have, or KindUnknown when there is none to check against. +func (c *CheckAdapter) checkExprAs(expr ast.Expression, slot, mf string, target exprcheck.TypeKind, r *Result) { // The captured source can carry the trailing layout of the statement it was // lifted from ("'Open'\n "), which the lexer would then have to recover // from. Trim before parsing rather than teaching every rule about it. @@ -234,6 +246,7 @@ func (c *CheckAdapter) checkExpr(expr ast.Expression, slot, mf string, r *Result Catalog: c.catalog, Entities: c.entities, Scope: c.kinds, + Target: target, }) r.Hints = append(r.Hints, hints...) } @@ -305,3 +318,52 @@ func (c *CheckAdapter) checkSendEmail(n *ast.SendEmailStmt, mf string, r *Result c.checkExpr(n.Auth.Password, "SendEmailStmt.Credential", mf, r) } } + +// primitiveTarget is the kind an expression must have to be stored in, or +// returned as, a value of type t — for the six primitives only. An enumeration, +// an entity, a list or anything unusual is KindUnknown, which checks nothing: +// those need the catalog, and a wrong answer here blocks a script that builds. +// The mapping is exact on purpose (no AutoNumber, no Date): only the six +// spellings measured on MxBuild are checked. +func primitiveTarget(t *ast.DataType) exprcheck.TypeKind { + if t == nil || t.EntityRef != nil || t.EnumRef != nil { + return exprcheck.KindUnknown + } + switch t.Kind { + case ast.TypeString: + return exprcheck.KindString + case ast.TypeInteger: + return exprcheck.KindInteger + case ast.TypeLong: + return exprcheck.KindLong + case ast.TypeDecimal: + return exprcheck.KindDecimal + case ast.TypeBoolean: + return exprcheck.KindBoolean + case ast.TypeDateTime: + return exprcheck.KindDateTime + } + return exprcheck.KindUnknown +} + +func returnDataType(rt *ast.MicroflowReturnType) *ast.DataType { + if rt == nil { + return nil + } + return &rt.Type +} + +// setTargetKind is the declared primitive kind of the variable a SET assigns. +// An attribute path (`$Order/Status`) is typed by the attribute, which the +// CHANGE slots own, and a variable the scope cannot type is silence. +func (c *CheckAdapter) setTargetKind(target string) exprcheck.TypeKind { + name := strings.TrimPrefix(target, "$") + if name == "" || strings.Contains(name, "/") || c.kinds == nil { + return exprcheck.KindUnknown + } + k, ok := c.kinds.Lookup(name) + if !ok { + return exprcheck.KindUnknown + } + return k +} diff --git a/mdl/exprcheck/adapters/target_kind_test.go b/mdl/exprcheck/adapters/target_kind_test.go new file mode 100644 index 0000000000..438782fb4b --- /dev/null +++ b/mdl/exprcheck/adapters/target_kind_test.go @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: Apache-2.0 + +package adapters + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +func src(s string) ast.Expression { return &ast.SourceExpr{Source: s} } + +func dt(k ast.DataTypeKind) ast.DataType { return ast.DataType{Kind: k} } + +func countE009(r *Result) int { + n := 0 + for _, h := range r.Hints { + if h.Code == "E009" { + n++ + } + } + return n +} + +// DECLARE, SET and RETURN each check the expression against the type the +// position was declared with, resolved from the AST rather than the catalog. +// Measured on MxBuild 11.16.1: all three are CE0117 for a mismatch +// (see exprcheck.TestTargetKindMatrix for the matrix itself). +func TestTargetKindWiredForDeclareSetReturn(t *testing.T) { + flow := func(body ...ast.MicroflowStatement) *ast.CreateMicroflowStmt { + return &ast.CreateMicroflowStmt{ + Name: ast.QualifiedName{Module: "M", Name: "F"}, + ReturnType: &ast.MicroflowReturnType{Type: dt(ast.TypeInteger)}, + Body: body, + } + } + cases := []struct { + name string + body []ast.MicroflowStatement + want int + }{ + {"declare mismatch", []ast.MicroflowStatement{ + &ast.DeclareStmt{Variable: "X", Type: dt(ast.TypeInteger), InitialValue: src("'abc'")}, + &ast.ReturnStmt{Value: src("$X")}, + }, 1}, + {"declare ok", []ast.MicroflowStatement{ + &ast.DeclareStmt{Variable: "X", Type: dt(ast.TypeInteger), InitialValue: src("1")}, + &ast.ReturnStmt{Value: src("$X")}, + }, 0}, + {"set mismatch", []ast.MicroflowStatement{ + &ast.DeclareStmt{Variable: "X", Type: dt(ast.TypeInteger), InitialValue: src("0")}, + &ast.MfSetStmt{Target: "X", Value: src("'abc'")}, + &ast.ReturnStmt{Value: src("$X")}, + }, 1}, + {"set widening Integer to Decimal is fine", []ast.MicroflowStatement{ + &ast.DeclareStmt{Variable: "D", Type: dt(ast.TypeDecimal), InitialValue: src("0")}, + &ast.MfSetStmt{Target: "D", Value: src("1")}, + &ast.ReturnStmt{Value: src("1")}, + }, 0}, + {"set Decimal into Integer", []ast.MicroflowStatement{ + &ast.DeclareStmt{Variable: "X", Type: dt(ast.TypeInteger), InitialValue: src("0")}, + &ast.MfSetStmt{Target: "X", Value: src("1.5")}, + &ast.ReturnStmt{Value: src("$X")}, + }, 1}, + {"return mismatch", []ast.MicroflowStatement{ + &ast.ReturnStmt{Value: src("'abc'")}, + }, 1}, + {"return in a branch", []ast.MicroflowStatement{ + &ast.IfStmt{ + Condition: src("true"), + ThenBody: []ast.MicroflowStatement{&ast.ReturnStmt{Value: src("true")}}, + ElseBody: []ast.MicroflowStatement{&ast.ReturnStmt{Value: src("1")}}, + }, + }, 1}, + // MDL041 already reports a COMPUTED Decimal assigned to an Integer by + // DECLARE or SET; reporting it here too would print one problem twice. + {"computed Decimal into Integer declare is MDL041's", []ast.MicroflowStatement{ + &ast.DeclareStmt{Variable: "X", Type: dt(ast.TypeInteger), InitialValue: src("1.5 * 2")}, + &ast.ReturnStmt{Value: src("$X")}, + }, 0}, + {"computed Decimal into Integer set is MDL041's", []ast.MicroflowStatement{ + &ast.DeclareStmt{Variable: "X", Type: dt(ast.TypeInteger), InitialValue: src("0")}, + &ast.MfSetStmt{Target: "X", Value: src("1 div 2")}, + &ast.ReturnStmt{Value: src("$X")}, + }, 0}, + // MDL041 does not look at a RETURN, a bare Decimal literal, or a Decimal variable. + {"computed Decimal returned as Integer is not MDL041's", []ast.MicroflowStatement{ + &ast.ReturnStmt{Value: src("1.5 * 2")}, + }, 1}, + {"Decimal variable into Integer declare is not MDL041's", []ast.MicroflowStatement{ + &ast.DeclareStmt{Variable: "D", Type: dt(ast.TypeDecimal), InitialValue: src("0")}, + &ast.DeclareStmt{Variable: "X", Type: dt(ast.TypeInteger), InitialValue: src("$D")}, + &ast.ReturnStmt{Value: src("$X")}, + }, 1}, + // An attribute path is typed by the attribute, not by a declared variable; + // the CHANGE/CREATE slots own that. + {"set of an attribute path is not this rule", []ast.MicroflowStatement{ + &ast.MfSetStmt{Target: "Order/Status", Value: src("'abc'")}, + &ast.ReturnStmt{Value: src("1")}, + }, 0}, + // A variable the scope cannot type stays silent. + {"set of an unknown variable", []ast.MicroflowStatement{ + &ast.MfSetStmt{Target: "Nope", Value: src("'abc'")}, + &ast.ReturnStmt{Value: src("1")}, + }, 0}, + // An entity-typed declare is not a primitive target. + {"entity declare is not checked", []ast.MicroflowStatement{ + &ast.DeclareStmt{Variable: "O", Type: ast.DataType{Kind: ast.TypeEntity, EntityRef: &ast.QualifiedName{Module: "M", Name: "E"}}, InitialValue: src("empty")}, + &ast.ReturnStmt{Value: src("1")}, + }, 0}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + r := NewCheckAdapter(nil).CheckMicroflow(flow(c.body...)) + if got := countE009(r); got != c.want { + t.Errorf("E009 count = %d, want %d: %+v", got, c.want, r.Hints) + } + }) + } +} + +// A flow with no return type (or Nothing) has nothing to check a RETURN against. +func TestTargetKindReturnWithoutType(t *testing.T) { + for _, rt := range []*ast.MicroflowReturnType{nil, {Type: dt(ast.TypeVoid)}} { + stmt := &ast.CreateMicroflowStmt{ + Name: ast.QualifiedName{Module: "M", Name: "F"}, + ReturnType: rt, + Body: []ast.MicroflowStatement{&ast.ReturnStmt{Value: src("'abc'")}}, + } + if got := countE009(NewCheckAdapter(nil).CheckMicroflow(stmt)); got != 0 { + t.Errorf("return type %+v: E009 count = %d, want 0", rt, got) + } + } +} + +// A nanoflow is the same walk with the same declared types. +func TestTargetKindNanoflow(t *testing.T) { + stmt := &ast.CreateNanoflowStmt{ + Name: ast.QualifiedName{Module: "M", Name: "N"}, + ReturnType: &ast.MicroflowReturnType{Type: dt(ast.TypeString)}, + Body: []ast.MicroflowStatement{&ast.ReturnStmt{Value: src("1")}}, + } + if got := countE009(NewCheckAdapter(nil).CheckNanoflow(stmt)); got != 1 { + t.Errorf("E009 count = %d, want 1", got) + } +} diff --git a/mdl/exprcheck/func_checker.go b/mdl/exprcheck/func_checker.go index 27c07976db..8a5025258d 100644 --- a/mdl/exprcheck/func_checker.go +++ b/mdl/exprcheck/func_checker.go @@ -93,7 +93,6 @@ var funcTable = map[string]funcSig{ "toLowerCase": {args: []TypeKind{KindString}, ret: KindString}, "replaceAll": {args: []TypeKind{KindString, KindString, KindString}, ret: KindString}, "replaceFirst": {args: []TypeKind{KindString, KindString, KindString}, ret: KindString}, - "replace": {args: []TypeKind{KindString, KindString, KindString}, ret: KindString}, "urlEncode": {args: []TypeKind{KindString}, ret: KindString}, "urlDecode": {args: []TypeKind{KindString}, ret: KindString}, diff --git a/mdl/exprcheck/interfaces.go b/mdl/exprcheck/interfaces.go index 08648399ee..47136ac21a 100644 --- a/mdl/exprcheck/interfaces.go +++ b/mdl/exprcheck/interfaces.go @@ -29,6 +29,13 @@ type Context struct { // `[...]` predicate. Neither is valid in an expression, so it is off // everywhere but parse_xpath (ako/mxcli#1439). XPath bool + + // Target is the primitive kind the whole expression must have, for a position + // that is typed by something other than the slot table: the variable a DECLARE + // introduces, the variable a SET assigns, the return type of the flow a RETURN + // leaves. The adapter resolves it from the declaration; KindUnknown (the zero + // value) means there is none, and only the six primitives are checked. + Target TypeKind } // IsSemanticEnabled reports whether Catalog and Slots are both wired so that diff --git a/mdl/exprcheck/parser.go b/mdl/exprcheck/parser.go index 8efab0c52e..14d83f5049 100644 --- a/mdl/exprcheck/parser.go +++ b/mdl/exprcheck/parser.go @@ -64,6 +64,7 @@ func (p *parserImpl) Parse(src string, ctx Context) (RobustExpr, []Hint) { hs = append(hs, checkValueLiterals(expr, ctx)...) hs = append(hs, checkSlotKind(expr, ctx)...) hs = append(hs, checkCallArgumentKind(expr, ctx)...) + hs = append(hs, checkTargetKind(expr, ctx)...) hs = append(hs, checkBareIdentifierValue(expr, ctx)...) hs = append(hs, checkStringFuncsComparedWithEmpty(expr, nil, ctx)...) return expr, hs diff --git a/mdl/exprcheck/slot_resolver.go b/mdl/exprcheck/slot_resolver.go index 901500ee9f..b00d7b1679 100644 --- a/mdl/exprcheck/slot_resolver.go +++ b/mdl/exprcheck/slot_resolver.go @@ -37,8 +37,11 @@ var staticExpectations = map[string]SlotConstraint{ "SendEmailStmt.Port": {Kind: KindInteger, AlsoAccepts: []TypeKind{KindLong}, Mxbuild: "CE9528 \"The expression for 'Server Port' should be of type Integer/Long\""}, "SendEmailStmt.Credential": {Kind: KindString, Mxbuild: "CE9528 \"The expression for '…' should be of type String\""}, "SendEmailStmt.TemplateParam": {Kind: KindString}, - "MfSetStmt.Value": {Kind: KindUnknown, ResolveBy: "TargetVariable"}, - "DeclareStmt.InitialValue": {Kind: KindUnknown, ResolveBy: "DeclareType"}, + // Resolved by the adapter, which hands the declared primitive type to the + // checker as Context.Target (checkTargetKind), not by a slot-path qualifier: + // DECLARE, SET and RETURN are typed by their own declaration. + "MfSetStmt.Value": {Kind: KindUnknown, ResolveBy: "TargetVariable"}, + "DeclareStmt.InitialValue": {Kind: KindUnknown, ResolveBy: "DeclareType"}, } type defaultSlotResolver struct{} diff --git a/mdl/exprcheck/target_kind.go b/mdl/exprcheck/target_kind.go new file mode 100644 index 0000000000..47d00d0333 --- /dev/null +++ b/mdl/exprcheck/target_kind.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 + +package exprcheck + +import "github.com/mendixlabs/mxcli/mdl/exprcheck/hints" + +// checkTargetKind compares an expression with the declared type of the variable +// it initialises, the variable it is assigned to, or the flow it is returned from. +// +// `declare $X Integer = 'abc'`, `set $X = 'abc'` and a `return 'abc'` in a flow +// that returns Integer all passed `mxcli check` and `exec`, and failed the build: +// CE0117 "Error(s) in expression" on MxBuild, CE0844 "The expression is of type +// String but should be of type Integer/Long" on Mendix CLI 2. The slot table had +// declared all three (`DeclareType`, `TargetVariable`, `MicroflowReturn`) and +// nothing resolved them, because checkSlotKind skips every slot with a ResolveBy +// — the same gap #2096 closed for a call argument. +// +// The predicate is the one for call arguments, and for the same reason: the three +// positions were measured as a 6 x 12 matrix on MxBuild 11.16.1 (216 microflows, +// one `mx check`) and are identical to each other and to what callArgAccepted +// already encodes — Integer and Long pass into each other and widen into Decimal, +// every other primitive mismatch is CE0117. See TestTargetKindMatrix. +// +// Both sides must be a known primitive. A target of enumeration, object or list +// type, an expression of unknown kind and `empty` are silence, not an error: +// exec refuses what check reports. `empty` for a Boolean is a measured CE0117 and +// stays unreported here exactly as it does for a call argument. +func checkTargetKind(expr RobustExpr, ctx Context) []Hint { + want := ctx.Target + if !primitiveKind(want) { + return nil + } + got := argKind(expr, ctx) + if !primitiveKind(got) || callArgAccepted(want, "", got, "") { + return nil + } + if coveredByDecimalIntoInteger(expr, ctx, want, got) { + return nil + } + wantName, gotName := typeKindName(want), typeKindName(got) + where := hintsLocation(ctx, expr.Pos()) + where.Context = "value for a " + wantName + " variable or return" + return []Hint{{ + Code: "E009", + Slug: "slot-type-mismatch", + Severity: hints.SeverityError, + Where: where, + YouWrote: "<" + gotName + ">", + Problem: "This position holds " + withArticle(wantName) + ", but the expression is " + + withArticle(gotName) + ". Mendix converts only from Integer or Long to Decimal (and " + + "between Integer and Long); anything else is CE0117 \"Error(s) in expression\".", + Fix: callArgFix(want, "", got), + }} +} + +func primitiveKind(k TypeKind) bool { + switch k { + case KindBoolean, KindString, KindInteger, KindLong, KindDecimal, KindDateTime: + return true + } + return false +} + +// coveredByDecimalIntoInteger reports whether MDL041 (mdl/executor, +// checkNumericAssignment) already reports this one. It flags a COMPUTED Decimal — +// arithmetic, or a Decimal-returning built-in such as secondsBetween() — assigned +// to an Integer or Long by DECLARE or SET, with a message that names the cause +// and without needing a project. Reporting it again here would print one problem +// twice. A bare Decimal literal or variable, and any RETURN, are not MDL041's, so +// they stay reported here. +func coveredByDecimalIntoInteger(expr RobustExpr, ctx Context, want, got TypeKind) bool { + if got != KindDecimal || (want != KindInteger && want != KindLong) { + return false + } + if base, _ := splitSlotQual(ctx.SlotPath); base != "DeclareStmt.InitialValue" && base != "MfSetStmt.Value" { + return false + } + switch n := unwrapParens(expr).(type) { + case *BinExpr: + switch n.Op { + case "+", "-", "*", "div", ":", "mod": + return true + } + case *CallExpr: + _, known := funcTable[n.Name] + return known + } + return false +} diff --git a/mdl/exprcheck/target_kind_test.go b/mdl/exprcheck/target_kind_test.go new file mode 100644 index 0000000000..02df414d31 --- /dev/null +++ b/mdl/exprcheck/target_kind_test.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 + +package exprcheck + +import "testing" + +// TestTargetKindMatrix pins checkTargetKind to what MxBuild 11.16.1 does for an +// expression assigned to a typed variable. +// +// The matrix was measured once and holds for all three positions that give an +// expression a type from outside the slot table — `declare $X T = e`, +// `set $X = e` and `return e` in a microflow that `returns T`: 216 microflows +// (3 positions x 6 types x 12 value shapes), one `mx check`, every rejection +// CE0117 "Error(s) in expression" and every position identical. +// +// target accepted +// String 'a' empty $String +// Integer 1 empty $Integer $Long +// Long 1 empty $Integer $Long +// Decimal 1 1.5 empty $Integer $Long $Decimal +// Boolean true $Boolean (empty is CE0117 too: see below) +// DateTime empty [%CurrentDateTime%] $DateTime +// +// `empty` for a Boolean target is the one measured rejection this check does +// NOT report, as for a call argument (call_argument.go): `empty` is Mendix's +// null and is accepted for every other type, and refusing it deserves its own +// measurement across versions rather than a corner of this rule. +func TestTargetKindMatrix(t *testing.T) { + vars := mapScope{ + "pString": KindString, "pInteger": KindInteger, "pLong": KindLong, + "pDecimal": KindDecimal, "pBoolean": KindBoolean, "pDateTime": KindDateTime, + } + shapes := []struct{ name, src string }{ + {"lstr", "'a'"}, {"lint", "1"}, {"ldec", "1.5"}, {"lbool", "true"}, + {"empty", "empty"}, {"tok", "[%CurrentDateTime%]"}, + {"pString", "$pString"}, {"pInteger", "$pInteger"}, {"pLong", "$pLong"}, + {"pDecimal", "$pDecimal"}, {"pBoolean", "$pBoolean"}, {"pDateTime", "$pDateTime"}, + } + targets := []struct { + kind TypeKind + name string + accepted []string + }{ + {KindString, "String", []string{"lstr", "empty", "pString"}}, + {KindInteger, "Integer", []string{"lint", "empty", "pInteger", "pLong"}}, + {KindLong, "Long", []string{"lint", "empty", "pInteger", "pLong"}}, + {KindDecimal, "Decimal", []string{"lint", "ldec", "empty", "pInteger", "pLong", "pDecimal"}}, + {KindBoolean, "Boolean", []string{"lbool", "pBoolean", "empty"}}, // empty: measured CE0117, deliberately silent + {KindDateTime, "DateTime", []string{"empty", "tok", "pDateTime"}}, + } + for _, tg := range targets { + acc := map[string]bool{} + for _, a := range tg.accepted { + acc[a] = true + } + for _, sh := range shapes { + _, hs := (&parserImpl{}).Parse(sh.src, Context{ + SlotPath: "DeclareStmt.InitialValue", Microflow: "Mx.F", Scope: vars, Target: tg.kind, + }) + var reported bool + for _, h := range hs { + if h.Code == "E009" { + reported = true + } + } + if want := !acc[sh.name]; reported != want { + t.Errorf("%s target, %s (%s): reported=%v, MxBuild says rejected=%v", tg.name, sh.name, sh.src, reported, want) + } + } + } +} + +// A target that is not one of the six primitives, or no target at all, checks +// nothing: an enumeration, an object or a list needs the catalog, and a position +// with no declared type has nothing to compare against. +func TestTargetKindSilentWithoutPrimitiveTarget(t *testing.T) { + for _, k := range []TypeKind{KindUnknown, KindAny, KindEnumeration, KindObject, KindList, KindBinary} { + _, hs := (&parserImpl{}).Parse("'a'", Context{SlotPath: "DeclareStmt.InitialValue", Target: k}) + for _, h := range hs { + if h.Code == "E009" { + t.Errorf("target %v: unexpected %s %q", k, h.Code, h.Problem) + } + } + } +} + +// An expression whose kind is not known is never reported: exec refuses what +// check reports, so a guess would block a script that builds. +func TestTargetKindSilentWhenKindUnknown(t *testing.T) { + _, hs := (&parserImpl{}).Parse("$nobody + $else", Context{SlotPath: "DeclareStmt.InitialValue", Target: KindInteger}) + for _, h := range hs { + if h.Code == "E009" { + t.Errorf("unexpected %s %q", h.Code, h.Problem) + } + } +} diff --git a/mdl/exprcheck/unknown_funcs.go b/mdl/exprcheck/unknown_funcs.go index 62e88905db..66bd573b28 100644 --- a/mdl/exprcheck/unknown_funcs.go +++ b/mdl/exprcheck/unknown_funcs.go @@ -2,7 +2,10 @@ package exprcheck -import "strings" +import ( + "sort" + "strings" +) // FuncRef identifies a function call found in an expression whose name is not a // known Mendix built-in — a hallucinated or misspelled function (e.g. randomInt). @@ -86,7 +89,14 @@ func walkCalls(e RobustExpr, fn func(*CallExpr)) { func nearestFunc(name string) string { lower := strings.ToLower(name) best, bestDist := "", 1<<30 + // Sorted, so a tie (replace is one edit from replaceAll and from replaceFirst) + // gives the same suggestion every run rather than whichever the map yielded. + cands := make([]string, 0, len(funcTable)) for cand := range funcTable { + cands = append(cands, cand) + } + sort.Strings(cands) + for _, cand := range cands { cl := strings.ToLower(cand) // A prefix relationship is a strong signal (randomInt / random). if strings.HasPrefix(lower, cl) || strings.HasPrefix(cl, lower) { diff --git a/mdl/exprcheck/unknown_funcs_test.go b/mdl/exprcheck/unknown_funcs_test.go index be428c8f3c..01bca83e5f 100644 --- a/mdl/exprcheck/unknown_funcs_test.go +++ b/mdl/exprcheck/unknown_funcs_test.go @@ -17,6 +17,12 @@ func TestUnknownFunctionCalls(t *testing.T) { {"$a + length($s)", "", ""}, // known nested {"if $a then floor($b) else ceil($c)", "", ""}, {"totallyMadeUpFn($x)", "totallyMadeUpFn", ""}, // no close match + // replace() is not a Mendix function: the string replacers are replaceAll + // and replaceFirst. MxBuild reports CE0117 and Mendix CLI 2 CE7811 + // "Unknown function 'replace'" (measured on 11.16.1). + {"replace($s, 'a', 'b')", "replace", "replaceAll"}, + {"replaceAll($s, 'a', 'b')", "", ""}, + {"replaceFirst($s, 'a', 'b')", "", ""}, } for _, c := range cases { got := UnknownFunctionCalls(c.src) diff --git a/mdl/visitor/visible_roles_split.go b/mdl/visitor/visible_roles_split.go new file mode 100644 index 0000000000..59714c37f2 --- /dev/null +++ b/mdl/visitor/visible_roles_split.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "regexp" + "strings" +) + +// leadingVisibleRolesRe is `roles (Mod.A, Mod.B) and ` at the start of a +// visibility expression. +var leadingVisibleRolesRe = regexp.MustCompile( + `(?is)^\s*roles\s*\(\s*([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+(?:\s*,\s*[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+)*)\s*\)\s+and\s+(.+?)\s*$`) + +// splitLeadingVisibleRoles splits `roles (…) and ` into the role +// names and the condition (ako/mxcli#2319). It declines when the condition +// has a top-level `or`: `roles (A) and x or y` reads (roles and x) or y, +// which is not "these roles, and this condition", and is left to be refused. +func splitLeadingVisibleRoles(expr string) (roles []string, cond string, ok bool) { + m := leadingVisibleRolesRe.FindStringSubmatch(expr) + if m == nil || hasTopLevelOr(m[2]) { + return nil, "", false + } + for _, r := range strings.Split(m[1], ",") { + roles = append(roles, strings.TrimSpace(r)) + } + return roles, m[2], true +} + +// hasTopLevelOr reports whether expr has an `or` outside parentheses and +// string literals. +func hasTopLevelOr(expr string) bool { + depth, inStr := 0, false + lower := strings.ToLower(expr) + for i := 0; i < len(expr); i++ { + c := expr[i] + switch { + case inStr: + if c == '\'' { + inStr = false + } + case c == '\'': + inStr = true + case c == '(': + depth++ + case c == ')': + depth-- + case depth == 0 && strings.HasPrefix(lower[i:], "or") && + (i == 0 || !isWordByte(expr[i-1])) && (i+2 == len(expr) || !isWordByte(expr[i+2])): + return true + } + } + return false +} + +func isWordByte(c byte) bool { + return c == '_' || c >= '0' && c <= '9' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' +} diff --git a/mdl/visitor/visitor_page_v3.go b/mdl/visitor/visitor_page_v3.go index ecb73657c2..74d5340a01 100644 --- a/mdl/visitor/visitor_page_v3.go +++ b/mdl/visitor/visitor_page_v3.go @@ -923,7 +923,16 @@ func parseWidgetPropertyV3(ctx parser.IWidgetPropertyV3Context, widget *ast.Widg if xc := propCtx.XpathConstraint(); xc != nil { widget.Properties["VisibleIf"] = buildConditionalExpression(xc) // MDL-DEPR081 } else if e := propCtx.Expression(); e != nil { - widget.Properties["VisibleIf"] = bareArgumentText(e) + text := bareArgumentText(e) + // `roles (Mod.A, …) and ` is both settings at once: + // Studio Pro keeps the roles beside the condition, and both + // apply (ako/mxcli#2319). Stored whole, the text was an invalid + // expression and the roles were lost. + if roles, cond, ok := splitLeadingVisibleRoles(text); ok { + widget.Properties["VisibleRoles"] = roles + text = cond + } + widget.Properties["VisibleIf"] = text } else if valCtx := propCtx.PropertyValueV3(); valCtx != nil { widget.Properties["Visible"] = buildPropertyValueV3(valCtx) } diff --git a/modelsdk/canon/identity.go b/modelsdk/canon/identity.go index 6e1f4e47bd..008a93577c 100644 --- a/modelsdk/canon/identity.go +++ b/modelsdk/canon/identity.go @@ -144,6 +144,14 @@ func Reconcile(contents, stored []byte, opts ...Option) (out []byte, unchanged b contents = transplantIDs(contents, storedDoc) } + // The translations of the texts the transplant paired, which the carry + // above could not pair: a Studio Pro flow's objects in another order, an + // unnamed activity's message, a source string the statement changed + // (ako/mxcli#2329). + if !o.contentsOwnTranslations { + contents = carryTranslationsByID(contents, stored) + } + // And the nested identity property the transplant does not cover: every // Workflows$* element carries a PersistentId that both engines re-mint on // every write, so without this a workflow document never equals itself and @@ -208,6 +216,10 @@ func reconcileInPasses(contents, stored []byte, opts ...Option) (out []byte, unc contents = transplantIDsInPasses(contents, stored) } + if !o.contentsOwnTranslations { + contents = carryTranslationsByID(contents, stored) + } + // And the nested identity property the transplant does not cover: every // Workflows$* element carries a PersistentId that both engines re-mint on // every write, so without this a workflow document never equals itself and diff --git a/modelsdk/canon/translations.go b/modelsdk/canon/translations.go index c2703fe8b2..a0e58a4c8c 100644 --- a/modelsdk/canon/translations.go +++ b/modelsdk/canon/translations.go @@ -3,6 +3,7 @@ package canon import ( + "bytes" "crypto/sha256" "sort" "strconv" @@ -102,7 +103,7 @@ func CarryTranslations(contents, stored []byte) []byte { storedByPath := textsByPath(storedDoc) newByPath := textsByPath(newDoc) - byPath := samePathSet(storedByPath, newByPath) + byPath := samePathSet(storedByPath, newByPath) && samePlaces(storedDoc, newDoc) _, storedByElement := textsByElement(storedDoc) newElementOf, _ := textsByElement(newDoc) @@ -608,3 +609,155 @@ func sortedPaths(m map[string]bson.D) []string { sort.Strings(out) return out } + +// carryTranslationsByID is the translation carry for texts the transplant has +// paired: a rebuilt Texts$Text holding the $ID of a stored one is that text, +// and takes back every language of it the rebuild did not write +// (ako/mxcli#2329, mendixlabs/mxcli#1872). +// +// CarryTranslations runs before the transplant and cannot use its pairing. +// Its three ways to pair a text all miss a rebuilt Studio Pro flow: the paths +// differ, because Studio Pro stores a flow's objects in drawing order and +// mxcli writes the parameters and the start event first; an activity has no +// name to address its message by; and the source string is no key where the +// statement changed it, or where two messages share a default text but not +// their translations. The transplant pairs elements structurally, across that +// reorder (pairLeftovers), so after it the $ID is the exact pairing. +// +// A pairing is not a guarantee — two activities of one kind can cross where +// the two orders differ — and a translation carried onto the wrong text is the +// silent failure CarryTranslations is built to avoid. So a text takes back its +// stored languages only when the pairing is confirmed: the flow object that +// owns it is drawn at the same place in both (a describe states every +// position), or every language the rebuild wrote has the stored text. +// +// The rules are then CarryTranslations': a language the rebuild wrote is never +// overwritten, and a carried translation keeps its stored $ID unless the +// document already holds it. +func carryTranslationsByID(contents, stored []byte) []byte { + if len(contents) == 0 || !bytes.Contains(stored, []byte("Texts$Text")) { + return contents + } + var storedDoc, newDoc bson.D + if bson.Unmarshal(stored, &storedDoc) != nil || bson.Unmarshal(contents, &newDoc) != nil { + return contents + } + type placedText struct { + text bson.D + place string + } + storedTexts := map[string]placedText{} + walkPlacedTexts(storedDoc, "", func(text bson.D, place string) { + if id, ok := docLookup(text, "$ID").(bson.Binary); ok { + storedTexts[string(id.Data)] = placedText{text, place} + } + }) + if len(storedTexts) == 0 { + return contents + } + used := elementIDs(newDoc) + changed := false + walkPlacedTexts(newDoc, "", func(text bson.D, place string) { + id, ok := docLookup(text, "$ID").(bson.Binary) + if !ok { + return + } + old, ok := storedTexts[string(id.Data)] + if !ok { + return + } + samePlace := place != "" && place == old.place + if !samePlace && !writtenLanguagesAgree(text, old.text) { + return + } + if mergeText(text, textTranslationElems(old.text), textLanguageOrder(old.text), blobToUUID(id.Data), used) { + changed = true + } + }) + if !changed { + return contents + } + out, err := bson.Marshal(newDoc) + if err != nil { + return contents + } + return out +} + +// walkPlacedTexts calls fn on every Texts$Text in v with the RelativeMiddlePoint +// of the nearest element above it that has one ("" if none does). +func walkPlacedTexts(v any, place string, fn func(text bson.D, place string)) { + switch n := v.(type) { + case bson.D: + if docLookup(n, "$Type") == "Texts$Text" { + fn(n, place) + return + } + if p, ok := docLookup(n, "RelativeMiddlePoint").(string); ok && p != "" { + place = p + } + for _, e := range n { + walkPlacedTexts(e.Value, place, fn) + } + case bson.A: + for _, e := range n { + walkPlacedTexts(e, place, fn) + } + } +} + +// writtenLanguagesAgree reports whether every language text holds is in stored +// with the same text, and text holds at least one. +func writtenLanguagesAgree(text, stored bson.D) bool { + have, want := textTranslations(text), textTranslations(stored) + if len(have) == 0 { + return false + } + for lang, t := range have { + if s, ok := want[lang]; !ok || s != t { + return false + } + } + return true +} + +// samePlaces reports whether every text at one path in the two documents is +// owned by a flow object drawn at the same place (walkPlacedTexts). A flow's +// objects are a set stored as a list, in Studio Pro's drawing order or in +// mxcli's, so the same set of paths does not mean the same texts: two +// messages stored in the other order sit at each other's path, and pairing by +// path moved one's translation onto the other (ako/mxcli#2329). +func samePlaces(stored, written bson.D) bool { + placesOf := func(doc bson.D) map[string]string { + out := map[string]string{} + var walk func(v any, path, place string) + walk = func(v any, path, place string) { + switch n := v.(type) { + case bson.D: + if docLookup(n, "$Type") == "Texts$Text" { + out[path] = place + return + } + if p, ok := docLookup(n, "RelativeMiddlePoint").(string); ok && p != "" { + place = p + } + for _, e := range n { + walk(e.Value, path+"/"+e.Key, place) + } + case bson.A: + for i, e := range n { + walk(e, path+"/"+strconv.Itoa(i), place) + } + } + } + walk(doc, "", "") + return out + } + sp, wp := placesOf(stored), placesOf(written) + for path, p := range wp { + if sp[path] != p { + return false + } + } + return true +} diff --git a/modelsdk/canon/transplant.go b/modelsdk/canon/transplant.go index f727d4412d..3cbdec04e7 100644 --- a/modelsdk/canon/transplant.go +++ b/modelsdk/canon/transplant.go @@ -3,6 +3,8 @@ package canon import ( + "sort" + "go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/x/bsonx/bsoncore" ) @@ -115,7 +117,7 @@ func idMappingWith(contents []byte, oldDoc bson.D) map[string][]byte { if err := bson.Unmarshal(contents, &newDoc); err != nil { return nil } - w := &pairer{pairs: map[string][]byte{}, claimed: map[string]bool{}} + w := &pairer{pairs: map[string][]byte{}, claimed: map[string]bool{}, storedNames: namedElementIDs(oldDoc)} w.pairValue(newDoc, oldDoc) if len(w.pairs) == 0 { return nil @@ -129,6 +131,14 @@ func idMappingWith(contents []byte, oldDoc bson.D) map[string][]byte { type pairer struct { pairs map[string][]byte claimed map[string]bool + // storedNames maps the stored document's named elements' ids to their + // names. A written element that carries one of those ids under the same + // name IS that stored element — moved to another list, or retyped from an + // association to a cross-association — and keeps its id rather than take + // the id of whatever the alignment put opposite it (ako/mxcli#2322: a + // converted cross-association took its neighbour's $ID, and with it the + // GUID mxcli had minted equal to it, and mx check reported CE9208). + storedNames map[string]string } func (p *pairer) record(newID string, oldID string, oldData []byte) { @@ -171,6 +181,12 @@ func (p *pairer) pairDoc(nd, od map[string]any) { } if nid, ndata, ok := binaryElementID(nd); ok { if oid, odata, ok := binaryElementID(od); ok && len(ndata) == len(odata) { + // Not the element opposite it: neither it nor anything under it + // takes that element's ids (its delete behaviour took the + // neighbour's, the second duplicate of #2322). + if p.keepsOwnID(nid, oid, nd) { + return + } p.record(nid, oid, odata) } } @@ -220,7 +236,7 @@ func alignSlices(news, olds []any) [][2]int { return positionalPairs(0, 0, len(news), len(olds)) } - anchors := lcsPairs(matchKeys(news), matchKeys(olds)) + anchors := lcsPairsPlaced(matchKeys(news), matchKeys(olds), places(news), places(olds)) var out [][2]int ni, oi := 0, 0 @@ -229,7 +245,65 @@ func alignSlices(news, olds []any) [][2]int { out = append(out, a) ni, oi = a[0]+1, a[1]+1 } - return append(out, positionalPairs(ni, oi, len(news), len(olds))...) + out = append(out, positionalPairs(ni, oi, len(news), len(olds))...) + return pairLeftovers(news, olds, out) +} + +// pairLeftovers adds to pairs the entries the alignment left with nothing +// opposite them, paired by matchKey where that key occurs exactly once among +// the leftovers on each side (ako/mxcli#2329, mendixlabs/mxcli#1872). +// +// The alignment is ordered, and some lists are not: Studio Pro stores a flow's +// objects in the order they were drawn, and mxcli writes a flow's parameters +// and start event first. Anchored on the longest common run of activities, +// the parameter and the start event fell in a gap with nothing — or an element +// of another $Type, which pairDoc refuses — opposite them, so a rebuild of a +// Studio Pro flow re-minted their $IDs and every carry keyed on the $ID missed +// them: the parameter's documentation was lost on `drop` + `create`. +// +// A positional pair of two elements of different $Types pairs nothing +// (pairDoc), so it is released here rather than holding both entries. A key +// that occurs twice among the leftovers on either side is ambiguous, and those +// entries stay unpaired rather than pair by guess. +func pairLeftovers(news, olds []any, pairs [][2]int) [][2]int { + usedNew, usedOld := map[int]bool{}, map[int]bool{} + kept := pairs[:0:0] + for _, p := range pairs { + nd, nok := asDoc(news[p[0]]) + od, ook := asDoc(olds[p[1]]) + if nok && ook && typeName(nd) != typeName(od) { + continue + } + kept = append(kept, p) + usedNew[p[0]], usedOld[p[1]] = true, true + } + leftovers := func(vs []any, used map[int]bool) map[string][]int { + out := map[string][]int{} + for i, v := range vs { + if used[i] { + continue + } + if _, ok := asDoc(v); !ok { + continue + } + k := matchKey(v) + out[k] = append(out[k], i) + } + return out + } + newLeft, oldLeft := leftovers(news, usedNew), leftovers(olds, usedOld) + var extra [][2]int + for k, ns := range newLeft { + if os := oldLeft[k]; len(ns) == 1 && len(os) == 1 { + extra = append(extra, [2]int{ns[0], os[0]}) + } + } + if len(extra) == 0 { + return kept + } + // In list order, so the pairing does not depend on map iteration. + sort.Slice(extra, func(i, j int) bool { return extra[i][0] < extra[j][0] }) + return append(kept, extra...) } // positionalPairs pairs [nStart,nEnd) with [oStart,oEnd) by offset. Used only @@ -281,6 +355,17 @@ func matchKey(v any) string { return key } +// places returns each entry's RelativeMiddlePoint, "" where it has none. +func places(vs []any) []string { + out := make([]string, len(vs)) + for i, v := range vs { + if d, ok := asDoc(v); ok { + out[i], _ = d["RelativeMiddlePoint"].(string) + } + } + return out +} + func matchKeys(vs []any) []string { out := make([]string, len(vs)) for i, v := range vs { @@ -292,28 +377,54 @@ func matchKeys(vs []any) []string { // lcsPairs returns a longest common subsequence of the two key sequences as // index pairs. func lcsPairs(a, b []string) [][2]int { + return lcsPairsPlaced(a, b, nil, nil) +} + +// lcsPairsPlaced is lcsPairs that, among the alignments matching as many keys, +// favours entries drawn at the same place: a flow object's RelativeMiddlePoint, +// in pa and pb ("" for an entry with none). +// +// Every activity of one kind has one key, so where Studio Pro's order of the +// objects differs from mxcli's, two show messages could cross: the rebuilt one +// took the identity, and with it the documentation and translations, of the +// other (ako/mxcli#2329). A describe states every object's position, so a +// rebuild of it puts each one where it was, and an element of one kind at one +// point is that element. A match scores 2, and 12 at the same place, so an +// alignment keeps the same-place pairs over a longer run of guesses; what the +// order then leaves out pairs by key afterwards (pairLeftovers). Where nothing +// is at its old place — an insertion that moved everything after it — every +// match scores the same and the alignment is the plain one. +func lcsPairsPlaced(a, b, pa, pb []string) [][2]int { n, m := len(a), len(b) + score := func(i, j int) int { + if a[i] != b[j] { + return -1 + } + if pa != nil && pb != nil && pa[i] != "" && pa[i] == pb[j] { + return 12 + } + return 2 + } table := make([][]int, n+1) for i := range table { table[i] = make([]int, m+1) } for i := n - 1; i >= 0; i-- { for j := m - 1; j >= 0; j-- { - if a[i] == b[j] { - table[i][j] = table[i+1][j+1] + 1 - continue + best := table[i+1][j] + if table[i][j+1] > best { + best = table[i][j+1] } - if table[i+1][j] >= table[i][j+1] { - table[i][j] = table[i+1][j] - } else { - table[i][j] = table[i][j+1] + if w := score(i, j); w > 0 && w+table[i+1][j+1] > best { + best = w + table[i+1][j+1] } + table[i][j] = best } } var out [][2]int for i, j := 0, 0; i < n && j < m; { switch { - case a[i] == b[j]: + case score(i, j) > 0 && table[i][j] == score(i, j)+table[i+1][j+1]: out = append(out, [2]int{i, j}) i, j = i+1, j+1 case table[i+1][j] >= table[i][j+1]: @@ -515,3 +626,40 @@ func patchDocument(doc bsoncore.Document, m map[string][]byte) bool { } return true } + +// keepsOwnID reports whether nd, a written element with id nid paired with the +// stored element oid, is instead the stored element that already holds nid: +// one with the same name. +func (p *pairer) keepsOwnID(nid, oid string, nd map[string]any) bool { + if nid == oid { + return false + } + name, held := p.storedNames[nid] + return held && name != "" && name == nd["Name"] +} + +// namedElementIDs maps the id of every element of doc that has a Name to it. +func namedElementIDs(doc bson.D) map[string]string { + out := map[string]string{} + var walk func(v any) + walk = func(v any) { + if d, ok := asDoc(v); ok { + if id, _, ok := binaryElementID(d); ok { + if name, ok := d["Name"].(string); ok && name != "" { + out[id] = name + } + } + for _, k := range sortedKeys(d) { + walk(d[k]) + } + return + } + if s, ok := asSlice(v); ok { + for _, e := range s { + walk(e) + } + } + } + walk(doc) + return out +} diff --git a/modelsdk/canon/transplant_keeps_stored_id_2322_test.go b/modelsdk/canon/transplant_keeps_stored_id_2322_test.go new file mode 100644 index 0000000000..8dbd54c0bd --- /dev/null +++ b/modelsdk/canon/transplant_keeps_stored_id_2322_test.go @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: Apache-2.0 + +package canon + +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// ako/mxcli#2322: moving Src.A2 to Dst converts Src.A3_A2 (FROM A3, TO A2) +// into a cross-module association that stays in Src, keeping its $ID. The +// rewrite of Src's domain model then paired it, by position, with the +// cross-association already in that list — Src.A2_A1, from moving A1 — and +// gave it A2_A1's $ID (and, on an mxcli-created model where GUID == $ID, its +// GUID). A3_A2's own identity was gone, the two associations shared one, and +// mx check reported CE9208 ("map to the same database table"); the next move +// refused the write as two elements with one $ID. +// +// An element that already carries a $ID the stored document holds is that +// element, whatever list it now sits in, and keeps it. +func assocDoc(id byte, typ, name string) bson.D { + return bson.D{{Key: "$ID", Value: id16(id)}, {Key: "$Type", Value: typ}, + {Key: "DeleteBehavior", Value: bson.D{{Key: "$ID", Value: id16(id + 1)}, {Key: "$Type", Value: "DomainModels$DeleteBehavior"}}}, + {Key: "GUID", Value: id16(id)}, {Key: "Name", Value: name}} +} + +func domainModelWith(assocs, cross []bson.D) []byte { + a, c := bson.A{int32(3)}, bson.A{int32(3)} + for _, x := range assocs { + a = append(a, x) + } + for _, x := range cross { + c = append(c, x) + } + b, _ := bson.Marshal(bson.D{{Key: "$ID", Value: id16(0xD0)}, {Key: "$Type", Value: "DomainModels$DomainModel"}, + {Key: "Associations", Value: a}, {Key: "CrossAssociations", Value: c}}) + return b +} + +func TestTransplant_ElementWithAStoredIDKeepsIt_Issue2322(t *testing.T) { + stored := domainModelWith( + []bson.D{assocDoc(0x32, "DomainModels$Association", "A3_A2"), assocDoc(0x43, "DomainModels$Association", "A4_A3")}, + []bson.D{assocDoc(0x21, "DomainModels$CrossAssociation", "A2_A1")}) + written := domainModelWith( + []bson.D{assocDoc(0x43, "DomainModels$Association", "A4_A3")}, + []bson.D{assocDoc(0x32, "DomainModels$CrossAssociation", "A3_A2")}) + out, _ := Reconcile(written, stored) + var d bson.D + if err := bson.Unmarshal(out, &d); err != nil { + t.Fatal(err) + } + walkElements(d, func(e bson.D) { + if docLookup(e, "Name") != "A3_A2" { + return + } + id, _ := docLookup(e, "$ID").(bson.Binary) + guid, _ := docLookup(e, "GUID").(bson.Binary) + if string(id.Data) != string(id16(0x32).Data) || string(guid.Data) != string(id16(0x32).Data) { + t.Errorf("A3_A2 $ID/GUID = %x/%x, want its own %x", id.Data, guid.Data, id16(0x32).Data) + } + db, _ := docLookup(e, "DeleteBehavior").(bson.D) + if dbID, _ := docLookup(db, "$ID").(bson.Binary); string(dbID.Data) != string(id16(0x33).Data) { + t.Errorf("A3_A2's delete behaviour $ID = %x, want its own %x", dbID.Data, id16(0x33).Data) + } + }) +} + +// CONTROL: a rebuilt element with a fresh $ID still takes the stored one of +// the element it stands for — the transplant's purpose. +func TestTransplant_FreshIDStillTakesTheStoredOne_Issue2322(t *testing.T) { + stored := domainModelWith(nil, []bson.D{assocDoc(0x21, "DomainModels$CrossAssociation", "A2_A1")}) + written := domainModelWith(nil, []bson.D{assocDoc(0x99, "DomainModels$CrossAssociation", "A2_A1")}) + var d bson.D + _ = bson.Unmarshal(TransplantIDs(written, stored), &d) + walkElements(d, func(e bson.D) { + if docLookup(e, "Name") == "A2_A1" { + if id, _ := docLookup(e, "$ID").(bson.Binary); string(id.Data) != string(id16(0x21).Data) { + t.Errorf("A2_A1 $ID = %x, want the stored %x", id.Data, id16(0x21).Data) + } + } + }) +} diff --git a/modelsdk/canon/transplant_unordered_2329_test.go b/modelsdk/canon/transplant_unordered_2329_test.go new file mode 100644 index 0000000000..67d3816a1d --- /dev/null +++ b/modelsdk/canon/transplant_unordered_2329_test.go @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: Apache-2.0 + +package canon + +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// ako/mxcli#2329 (mendixlabs/mxcli#1872): `drop microflow` + `create +// microflow` of a flow Studio Pro authored lost its parameters' documentation. +// The carry for it (unstatedFlowDefaults) puts the stored value back on the +// element of the same $ID, so it depends on TransplantIDs pairing the +// parameter. It did not: Studio Pro stores a flow's objects in the order they +// were drawn, mxcli writes the parameters and the start event first, and the +// alignment anchored on the longest common run of activities — leaving the +// parameter and the start event in a gap with nothing opposite. A flow mxcli +// had created was written in mxcli's order, so it kept its documentation; one +// from Studio Pro did not, whatever its content. + +func flowObj(id byte, typ string, props ...bson.E) bson.D { + return append(bson.D{{Key: "$ID", Value: id16(id)}, {Key: "$Type", Value: typ}}, props...) +} + +func activity(id byte, action string) bson.D { + return flowObj(id, "Microflows$ActionActivity", bson.E{Key: "Documentation", Value: ""}, + bson.E{Key: "Action", Value: flowObj(id+0x80, action, bson.E{Key: "ErrorHandlingType", Value: "Rollback"})}) +} + +func parameter(id byte, doc string) bson.D { + return flowObj(id, "Microflows$MicroflowParameter", bson.E{Key: "Documentation", Value: doc}, + bson.E{Key: "Name", Value: "AccountPasswordData"}, + bson.E{Key: "VariableType", Value: flowObj(id+0x80, "DataTypes$ObjectType", + bson.E{Key: "Entity", Value: "Administration.AccountPasswordData"})}) +} + +func flowWithObjects(objs ...bson.D) []byte { + list := bson.A{int32(3)} + for _, o := range objs { + list = append(list, o) + } + b, _ := bson.Marshal(bson.D{ + {Key: "$ID", Value: id16(0xAA)}, {Key: "$Type", Value: "Microflows$Microflow"}, + {Key: "ObjectCollection", Value: bson.D{{Key: "$ID", Value: id16(0xAB)}, + {Key: "$Type", Value: "Microflows$MicroflowObjectCollection"}, {Key: "Objects", Value: list}}}, + }) + return b +} + +func parameterOf(t *testing.T, unit []byte) bson.D { + t.Helper() + var d bson.D + if err := bson.Unmarshal(unit, &d); err != nil { + t.Fatal(err) + } + var p bson.D + walkElements(d, func(e bson.D) { + if docLookup(e, "$Type") == "Microflows$MicroflowParameter" { + p = e + } + }) + if p == nil { + t.Fatal("no parameter in the unit") + } + return p +} + +// Studio Pro's order: the activities first, the parameter and the start event +// last. mxcli's: the parameter and the start event first. +func storedStudioProOrder(doc string) []byte { + return flowWithObjects( + activity(0x10, "Microflows$RetrieveAction"), + activity(0x11, "Microflows$ChangeAction"), + activity(0x12, "Microflows$ShowMessageAction"), + parameter(0x13, doc), + flowObj(0x14, "Microflows$StartEvent")) +} + +func writtenMxcliOrder(doc string) []byte { + return flowWithObjects( + parameter(0x23, doc), + flowObj(0x24, "Microflows$StartEvent"), + activity(0x20, "Microflows$RetrieveAction"), + activity(0x21, "Microflows$ChangeAction"), + activity(0x22, "Microflows$ShowMessageAction")) +} + +func TestReconcile_ParameterDocSurvivesAReorderedRebuild_Issue2329(t *testing.T) { + out, _ := Reconcile(writtenMxcliOrder(""), storedStudioProOrder("X DB")) + p := parameterOf(t, out) + if got, _ := docLookup(p, "Documentation").(string); got != "X DB" { + t.Errorf("parameter documentation = %q, want the stored %q", got, "X DB") + } + if id, _ := docLookup(p, "$ID").(bson.Binary); string(id.Data) != string(id16(0x13).Data) { + t.Errorf("parameter $ID was not carried from the stored one") + } +} + +// The start event is the other element the reorder left unpaired; it keeps its +// identity too. +func TestTransplant_StartEventPairsAcrossAReorder_Issue2329(t *testing.T) { + out := TransplantIDs(writtenMxcliOrder(""), storedStudioProOrder("")) + var d bson.D + _ = bson.Unmarshal(out, &d) + found := false + walkElements(d, func(e bson.D) { + if docLookup(e, "$Type") == "Microflows$StartEvent" { + id, _ := docLookup(e, "$ID").(bson.Binary) + found = string(id.Data) == string(id16(0x14).Data) + } + }) + if !found { + t.Error("start event did not keep the stored $ID") + } +} + +// CONTROL: a documentation the write states is not replaced. +func TestReconcile_WrittenParameterDocLands_Issue2329(t *testing.T) { + out, _ := Reconcile(writtenMxcliOrder("new"), storedStudioProOrder("X DB")) + if got, _ := docLookup(parameterOf(t, out), "Documentation").(string); got != "new" { + t.Errorf("parameter documentation = %q, want the written %q", got, "new") + } +} + +// CONTROL: an entry whose key occurs twice among the leftovers is ambiguous +// and pairs with nothing, rather than with a guess. +func TestTransplant_AmbiguousLeftoversStayUnpaired_Issue2329(t *testing.T) { + stored := flowWithObjects( + activity(0x10, "Microflows$RetrieveAction"), + activity(0x11, "Microflows$ChangeAction"), + activity(0x12, "Microflows$ShowMessageAction"), + flowObj(0x13, "Microflows$EndEvent"), + flowObj(0x14, "Microflows$EndEvent")) + written := flowWithObjects( + flowObj(0x23, "Microflows$EndEvent"), + flowObj(0x24, "Microflows$EndEvent"), + activity(0x20, "Microflows$RetrieveAction"), + activity(0x21, "Microflows$ChangeAction"), + activity(0x22, "Microflows$ShowMessageAction")) + var d bson.D + _ = bson.Unmarshal(TransplantIDs(written, stored), &d) + walkElements(d, func(e bson.D) { + if docLookup(e, "$Type") == "Microflows$EndEvent" { + id, _ := docLookup(e, "$ID").(bson.Binary) + if s := string(id.Data); s == string(id16(0x13).Data) || s == string(id16(0x14).Data) { + t.Errorf("an end event took a stored $ID by guess") + } + } + }) +} + +// The same reorder with the show messages' translations (#2329's second +// symptom). The translation carry runs before the transplant and pairs texts by +// place, by owning element or by source string. Reordered, the places differ; +// an activity has no name; and two messages with one default text but +// different translations make the source ambiguous, so nothing was carried and +// the rebuild kept only the language MDL writes. +func message(id byte, place string, texts ...[2]string) bson.D { + items := bson.A{int32(3)} + for i, l := range texts { + items = append(items, flowObj(id+0x40+byte(i), "Texts$Translation", + bson.E{Key: "LanguageCode", Value: l[0]}, bson.E{Key: "Text", Value: l[1]})) + } + return flowObj(id, "Microflows$ActionActivity", bson.E{Key: "Documentation", Value: ""}, + bson.E{Key: "RelativeMiddlePoint", Value: place}, + bson.E{Key: "Action", Value: flowObj(id+0x80, "Microflows$ShowMessageAction", + bson.E{Key: "Template", Value: flowObj(id+0x20, "Microflows$TextTemplate", + bson.E{Key: "Text", Value: flowObj(id+0x30, "Texts$Text", bson.E{Key: "Items", Value: items})})})}) +} + +func messageLanguages(t *testing.T, unit []byte) []map[string]string { + t.Helper() + var d bson.D + if err := bson.Unmarshal(unit, &d); err != nil { + t.Fatal(err) + } + var out []map[string]string + walkElements(d, func(e bson.D) { + if docLookup(e, "$Type") == "Texts$Text" { + out = append(out, textTranslations(e)) + } + }) + return out +} + +func TestReconcile_MessageTranslationsSurviveAReorderedRebuild_Issue2329(t *testing.T) { + stored := flowWithObjects( + message(0x01, "100;100", [2]string{"en_US", "Done."}, [2]string{"nl_NL", "Klaar."}, [2]string{"en_GB", "Done, mate."}), + message(0x02, "300;100", [2]string{"en_US", "Done."}, [2]string{"nl_NL", "Gereed."}), + parameter(0x13, ""), + flowObj(0x14, "Microflows$StartEvent")) + written := flowWithObjects( + parameter(0x23, ""), + flowObj(0x24, "Microflows$StartEvent"), + message(0x05, "100;100", [2]string{"en_US", "Done."}), + message(0x06, "300;100", [2]string{"en_US", "Done."})) + out, _ := Reconcile(written, stored) + got := messageLanguages(t, out) + if len(got) != 2 { + t.Fatalf("texts = %v, want two", got) + } + if got[0]["nl_NL"] != "Klaar." || got[0]["en_GB"] != "Done, mate." { + t.Errorf("first message = %v, want its stored nl_NL and en_GB back", got[0]) + } + if got[1]["nl_NL"] != "Gereed." || got[1]["en_GB"] != "" { + t.Errorf("second message = %v, want its own stored nl_NL and no en_GB", got[1]) + } +} + +// CONTROL: a language the write states is the write's. +func TestReconcile_WrittenLanguageLandsAfterReorder_Issue2329(t *testing.T) { + stored := flowWithObjects( + message(0x01, "100;100", [2]string{"en_US", "Done."}, [2]string{"nl_NL", "Klaar."}), + activity(0x10, "Microflows$RetrieveAction"), + parameter(0x13, "")) + written := flowWithObjects( + parameter(0x23, ""), + message(0x05, "100;100", [2]string{"en_US", "Finished."}), + activity(0x20, "Microflows$RetrieveAction")) + out, _ := Reconcile(written, stored) + got := messageLanguages(t, out) + if len(got) != 1 || got[0]["en_US"] != "Finished." || got[0]["nl_NL"] != "Klaar." { + t.Errorf("message = %v, want the written en_US and the stored nl_NL", got) + } +} + +// The crossing the place breaks: Studio Pro stored the two messages in one +// order, the rebuild writes them in the other. Without the place, the +// alignment paired each with the other, and the other's translation was +// carried onto it. +func TestReconcile_CrossedMessagesPairByPlace_Issue2329(t *testing.T) { + stored := flowWithObjects( + message(0x01, "100;100", [2]string{"en_US", "No match."}, [2]string{"nl_NL", "Geen match."}), + activity(0x10, "Microflows$RetrieveAction"), + message(0x02, "300;100", [2]string{"en_US", "Updated."}, [2]string{"nl_NL", "Bijgewerkt."})) + written := flowWithObjects( + message(0x06, "300;100", [2]string{"en_US", "Updated!"}), + activity(0x20, "Microflows$RetrieveAction"), + message(0x05, "100;100", [2]string{"en_US", "No match."})) + out, _ := Reconcile(written, stored) + got := messageLanguages(t, out) + if len(got) != 2 || got[0]["nl_NL"] != "Bijgewerkt." || got[1]["nl_NL"] != "Geen match." { + t.Errorf("messages = %v, want each its own stored nl_NL", got) + } +} + +// CONTROL: a text whose pairing is unconfirmed — moved, and its source +// changed — takes nothing rather than a guess. +func TestReconcile_UnconfirmedPairingCarriesNothing_Issue2329(t *testing.T) { + stored := flowWithObjects( + message(0x01, "100;100", [2]string{"en_US", "Done."}, [2]string{"nl_NL", "Klaar."}), + parameter(0x13, "")) + written := flowWithObjects( + parameter(0x23, ""), + message(0x05, "500;500", [2]string{"en_US", "Finished."})) + out, _ := Reconcile(written, stored) + if got := messageLanguages(t, out); len(got) != 1 || len(got[0]) != 1 { + t.Errorf("message = %v, want only the written en_US", got) + } +} diff --git a/sdk/javaactions/core_import_2310_test.go b/sdk/javaactions/core_import_2310_test.go new file mode 100644 index 0000000000..ef43e066b2 --- /dev/null +++ b/sdk/javaactions/core_import_2310_test.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: Apache-2.0 + +package javaactions + +import ( + "strings" + "testing" +) + +// ako/mxcli#2310: a Java action whose body calls Core.getConfiguration() but +// states no `import com.mendix.core.Core;` was written without it, and passed +// check, exec and docker check (mx check does not compile Java); only the real +// build failed: "cannot find symbol: variable Core". Core is the Mendix runtime +// API almost every action body reaches for, so the generated file imports it +// when the code uses it. mxbuild keeps an import it finds (it never drops one), +// so its regeneration keeps this one too. + +func TestGenerateSource_ImportsCoreWhenTheCodeUsesIt_Issue2310(t *testing.T) { + src := GenerateSource("Helpdesk", "RunSparql", + "File resources = Core.getConfiguration().getResourcesPath();\nreturn true;", + nil, &BooleanType{}, nil, "", "") + if !strings.Contains(src, "import com.mendix.core.Core;") { + t.Errorf("generated source lacks the Core import:\n%s", src) + } +} + +// The extra code section counts too. +func TestGenerateSource_ImportsCoreForExtraCode_Issue2310(t *testing.T) { + src := GenerateSource("M", "A", "return true;", nil, &BooleanType{}, nil, + "private static void log() { Core.getLogger(\"M\").info(\"x\"); }", "") + if !strings.Contains(src, "import com.mendix.core.Core;") { + t.Errorf("generated source lacks the Core import for extra code:\n%s", src) + } +} + +// CONTROLS: no import when the code does not use Core — a string or a comment +// naming it, a longer identifier — or when it imports a Core of its own. +func TestGenerateSource_NoCoreImportOtherwise_Issue2310(t *testing.T) { + for _, tc := range []struct { + name, code string + imports []string + }{ + {"no use", "return true;", nil}, + {"in a string", "String s = \"Core.x\";\nreturn true;", nil}, + {"in a comment", "// Core.getConfiguration()\nreturn true;", nil}, + {"longer name", "MyCore.run();\nreturn true;", nil}, + {"own Core", "Core.run();\nreturn true;", []string{"import my.pkg.Core;"}}, + {"wildcard", "Core.run();\nreturn true;", []string{"import com.mendix.core.*;"}}, + } { + src := GenerateSource("M", "A", tc.code, nil, &BooleanType{}, tc.imports, "", "") + if strings.Contains(src, "import com.mendix.core.Core;") { + t.Errorf("%s: Core imported:\n%s", tc.name, src) + } + } +} diff --git a/sdk/javaactions/source.go b/sdk/javaactions/source.go index f2c17c4146..d47279e0d1 100644 --- a/sdk/javaactions/source.go +++ b/sdk/javaactions/source.go @@ -4,6 +4,7 @@ package javaactions import ( "fmt" + "regexp" "sort" "strings" ) @@ -53,7 +54,7 @@ func GenerateSource(moduleName, actionName string, userCode string, params []*Ja sb.WriteString(fmt.Sprintf("package %s.actions;\n\n", moduleNameLower)) - for _, imp := range completeImports(imports, params, returnType) { + for _, imp := range completeImports(withCoreImport(imports, userCode, extraCode), params, returnType) { sb.WriteString(imp) sb.WriteString("\n") } @@ -359,3 +360,65 @@ func lowerFirst(s string) string { } return strings.ToLower(s[:1]) + s[1:] } + +// importCore is the Mendix runtime API's entry point, which nearly every +// action body reaches for (Core.getConfiguration, Core.getLogger, …). +const importCore = "import com.mendix.core.Core;" + +// coreUseRe is a use of the class Core: `Core.` as a word of its own. +var coreUseRe = regexp.MustCompile(`(^|[^A-Za-z0-9_$.])Core\s*\.`) + +// withCoreImport returns imports with importCore appended when the user code +// or the extra code uses Core and the list imports no Core of its own +// (ako/mxcli#2310). Without it the file compiled only once the author added +// the line: check, exec and docker check passed — mx check does not compile +// Java — and the build failed with "cannot find symbol: variable Core". +// mxbuild never drops an import it finds, so its regeneration keeps the line. +func withCoreImport(imports []string, userCode, extraCode string) []string { + if !coreUseRe.MatchString(javaCodeOnly(userCode)) && !coreUseRe.MatchString(javaCodeOnly(extraCode)) { + return imports + } + for _, imp := range imports { + imp = strings.TrimSpace(imp) + if strings.HasSuffix(imp, ".Core;") || imp == "import com.mendix.core.*;" { + return imports + } + } + return append(append([]string(nil), imports...), importCore) +} + +// javaCodeOnly is src with its comments and string and character literals +// blanked, so a name inside one is not read as code. +func javaCodeOnly(src string) string { + out := []byte(src) + for i := 0; i < len(out); i++ { + switch { + case out[i] == '/' && i+1 < len(out) && out[i+1] == '/': + for ; i < len(out) && out[i] != '\n'; i++ { + out[i] = ' ' + } + case out[i] == '/' && i+1 < len(out) && out[i+1] == '*': + for ; i < len(out) && !(out[i] == '*' && i+1 < len(out) && out[i+1] == '/'); i++ { + out[i] = ' ' + } + if i+1 < len(out) { + out[i], out[i+1] = ' ', ' ' + i++ + } + case out[i] == '"' || out[i] == '\'': + q := out[i] + out[i] = ' ' + for i++; i < len(out) && out[i] != q; i++ { + if out[i] == '\\' && i+1 < len(out) { + out[i] = ' ' + i++ + } + out[i] = ' ' + } + if i < len(out) { + out[i] = ' ' + } + } + } + return string(out) +}