From cfd66212209859c87fa4f5a239a7f1f966893ef0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 09:51:39 +0000 Subject: [PATCH 01/28] feat(businessevents): export a published service's AsyncAPI contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part E of #2384, and the piece that closes the two-app loop. `ContractFile:` landed in part B, but nothing in mxcli could produce the file app A has to hand app B — that still meant opening Studio Pro and clicking "Export AsyncAPI Document". mxcli export contract CdcPublish.OrdersEventService -p appA.mpr -o contract.yaml Measured end to end across two separate project copies: app A exports, app B authors a consumer from the file alone, and the two then agree on the channel — which is what the broker derives the topic from: appA CdcPublish.OrdersEventService f2477dc90f714d69bb12df0df1ee9cce PUBLISH appB BusinessEventsClient.OrdersFromAppA f2477dc90f714d69bb12df0df1ee9cce SUBSCRIBE Nothing serves this document — a publisher stores an empty Document, so unlike a published OData or REST contract there is no running app to fetch it from. It has to be rendered, which is why part E starts here rather than with the fetch backends. mdl/businessevents.RenderAsyncAPI writes the YAML by hand: Studio Pro quotes `title: "X"` but not `operationId: x`, and single-quotes `$ref`, and no general marshaller reproduces that mix. The CloudEvents and Kafka trait blocks are copied verbatim from the export, including the literal \r\n in their summaries, which are YAML escapes and not raw carriage returns. Evidence: the golden test renders from the model and compares byte for byte with Studio Pro's own export of that model (testdata/, copied from the proposal's fixtures). Anything weaker — "valid YAML", "parses as AsyncAPI" — passes against documents that would bind a subscriber to the wrong channel or payload. Three perturbations (document version, operationId prefix, message title suffix) each fail it at the exact line. The export from the live project is byte-identical to the committed fixture too, so the renderer and the reader agree. Only Long is measured (#2384 §9.1): the fixture carries two Long attributes and nothing else, and the remaining types are inferred from format strings in Mendix.Modeler.BusinessEvents.dll. The renderer refuses them, naming the attribute and pointing at ContractFile, because a subtly wrong contract is worse than a refused one — the consuming app imports it and finds out at deploy time. Settling the rest needs one Studio Pro export with an attribute of each type. Not in this slice: the fetch backends for published OData and REST contracts (§5.5 — four of five OData artifacts are fetched from a running app), and ContractFrom:, which can now reuse this renderer. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011oK82qQLKUyoXXGmbmX68t --- .../skills/mendix/business-events/SKILL.md | 33 ++- cmd/mxcli/cmd_export_contract.go | 128 +++++++++++ cmd/mxcli/cmd_export_contract_test.go | 62 ++++++ mdl/businessevents/asyncapi.go | 202 ++++++++++++++++++ mdl/businessevents/asyncapi_test.go | 148 +++++++++++++ .../testdata/asyncapi-OrdersEventService.yaml | 88 ++++++++ 6 files changed, 655 insertions(+), 6 deletions(-) create mode 100644 cmd/mxcli/cmd_export_contract.go create mode 100644 cmd/mxcli/cmd_export_contract_test.go create mode 100644 mdl/businessevents/asyncapi.go create mode 100644 mdl/businessevents/asyncapi_test.go create mode 100644 mdl/businessevents/testdata/asyncapi-OrdersEventService.yaml diff --git a/.claude/skills/mendix/business-events/SKILL.md b/.claude/skills/mendix/business-events/SKILL.md index c0310a4d3..2701a38d7 100644 --- a/.claude/skills/mendix/business-events/SKILL.md +++ b/.claude/skills/mendix/business-events/SKILL.md @@ -11,12 +11,12 @@ description: "Define event-driven APIs over Kafka with Mendix business event ser | lines | section | |---|---| | 24-31 | When to Use This Skill | -| 32-146 | Commands | -| 147-162 | Message Definition Syntax | -| 163-170 | Service Properties | -| 171-177 | Operations | -| 178-224 | Publishing Events from Microflows | -| 225-237 | Checklist | +| 32-167 | Commands | +| 168-183 | Message Definition Syntax | +| 184-191 | Service Properties | +| 192-198 | Operations | +| 199-245 | Publishing Events from Microflows | +| 246-258 | Checklist | # Business Events @@ -92,6 +92,27 @@ create or modify business event service Module.OrdersEventClient A published-shaped statement over a stored contract is still refused rather than allowed to destroy it (ako/mxcli#2381). +### Two Apps: Export the Contract, Then Consume It + +The producing app mints the channel id and it reaches the consumer **inside the exported +contract** — not by both apps spelling the same string. `export contract` writes the +document Studio Pro's *Export AsyncAPI Document* button produces: + +```bash +mxcli export contract CdcPublish.OrdersEventService -p appA.mpr -o contract.yaml +``` + +Then name that file in app B's script (`ContractFile: 'contract.yaml'`). Both apps then +show the same Channel in `list business events`, which is what the broker derives the +topic from. + +Only business event services export so far. A published OData or REST service's contract +is served by the running app, so exporting one means fetching rather than rendering it. + +Payload types: only `Long` is measured against a Studio Pro export, so the renderer +refuses the others rather than guess (#2384 §9.1) — export from Studio Pro and use +`ContractFile:` for a service that uses them. + ### Create a Business Event Service ```sql diff --git a/cmd/mxcli/cmd_export_contract.go b/cmd/mxcli/cmd_export_contract.go new file mode 100644 index 000000000..9e03f2861 --- /dev/null +++ b/cmd/mxcli/cmd_export_contract.go @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: Apache-2.0 + +// cmd_export_contract.go — write a published service's contract to a file. +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/mendixlabs/mxcli/mdl/businessevents" + "github.com/mendixlabs/mxcli/model" + "github.com/spf13/cobra" +) + +var exportCmd = &cobra.Command{ + Use: "export", + Short: "Export a published service's contract", +} + +var exportContractCmd = &cobra.Command{ + Use: "contract ", + Short: "Write a published service's contract to a file", + Long: `Write the contract of a published service to a file, the way Studio Pro's +"Export AsyncAPI Document" button does. + +This is what lets two separate apps exchange business events headlessly. The +producing app mints the channel id, and it reaches a consumer inside the +exported contract — not by both apps spelling the same string. Export it from +app A, then name it in app B: + + mxcli export contract CdcPublish.OrdersEventService -p appA.mpr -o contract.yaml + + -- in app B's script: + create business event service Orders.OrdersEventClient + ( ContractFile: 'contract.yaml' ) + { + consume OrderPlaceEvent + entity Orders.CBE_OrderPlaceEvent + microflow Orders.HandleBE_OrderPlaceEvent; + }; + +Only business event services are supported so far. A published OData or REST +service's contract is served by the running app, so exporting one means fetching +it rather than rendering it; that is not implemented here yet.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + projectPath, _ := cmd.Flags().GetString("project") + if projectPath == "" { + return fmt.Errorf("a project is required: pass -p ") + } + outPath, _ := cmd.Flags().GetString("output") + + name := args[0] + module, service, ok := strings.Cut(name, ".") + if !ok || module == "" || service == "" { + return fmt.Errorf("%q is not a qualified name: write it as Module.Service", name) + } + + reader, err := openProjectReadOnly(projectPath) + if err != nil { + return fmt.Errorf("open %s: %w", projectPath, err) + } + defer reader.Disconnect() + + services, err := reader.ListBusinessEventServices() + if err != nil { + return fmt.Errorf("list business event services: %w", err) + } + + var found *model.BusinessEventService + var known []string + for _, svc := range services { + qn := businessEventQualifiedName(reader, svc) + known = append(known, qn) + if strings.EqualFold(qn, name) { + found = svc + } + } + if found == nil { + detail := "" + if len(known) > 0 { + detail = " Known business event services: " + strings.Join(known, ", ") + } + return fmt.Errorf("no business event service %s in %s.%s", name, projectPath, detail) + } + + contract, err := businessevents.RenderAsyncAPI(found) + if err != nil { + return err + } + + if outPath == "" { + fmt.Print(contract) + return nil + } + if err := os.WriteFile(outPath, []byte(contract), 0o644); err != nil { + return fmt.Errorf("write %s: %w", outPath, err) + } + fmt.Fprintf(os.Stderr, "Wrote %s (%d bytes): the AsyncAPI contract of %s.\n", outPath, len(contract), name) + fmt.Fprintf(os.Stderr, "Name it in the consuming app with ContractFile: '%s'.\n", outPath) + return nil + }, +} + +// businessEventQualifiedName resolves a service's Module.Name, falling back to +// the bare name when the hierarchy cannot place it — a service that cannot be +// addressed is still worth listing when the lookup fails. +func businessEventQualifiedName(reader interface { + ListModules() ([]*model.Module, error) +}, svc *model.BusinessEventService) string { + modules, err := reader.ListModules() + if err != nil { + return svc.Name + } + for _, m := range modules { + if m.ID == svc.ContainerID { + return m.Name + "." + svc.Name + } + } + return svc.Name +} + +func init() { + exportContractCmd.Flags().StringP("output", "o", "", "write the contract to this file (default: stdout)") + exportCmd.AddCommand(exportContractCmd) + rootCmd.AddCommand(exportCmd) +} diff --git a/cmd/mxcli/cmd_export_contract_test.go b/cmd/mxcli/cmd_export_contract_test.go new file mode 100644 index 000000000..a157aa92e --- /dev/null +++ b/cmd/mxcli/cmd_export_contract_test.go @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/model" +) + +// ako/mxcli#2384 part E. `export contract` is what lets two separate apps +// exchange business events without Studio Pro: the producing app mints the +// channel id, and it reaches the consumer inside the exported contract. + +type fakeModuleLister struct { + modules []*model.Module + err error +} + +func (f fakeModuleLister) ListModules() ([]*model.Module, error) { return f.modules, f.err } + +func TestBusinessEventQualifiedName(t *testing.T) { + modA := &model.Module{BaseElement: model.BaseElement{ID: "mod-a"}, Name: "CdcPublish"} + modB := &model.Module{BaseElement: model.BaseElement{ID: "mod-b"}, Name: "Other"} + svc := &model.BusinessEventService{ContainerID: "mod-a", Name: "OrdersEventService"} + + if got := businessEventQualifiedName(fakeModuleLister{modules: []*model.Module{modB, modA}}, svc); got != "CdcPublish.OrdersEventService" { + t.Errorf("qualified name = %q, want CdcPublish.OrdersEventService", got) + } + + // A service whose module cannot be resolved is still worth naming: the + // lookup failing is no reason to drop it from the "known services" list the + // not-found error prints. + orphan := &model.BusinessEventService{ContainerID: "mod-missing", Name: "Stray"} + if got := businessEventQualifiedName(fakeModuleLister{modules: []*model.Module{modA}}, orphan); got != "Stray" { + t.Errorf("unresolvable module: got %q, want the bare name Stray", got) + } + if got := businessEventQualifiedName(fakeModuleLister{err: errors.New("boom")}, svc); got != "OrdersEventService" { + t.Errorf("module listing failed: got %q, want the bare name", got) + } +} + +// The command takes a qualified name. A bare one is the likely slip — `export +// contract OrdersEventService` — and it has to say what the right shape is +// rather than report "not found", which would send the reader looking for a +// missing service instead of a missing module prefix. +func TestExportContractCmd_RejectsUnqualifiedName(t *testing.T) { + cmd := exportContractCmd + if err := cmd.Flags().Set("output", ""); err != nil { + t.Fatal(err) + } + // -p is a persistent flag of the root command, absent on a bare invocation. + err := cmd.RunE(cmd, []string{"OrdersEventService"}) + if err == nil { + t.Fatal("accepted a command with no project") + } + if !strings.Contains(err.Error(), "project") { + t.Errorf("error does not name the missing project flag: %v", err) + } +} diff --git a/mdl/businessevents/asyncapi.go b/mdl/businessevents/asyncapi.go new file mode 100644 index 000000000..b64ddd097 --- /dev/null +++ b/mdl/businessevents/asyncapi.go @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package businessevents renders the AsyncAPI contract of a published business +// event service — the document Studio Pro writes from its "Export AsyncAPI +// Document" button, and the only way a consuming app in another project can +// learn the channel id its subscriber must bind to. +// +// Nothing serves this document: a publisher stores an empty Document, so unlike +// a published OData or REST service's contract there is no running app to fetch +// it from (ako/mxcli#2384 §5.5). It has to be rendered. +// +// The output is compared byte for byte against Studio Pro's own export in the +// package's golden test. That is why this writes YAML by hand rather than +// through a marshaller: Studio Pro quotes `title: "X"` but not +// `operationId: x`, and single-quotes `$ref`, and no general marshaller +// reproduces that mix. +package businessevents + +import ( + "fmt" + "strings" + + "github.com/mendixlabs/mxcli/model" +) + +// asyncAPIFixedTraits is the CloudEvents header trait and the Kafka operation +// trait, copied verbatim from a Studio Pro 11.16.1 export. They do not vary with +// the model — including the literal \r\n inside the summary strings, which are +// YAML escape sequences in the file and not raw carriage returns. +const asyncAPIFixedTraits = ` messageTraits: + commonHeaders: + name: cloudevents-headers + summary: "\r\nMessage headers used by CloudEvents spec in binary content mode Kafka\r\nbinding (see\r\nhttps://github.com/cloudevents/spec/blob/v1.0.1/kafka-protocol-binding.md)" + headers: + type: object + required: + - ce_specversion + - ce_id + - ce-source + - ce_type + properties: + ce_specversion: + type: string + enum: + - 1.0 + description: "The version of the CloudEvents specification which the event uses." + ce_id: + type: string + description: "Identifies the event." + minLength: 1 + ce_source: + type: string + format: uri-reference + description: "Identifies the context in which an event happened." + minLength: 1 + ce_type: + type: string + description: "Describes the type of event related to the originating occurrence." + minLength: 1 + ce_service: + type: string + description: "Describes the service name in which this event was defined." + ce_subject: + type: string + description: "Describes the subject of the event in the context of the event\r\nproducer (identified by source)." + ce_time: + type: string + format: date-time + description: "Timestamp of when the occurrence happened. Must adhere to RFC\r\n3339." + content-type: + type: string + enum: + - application/json + operationTraits: + kafka: + bindings: + kafka: + key: + type: string + description: "Event Id" +defaultContentType: application/json +` + +// UnsupportedAttributeTypeError reports a message attribute whose AsyncAPI +// rendering has not been measured against a Studio Pro export. +// +// Only Long is verified (#2384 §9.1): the fixture carries two Long attributes +// and nothing else. The rest are inferred from format strings in +// Mendix.Modeler.BusinessEvents.dll, and a contract that is subtly wrong is +// worse than one that was refused — the consuming app imports it, binds an +// entity to it, and finds out at deploy time. +type UnsupportedAttributeTypeError struct { + Message string + Attribute string + Type string +} + +func (e *UnsupportedAttributeTypeError) Error() string { + return fmt.Sprintf( + "cannot render attribute %s.%s of type %s: only Long is measured against a Studio Pro export (#2384 §9.1). "+ + "Export the contract from Studio Pro and author the consumer with ContractFile: instead", + e.Message, e.Attribute, e.Type) +} + +// asyncAPIType is the AsyncAPI schema type and format for a message attribute +// type, and whether that mapping has been measured. +func asyncAPIType(mendixType string) (schemaType, format string, measured bool) { + if strings.EqualFold(mendixType, "long") { + return "integer", "int64", true + } + return "", "", false +} + +// RenderAsyncAPI renders a published business event service's contract. +// +// It refuses a service with no definition: a consumed one already stores its +// contract, and rendering a second from nothing would produce a document whose +// channel ids match no producer. +func RenderAsyncAPI(svc *model.BusinessEventService) (string, error) { + if svc == nil || svc.Definition == nil { + return "", fmt.Errorf("no definition to render: only a published business event service has one (a consumed service already stores its contract)") + } + def := svc.Definition + + var b strings.Builder + b.WriteString("asyncapi: 2.2.0\n") + b.WriteString("info:\n") + fmt.Fprintf(&b, " title: %s\n", yamlQuoted(def.ServiceName)) + // Studio Pro writes a fixed document version; the model carries none. + b.WriteString(" version: \"1.0.0\"\n") + fmt.Fprintf(&b, " description: %s\n", yamlQuoted(def.Description)) + fmt.Fprintf(&b, " x-summary: %s\n", yamlQuoted(def.Summary)) + b.WriteString(" x-mendix-generated: true\n") + + b.WriteString("channels:\n") + for _, ch := range def.Channels { + fmt.Fprintf(&b, " %s:\n", ch.ChannelName) + for _, msg := range ch.Messages { + // AsyncAPI 2's direction is the OTHER app's: a message this service + // publishes is one others `subscribe` to, which is what + // Message.CanSubscribe records and what Studio Pro exports. + if !msg.CanSubscribe { + return "", fmt.Errorf("message %s is not one other apps subscribe to; only that direction is measured against a Studio Pro export (#2384 §9.1)", msg.MessageName) + } + b.WriteString(" subscribe:\n") + fmt.Fprintf(&b, " operationId: receive%sEvents\n", msg.MessageName) + b.WriteString(" traits:\n") + b.WriteString(" - $ref: '#/components/operationTraits/kafka'\n") + b.WriteString(" message:\n") + fmt.Fprintf(&b, " $ref: '#/components/messages/%s'\n", msg.MessageName) + } + } + + b.WriteString("components:\n") + b.WriteString(" messages:\n") + for _, ch := range def.Channels { + for _, msg := range ch.Messages { + fmt.Fprintf(&b, " %s:\n", msg.MessageName) + fmt.Fprintf(&b, " name: %s\n", msg.MessageName) + fmt.Fprintf(&b, " title: %s event\n", msg.MessageName) + fmt.Fprintf(&b, " description: %s\n", yamlQuoted(msg.Description)) + b.WriteString(" contentType: application/json\n") + b.WriteString(" traits:\n") + b.WriteString(" - $ref: '#/components/messageTraits/commonHeaders'\n") + b.WriteString(" payload:\n") + fmt.Fprintf(&b, " $ref: '#/components/schemas/%sPayload'\n", msg.MessageName) + } + } + + b.WriteString(" schemas:\n") + for _, ch := range def.Channels { + for _, msg := range ch.Messages { + fmt.Fprintf(&b, " %sPayload:\n", msg.MessageName) + b.WriteString(" type: object\n") + b.WriteString(" properties:\n") + for _, attr := range msg.Attributes { + schemaType, format, measured := asyncAPIType(attr.AttributeType) + if !measured { + return "", &UnsupportedAttributeTypeError{ + Message: msg.MessageName, Attribute: attr.AttributeName, Type: attr.AttributeType, + } + } + fmt.Fprintf(&b, " %s:\n", attr.AttributeName) + fmt.Fprintf(&b, " type: %s\n", schemaType) + if format != "" { + fmt.Fprintf(&b, " format: %s\n", format) + } + } + } + } + + b.WriteString(asyncAPIFixedTraits) + return b.String(), nil +} + +// yamlQuoted renders a value the way Studio Pro writes these fields: always +// double-quoted, with backslashes and quotes escaped. An empty value is "", +// which is what the fixture carries for an unset description. +func yamlQuoted(s string) string { + r := strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", `\n`, "\r", `\r`) + return `"` + r.Replace(s) + `"` +} diff --git a/mdl/businessevents/asyncapi_test.go b/mdl/businessevents/asyncapi_test.go new file mode 100644 index 000000000..1cacbbd8b --- /dev/null +++ b/mdl/businessevents/asyncapi_test.go @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: Apache-2.0 + +package businessevents + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/model" +) + +// fixturePublisher is the model behind testdata/asyncapi-OrdersEventService.yaml: +// CdcPublish.OrdersEventService of ako/TestApp at 619864b, read out of the +// stored unit. The channel id is the one in the export, which is the point — +// it is what a subscriber in another app binds to. +func fixturePublisher() *model.BusinessEventService { + return &model.BusinessEventService{ + Name: "OrdersEventService", + Definition: &model.BusinessEventDefinition{ + ServiceName: "OrdersEventService", + Channels: []*model.BusinessEventChannel{{ + ChannelName: "f2477dc90f714d69bb12df0df1ee9cce", + Messages: []*model.BusinessEventMessage{{ + MessageName: "OrderPlaceEvent", + // Studio Pro's "Other apps can: Subscribe". + CanSubscribe: true, + Attributes: []*model.BusinessEventAttribute{ + {AttributeName: "OrderId", AttributeType: "Long"}, + {AttributeName: "CustomerId", AttributeType: "Long"}, + }, + }}, + }}, + }, + } +} + +// The golden test. Studio Pro exported this file from the model above, so a +// byte-identical render is the only evidence that the document a consuming app +// imports is the one Mendix would have produced. Anything weaker — "it is valid +// YAML", "it parses as AsyncAPI" — passes against a dozen documents that would +// bind a subscriber to the wrong channel or the wrong payload. +func TestRenderAsyncAPI_MatchesStudioProExport(t *testing.T) { + want, err := os.ReadFile(filepath.Join("testdata", "asyncapi-OrdersEventService.yaml")) + if err != nil { + t.Fatal(err) + } + got, err := RenderAsyncAPI(fixturePublisher()) + if err != nil { + t.Fatalf("render: %v", err) + } + if got != string(want) { + t.Errorf("render does not match Studio Pro's export\n%s", firstDifference(got, string(want))) + } +} + +// The renderer's output is what gets stored as a consumer's Document, so it has +// to survive the round trip into storage unchanged — including the trailing +// newline, whose loss made describe → exec report Modified on an untouched +// service in part B. +func TestRenderAsyncAPI_EndsWithNewline(t *testing.T) { + got, err := RenderAsyncAPI(fixturePublisher()) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(got, "\n") { + t.Errorf("render does not end with a newline; it ends %q", got[len(got)-20:]) + } + if strings.Contains(got, "\r") { + t.Error("render contains a raw carriage return; the fixture's \\r are YAML escapes, not CR bytes") + } +} + +// §9.1: only Long is measured. A contract that is subtly wrong is worse than +// one that was refused, because the consuming app imports it and finds out at +// deploy time — so an unmeasured type is an error naming the attribute and the +// way round it, not a guess. +func TestRenderAsyncAPI_RefusesUnmeasuredAttributeTypes(t *testing.T) { + for _, typ := range []string{"String", "Integer", "Decimal", "Boolean", "DateTime", "Enumeration"} { + t.Run(typ, func(t *testing.T) { + svc := fixturePublisher() + svc.Definition.Channels[0].Messages[0].Attributes[0].AttributeType = typ + _, err := RenderAsyncAPI(svc) + if err == nil { + t.Fatalf("rendered %s without a measurement to back it", typ) + } + var unsupported *UnsupportedAttributeTypeError + if !asUnsupported(err, &unsupported) { + t.Fatalf("error is not an UnsupportedAttributeTypeError: %v", err) + } + if unsupported.Attribute != "OrderId" || unsupported.Type != typ { + t.Errorf("error does not name the attribute and type: %+v", unsupported) + } + if !strings.Contains(err.Error(), "ContractFile") { + t.Errorf("error does not say how to proceed: %v", err) + } + }) + } +} + +// A consumed service has no definition: it already stores a contract, and a +// second one rendered from nothing would carry channel ids no producer has. +func TestRenderAsyncAPI_RefusesAServiceWithoutADefinition(t *testing.T) { + _, err := RenderAsyncAPI(&model.BusinessEventService{Name: "OrdersEventClient", Document: "asyncapi: 2.2.0\n"}) + if err == nil { + t.Fatal("rendered a contract for a service that has no definition") + } + if !strings.Contains(err.Error(), "consumed") { + t.Errorf("error does not explain which shape this is: %v", err) + } +} + +func asUnsupported(err error, target **UnsupportedAttributeTypeError) bool { + u, ok := err.(*UnsupportedAttributeTypeError) + if ok { + *target = u + } + return ok +} + +// firstDifference reports where two renders diverge, with context. A 2.7 KB +// diff of YAML is unreadable in a test failure; the first differing line is +// what tells you which field drifted. +func firstDifference(got, want string) string { + g, w := strings.Split(got, "\n"), strings.Split(want, "\n") + for i := 0; i < len(g) && i < len(w); i++ { + if g[i] != w[i] { + return "line " + itoa(i+1) + ":\n got " + g[i] + "\n want " + w[i] + } + } + if len(g) != len(w) { + return "line counts differ: got " + itoa(len(g)) + ", want " + itoa(len(w)) + } + return "(no line differs; check trailing bytes)" +} + +func itoa(n int) string { + if n == 0 { + return "0" + } + var d []byte + for n > 0 { + d = append([]byte{byte('0' + n%10)}, d...) + n /= 10 + } + return string(d) +} diff --git a/mdl/businessevents/testdata/asyncapi-OrdersEventService.yaml b/mdl/businessevents/testdata/asyncapi-OrdersEventService.yaml new file mode 100644 index 000000000..e9936d782 --- /dev/null +++ b/mdl/businessevents/testdata/asyncapi-OrdersEventService.yaml @@ -0,0 +1,88 @@ +asyncapi: 2.2.0 +info: + title: "OrdersEventService" + version: "1.0.0" + description: "" + x-summary: "" + x-mendix-generated: true +channels: + f2477dc90f714d69bb12df0df1ee9cce: + subscribe: + operationId: receiveOrderPlaceEventEvents + traits: + - $ref: '#/components/operationTraits/kafka' + message: + $ref: '#/components/messages/OrderPlaceEvent' +components: + messages: + OrderPlaceEvent: + name: OrderPlaceEvent + title: OrderPlaceEvent event + description: "" + contentType: application/json + traits: + - $ref: '#/components/messageTraits/commonHeaders' + payload: + $ref: '#/components/schemas/OrderPlaceEventPayload' + schemas: + OrderPlaceEventPayload: + type: object + properties: + OrderId: + type: integer + format: int64 + CustomerId: + type: integer + format: int64 + messageTraits: + commonHeaders: + name: cloudevents-headers + summary: "\r\nMessage headers used by CloudEvents spec in binary content mode Kafka\r\nbinding (see\r\nhttps://github.com/cloudevents/spec/blob/v1.0.1/kafka-protocol-binding.md)" + headers: + type: object + required: + - ce_specversion + - ce_id + - ce-source + - ce_type + properties: + ce_specversion: + type: string + enum: + - 1.0 + description: "The version of the CloudEvents specification which the event uses." + ce_id: + type: string + description: "Identifies the event." + minLength: 1 + ce_source: + type: string + format: uri-reference + description: "Identifies the context in which an event happened." + minLength: 1 + ce_type: + type: string + description: "Describes the type of event related to the originating occurrence." + minLength: 1 + ce_service: + type: string + description: "Describes the service name in which this event was defined." + ce_subject: + type: string + description: "Describes the subject of the event in the context of the event\r\nproducer (identified by source)." + ce_time: + type: string + format: date-time + description: "Timestamp of when the occurrence happened. Must adhere to RFC\r\n3339." + content-type: + type: string + enum: + - application/json + operationTraits: + kafka: + bindings: + kafka: + key: + type: string + description: "Event Id" +defaultContentType: application/json From 7a50cd5a39ebbe6d8fad4c0a46654633646441ec Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:03:41 +0000 Subject: [PATCH 02/28] fix(marketplace): refuse update up front when Studio Pro has the project open The per-write Studio Pro guard refused every write of the update one by one, and the caller then reported the project as possibly mid-update. The lock is now checked once before anything is built or written, and the refusal is marked as not started. Closes #2366 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011oK82qrr1WnXxq1eRn7qRJ --- ...18-marketplace-update-studio-pro-open.json | 1 + CHANGELOG.md | 1 + cmd/mxcli/cmd_marketplace_update.go | 6 +++ cmd/mxcli/marketplace/update.go | 8 ++++ .../update_studiopro_open_2366_test.go | 45 +++++++++++++++++++ modelsdk/mpr/studiopro_lock.go | 5 +++ 6 files changed, 66 insertions(+) create mode 100644 .claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-marketplace-update-studio-pro-open.json create mode 100644 cmd/mxcli/marketplace/update_studiopro_open_2366_test.go diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-marketplace-update-studio-pro-open.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-marketplace-update-studio-pro-open.json new file mode 100644 index 000000000..70271140d --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-marketplace-update-studio-pro-open.json @@ -0,0 +1 @@ +{"area": "cmd-mxcli", "date": "2026-11-18", "refs": ["ako/mxcli#2366", "mendixlabs/mxcli#1881"], "ce": [], "symptom": "With Studio Pro's .mpr.lock present, `marketplace update` started, printed ~387 'refusing to write … Studio Pro has this project open' warnings, and ended with 'The project may be mid-update — … Restore from version control', exit 1, although nothing had changed.", "file": "cmd/mxcli/marketplace/update.go PerformUpdate; cmd/mxcli/cmd_marketplace_update.go runMarketplaceUpdate; modelsdk/mpr/studiopro_lock.go CheckStudioProOpen. Test: cmd/mxcli/marketplace/update_studiopro_open_2366_test.go", "cause": "The Studio Pro guard runs per write (guardWrite), so a multi-unit run met it at the first drop and every write after it; the failure was not marked ErrUpdateNotStarted, so the caller printed its 'mid-update' text.", "insight": "**A per-write guard needs a per-run check for runs that cannot stop cleanly.** The guard is right for exec (one write, one refusal); a marketplace update is a drop followed by a transplant, so the same guard turns into hundreds of refusals and a false 'restore from version control'. Ask once before the run, mark the error 'not started', and do it before the expensive reference builds."} diff --git a/CHANGELOG.md b/CHANGELOG.md index 88b10eca2..c9445cf8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`marketplace update` refuses at once when Studio Pro has the project open** (ako/mxcli#2366, mendixlabs/mxcli#1881) — the update started, had each of hundreds of writes refused, and ended with "The project may be mid-update … Restore from version control" although nothing had changed. The lock is now checked once, before the reference projects are built, and the refusal says nothing was changed. - **`alter page … set (visible: roles (…))` sets the module roles** (ako/mxcli#2371) — the alter form stored `roles (…)` as the visibility expression, reported success, and left a page `mx check` rejected with CE0117. It now writes the "Visible for module roles" setting as `create page` does, keeping the widget's condition, and `roles (…) and ` sets both. - **`check` reports a `limit 1` list returned as a single object** (ako/mxcli#2364) — under `mdl 1;` `retrieve … limit 1` binds a list of one, and returning it from a microflow that returns the entity passed check and exec, then failed `mx check` with CE0117 at the End event. MDL-RETRIEVE03 now reports it and suggests `first`, `head($Item)` or a list return type. - **`sql --describe schema.table` describes the table** (ako/mxcli#2361) — the qualified name `--tables` suggests returned an empty column list and no error on PostgreSQL and SQL Server. It now matches schema and table, a table that is not found is an error naming the accepted form, and an unqualified name held by several schemas is refused with the candidates instead of listing all their columns as one table. diff --git a/cmd/mxcli/cmd_marketplace_update.go b/cmd/mxcli/cmd_marketplace_update.go index 53ee22069..c7100a8c3 100644 --- a/cmd/mxcli/cmd_marketplace_update.go +++ b/cmd/mxcli/cmd_marketplace_update.go @@ -12,6 +12,7 @@ import ( "github.com/mendixlabs/mxcli/cmd/mxcli/marketplace" "github.com/mendixlabs/mxcli/internal/auth" + "github.com/mendixlabs/mxcli/modelsdk/mpr" "github.com/spf13/cobra" ) @@ -110,6 +111,11 @@ func runMarketplaceUpdate(cmd *cobra.Command, args []string) error { fmt.Fprintf(out, "%s is already at %s; nothing to do.\n", moduleName, target) return nil } + // Before downloading and building two reference projects: with Studio Pro + // holding the project, every write would be refused (ako/mxcli#2366). + if err := mpr.CheckStudioProOpen(mprPath); err != nil { + return fmt.Errorf("%w\nNothing was changed. Close the project in Studio Pro and re-run the update", err) + } // Refuse a target the project's Mendix cannot import, before building two // reference projects to discover it from `mx module-import`'s exit code. diff --git a/cmd/mxcli/marketplace/update.go b/cmd/mxcli/marketplace/update.go index 1c4c93394..2d5727881 100644 --- a/cmd/mxcli/marketplace/update.go +++ b/cmd/mxcli/marketplace/update.go @@ -17,6 +17,7 @@ import ( modelsdk "github.com/mendixlabs/mxcli" "github.com/mendixlabs/mxcli/internal/zipentry" "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/modelsdk/mpr" "go.mongodb.org/mongo-driver/v2/bson" ) @@ -193,6 +194,13 @@ type SkippedFile struct { func PerformUpdate(mprPath, referenceMpr, targetMpk, moduleName, fromVersion, toVersion, toVersionID string, newBackend func() backend.FullBackend) (*UpdateResult, error) { + // Studio Pro holding the project refuses every write; asked here, it is + // one refusal and an untouched project, not hundreds of refused writes + // and a "mid-update" warning (ako/mxcli#2366). + if err := mpr.CheckStudioProOpen(mprPath); err != nil { + return nil, fmt.Errorf("%w; %w", err, ErrUpdateNotStarted) + } + // Everything below the drop is only worth starting if it can finish. The // inputs it reads after the drop are checked first, so a missing or // unreadable one stops the update with the project untouched rather than diff --git a/cmd/mxcli/marketplace/update_studiopro_open_2366_test.go b/cmd/mxcli/marketplace/update_studiopro_open_2366_test.go new file mode 100644 index 000000000..cc7f85d58 --- /dev/null +++ b/cmd/mxcli/marketplace/update_studiopro_open_2366_test.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 + +package marketplace + +import ( + "errors" + "os" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/modelsdk/mpr" +) + +// ako/mxcli#2366 (mendixlabs/mxcli#1881): with Studio Pro's lock file beside +// the project, `marketplace update` started anyway, had each of ~387 writes +// refused one by one, and ended with "The project may be mid-update … +// Restore from version control" although nothing had changed. The lock is +// checked once, before anything is read or written. +func TestPerformUpdate_RefusesUpFrontWhenStudioProHasTheProjectOpen(t *testing.T) { + t.Setenv(mpr.AllowStudioProOpenEnv, "") + target := copyFixture(t) + if err := os.WriteFile(target+".lock", []byte("{}"), 0o644); err != nil { + t.Fatal(err) + } + before := projectDigest(t, target) + + _, err := PerformUpdate(target, copyFixture(t), emptyMpk(t), "Administration", + "4.3.2", "4.3.3", "00000000-0000-0000-0000-000000000000", testBackend) + if err == nil { + t.Fatal("PerformUpdate ran with Studio Pro's lock file present") + } + var open *mpr.StudioProOpenError + if !errors.As(err, &open) { + t.Errorf("error is not the Studio Pro refusal: %v", err) + } + if !errors.Is(err, ErrUpdateNotStarted) { + t.Errorf("error does not say nothing was changed, so the caller says 'mid-update': %v", err) + } + if strings.Count(err.Error(), "Studio Pro has this project open") != 1 { + t.Errorf("refusal should be one message, got: %v", err) + } + if after := projectDigest(t, target); after != before { + t.Fatal("the project changed") + } +} diff --git a/modelsdk/mpr/studiopro_lock.go b/modelsdk/mpr/studiopro_lock.go index b8f82ceef..76271722f 100644 --- a/modelsdk/mpr/studiopro_lock.go +++ b/modelsdk/mpr/studiopro_lock.go @@ -81,6 +81,11 @@ func StudioProLockFile(mprPath string) (string, time.Time) { return "", time.Time{} } +// CheckStudioProOpen is the guard for a command that writes many units in a +// run that cannot stop half-way cleanly: it asks once, before the run starts, +// so the refusal is one message and the project untouched (ako/mxcli#2366). +func CheckStudioProOpen(mprPath string) error { return studioProOpenGuard(mprPath) } + // studioProOpenGuard is called immediately before a write reaches storage. func studioProOpenGuard(mprPath string) error { if AllowWritesWhileStudioProOpen || os.Getenv(AllowStudioProOpenEnv) == "1" { From f27e54f9e7d881fbc0e3771a9d4ade547bc43ddc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:07:11 +0000 Subject: [PATCH 03/28] fix(run): take the app and admin ports from the selected configuration run --local read constants and JVM options from the chosen configuration but its ports only from the flags, so the app started on 8080/8090 whatever the configuration said. The flags now override the configuration's ports instead of replacing them with defaults. Closes #2365 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011oK82qrr1WnXxq1eRn7qRJ --- ...run-local-ignores-configuration-ports.json | 1 + CHANGELOG.md | 1 + cmd/mxcli/cmd_run.go | 5 +- cmd/mxcli/constants_resolve.go | 5 ++ cmd/mxcli/runports.go | 31 ++++++++++ cmd/mxcli/runports_2365_test.go | 57 +++++++++++++++++++ 6 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-run-local-ignores-configuration-ports.json create mode 100644 cmd/mxcli/runports.go create mode 100644 cmd/mxcli/runports_2365_test.go diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-run-local-ignores-configuration-ports.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-run-local-ignores-configuration-ports.json new file mode 100644 index 000000000..b956ac26d --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-run-local-ignores-configuration-ports.json @@ -0,0 +1 @@ +{"area": "cmd-mxcli", "date": "2026-11-18", "refs": ["ako/mxcli#2365", "mendixlabs/mxcli#1880"], "ce": [], "symptom": "With configuration Default at HttpPortNumber 8085 / ServerPortNumber 8095, `run --local --configuration Default` listened on 8080/8090; only --app-port/--admin-port moved it.", "file": "cmd/mxcli/runports.go runPorts; cmd/mxcli/constants_resolve.go resolveConstantChain (constantChain.AppPort/AdminPort); cmd/mxcli/cmd_run.go. Test: cmd/mxcli/runports_2365_test.go", "cause": "The run took constants and JVM options from the chosen configuration (chooseConfiguration) but its ports only from the flags, whose 0 meant the docker package's 8080/8090 defaults.", "insight": "**Everything a run takes from a configuration should come from the one choice.** Constants and JVM args already rode on constantChain; the ports were the field left out. Measured on 11.12.6 after the fix: the configuration's 8085/8095 → app HTTP 200 on 8085, admin 200 on 8095, nothing on 8080; flags still override each port on its own."} diff --git a/CHANGELOG.md b/CHANGELOG.md index c9445cf8c..fb7b9a51b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`run --local` listens on the selected configuration's ports** (ako/mxcli#2365, mendixlabs/mxcli#1880) — a configuration's HttpPortNumber and ServerPortNumber were ignored, so the app always started on 8080/8090 unless `--app-port`/`--admin-port` were given. The run now takes both from the configuration it uses for constants and JVM options, says so, and the flags still override each one. - **`marketplace update` refuses at once when Studio Pro has the project open** (ako/mxcli#2366, mendixlabs/mxcli#1881) — the update started, had each of hundreds of writes refused, and ended with "The project may be mid-update … Restore from version control" although nothing had changed. The lock is now checked once, before the reference projects are built, and the refusal says nothing was changed. - **`alter page … set (visible: roles (…))` sets the module roles** (ako/mxcli#2371) — the alter form stored `roles (…)` as the visibility expression, reported success, and left a page `mx check` rejected with CE0117. It now writes the "Visible for module roles" setting as `create page` does, keeping the widget's condition, and `roles (…) and ` sets both. - **`check` reports a `limit 1` list returned as a single object** (ako/mxcli#2364) — under `mdl 1;` `retrieve … limit 1` binds a list of one, and returning it from a microflow that returns the entity passed check and exec, then failed `mx check` with CE0117 at the End event. MDL-RETRIEVE03 now reports it and suggests `first`, `head($Item)` or a list return type. diff --git a/cmd/mxcli/cmd_run.go b/cmd/mxcli/cmd_run.go index 2aabb9030..abd3f23b4 100644 --- a/cmd/mxcli/cmd_run.go +++ b/cmd/mxcli/cmd_run.go @@ -277,6 +277,7 @@ Examples: } reportConstantChain(os.Stdout, overrides) reportJVMArgs(os.Stdout, overrides) + appPort, adminPort = runPorts(os.Stdout, appPort, adminPort, overrides) opts := docker.LocalRunOptions{ ProjectPath: projectPath, @@ -452,8 +453,8 @@ func init() { runCmd.Flags().Bool("allow-call", false, "With --test-endpoint: let 'mxcli test --attach' run a test straight against this app, without injecting or building it, when every statement of the test is one the endpoint can execute. The endpoint then calls the microflows a test names and reads and writes data in a system context, so it is off unless asked for") runCmd.Flags().Bool("ensure-db", false, "Provision the local Postgres + app database if missing (fresh-session bootstrap)") runCmd.Flags().Bool("setup", false, "Prepare prerequisites (cache MxBuild+runtime, ensure DB) and exit without booting — for a SessionStart hook") - runCmd.Flags().Int("app-port", 0, "HTTP port for the app (default 8080)") - runCmd.Flags().Int("admin-port", 0, "M2EE admin API port (default 8090)") + runCmd.Flags().Int("app-port", 0, "HTTP port for the app (default: the configuration's, else 8080)") + runCmd.Flags().Int("admin-port", 0, "M2EE admin API port (default: the configuration's, else 8090)") runCmd.Flags().String("listen-addr", "", "Address the app binds to (default 127.0.0.1, loopback only). Must be an IP address. Set 0.0.0.0 (or :: for IPv6 too) to reach the app from other machines on the network; the URLs it is reachable at are printed once it runs. If the app builds absolute URLs (SSO redirects, links in emails), pair it with a reachable root URL — an ApplicationRootUrl naming a real host in the model, or --runtime-setting ApplicationRootUrl=http://:/ — because the runtime does not derive one from this address (a warning says so). The M2EE admin API is unaffected and always stays on 127.0.0.1.") runCmd.Flags().Int("serve-port", 0, "mxbuild --serve port (default 6543)") runCmd.Flags().String("mxbuild-path", "", "Path to the mxbuild to build with, overriding resolution (Studio Pro's bundled mxbuild on macOS/Windows, the cached CDN download on Linux)") diff --git a/cmd/mxcli/constants_resolve.go b/cmd/mxcli/constants_resolve.go index a6e85a260..e697049e1 100644 --- a/cmd/mxcli/constants_resolve.go +++ b/cmd/mxcli/constants_resolve.go @@ -48,6 +48,10 @@ type constantChain struct { // the chain because they come from the one configuration choice and the one // project read; a second read would cost a second open of the project. JVMArgs []string + // AppPort and AdminPort are the configuration's HTTP and admin ports, 0 + // where it sets none; the --app-port and --admin-port flags override them + // (ako/mxcli#2365). + AppPort, AdminPort int } // parseConstantFlags turns repeated `--constant Module.Name=value` into a map. @@ -231,6 +235,7 @@ func resolveConstantChain(ps *model.ProjectSettings, want string, flags, machine return constantChain{}, err } chain.JVMArgs = args + chain.AppPort, chain.AdminPort = chosen.HttpPortNumber, chosen.ServerPortNumber } var unknown []string diff --git a/cmd/mxcli/runports.go b/cmd/mxcli/runports.go new file mode 100644 index 000000000..78a6b4424 --- /dev/null +++ b/cmd/mxcli/runports.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "io" + "strings" +) + +// runPorts is the app and admin ports of a run: each flag that was given, +// else the selected configuration's port, else 0 (the run's own default). +// Studio Pro starts an app on its configuration's ports, and `run --local` +// listened on 8080/8090 whatever the configuration said (ako/mxcli#2365). +func runPorts(w io.Writer, appFlag, adminFlag int, c constantChain) (app, admin int) { + app, admin = appFlag, adminFlag + var from []string + if app == 0 && c.AppPort > 0 { + app = c.AppPort + from = append(from, fmt.Sprintf("app %d", app)) + } + if admin == 0 && c.AdminPort > 0 { + admin = c.AdminPort + from = append(from, fmt.Sprintf("admin %d", admin)) + } + if len(from) > 0 { + fmt.Fprintf(w, "Ports from configuration %q: %s (override with --app-port / --admin-port)\n", + c.Configuration, strings.Join(from, ", ")) + } + return app, admin +} diff --git a/cmd/mxcli/runports_2365_test.go b/cmd/mxcli/runports_2365_test.go new file mode 100644 index 000000000..ac91d7db2 --- /dev/null +++ b/cmd/mxcli/runports_2365_test.go @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/model" +) + +// ako/mxcli#2365 (mendixlabs/mxcli#1880): with the selected configuration +// saying HttpPortNumber 8085 and ServerPortNumber 8095, `run --local +// --configuration Default` listened on 8080 and 8090; only --app-port and +// --admin-port moved it. The configuration's ports are the run's ports, and +// the flags override them. + +func portsCfg(name string, http, admin int) *model.ServerConfiguration { + return &model.ServerConfiguration{Name: name, HttpPortNumber: http, ServerPortNumber: admin} +} + +func TestResolveConstantChain_CarriesTheConfigurationsPorts_Issue2365(t *testing.T) { + ps := settingsWith(portsCfg("Default", 8085, 8095), portsCfg("Other", 9000, 9001)) + chain, err := resolveConstantChain(ps, "Default", nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if chain.AppPort != 8085 || chain.AdminPort != 8095 { + t.Errorf("ports = %d/%d, want Default's 8085/8095", chain.AppPort, chain.AdminPort) + } +} + +func TestRunPorts_Issue2365(t *testing.T) { + chain := constantChain{Configuration: "Default", AppPort: 8085, AdminPort: 8095} + var out strings.Builder + + app, admin := runPorts(&out, 0, 0, chain) + if app != 8085 || admin != 8095 { + t.Errorf("no flags: ports = %d/%d, want the configuration's 8085/8095", app, admin) + } + if !strings.Contains(out.String(), `"Default"`) { + t.Errorf("the run does not say where its ports came from: %q", out.String()) + } + + // CONTROLS: flags win, each on its own; a configuration with no ports + // leaves the defaults (0 = the run's own default) alone. + if app, admin := runPorts(&out, 8100, 0, chain); app != 8100 || admin != 8095 { + t.Errorf("--app-port 8100: ports = %d/%d, want 8100/8095", app, admin) + } + if app, admin := runPorts(&out, 0, 8200, chain); app != 8085 || admin != 8200 { + t.Errorf("--admin-port 8200: ports = %d/%d, want 8085/8200", app, admin) + } + out.Reset() + if app, admin := runPorts(&out, 0, 0, constantChain{Configuration: "Default"}); app != 0 || admin != 0 || out.Len() != 0 { + t.Errorf("no configured ports: %d/%d %q, want 0/0 and nothing said", app, admin, out.String()) + } +} From d862964110ceaa346902bdcd08fcd65e3652f7c1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:08:40 +0000 Subject: [PATCH 04/28] fix(run): close the diagnostic session when a run command exits run, run status/stop/wait/restart exited through os.Exit, which skips runCLI's session close, so diag loop-report counted every call as a vanished process with no time. They now exit through exitProcess, which records session_end with the exit code first. Closes #2360 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011oK82qrr1WnXxq1eRn7qRJ --- ...11-18-run-lifecycle-unclosed-sessions.json | 1 + CHANGELOG.md | 1 + cmd/mxcli/cmd_run.go | 28 +++++------ cmd/mxcli/cmd_run_lifecycle.go | 20 ++++---- cmd/mxcli/diag_loop_report.go | 8 ++-- cmd/mxcli/exit_process.go | 25 ++++++++++ cmd/mxcli/exit_process_2360_test.go | 48 +++++++++++++++++++ 7 files changed, 104 insertions(+), 27 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-run-lifecycle-unclosed-sessions.json create mode 100644 cmd/mxcli/exit_process.go create mode 100644 cmd/mxcli/exit_process_2360_test.go diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-run-lifecycle-unclosed-sessions.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-run-lifecycle-unclosed-sessions.json new file mode 100644 index 000000000..c1b0f285c --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-run-lifecycle-unclosed-sessions.json @@ -0,0 +1 @@ +{"area": "cmd-mxcli", "date": "2026-11-18", "refs": ["ako/mxcli#2360"], "ce": [], "symptom": "diag loop-report over a ~20 h agent session listed every `run wait` (189), `run restart` (150), `run stop` and `run status` call as UNCLOSED with 0.0s total, and blamed 'a process that vanished: killed', although they exited normally and `run wait` alone accounted for over an hour.", "file": "cmd/mxcli/exit_process.go exitProcess; cmd/mxcli/cmd_run.go, cmd/mxcli/cmd_run_lifecycle.go (os.Exit → exitProcess); cmd/mxcli/diag_loop_report.go help. Test: cmd/mxcli/exit_process_2360_test.go", "cause": "runCLI closes the diagnostic session (session_end with duration and exit code) when the root command returns; the run verbs left through os.Exit inside their Run funcs — on exit code 0 too — so no session_end was written.", "insight": "**Instrumentation that lives on the return path misses every os.Exit, including the successful ones.** The earlier fix (closing on the error path) assumed os.Exit meant failure; the lifecycle verbs use the exit code as their API and exit 0 through os.Exit. Route exits through one helper that closes the session, and keep a source test on the files that must not call os.Exit. Measured after: run status (exit 3) and run stop (exit 0) each write session_end; loop-report shows 0 unclosed. 271 other os.Exit sites in package main remain."} diff --git a/CHANGELOG.md b/CHANGELOG.md index fb7b9a51b..f8a18ef85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`diag loop-report` measures `run status`, `stop`, `wait` and `restart`** (ako/mxcli#2360) — these commands exited through `os.Exit`, which skipped the session close even on success, so every call was counted as a vanished process with no time and the hours spent waiting for and restarting the app were invisible. They now close their session with the exit code on every exit. - **`run --local` listens on the selected configuration's ports** (ako/mxcli#2365, mendixlabs/mxcli#1880) — a configuration's HttpPortNumber and ServerPortNumber were ignored, so the app always started on 8080/8090 unless `--app-port`/`--admin-port` were given. The run now takes both from the configuration it uses for constants and JVM options, says so, and the flags still override each one. - **`marketplace update` refuses at once when Studio Pro has the project open** (ako/mxcli#2366, mendixlabs/mxcli#1881) — the update started, had each of hundreds of writes refused, and ended with "The project may be mid-update … Restore from version control" although nothing had changed. The lock is now checked once, before the reference projects are built, and the refusal says nothing was changed. - **`alter page … set (visible: roles (…))` sets the module roles** (ako/mxcli#2371) — the alter form stored `roles (…)` as the visibility expression, reported success, and left a page `mx check` rejected with CE0117. It now writes the "Visible for module roles" setting as `create page` does, keeping the widget's condition, and `roles (…) and ` sets both. diff --git a/cmd/mxcli/cmd_run.go b/cmd/mxcli/cmd_run.go index abd3f23b4..441c9cf8b 100644 --- a/cmd/mxcli/cmd_run.go +++ b/cmd/mxcli/cmd_run.go @@ -131,7 +131,7 @@ Examples: // the user should learn that before waiting out a runtime start. if !docker.TunnelSupported() { fmt.Fprintf(os.Stderr, "Error: %v\n", docker.ErrTunnelUnsupported) - os.Exit(1) + exitProcess(1) } local = true // Present a per-user hub API key to an authenticated hub (MXCLI_HUB_KEY @@ -141,12 +141,12 @@ Examples: } if !local { fmt.Fprintln(os.Stderr, "Error: only --local is supported for now (use 'mxcli docker run' for the container workflow)") - os.Exit(1) + exitProcess(1) } projectPath, _ := cmd.Flags().GetString("project") if projectPath == "" { fmt.Fprintln(os.Stderr, "Error: --project (-p) is required") - os.Exit(1) + exitProcess(1) } // MxBuild requires an absolute project path; resolve a relative -p here // rather than surfacing MxBuild's raw "path should be an absolute path" @@ -174,7 +174,7 @@ Examples: secrets, err := resolveRunSecrets(cmd) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) + exitProcess(1) } hubSecret := secrets["hub-secret"] @@ -186,7 +186,7 @@ Examples: listenAddr, _ := cmd.Flags().GetString("listen-addr") if err := docker.ValidateListenAddr(listenAddr); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) + exitProcess(1) } // --detach: start this same command as a background service and return @@ -199,14 +199,14 @@ Examples: if detach, _ := cmd.Flags().GetBool("detach"); detach && !detachedChild { if setupOnly { fmt.Fprintln(os.Stderr, "Error: --detach has nothing to do with --setup (which never boots the app)") - os.Exit(1) + exitProcess(1) } - os.Exit(runDetach(projectPath, detachArgs(os.Args[1:], projectPath), os.Stdout)) + exitProcess(runDetach(projectPath, detachArgs(os.Args[1:], projectPath), os.Stdout)) } if !setupOnly { if err := refuseIfRunning(projectPath); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) + exitProcess(1) } } @@ -220,7 +220,7 @@ Examples: allowCall, _ := cmd.Flags().GetBool("allow-call") if allowCall && !testEndpoint { fmt.Fprintln(os.Stderr, "Error: --allow-call switches on a route of the test endpoint; it needs --test-endpoint") - os.Exit(1) + exitProcess(1) } if allowCall && hub != "" { // The tunnel client reaches the app from loopback, so the endpoint's @@ -268,12 +268,12 @@ Examples: constantFlags, err := parseConstantFlags(constantArgs) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) + exitProcess(1) } overrides, err := constantChainFor(projectPath, configuration, constantFlags) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) + exitProcess(1) } reportConstantChain(os.Stdout, overrides) reportJVMArgs(os.Stdout, overrides) @@ -339,7 +339,7 @@ Examples: if testEndpoint { if setupOnly { fmt.Fprintln(os.Stderr, "Error: --test-endpoint has nothing to do with --setup (which never boots the app)") - os.Exit(1) + exitProcess(1) } // Ignored before the endpoint's Java source exists, so a commit made // while the app runs cannot pick it up (ako/mxcli#2033). @@ -350,7 +350,7 @@ Examples: hosted, err = testrunner.InstallHostedEndpoint(projectPath, allowCall, os.Stdout) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) + exitProcess(1) } // Ctrl-C is the normal way to stop a dev loop, and it must not leave the // endpoint in the project. RunLocal returns on SIGINT, so the deferred @@ -425,7 +425,7 @@ Examples: fmt.Fprintf(os.Stderr, "Error: %v\n", err) // os.Exit skips deferred calls, so remove explicitly here. hosted.Remove() - os.Exit(1) + exitProcess(1) } }, } diff --git a/cmd/mxcli/cmd_run_lifecycle.go b/cmd/mxcli/cmd_run_lifecycle.go index 2a294dbc8..bdfed6351 100644 --- a/cmd/mxcli/cmd_run_lifecycle.go +++ b/cmd/mxcli/cmd_run_lifecycle.go @@ -82,7 +82,7 @@ Exit codes: 0 running, 3 not running, 4 still starting.`, mxcli run status -p app.mpr || mxcli run --local --watch --detach -p app.mpr`, Run: func(cmd *cobra.Command, args []string) { projectPath := lifecycleProject(cmd) - os.Exit(runStatus(projectPath, os.Stdout)) + exitProcess(runStatus(projectPath, os.Stdout)) }, } @@ -105,7 +105,7 @@ runs in.`, Example: ` mxcli run stop -p app.mpr`, Run: func(cmd *cobra.Command, args []string) { projectPath := lifecycleProject(cmd) - os.Exit(runStop(projectPath, os.Stdout)) + exitProcess(runStop(projectPath, os.Stdout)) }, } @@ -140,9 +140,9 @@ started without --watch never applies a change: wait says so and exits 1.`, ready, _ := cmd.Flags().GetBool("ready") since, _ := cmd.Flags().GetInt("since") if ready { - os.Exit(runWaitReady(projectPath, timeout, os.Stdout)) + exitProcess(runWaitReady(projectPath, timeout, os.Stdout)) } - os.Exit(runWait(projectPath, timeout, since, os.Stdout)) + exitProcess(runWait(projectPath, timeout, since, os.Stdout)) }, } @@ -169,25 +169,25 @@ shell.`, st, err := docker.ReadRunState(projectPath) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(runExitFailed) + exitProcess(runExitFailed) } noteForeignRunState(projectPath, os.Stderr) if st == nil || len(st.Args) == 0 { fmt.Fprintf(os.Stderr, "nothing to restart: no run of this project is recorded\n"+ " start one: mxcli run --local --watch --detach -p %s\n", projectPath) - os.Exit(runExitNotRunning) + exitProcess(runExitNotRunning) } // Secrets given on the command line are not stored with the run, so a // restart would hand the app the placeholder instead of the value. Say so // before stopping anything. if hasRedactedArgs(st.Args) { fmt.Fprint(os.Stderr, restartSecretRefusal(projectPath, redactedArgNames(st.Args))) - os.Exit(runExitFailed) + exitProcess(runExitFailed) } if code := runStop(projectPath, os.Stdout); code != runExitOK { - os.Exit(code) + exitProcess(code) } - os.Exit(runDetach(projectPath, st.Args, os.Stdout)) + exitProcess(runDetach(projectPath, st.Args, os.Stdout)) }, } @@ -203,7 +203,7 @@ func lifecycleProject(cmd *cobra.Command) string { projectPath, _ := cmd.Flags().GetString("project") if projectPath == "" { fmt.Fprintln(os.Stderr, "Error: --project (-p) is required") - os.Exit(runExitFailed) + exitProcess(runExitFailed) } if abs, err := filepath.Abs(projectPath); err == nil { projectPath = abs diff --git a/cmd/mxcli/diag_loop_report.go b/cmd/mxcli/diag_loop_report.go index 834c150a1..c94207f85 100644 --- a/cmd/mxcli/diag_loop_report.go +++ b/cmd/mxcli/diag_loop_report.go @@ -535,9 +535,11 @@ Every mxcli invocation writes a session record naming its argv, so the shape of an agent's loop is already on disk. This reports it: which commands were run, how often, how long they took, and how many did not exit cleanly. -A run that fails exits through os.Exit and writes no summary record, so it is -counted as "did not close" rather than as a failure — the report says which it -is measuring rather than presenting one as the other. +A run that is killed, or that leaves through a bare os.Exit inside a command, +writes no summary record, so it is counted as "did not close" rather than as a +failure — the report says which it is measuring rather than presenting one as +the other. The run commands (run, run status/stop/wait/restart) close their +session on every exit. It counts mxcli PROCESSES, not model calls — one shell command can run several. Read it to find which command dominates a loop, and re-run it after a change to diff --git a/cmd/mxcli/exit_process.go b/cmd/mxcli/exit_process.go new file mode 100644 index 000000000..d2faac51b --- /dev/null +++ b/cmd/mxcli/exit_process.go @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + + "github.com/mendixlabs/mxcli/mdl/diaglog" +) + +// exitProcess ends the process with code after closing the diagnostic +// session, which records the run's duration and exit code. A bare os.Exit +// skips runCLI's close, so `diag loop-report` counted the run as a vanished +// process with no time at all — every `run status/stop/wait/restart`, clean +// exits included (ako/mxcli#2360). +func exitProcess(code int) { + closeSession(code) + osExit(code) +} + +// Seams for tests. +var ( + closeSession = diaglog.CloseCurrentWithExit + osExit = os.Exit +) diff --git a/cmd/mxcli/exit_process_2360_test.go b/cmd/mxcli/exit_process_2360_test.go new file mode 100644 index 000000000..3e3867ebe --- /dev/null +++ b/cmd/mxcli/exit_process_2360_test.go @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "regexp" + "strings" + "testing" +) + +// ako/mxcli#2360: `run status/stop/wait/restart` left through os.Exit, which +// skips runCLI's session close, so even a clean exit wrote no session_end and +// `diag loop-report` counted 189 `run wait` calls as vanished processes with +// 0.0s between them, while an hour of waiting went unmeasured. + +func TestExitProcess_ClosesTheSessionWithTheCode_Issue2360(t *testing.T) { + var closed, exited []int + defer func(c func(int), e func(int)) { closeSession, osExit = c, e }(closeSession, osExit) + closeSession = func(code int) { closed = append(closed, code) } + osExit = func(code int) { exited = append(exited, code) } + + exitProcess(0) + exitProcess(3) + if len(closed) != 2 || closed[0] != 0 || closed[1] != 3 { + t.Errorf("session closed with %v, want [0 3]", closed) + } + if len(exited) != 2 || exited[0] != 0 || exited[1] != 3 { + t.Errorf("exited with %v, want [0 3]", exited) + } +} + +// The run commands are where an agent's loop spends its waiting time; none of +// them may leave without closing the session. +func TestRunCommandsDoNotCallOsExit_Issue2360(t *testing.T) { + raw := regexp.MustCompile(`\bos\.Exit\(`) + for _, f := range []string{"cmd_run.go", "cmd_run_lifecycle.go"} { + b, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for i, line := range strings.Split(string(b), "\n") { + if raw.MatchString(line) && !strings.HasPrefix(strings.TrimSpace(line), "//") { + t.Errorf("%s:%d calls os.Exit, which skips the session close: use exitProcess\n\t%s", f, i+1, strings.TrimSpace(line)) + } + } + } +} From 572139248d7e1c1d05ccea6fcca49fcfea3783d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:11:41 +0000 Subject: [PATCH 05/28] fix(rest): preview an OpenAPI query parameter as the import stores it Rest$QueryParameter has no DataType, so the import stores every query parameter as text and describe reads it back as String, while the contract preview printed the spec's type. The preview now shows what the import writes. Closes #2353 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011oK82qrr1WnXxq1eRn7qRJ --- ...1-18-openapi-query-param-type-preview.json | 1 + CHANGELOG.md | 1 + mdl/executor/cmd_rest_clients.go | 7 ++- mdl/executor/openapi_query_type_2353_test.go | 50 +++++++++++++++++++ 4 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-11-18-openapi-query-param-type-preview.json create mode 100644 mdl/executor/openapi_query_type_2353_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-openapi-query-param-type-preview.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-openapi-query-param-type-preview.json new file mode 100644 index 000000000..7c0127e72 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-openapi-query-param-type-preview.json @@ -0,0 +1 @@ +{"area": "mdl-executor", "date": "2026-11-18", "refs": ["ako/mxcli#2353"], "ce": [], "symptom": "`describe contract operation from openapi` showed `Query: ($limit: Integer)` for an integer query parameter, but after `create or modify consumed rest service … ( OpenAPI: … )` describe showed `Query: ($limit: String)`; path parameters kept Integer in both.", "file": "mdl/executor/cmd_rest_clients.go convertOpenAPIToModel. Test: mdl/executor/openapi_query_type_2353_test.go", "cause": "Rest$QueryParameter has no DataType in the metamodel (Name, TestValue, ParameterUsage only), so the writer cannot store a query type and describe reads it back as String; the preview printed the spec's type from the in-memory model instead.", "insight": "**When the preview and the write disagree, check the metamodel before 'fixing' the write.** The import was right — a consumed REST query parameter is text in Mendix — and the preview was the one promising a type no storage can hold. Measured on 11.12.6: preview and imported describe now both show `$limit: String`, `$petId: Integer`. An MDL `Query: ($x: Integer)` is still stored as String without a word; not addressed here."} diff --git a/CHANGELOG.md b/CHANGELOG.md index f8a18ef85..db7bcce92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`describe contract operation from openapi` shows query parameters as they are imported** (ako/mxcli#2353) — the preview showed an integer query parameter as `Integer` while the imported service described it as `String`. Mendix stores no type for a consumed REST query parameter (it is text), so the preview now shows `String`, matching the import; path parameters keep the spec's type. - **`diag loop-report` measures `run status`, `stop`, `wait` and `restart`** (ako/mxcli#2360) — these commands exited through `os.Exit`, which skipped the session close even on success, so every call was counted as a vanished process with no time and the hours spent waiting for and restarting the app were invisible. They now close their session with the exit code on every exit. - **`run --local` listens on the selected configuration's ports** (ako/mxcli#2365, mendixlabs/mxcli#1880) — a configuration's HttpPortNumber and ServerPortNumber were ignored, so the app always started on 8080/8090 unless `--app-port`/`--admin-port` were given. The run now takes both from the configuration it uses for constants and JVM options, says so, and the flags still override each one. - **`marketplace update` refuses at once when Studio Pro has the project open** (ako/mxcli#2366, mendixlabs/mxcli#1881) — the update started, had each of hundreds of writes refused, and ended with "The project may be mid-update … Restore from version control" although nothing had changed. The lock is now checked once, before the reference projects are built, and the refusal says nothing was changed. diff --git a/mdl/executor/cmd_rest_clients.go b/mdl/executor/cmd_rest_clients.go index ad4f24f1d..501bb61cf 100644 --- a/mdl/executor/cmd_rest_clients.go +++ b/mdl/executor/cmd_rest_clients.go @@ -1006,9 +1006,12 @@ func convertOpenAPIToModel(parsed *openapi.ConsumedRestService, containerID mode }) } for _, q := range op.QueryParameters { + // No DataType: Rest$QueryParameter has none, so the import stores + // every query parameter as text, and the preview built from this + // model has to say what the import writes, not what the spec + // declares (ako/mxcli#2353). modelOp.QueryParameters = append(modelOp.QueryParameters, &model.RestClientParameter{ - Name: q.Name, - DataType: q.DataType, + Name: q.Name, }) } for _, h := range op.Headers { diff --git a/mdl/executor/openapi_query_type_2353_test.go b/mdl/executor/openapi_query_type_2353_test.go new file mode 100644 index 000000000..14c31b63a --- /dev/null +++ b/mdl/executor/openapi_query_type_2353_test.go @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +// ako/mxcli#2353: `describe contract operation from openapi` showed +// `Query: ($limit: Integer)` while the imported service described back as +// `Query: ($limit: String)`. Rest$QueryParameter has no DataType — a consumed +// REST query parameter is text in the model — so the import cannot keep the +// spec's type, and the preview must show what the import writes. +const queryAndPathSpec = `{ + "openapi": "3.0.0", + "info": { "title": "Pets", "version": "1.0.0" }, + "servers": [{ "url": "https://api.example.com" }], + "paths": { + "/pets": { "get": { "operationId": "listPets", + "parameters": [ { "name": "limit", "in": "query", "schema": { "type": "integer" } } ], + "responses": { "200": { "description": "OK" } } } }, + "/pets/{petId}": { "get": { "operationId": "getPet", + "parameters": [ { "name": "petId", "in": "path", "required": true, "schema": { "type": "integer" } } ], + "responses": { "200": { "description": "OK" } } } } + } +}` + +func TestDescribeContractFromOpenAPI_QueryTypeIsWhatTheImportWrites_Issue2353(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(queryAndPathSpec)) + })) + defer srv.Close() + + ctx, buf := newMockCtx(t) + assertNoError(t, describeContractFromOpenAPI(ctx, &ast.DescribeContractFromOpenAPIStmt{SpecPath: srv.URL})) + out := buf.String() + if !strings.Contains(out, "$limit: String") { + t.Errorf("preview does not show the query parameter as the import stores it (String):\n%s", out) + } + // CONTROL: a path parameter has a stored DataType and keeps the spec's. + if !strings.Contains(out, "$petId: Integer") { + t.Errorf("path parameter lost its type:\n%s", out) + } +} From 0535f43f8a72678b84cad24249d099b14b2592a7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:14:02 +0000 Subject: [PATCH 06/28] fix(odata): name a colliding navigation association _ create external entities fell back to a numeric suffix when a navigation property's name was already an entity, so Product.Category became Category_2. The fallback is now Product_Category, with a number only if that is taken too. Closes #2354 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011oK82qrr1WnXxq1eRn7qRJ --- ...8-external-entities-assoc-suffix-name.json | 1 + CHANGELOG.md | 1 + docs-site/src/reference/odata/README.md | 6 +++ mdl/executor/cmd_contract.go | 18 +++++++-- mdl/executor/external_assoc_name_2354_test.go | 37 +++++++++++++++++++ 5 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-11-18-external-entities-assoc-suffix-name.json create mode 100644 mdl/executor/external_assoc_name_2354_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-external-entities-assoc-suffix-name.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-external-entities-assoc-suffix-name.json new file mode 100644 index 000000000..f29d417ad --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-external-entities-assoc-suffix-name.json @@ -0,0 +1 @@ +{"area": "mdl-executor", "date": "2026-11-18", "refs": ["ako/mxcli#2354"], "ce": ["CE0065"], "symptom": "`create external entities` from an OData v4 contract with Product.Category → Category named the association `Integ.Category_2`, a suffix suggesting a clash nothing in the module explained.", "file": "mdl/executor/cmd_contract.go uniqueAssocName (now takes the parent entity). Test: mdl/executor/external_assoc_name_2354_test.go", "cause": "Associations are named after the navigation property; `Category` collides with the entity Category (CE0065 forbids it), and the only fallback was a numeric suffix.", "insight": "**A collision fallback should add meaning, not a counter.** `_` is unique wherever the parent is and reads as the Mendix convention; a counter depends on import order. Re-import dedup matches by RemoteParentNavigationProperty, so existing suffixed names are left alone (verified: second run 0 created, 1 association). Measured on 11.12.6: Integ.Product_Category, mx check 0 errors."} diff --git a/CHANGELOG.md b/CHANGELOG.md index db7bcce92..1bf0acefa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **An imported navigation association that would clash is named `_`** (ako/mxcli#2354) — `create external entities` named Product's `Category` navigation `Category_2`, because the name collides with the entity Category. It is now `Product_Category`, with a number only if that is taken too. Names that do not collide are unchanged, and a re-import still recognises existing associations by their navigation property. - **`describe contract operation from openapi` shows query parameters as they are imported** (ako/mxcli#2353) — the preview showed an integer query parameter as `Integer` while the imported service described it as `String`. Mendix stores no type for a consumed REST query parameter (it is text), so the preview now shows `String`, matching the import; path parameters keep the spec's type. - **`diag loop-report` measures `run status`, `stop`, `wait` and `restart`** (ako/mxcli#2360) — these commands exited through `os.Exit`, which skipped the session close even on success, so every call was counted as a vanished process with no time and the hours spent waiting for and restarting the app were invisible. They now close their session with the exit code on every exit. - **`run --local` listens on the selected configuration's ports** (ako/mxcli#2365, mendixlabs/mxcli#1880) — a configuration's HttpPortNumber and ServerPortNumber were ignored, so the app always started on 8080/8090 unless `--app-port`/`--admin-port` were given. The run now takes both from the configuration it uses for constants and JVM options, says so, and the flags still override each one. diff --git a/docs-site/src/reference/odata/README.md b/docs-site/src/reference/odata/README.md index 4e1844488..ff22631b0 100644 --- a/docs-site/src/reference/odata/README.md +++ b/docs-site/src/reference/odata/README.md @@ -89,6 +89,12 @@ Two more consequences: ### Navigation properties become associations Each navigation property whose target is also imported becomes an association. +It is named after the navigation property. When that name is already an entity +or an association in the module — a `Category` navigation beside the entity +`Category` — it is named `_` (`Product_Category`), +and only if that is taken too does a number follow (`Product_Category_2`). A +re-import recognises the associations it made by their navigation property, not +their name, so an existing name is never changed. A navigation property with a `Partner` is one end of a bidirectional relationship, and Mendix does not accept its ends as separate associations (`CE6618` "Navigation '…' is not available in the OData service"): diff --git a/mdl/executor/cmd_contract.go b/mdl/executor/cmd_contract.go index ab87412ec..5cb4420ff 100644 --- a/mdl/executor/cmd_contract.go +++ b/mdl/executor/cmd_contract.go @@ -1284,7 +1284,7 @@ func createNavigationAssociations( // not collide with any entity name (CE0065). When the OData // nav property name collides with an existing entity, append // a numeric suffix. - assocName := uniqueAssocName(np.Name, dm, existingAssocs) + assocName := uniqueAssocName(np.Name, parentEnt.Name, dm, existingAssocs) assocType := domainmodel.AssociationTypeReference if isMany { @@ -1452,9 +1452,12 @@ func indexExistingAssociations(dm *domainmodel.DomainModel) (byName, byNav map[a } // uniqueAssocName returns a Mendix-safe association name for an OData nav -// property. If the requested name collides with an existing entity name OR an -// already-created association name, append a numeric suffix. -func uniqueAssocName(base string, dm *domainmodel.DomainModel, existingAssocs map[assocKey]bool) string { +// property: the nav property's own name, unless it collides with an entity or +// an association already in the module (CE0065) — then `_`, +// the Mendix convention, and a numeric suffix only if that is taken as well. +// The suffix used to come first, so Product's nav `Category` became +// `Category_2` beside the entity Category (ako/mxcli#2354). +func uniqueAssocName(base, parent string, dm *domainmodel.DomainModel, existingAssocs map[assocKey]bool) string { collides := func(name string) bool { for _, ent := range dm.Entities { if ent.Name == name { @@ -1471,6 +1474,13 @@ func uniqueAssocName(base string, dm *domainmodel.DomainModel, existingAssocs ma if !collides(base) { return base } + if parent != "" { + qualified := parent + "_" + base + if !collides(qualified) { + return qualified + } + base = qualified + } for i := 2; i < 100; i++ { candidate := fmt.Sprintf("%s_%d", base, i) if !collides(candidate) { diff --git a/mdl/executor/external_assoc_name_2354_test.go b/mdl/executor/external_assoc_name_2354_test.go new file mode 100644 index 000000000..7921d0c7e --- /dev/null +++ b/mdl/executor/external_assoc_name_2354_test.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// ako/mxcli#2354: `create external entities` named Product's navigation +// property `Category` as the association `Category_2` — the nav name collides +// with the entity Category, and the fallback was a numeric suffix. A colliding +// name now takes the Mendix convention, `_`. +func TestUniqueAssocName_Issue2354(t *testing.T) { + dm := &domainmodel.DomainModel{Entities: []*domainmodel.Entity{{Name: "Product"}, {Name: "Category"}}} + none := map[assocKey]bool{} + + if got := uniqueAssocName("Category", "Product", dm, none); got != "Product_Category" { + t.Errorf("nav Category on Product = %q, want Product_Category", got) + } + // CONTROL: a nav name that collides with nothing is kept as it is. + if got := uniqueAssocName("Supplier", "Product", dm, none); got != "Supplier" { + t.Errorf("nav Supplier = %q, want Supplier", got) + } + // Two entities with the same nav name: the second is told apart by its + // parent, not by a counter. + taken := map[assocKey]bool{{parent: "Product", name: "season"}: true} + if got := uniqueAssocName("season", "Race", dm, taken); got != "Race_season" { + t.Errorf("second season = %q, want Race_season", got) + } + // Only when the parent-qualified name is taken too does a number appear. + taken = map[assocKey]bool{{parent: "Product", name: "Product_Category"}: true} + if got := uniqueAssocName("Category", "Product", dm, taken); got != "Product_Category_2" { + t.Errorf("both taken = %q, want Product_Category_2", got) + } +} From a767b95c7c14b7dd57099b9e5794b2a158dc0ae7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:17:57 +0000 Subject: [PATCH 07/28] check: refuse a mapping custom handler that does not name exactly its microflow's parameters (CE1613, CE0277) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011oK82tETa46YQmMS58pQjL --- ...ping-handler-parameters-ce1613-ce0277.json | 1 + CHANGELOG.md | 1 + cmd/mxcli/syntax/features_integration.go | 2 +- ...mapping-handler-missing-parameter.fail.mdl | 45 ++++ ...mapping-handler-unknown-parameter.fail.mdl | 40 ++++ .../mapping-handler-unknown-parameter.mdl | 37 ++++ .../validate_mapping_handler_params.go | 205 ++++++++++++++++++ .../validate_mapping_handler_params_test.go | 152 +++++++++++++ mdl/executor/validate_program.go | 2 + 9 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-11-18-mapping-handler-parameters-ce1613-ce0277.json create mode 100644 mdl-examples/bug-tests/mapping-handler-missing-parameter.fail.mdl create mode 100644 mdl-examples/bug-tests/mapping-handler-unknown-parameter.fail.mdl create mode 100644 mdl-examples/bug-tests/mapping-handler-unknown-parameter.mdl create mode 100644 mdl/executor/validate_mapping_handler_params.go create mode 100644 mdl/executor/validate_mapping_handler_params_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-mapping-handler-parameters-ce1613-ce0277.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-mapping-handler-parameters-ce1613-ce0277.json new file mode 100644 index 000000000..be3811d97 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-mapping-handler-parameters-ce1613-ce0277.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-18", "refs": ["ako/mxcli#1258"], "ce": ["CE1613", "CE0277", "CE0282"], "symptom": "An import mapping's custom object handler `find … by M.MF ( P: member )` that names a parameter the microflow does not declare (or the right name in another case) passed `check` and `exec` and was written; `mx check` reported CE1613 \"The selected parameter 'M.MF.P' no longer exists\". Naming only some of the microflow's parameters was CE0277 \"Parameter mapping of microflow for obtaining objects needs to be refreshed\".", "file": "mdl/executor/validate_mapping_handler_params.go ValidateMappingHandlerParams (MDL104), called from validate_program.go with the lazy project opener. Tests: validate_mapping_handler_params_test.go; fixtures mdl-examples/bug-tests/mapping-handler-unknown-parameter(.fail).mdl, mapping-handler-missing-parameter.fail.mdl", "cause": "buildCustomHandler qualifies each parameter against the microflow (`Microflow + \".\" + Parameter`) and never looks the name up, and nothing counts the parameters. CE0281 itself, the code this was started from, cannot be produced by hand: exec refuses a `by` member that is not in the JSON structure (addCustomHandlerValueElements), so mx check never sees it; check does not mirror that refusal and it was left alone because the mirror would duplicate the builder's JSON-path tracking.", "insight": "**One mapping per project when measuring a build that stages its errors: a shared project made two real errors read as clean.** My first matrix put nine mappings in one project; three were CE1613 (an unresolved reference, stage 0, which hides every other error in every document), so `Index: code` (really CE0282) and one-of-two parameters (really CE0277) both showed '-' and I wrote 'leaving a parameter out is clean' into the rule's header and a test. It surfaced only because the control fixture, applied alone, reported 1 error. Re-measured isolated: names exact and all named clean; unknown, wrong-case or extra name CE1613; one or none of the parameters named CE0277; none named on a zero-parameter microflow clean; a String member into an Integer parameter CE0282 (not covered). The two codes need two fail fixtures for the same reason: in one project CE1613 hides the CE0277. An unknown name is judged alone, as the build does. The same staging that the documented stage table already records was the trap, so run the control and every fail fixture in their own project before trusting a count. Sweep: 1,325 repo .mdl files, 0 MDL104 hits. Control: stubbing the rule fails 12 sub-tests."} diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fae108c4..57c7b84e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`check` rejects an import mapping's custom handler that does not name exactly its microflow's parameters** (ako/mxcli#1258) — `find … by M.MF ( P: member )` with a parameter the microflow does not declare (or the right name in another case) passed `check` and `exec` and the build reported CE1613 "The selected parameter … no longer exists"; leaving a parameter out was CE0277 "Parameter mapping of microflow for obtaining objects needs to be refreshed". New MDL104, which reads the microflow from the script or the project; a microflow it cannot find is not judged, and an unknown name is reported before a missing one, as the build does. A parameter fed a member of the wrong type (CE0282) is not covered. Measured on MxBuild 11.16.1, one mapping per project. - **`check` rejects `rest call … returns mapping M as list of E` over an object-rooted mapping** (ako/mxcli#1258) — an import mapping whose root is one object returns one object, and declaring the call's result as a list passed `check` and `exec` and the build reported CE0243 "The mapping used to return a value of type 'List of …', but now returns a value of type '…'". New MDL103, which reads the mapping's shape from the script or the project (the same resolver as MDL-MAP04); a mapping whose shape cannot be established is not judged. `as Module.E` is fine on either root — over an array it binds one object out of the list. Measured on MxBuild 11.16.1. - **`check` rejects an `alter entity … modify attribute` that leaves a published OData attribute published as the wrong type** (ako/mxcli#1258) — changing the type of an attribute a published service exposes (String to Integer, Decimal to Integer, Boolean or DateTime to String, String to Decimal) passed `check` and `exec` and the build reported CE5016 "Attribute '…' has type …, but is published as …", one per attribute. New MDL102, which reads the service from the script and from the project. A change inside one EDM type (String(50) to String(80); Integer to Long) is fine, and so is a `create or modify published odata service` AFTER the alter, which re-derives the types — before the alter it does not help. Measured on MxBuild 11.16.1. - **`check` rejects a primitive where a list operation needs an object or a list** (ako/mxcli#1258) — `$R = UNION $L WITH $S`, and the same for `INTERSECT`, `SUBTRACT`, `EQUALS` and `CONTAINS $S IN $L`, with a String, Integer, Long, Decimal, Boolean or DateTime variable as the operand that is not the list, passed `check` and `exec` and the build reported CE0023 "Selected variable '…' must be of type Object or List." New MDL101, in microflows and nanoflows. An Object or a List there is fine, `contains($Hay, $Needle)` over two Strings is still the string function, and an enumeration parameter is not judged (it reads as an entity type until the project says otherwise). Measured on MxBuild 11.16.1. diff --git a/cmd/mxcli/syntax/features_integration.go b/cmd/mxcli/syntax/features_integration.go index 760c76aed..9b9925910 100644 --- a/cmd/mxcli/syntax/features_integration.go +++ b/cmd/mxcli/syntax/features_integration.go @@ -907,7 +907,7 @@ DESCRIBE DATABASE CONNECTION Ops.Erp;`, " by calling this microflow. Parameter sources: parent (the enclosing\n" + " mapped object), parameter (the mapping's own input object),\n" + " parent(2) (an ancestor N levels up), or a member path (a value from\n" + - " the payload). modelsdk engine only.\n\n" + + " the payload). modelsdk engine only.\n The handler must name EXACTLY the microflow's parameters, by their own\n names (case-sensitive): an unknown name is CE1613, and leaving a parameter\n out is CE0277 (mxcli check: MDL104).\n\n" + "Value transform (Attr = Module.MF(jsonField)):\n" + " The value passes through a microflow on its way to the attribute.\n" + " The stored element carries only the microflow — its input IS the\n" + diff --git a/mdl-examples/bug-tests/mapping-handler-missing-parameter.fail.mdl b/mdl-examples/bug-tests/mapping-handler-missing-parameter.fail.mdl new file mode 100644 index 000000000..bc1687005 --- /dev/null +++ b/mdl-examples/bug-tests/mapping-handler-missing-parameter.fail.mdl @@ -0,0 +1,45 @@ +mdl 1; +-- NEGATIVE TEST — `mxcli check` must REFUSE this file (MDL104), with no project. +-- +-- An import mapping's custom object handler (`find … by M.MF ( … )`) that leaves a +-- parameter of the microflow out. `check` and `exec` accepted it, and the build +-- reported +-- +-- [CE0277] "Parameter mapping of microflow for obtaining objects needs to be refreshed." +-- +-- Measured on MxBuild 11.16.1: one of two parameters named, and none named on a +-- one-parameter microflow, are both CE0277; naming every parameter is clean, and so is +-- naming none on a microflow that has none (see mapping-handler-unknown-parameter.mdl). +-- Kept apart from the unknown-name fixture: CE1613 hides every other error. +create persistent entity MyFirstModule.HmFailRoot (Name: string(50)); +create persistent entity MyFirstModule.HmFailChild (Code: string(50)); +create association MyFirstModule.HmFailChild_HmFailRoot from MyFirstModule.HmFailChild to MyFirstModule.HmFailRoot type reference; +create json structure MyFirstModule.JS_HmFail snippet '{"name": "n", "items": [{"code": "c", "idx": 1}]}'; +create microflow MyFirstModule.HmFailTwo ($A: integer, $B: integer) returns MyFirstModule.HmFailChild begin + @start(100, 200) + @position(400, 200) + return empty; +end; +create microflow MyFirstModule.HmFailOne ($Index: integer) returns MyFirstModule.HmFailChild begin + @start(100, 200) + @position(400, 200) + return empty; +end; + +create import mapping MyFirstModule.IMM_HmFailOneOfTwo with json structure MyFirstModule.JS_HmFail { + create MyFirstModule.HmFailRoot { + Name = name, + find MyFirstModule.HmFailChild_HmFailRoot/MyFirstModule.HmFailChild by MyFirstModule.HmFailTwo ( A: idx ) = items { + Code = code + } + } +}; + +create import mapping MyFirstModule.IMM_HmFailNoneNamed with json structure MyFirstModule.JS_HmFail { + create MyFirstModule.HmFailRoot { + Name = name, + find MyFirstModule.HmFailChild_HmFailRoot/MyFirstModule.HmFailChild by MyFirstModule.HmFailOne () = items { + Code = code + } + } +}; diff --git a/mdl-examples/bug-tests/mapping-handler-unknown-parameter.fail.mdl b/mdl-examples/bug-tests/mapping-handler-unknown-parameter.fail.mdl new file mode 100644 index 000000000..d96ec3089 --- /dev/null +++ b/mdl-examples/bug-tests/mapping-handler-unknown-parameter.fail.mdl @@ -0,0 +1,40 @@ +mdl 1; +-- NEGATIVE TEST — `mxcli check` must REFUSE this file (MDL104), with no project. +-- +-- An import mapping's custom object handler (`find … by M.MF ( P: member )`) that +-- names a parameter the microflow does not declare. `check` and `exec` accepted it +-- and wrote a handler whose parameter points at nothing; the build reported +-- +-- [CE1613] "The selected parameter 'M.MF.P' no longer exists." +-- +-- Measured on MxBuild 11.16.1: an unknown name and the right name in another case +-- (names are case-sensitive) are both CE1613. A handler that leaves a parameter OUT is +-- CE0277 instead — see mapping-handler-missing-parameter.fail.mdl, kept apart because +-- CE1613 is an unresolved reference and hides every other error in the project. +create persistent entity MyFirstModule.HpFailRoot (Name: string(50)); +create persistent entity MyFirstModule.HpFailChild (Code: string(50)); +create association MyFirstModule.HpFailChild_HpFailRoot from MyFirstModule.HpFailChild to MyFirstModule.HpFailRoot type reference; +create json structure MyFirstModule.JS_HpFail snippet '{"name": "n", "items": [{"code": "c", "idx": 1}]}'; +create microflow MyFirstModule.HpFailByIndex ($Index: integer) returns MyFirstModule.HpFailChild begin + @start(100, 200) + @position(400, 200) + return empty; +end; + +create import mapping MyFirstModule.IMM_HpFailUnknown with json structure MyFirstModule.JS_HpFail { + create MyFirstModule.HpFailRoot { + Name = name, + find MyFirstModule.HpFailChild_HpFailRoot/MyFirstModule.HpFailChild by MyFirstModule.HpFailByIndex ( Nope: idx ) = items { + Code = code + } + } +}; + +create import mapping MyFirstModule.IMM_HpFailCase with json structure MyFirstModule.JS_HpFail { + create MyFirstModule.HpFailRoot { + Name = name, + find MyFirstModule.HpFailChild_HpFailRoot/MyFirstModule.HpFailChild by MyFirstModule.HpFailByIndex ( index: idx ) = items { + Code = code + } + } +}; diff --git a/mdl-examples/bug-tests/mapping-handler-unknown-parameter.mdl b/mdl-examples/bug-tests/mapping-handler-unknown-parameter.mdl new file mode 100644 index 000000000..c36f4c1ea --- /dev/null +++ b/mdl-examples/bug-tests/mapping-handler-unknown-parameter.mdl @@ -0,0 +1,37 @@ +mdl 1; +-- Control for mapping-handler-unknown-parameter.fail.mdl and +-- mapping-handler-missing-parameter.fail.mdl: a custom handler that names exactly the +-- microflow's parameters, and one on a microflow that has none. Builds with 0 errors +-- on MxBuild 11.16.1. +create persistent entity MyFirstModule.HpOkRoot (Name: string(50)); +create persistent entity MyFirstModule.HpOkChild (Code: string(50)); +create association MyFirstModule.HpOkChild_HpOkRoot from MyFirstModule.HpOkChild to MyFirstModule.HpOkRoot type reference; +create json structure MyFirstModule.JS_HpOk snippet '{"name": "n", "items": [{"code": "c", "idx": 1}]}'; +create microflow MyFirstModule.HpOkTwo ($A: integer, $B: integer) returns MyFirstModule.HpOkChild begin + @start(100, 200) + @position(400, 200) + return empty; +end; +create microflow MyFirstModule.HpOkNone () returns MyFirstModule.HpOkChild begin + @start(100, 200) + @position(400, 200) + return empty; +end; + +create import mapping MyFirstModule.IMM_HpOkBoth with json structure MyFirstModule.JS_HpOk { + create MyFirstModule.HpOkRoot { + Name = name, + find MyFirstModule.HpOkChild_HpOkRoot/MyFirstModule.HpOkChild by MyFirstModule.HpOkTwo ( A: idx, B: idx ) = items { + Code = code + } + } +}; + +create import mapping MyFirstModule.IMM_HpOkNone with json structure MyFirstModule.JS_HpOk { + create MyFirstModule.HpOkRoot { + Name = name, + find MyFirstModule.HpOkChild_HpOkRoot/MyFirstModule.HpOkChild by MyFirstModule.HpOkNone () = items { + Code = code + } + } +}; diff --git a/mdl/executor/validate_mapping_handler_params.go b/mdl/executor/validate_mapping_handler_params.go new file mode 100644 index 000000000..5c2a629ce --- /dev/null +++ b/mdl/executor/validate_mapping_handler_params.go @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "sort" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// mappingHandlerParamRule is the rule ID for "an import mapping's custom handler +// names a parameter its microflow does not have". +const mappingHandlerParamRule = "MDL104" + +// A custom object handler (`find … by M.MF ( P: member )`) must name exactly the +// microflow's parameters. Measured on MxBuild 11.16.1, ONE import mapping per +// project — a shared project is wrong, because CE1613 is an unresolved reference +// and hides every other error, which first made two of these look clean: +// +// the microflow's own parameter names, all of them clean +// a name it does not have CE1613 "The selected parameter 'M.MF.P' no longer exists." +// the right name in another case CE1613 (names are case-sensitive) +// a valid name plus an unknown one CE1613 +// one of two parameters left out CE0277 "Parameter mapping of microflow for obtaining objects needs to be refreshed." +// none named on a one-parameter microflow CE0277 +// none named on a microflow with no parameters clean +// a name on a microflow with no parameters CE1613 +// +// check and exec accepted every rejected case and wrote the mapping. The build +// reports CE1613 before CE0277, so a handler that has an unknown name is judged on +// that alone; the missing parameters are reported once the names are right. +// +// A microflow that cannot be found — not the script's, not the project's, or no +// project — is not judged: exec already refuses one it cannot find, and an +// unanswered question is not "no". A parameter's TYPE against the member it is +// fed is a different error (CE0282) and is not judged here. +// +// This is NOT CE0281 ("Unknown value element … selected for parameter"), which the +// same construct raises for a MEMBER that is not in the JSON structure: exec +// refuses that one itself (mapping_customhandler.go), so mx check never sees it. +func ValidateMappingHandlerParams(prog *ast.Program, open func() backend.FullBackend) []linter.Violation { + if prog == nil { + return nil + } + script := map[string]map[string]bool{} // lower-cased microflow QN -> its parameter names, exactly as written + for _, stmt := range prog.Statements { + if mf, ok := stmt.(*ast.CreateMicroflowStmt); ok { + names := map[string]bool{} + for _, p := range mf.Parameters { + names[p.Name] = true + } + script[strings.ToLower(mf.Name.String())] = names + } + } + + var project map[string]map[string]bool + loaded := false + projectParams := func(qn string) (map[string]bool, bool) { + if !loaded { + loaded = true + project = loadProjectMicroflowParams(open) + } + names, ok := project[strings.ToLower(qn)] + return names, ok + } + declared := func(qn string) (map[string]bool, bool) { + if names, ok := script[strings.ToLower(qn)]; ok { + return names, true + } + return projectParams(qn) + } + + var out []linter.Violation + for _, stmt := range prog.Statements { + m, ok := stmt.(*ast.CreateImportMappingStmt) + if !ok || m.RootElement == nil { + continue + } + var walk func(e *ast.ImportMappingElementDef) + walk = func(e *ast.ImportMappingElementDef) { + if h := e.CustomHandler; h != nil && h.Microflow != "" { + qn := qualifyIn(m.Name.Module, h.Microflow) + if names, known := declared(qn); known { + out = append(out, handlerParamViolations(m.Name, h, qn, names)...) + } + } + for _, c := range e.Children { + if c != nil { + walk(c) + } + } + } + walk(m.RootElement) + } + return out +} + +// loadProjectMicroflowParams reads every project microflow's parameter names, +// keyed by lower-cased qualified name. Nothing is returned when no project can +// be read, which leaves every project microflow unjudged. +func loadProjectMicroflowParams(open func() backend.FullBackend) map[string]map[string]bool { + out := map[string]map[string]bool{} + if open == nil { + return out + } + b := open() + if b == nil { + return out + } + modules, err := b.ListModules() + if err != nil { + return out + } + names := map[string]string{} + for _, mod := range modules { + if mod != nil { + names[string(mod.ID)] = mod.Name + } + } + mfs, err := b.ListMicroflows() + if err != nil { + return out + } + for _, mf := range mfs { + mod := names[string(mf.ContainerID)] + if mf == nil || mod == "" { + continue + } + params := map[string]bool{} + for _, p := range mf.Parameters { + if p != nil { + params[p.Name] = true + } + } + out[strings.ToLower(mod+"."+mf.Name)] = params + } + return out +} + +// handlerParamList is a microflow's parameter names, sorted, for the suggestion. +func handlerParamList(m map[string]bool) string { + if len(m) == 0 { + return "(it declares none)" + } + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return strings.Join(keys, ", ") +} + +// handlerParamViolations judges one handler against its microflow's parameters. +func handlerParamViolations(mapping ast.QualifiedName, h *ast.MappingCustomHandlerDef, qn string, declared map[string]bool) []linter.Violation { + loc := linter.Location{Module: mapping.Module, DocumentType: "import mapping", DocumentName: mapping.Name} + var out []linter.Violation + named := map[string]bool{} + for _, p := range h.Parameters { + if p == nil { + continue + } + named[p.Parameter] = true + if declared[p.Parameter] { + continue + } + out = append(out, linter.Violation{ + RuleID: mappingHandlerParamRule, + Severity: linter.SeverityError, + Message: fmt.Sprintf( + "import mapping %s: the handler `by %s` names a parameter %q, which %s does not declare — "+ + "mxbuild reports CE1613 \"The selected parameter '%s.%s' no longer exists\" "+ + "(parameter names are case-sensitive)", + mapping.String(), h.Microflow, p.Parameter, qn, qn, p.Parameter), + Location: loc, + Suggestion: fmt.Sprintf("Use %s's own parameter names: %s.", qn, handlerParamList(declared)), + }) + } + if len(out) > 0 { + return out // CE1613 comes first; the missing ones are reported once the names are right + } + var missing []string + for name := range declared { + if !named[name] { + missing = append(missing, name) + } + } + if len(missing) == 0 { + return nil + } + sort.Strings(missing) + return []linter.Violation{{ + RuleID: mappingHandlerParamRule, + Severity: linter.SeverityError, + Message: fmt.Sprintf( + "import mapping %s: the handler `by %s` does not name %s of %s's parameters — mxbuild reports "+ + "CE0277 \"Parameter mapping of microflow for obtaining objects needs to be refreshed\"", + mapping.String(), h.Microflow, strings.Join(missing, ", "), qn), + Location: loc, + Suggestion: fmt.Sprintf("Name every parameter of %s: %s (each as `Name: source`).", qn, handlerParamList(declared)), + }} +} diff --git a/mdl/executor/validate_mapping_handler_params_test.go b/mdl/executor/validate_mapping_handler_params_test.go new file mode 100644 index 000000000..3ef34ea6e --- /dev/null +++ b/mdl/executor/validate_mapping_handler_params_test.go @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// A custom object handler (`find … by M.MF ( P: member )`) must name exactly the +// microflow's parameters. Measured on MxBuild 11.16.1, ONE import mapping per +// project (a shared project is wrong: CE1613 is an unresolved reference and hides +// every other error, which first made two of these look clean): +// +// the microflow's own parameter names, all of them clean +// a name it does not have (Nope) CE1613 <- check and exec accepted it +// the right name in another case (index) CE1613 (names are case-sensitive) +// a valid name plus an unknown one (Extra) CE1613 +// one of two parameters left out CE0277 "needs to be refreshed" +// none named on a one-parameter microflow CE0277 +// none named on a microflow with no parameters clean +// a name on a microflow with no parameters CE1613 +// +// `check` and `exec` accepted every rejected case and wrote the mapping. The +// build reports CE1613 before CE0277, so a handler with an unknown name is judged +// on that alone. +const handlerParamsScript = `mdl 1; +create module Hp; +create persistent entity Hp.Root (Name: string(50)); +create persistent entity Hp.Child (Code: string(50)); +create association Hp.Child_Root from Hp.Child to Hp.Root type reference; +create json structure Hp.JS sample '{"name": "n", "items": [{"code": "c", "idx": 1}]}'; +create microflow Hp.ByIndex ($Index: integer) returns Hp.Child begin + return empty; +end; +create microflow Hp.Two ($A: integer, $B: integer) returns Hp.Child begin + return empty; +end; +create microflow Hp.NoParams () returns Hp.Child begin + return empty; +end; +` + +func handlerMapping(mf, params string) string { + return handlerParamsScript + `create import mapping Hp.IMM with json structure Hp.JS { + create Hp.Root { + Name = name, + find Hp.Child_Root/Hp.Child by ` + mf + ` ( ` + params + ` ) = items { Code = code } + } +}; +` +} + +func TestValidateMappingHandlerParams_Script(t *testing.T) { + cases := []struct { + name, mf, params string + want int + }{ + {"the microflow's own parameter (control)", "Hp.ByIndex", "Index: idx", 0}, + {"a name the microflow does not have", "Hp.ByIndex", "Nope: idx", 1}, + {"the right name in another case", "Hp.ByIndex", "index: idx", 1}, + {"a valid name plus an unknown one", "Hp.ByIndex", "Index: idx, Extra: idx", 1}, + {"one of two parameters left out", "Hp.Two", "A: idx", 1}, + {"none named on a one-parameter microflow", "Hp.ByIndex", "", 1}, + {"none named on a microflow with no parameters (control)", "Hp.NoParams", "", 0}, + {"a name on a microflow with no parameters", "Hp.NoParams", "Extra: idx", 1}, + {"an unknown name AND a missing parameter: judged on the unknown name alone", "Hp.Two", "Nope: idx", 1}, + {"both parameters named (control)", "Hp.Two", "A: idx, B: idx", 0}, + {"two wrong names, two violations", "Hp.Two", "X: idx, Y: idx", 2}, + {"a microflow that is not known here (control)", "Other.Elsewhere", "Whatever: idx", 0}, + {"parent source judged by name too", "Hp.ByIndex", "Nope: parent", 1}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := ValidateMappingHandlerParams(mustBuild(t, handlerMapping(c.mf, c.params)), nil) + if len(got) != c.want { + t.Fatalf("violations=%v, want %d", rulesOf(got), c.want) + } + if c.want > 0 { + v := got[0] + wantCode := "CE1613" + if c.name == "one of two parameters left out" || c.name == "none named on a one-parameter microflow" { + wantCode = "CE0277" + } + if v.RuleID != mappingHandlerParamRule || !strings.Contains(v.Message, wantCode) || + !strings.Contains(v.Message, c.mf) || v.Location.DocumentName != "IMM" { + t.Errorf("rule, CE code, microflow or location wrong: %+v", v) + } + } + }) + } +} + +// The handler on a nested element and on the item of a two-level array is the +// same handler. +func TestValidateMappingHandlerParams_NestedElements(t *testing.T) { + src := handlerParamsScript + `create import mapping Hp.IMM2 with json structure Hp.JS { + create Hp.Root { + Name = name, + create Hp.Child_Root/Hp.Child = items { + Code = code, + find Hp.Child_Root/Hp.Child by Hp.ByIndex ( Nope: idx ) = meta { Code = code } + } + } +}; +` + if got := ValidateMappingHandlerParams(mustBuild(t, src), nil); len(got) != 1 { + t.Errorf("a handler two levels down: want one violation, got %v", rulesOf(got)) + } +} + +// The microflow lives in the project, not the script. +func TestValidateMappingHandlerParams_ProjectMicroflow(t *testing.T) { + mod := &model.Module{BaseElement: model.BaseElement{ID: "mod-pj"}, Name: "Pj"} + mf := µflows.Microflow{ + BaseElement: model.BaseElement{ID: "mf1"}, ContainerID: mod.ID, Name: "Resolve", + Parameters: []*microflows.MicroflowParameter{{Name: "Index"}}, + } + mb := &mock.MockBackend{ + IsConnectedFunc: func() bool { return true }, + ListModulesFunc: func() ([]*model.Module, error) { return []*model.Module{mod}, nil }, + ListMicroflowsFunc: func() ([]*microflows.Microflow, error) { return []*microflows.Microflow{mf}, nil }, + } + open := func() backend.FullBackend { return mb } + src := func(params string) string { + return `mdl 1; +create import mapping Pj.IMM with json structure Pj.JS { + create Pj.Root { + Name = name, + find Pj.Child_Root/Pj.Child by Pj.Resolve ( ` + params + ` ) = items { Code = code } + } +}; +` + } + if got := ValidateMappingHandlerParams(mustBuild(t, src("Nope: idx")), open); len(got) != 1 { + t.Errorf("an unknown parameter of a project microflow: want one violation, got %v", rulesOf(got)) + } + if got := ValidateMappingHandlerParams(mustBuild(t, src("")), open); len(got) != 1 || !strings.Contains(got[0].Message, "CE0277") { + t.Errorf("a project microflow's parameter left unnamed: want one CE0277, got %v", rulesOf(got)) + } + if got := ValidateMappingHandlerParams(mustBuild(t, src("Index: idx")), open); len(got) != 0 { + t.Errorf("control: the project microflow's own parameter was reported: %v", rulesOf(got)) + } + if got := ValidateMappingHandlerParams(mustBuild(t, src("Nope: idx")), nil); len(got) != 0 { + t.Errorf("no project: a microflow nothing knows must not be judged: %v", rulesOf(got)) + } +} diff --git a/mdl/executor/validate_program.go b/mdl/executor/validate_program.go index 33ca539a2..cfa9cded4 100644 --- a/mdl/executor/validate_program.go +++ b/mdl/executor/validate_program.go @@ -423,6 +423,8 @@ func ValidateProgram(prog *ast.Program, projectPath string) []linter.Violation { violations = append(violations, ValidateImportMappingRange(prog, projectPath)...) // A REST call declared to return a list from a mapping that returns one object (CE0243). violations = append(violations, ValidateRestMappingResult(prog, projectPath)...) + // A custom object handler naming a parameter its microflow does not declare (CE1613). + violations = append(violations, ValidateMappingHandlerParams(prog, project)...) // Flag a REST client operation whose Body/Response mapping clause has no // `{ ... }` body — Mendix cannot reference a mapping document from an From 8da4ec98ebc47f76dccf9cdd13c68840a3ca52c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:22:31 +0000 Subject: [PATCH 08/28] fix(marketplace): compare against a reference with widget definitions and install's repairs The reference cache dropped widgets/, so a cached reference described pluggable widgets generically, and the reference never got the widget and design-property repairs install runs on the app. Both read as local edits of an untouched module. The cache now keeps widgets/ (format f3) and a freshly built reference is repaired before it is cached. Closes #2367 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011oK82qrr1WnXxq1eRn7qRJ --- ...rketplace-diff-widget-false-positives.json | 1 + CHANGELOG.md | 1 + cmd/mxcli/cmd_marketplace_diff.go | 5 ++ cmd/mxcli/marketplace/refcache.go | 10 +++- .../marketplace/refcache_widgets_2367_test.go | 35 ++++++++++++++ cmd/mxcli/marketplace_postinstall.go | 19 ++++++++ .../marketplace_reference_repair_2367_test.go | 46 +++++++++++++++++++ 7 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-marketplace-diff-widget-false-positives.json create mode 100644 cmd/mxcli/marketplace/refcache_widgets_2367_test.go create mode 100644 cmd/mxcli/marketplace_reference_repair_2367_test.go diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-marketplace-diff-widget-false-positives.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-marketplace-diff-widget-false-positives.json new file mode 100644 index 000000000..958b87bd5 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-18-marketplace-diff-widget-false-positives.json @@ -0,0 +1 @@ +{"area": "cmd-mxcli", "date": "2026-11-18", "refs": ["ako/mxcli#2367", "mendixlabs/mxcli#1882"], "ce": [], "symptom": "Right after `marketplace install` of Workflow Commons 4.10.0, `marketplace diff` reported 2 snippets as locally changed on the run that built the reference and 8 on every cached run (all with pluggable widgets: Timeline, Pop-up menu); `update` then refused without --force and --save-edits saved them as edits.", "file": "cmd/mxcli/marketplace/refcache.go isModelFile (+widgets/) and refEntryFormat f3; cmd/mxcli/marketplace_postinstall.go repairReference; cmd/mxcli/cmd_marketplace_diff.go referenceFor. Tests: cmd/mxcli/marketplace/refcache_widgets_2367_test.go, cmd/mxcli/marketplace_reference_repair_2367_test.go", "cause": "Two differences between the app and its baseline: (1) the reference cache kept the model and action sources but not widgets/, so a cached reference described pluggable widgets generically (`pluggablewidget '…Timeline' …` with the definition's defaults) while the app described `timeline …`; (2) install runs mx update-widgets / rename-design-properties on the app, never on the reference, so the repair's rewrites of stored props read as edits even on a fresh build.", "insight": "**A baseline has to go through every step the subject went through, and its cache has to keep every input DESCRIBE reads.** The 2-vs-8 split between the first and cached runs was the tell: two causes, one in the build and one in the cache. Measured on 11.12.6 with Workflow Commons 4.10.0: untouched install → fresh reference 0 changed (was 2), cached 0 (was 8); with one real `alter snippet` edit → exactly that snippet."} diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bf0acefa..533f3bea6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`marketplace diff` and `update` no longer report untouched pluggable-widget snippets as edits** (ako/mxcli#2367, mendixlabs/mxcli#1882) — right after an install, Workflow Commons showed 2 changed snippets on the first `diff` and 8 on every later one, so `update` needed `--force`, which also dropped the guard for real edits. The reference cache now keeps the widget definitions, and a reference gets the same widget and design-property repairs install runs on the app. Measured: 0 false edits on both runs, and a real edit is still reported. Cached references from earlier versions are rebuilt once. - **An imported navigation association that would clash is named `_`** (ako/mxcli#2354) — `create external entities` named Product's `Category` navigation `Category_2`, because the name collides with the entity Category. It is now `Product_Category`, with a number only if that is taken too. Names that do not collide are unchanged, and a re-import still recognises existing associations by their navigation property. - **`describe contract operation from openapi` shows query parameters as they are imported** (ako/mxcli#2353) — the preview showed an integer query parameter as `Integer` while the imported service described it as `String`. Mendix stores no type for a consumed REST query parameter (it is text), so the preview now shows `String`, matching the import; path parameters keep the spec's type. - **`diag loop-report` measures `run status`, `stop`, `wait` and `restart`** (ako/mxcli#2360) — these commands exited through `os.Exit`, which skipped the session close even on success, so every call was counted as a vanished process with no time and the hours spent waiting for and restarting the app were invisible. They now close their session with the exit code on every exit. diff --git a/cmd/mxcli/cmd_marketplace_diff.go b/cmd/mxcli/cmd_marketplace_diff.go index 6cc60b4e2..7f4c997dc 100644 --- a/cmd/mxcli/cmd_marketplace_diff.go +++ b/cmd/mxcli/cmd_marketplace_diff.go @@ -242,6 +242,11 @@ func referenceFor(ctx context.Context, client *mp.Client, v *mp.Version, if err != nil { return "", "", "", err } + // The app's copy of the module was repaired by install; the reference + // has to be too, or the repair's rewrites read as edits (ako/mxcli#2367). + if err := repairReference(mprPath, mendixVersion); err != nil { + return "", "", "", err + } marketplace.CacheReference(v.VersionID, mendixVersion, refDir, mpkPath) return mprPath, pkgModule, mpkPath, nil } diff --git a/cmd/mxcli/marketplace/refcache.go b/cmd/mxcli/marketplace/refcache.go index 22502fe4d..9f2450dff 100644 --- a/cmd/mxcli/marketplace/refcache.go +++ b/cmd/mxcli/marketplace/refcache.go @@ -96,7 +96,10 @@ func refCacheDir(versionID, mendixVersion string) (string, error) { // "f2" added javascriptsource/ and javasource/ (mendixlabs/mxcli#1441). Entries // with no prefix were model-only; they sit beside the new ones under ref/, are // never looked up again, and age out through pruneRefCache. -const refEntryFormat = "f2" +// +// "f3" added widgets/, and a reference is now widget-repaired before it is +// cached (ako/mxcli#2367). +const refEntryFormat = "f3" // safeKey makes a path component out of a version string or UUID. Anything that // is not plainly alphanumeric becomes '-', so a malformed version from the API @@ -303,7 +306,10 @@ func publishToCache(buildDir, cacheDir string) error { // but a reference built from a protected module's tree (moduletree.go) skips // module-import and keeps whatever format the blank project was created in. func isModelFile(rel string) bool { - for _, dir := range []string{"mprcontents", "javascriptsource", "javasource"} { + // widgets/ holds the pluggable widget definitions DESCRIBE renders a + // widget by; without them it falls back to the generic form, and every + // snippet with a pluggable widget read as edited (ako/mxcli#2367). + for _, dir := range []string{"mprcontents", "javascriptsource", "javasource", "widgets"} { if rel == dir || strings.HasPrefix(rel, dir+string(filepath.Separator)) { return true } diff --git a/cmd/mxcli/marketplace/refcache_widgets_2367_test.go b/cmd/mxcli/marketplace/refcache_widgets_2367_test.go new file mode 100644 index 000000000..27ade4fcc --- /dev/null +++ b/cmd/mxcli/marketplace/refcache_widgets_2367_test.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 + +package marketplace + +import ( + "path/filepath" + "testing" +) + +// ako/mxcli#2367 (mendixlabs/mxcli#1882): a cached reference held the model +// but not widgets/, so DESCRIBE rendered its pluggable widgets in the generic +// form (`pluggablewidget 'com.mendix.widget.web.timeline.Timeline' …`, with the +// definition's defaults) while the app, which has the definitions, rendered +// `timeline …` — and `marketplace diff` reported every such snippet of an +// untouched module as locally changed. +func TestIsModelFile_KeepsWidgetDefinitions_Issue2367(t *testing.T) { + for _, rel := range []string{ + filepath.Join("widgets", "com.mendix.widget.web.Timeline.mpk"), + "widgets", + } { + if !isModelFile(rel) { + t.Errorf("isModelFile(%q) = false: DESCRIBE needs the widget definitions", rel) + } + } + // CONTROL: the rest of the project stays out of the cache. + for _, rel := range []string{filepath.Join("theme", "web", "main.scss"), filepath.Join("deployment", "x")} { + if isModelFile(rel) { + t.Errorf("isModelFile(%q) = true", rel) + } + } + // An entry written without widgets/ must not be served again. + if refEntryFormat == "f2" { + t.Error("refEntryFormat still f2: entries cached without widgets/ would keep being served") + } +} diff --git a/cmd/mxcli/marketplace_postinstall.go b/cmd/mxcli/marketplace_postinstall.go index 61e4a7bab..bb58bd1f1 100644 --- a/cmd/mxcli/marketplace_postinstall.go +++ b/cmd/mxcli/marketplace_postinstall.go @@ -104,3 +104,22 @@ func printManualRepairs(out io.Writer, mprPath string, repairs []postInstallRepa } fmt.Fprintf(out, " mxcli docker check -p %s\n", mprPath) } + +// repairReference runs on a freshly built reference project the repairs a +// module install runs on the app, so `marketplace diff` compares like with +// like. Without them every snippet whose stored widgets the repair rewrites +// read as a local edit of an untouched module (ako/mxcli#2367). A failure is +// an error: a half-repaired reference reports false edits, and must not be +// cached. +func repairReference(mprPath, mendixVer string) error { + mxPath, err := postInstallResolveMx(mendixVer) + if err != nil { + return fmt.Errorf("locate mx for Mendix %s to repair the reference: %w", mendixVer, err) + } + for _, r := range moduleRepairs { + if _, err := postInstallRunTool(mxPath, mprPath, r.subcommand, io.Discard, io.Discard); err != nil { + return fmt.Errorf("repair the reference project (%s, as install does on the app): %w", r.subcommand, err) + } + } + return nil +} diff --git a/cmd/mxcli/marketplace_reference_repair_2367_test.go b/cmd/mxcli/marketplace_reference_repair_2367_test.go new file mode 100644 index 000000000..c3f2e39b1 --- /dev/null +++ b/cmd/mxcli/marketplace_reference_repair_2367_test.go @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "errors" + "io" + "slices" + "testing" + + "github.com/mendixlabs/mxcli/cmd/mxcli/docker" +) + +// ako/mxcli#2367: `marketplace install` repairs the app it writes into (mx +// update-widgets, rename-design-properties), but the reference `diff` compares +// against was built without them, so snippets whose stored widget props the +// repair rewrites (`popupmenu … clippingStrategy: 'absolute'`) read as edits. +// The reference gets the same repairs, in the same order. +func TestRepairReference_RunsTheInstallRepairs_Issue2367(t *testing.T) { + var ran []string + defer func(r func(string) (string, error), t func(string, string, string, io.Writer, io.Writer) (*docker.HarvestResult, error)) { + postInstallResolveMx, postInstallRunTool = r, t + }(postInstallResolveMx, postInstallRunTool) + postInstallResolveMx = func(string) (string, error) { return "/mx", nil } + postInstallRunTool = func(_, mpr, sub string, _, _ io.Writer) (*docker.HarvestResult, error) { + ran = append(ran, sub+" "+mpr) + return nil, nil + } + + if err := repairReference("/ref/PackageRef.mpr", "11.12.6"); err != nil { + t.Fatal(err) + } + want := []string{"update-widgets /ref/PackageRef.mpr", "rename-design-properties /ref/PackageRef.mpr"} + if !slices.Equal(ran, want) { + t.Errorf("ran %v, want %v (what install runs on the app)", ran, want) + } + + // A repair that fails leaves a reference that would report false edits: + // it is an error, so the reference is not cached. + postInstallRunTool = func(_, _, _ string, _, _ io.Writer) (*docker.HarvestResult, error) { + return nil, errors.New("boom") + } + if err := repairReference("/ref/PackageRef.mpr", "11.12.6"); err == nil { + t.Error("a failed repair was not reported") + } +} From 654fd524d4f925490023d5e52ba4b8890514e113 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:33:01 +0000 Subject: [PATCH 09/28] fixtures: write json structure samples as `sample`, not the deprecated `snippet` Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011oK82tETa46YQmMS58pQjL --- .../bug-tests/mapping-handler-missing-parameter.fail.mdl | 2 +- .../bug-tests/mapping-handler-unknown-parameter.fail.mdl | 2 +- mdl-examples/bug-tests/mapping-handler-unknown-parameter.mdl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mdl-examples/bug-tests/mapping-handler-missing-parameter.fail.mdl b/mdl-examples/bug-tests/mapping-handler-missing-parameter.fail.mdl index bc1687005..249f7a356 100644 --- a/mdl-examples/bug-tests/mapping-handler-missing-parameter.fail.mdl +++ b/mdl-examples/bug-tests/mapping-handler-missing-parameter.fail.mdl @@ -14,7 +14,7 @@ mdl 1; create persistent entity MyFirstModule.HmFailRoot (Name: string(50)); create persistent entity MyFirstModule.HmFailChild (Code: string(50)); create association MyFirstModule.HmFailChild_HmFailRoot from MyFirstModule.HmFailChild to MyFirstModule.HmFailRoot type reference; -create json structure MyFirstModule.JS_HmFail snippet '{"name": "n", "items": [{"code": "c", "idx": 1}]}'; +create json structure MyFirstModule.JS_HmFail sample '{"name": "n", "items": [{"code": "c", "idx": 1}]}'; create microflow MyFirstModule.HmFailTwo ($A: integer, $B: integer) returns MyFirstModule.HmFailChild begin @start(100, 200) @position(400, 200) diff --git a/mdl-examples/bug-tests/mapping-handler-unknown-parameter.fail.mdl b/mdl-examples/bug-tests/mapping-handler-unknown-parameter.fail.mdl index d96ec3089..57826e0a4 100644 --- a/mdl-examples/bug-tests/mapping-handler-unknown-parameter.fail.mdl +++ b/mdl-examples/bug-tests/mapping-handler-unknown-parameter.fail.mdl @@ -14,7 +14,7 @@ mdl 1; create persistent entity MyFirstModule.HpFailRoot (Name: string(50)); create persistent entity MyFirstModule.HpFailChild (Code: string(50)); create association MyFirstModule.HpFailChild_HpFailRoot from MyFirstModule.HpFailChild to MyFirstModule.HpFailRoot type reference; -create json structure MyFirstModule.JS_HpFail snippet '{"name": "n", "items": [{"code": "c", "idx": 1}]}'; +create json structure MyFirstModule.JS_HpFail sample '{"name": "n", "items": [{"code": "c", "idx": 1}]}'; create microflow MyFirstModule.HpFailByIndex ($Index: integer) returns MyFirstModule.HpFailChild begin @start(100, 200) @position(400, 200) diff --git a/mdl-examples/bug-tests/mapping-handler-unknown-parameter.mdl b/mdl-examples/bug-tests/mapping-handler-unknown-parameter.mdl index c36f4c1ea..9eef0c18b 100644 --- a/mdl-examples/bug-tests/mapping-handler-unknown-parameter.mdl +++ b/mdl-examples/bug-tests/mapping-handler-unknown-parameter.mdl @@ -6,7 +6,7 @@ mdl 1; create persistent entity MyFirstModule.HpOkRoot (Name: string(50)); create persistent entity MyFirstModule.HpOkChild (Code: string(50)); create association MyFirstModule.HpOkChild_HpOkRoot from MyFirstModule.HpOkChild to MyFirstModule.HpOkRoot type reference; -create json structure MyFirstModule.JS_HpOk snippet '{"name": "n", "items": [{"code": "c", "idx": 1}]}'; +create json structure MyFirstModule.JS_HpOk sample '{"name": "n", "items": [{"code": "c", "idx": 1}]}'; create microflow MyFirstModule.HpOkTwo ($A: integer, $B: integer) returns MyFirstModule.HpOkChild begin @start(100, 200) @position(400, 200) From 3173c68416978114e4b4be83b034898672621afb Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 10:44:41 +0000 Subject: [PATCH 10/28] test: follow the reference cache keeping widgets/ and the _ name TestModelOnlyCopyKeepsWhatIsRead pinned widgets/ out of the reference cache, which #2367 keeps on purpose; the splice test named the colliding Unit navigation Unit_2, which #2354 now names QuantityParameterType_Unit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011oK82qrr1WnXxq1eRn7qRJ --- cmd/mxcli/marketplace/refcache_test.go | 12 ++++++++---- .../external_action_include_splice_pedapp_test.go | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/mxcli/marketplace/refcache_test.go b/cmd/mxcli/marketplace/refcache_test.go index 1c5f1dce1..bc3451136 100644 --- a/cmd/mxcli/marketplace/refcache_test.go +++ b/cmd/mxcli/marketplace/refcache_test.go @@ -281,9 +281,10 @@ func TestPruneDisabledByZero(t *testing.T) { // javascriptsource/ and javasource/ are read by DESCRIBE; everything else in a // reference project is bulk nothing reads. // -// The negative half matters as much as the positive: if widgets/ started being -// copied again the cache would quietly double in size, and if the .mpr stopped -// being copied every hit would serve an unreadable reference. +// The negative half matters as much as the positive: if the theme trees started +// being copied the cache would quietly grow, and if the .mpr stopped being +// copied every hit would serve an unreadable reference. widgets/ is kept: its +// definitions decide how DESCRIBE renders a pluggable widget (ako/mxcli#2367). func TestModelOnlyCopyKeepsWhatIsRead(t *testing.T) { src, dst := t.TempDir(), filepath.Join(t.TempDir(), "out") @@ -313,7 +314,10 @@ func TestModelOnlyCopyKeepsWhatIsRead(t *testing.T) { if got := readFile(t, filepath.Join(dst, "javasource", "mod", "actions", "A.java")); got != "java" { t.Errorf("javasource action = %q, want %q", got, "java") } - for _, bulk := range []string{"widgets", "themesource", "theme-cache"} { + if got := readFile(t, filepath.Join(dst, "widgets", "big.mpk")); got != "bulk" { + t.Errorf("widgets/ definition = %q, want it kept (ako/mxcli#2367)", got) + } + for _, bulk := range []string{"themesource", "theme-cache"} { if _, err := os.Stat(filepath.Join(dst, bulk)); !os.IsNotExist(err) { t.Errorf("%s/ was copied into the cache entry; nothing reads it", bulk) } diff --git a/mdl/executor/external_action_include_splice_pedapp_test.go b/mdl/executor/external_action_include_splice_pedapp_test.go index 6e2dd4005..3bc9f40d5 100644 --- a/mdl/executor/external_action_include_splice_pedapp_test.go +++ b/mdl/executor/external_action_include_splice_pedapp_test.go @@ -42,7 +42,7 @@ end;` } return out.String() } - const tree = " include associations (MyFirstModule.Quantity (MyFirstModule.Unit_2))" + const tree = " include associations (MyFirstModule.Quantity (MyFirstModule.QuantityParameterType_Unit))" if err := agreeExec(t, exec, flow(tree)); err != nil { t.Fatalf("create: %v\n%s", err, out.String()) } From edb32b25d7181243d8094d51dc7bdbf69e6ab5cf Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 11:06:00 +0000 Subject: [PATCH 11/28] fix(modelsdk): CDC baseline GUIDs read back empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part C of #2384 begins with its prerequisite: the vendored gen cannot read a change data capture service. Measured on ako/TestApp 619864b (Mendix 11.16.1), 8 of 8 GUIDs in the baseline came back as the empty string — EntityGuid on each source, PropertyGuid and AssociatedEntityGuid on each property — while everything else decoded correctly, so nothing looked wrong unless the GUIDs were printed. Those GUIDs are the DATABASE's identity for the tracked entity and its members: mendixsystem$entity.id and mendixsystem$attribute.id hold them verbatim. A write path built on an empty read would detach every tracked entity from its table — the failure class CLAUDE.md puts first, and one that no mx check and no build reports. Cause: the gen typed EntityGuid/PropertyGuid as Primitive[string] with DecodeString, but Mendix stores every GUID as BSON binary subtype 0, and DecodeString does `s, _ := val.StringValueOK()` — discarding the ok flag, so a binary value silently yields "". They are now ByIdRef[element.Element], decoded the way domainmodels already decodes ParentPointer (StringValueOK, then BinaryOK + codec.BinaryToUUID). ByIdRef also encodes correctly, since the codec turns a non-empty element.ID into binary. The accessors keep their string signatures, so no caller changes. #2384 §4 listed four properties missing from the gen outright; all four are added here with sequential Bind indices and SetProperties entries: EntityChangeDataCaptureService AppName EntityChangeSource VersionedSourcePointer Attribute/AssociationProperty VersionedPropertyPointer VersionedProperty AssociatedEntityGuid The fifth defect was in a property that was already present, which is why the schema diff behind §4 did not find it — the values had to be printed. With this the fixture reads completely: AppName=TestApp, every GUID resolved, Order_Customer's AssociatedEntityGuid equal to Customer's EntityGuid (confirming §4's claim that it is the TO entity), and every live->baseline pointer intact. types.go is generated and says DO NOT EDIT, but the generator is not in-tree, so handadded_test.go guards the additions: a regeneration that drops them fails loudly instead of quietly returning "". Control: restoring DecodeString makes each test fail with `= ""`, the reported symptom. The whole modelsdk suite, including the codec round-trip guards, stays green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011oK82qQLKUyoXXGmbmX68t --- ...18-cdc-baseline-guids-read-back-empty.json | 1 + .../gen/changedatacapture/handadded_test.go | 176 ++++++++++++++++++ modelsdk/gen/changedatacapture/types.go | 173 +++++++++++++++-- 3 files changed, 332 insertions(+), 18 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/modelsdk/2026-11-18-cdc-baseline-guids-read-back-empty.json create mode 100644 modelsdk/gen/changedatacapture/handadded_test.go diff --git a/.claude/skills/fix-issue/findings/modelsdk/2026-11-18-cdc-baseline-guids-read-back-empty.json b/.claude/skills/fix-issue/findings/modelsdk/2026-11-18-cdc-baseline-guids-read-back-empty.json new file mode 100644 index 000000000..8116688c4 --- /dev/null +++ b/.claude/skills/fix-issue/findings/modelsdk/2026-11-18-cdc-baseline-guids-read-back-empty.json @@ -0,0 +1 @@ +{"area":"modelsdk/gen","date":"2026-11-18","symptom":"Every GUID in a ChangeDataCapture$EntityChangeDataCaptureService read back as the empty string: EntityGuid on each baseline source, PropertyGuid and AssociatedEntityGuid on each baseline property. The rest of the document decoded fine — exposed names, Type enums, the live sources, the entity references — so the failure was invisible except by printing the GUIDs. Measured on ako/TestApp 619864b (Mendix 11.16.1): 8 of 8 GUID fields empty.","cause":"The vendored gen typed EntityGuid/PropertyGuid as property.Primitive[string] with property.DecodeString, but Mendix stores every GUID as BSON binary subtype 0. DecodeString does `s, _ := val.StringValueOK()` and DISCARDS the ok flag, so a binary value yields \"\" with no error. Four further properties Studio Pro writes were absent from the gen entirely (AppName, VersionedSourcePointer, VersionedPropertyPointer, AssociatedEntityGuid) because they are not in the vendored reflection data.","file":"modelsdk/gen/changedatacapture/types.go (hand-edited: the generator is not in-tree); guard tests in modelsdk/gen/changedatacapture/handadded_test.go","fix":"Type every binary GUID as property.ByIdRef[element.Element] and decode it the way domainmodels decodes ParentPointer — StringValueOK first, then BinaryOK + codec.BinaryToUUID. ByIdRef also encodes correctly: the codec turns a non-empty element.ID into binary via idToBinary. Accessors keep their string signatures (EntityGuid() string) so no caller changes. The four missing properties are added to the struct, the accessors, init (sequential Bind index + SetProperties) and InitFromRaw.","test":"modelsdk/gen/changedatacapture/handadded_test.go: binary decode per GUID field, the live->baseline pointers, AppName, and a round-trip pinning codec.BinaryToUUID against mpr.IDToBsonBinary on the fixture's real GUIDs. Control: restoring DecodeString makes each fail with `= \"\"`, which is the reported symptom.","insight":"A decode helper that drops its ok flag turns a type mismatch into a plausible empty value, and an empty GUID is indistinguishable from an unset one — nothing downstream can tell. Grep gen for Primitive[string] on any property whose name ends in Guid before trusting a read. The subject matters as much as the field: only a Studio Pro-authored document has GUIDs that differ from $ID, so an mxcli-created one would have shown nothing wrong. And the proposal that mapped this document listed FOUR missing properties from the HAM schema; the fifth defect was in a property that was present, which is why a schema diff alone was not enough — the values had to be printed.","refs":["ako/mxcli#2384","docs-wiki/bug-patterns/rewrite-drops-unauthored-state.md","CLAUDE.md#a-guid-is-the-databases-identity"]} diff --git a/modelsdk/gen/changedatacapture/handadded_test.go b/modelsdk/gen/changedatacapture/handadded_test.go new file mode 100644 index 000000000..c8d1a62b1 --- /dev/null +++ b/modelsdk/gen/changedatacapture/handadded_test.go @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: Apache-2.0 + +package changedatacapture + +import ( + "testing" + + "github.com/mendixlabs/mxcli/modelsdk/codec" + "github.com/mendixlabs/mxcli/modelsdk/element" + mmpr "github.com/mendixlabs/mxcli/modelsdk/mpr" + "go.mongodb.org/mongo-driver/v2/bson" +) + +// This file guards the properties added to types.go by hand (ako/mxcli#2384 §4). +// +// types.go is generated and says DO NOT EDIT, but the generator is not in-tree, +// so the additions live in the checked-in file. A regeneration — or anyone +// "restoring" the file — drops them silently: the document still decodes, the +// structure still looks right, and only the GUIDs are gone. These tests make +// that loud. + +const ( + entityGUID = "057964e1-fd78-4e74-bd66-c74a06131511" + attrGUID = "f3bcaa9f-4ed8-46de-9b7d-dad81425bc33" + assocGUID = "bae3f172-aebb-49a6-ad92-783f53cf5014" + zeroGUID = "00000000-0000-0000-0000-000000000000" + pointerID = "70e40d90-b3a5-4cd2-acfa-ab8158b9bc89" +) + +// A GUID is stored as BSON binary. Decoded through property.DecodeString — the +// generator's default for a string-typed field — StringValueOK() returns +// ("", false) and the false is discarded, so every GUID read back EMPTY on every +// Studio Pro-authored document, with no error anywhere. +// +// That is the CDC baseline's whole purpose: mendixsystem$entity.id and +// mendixsystem$attribute.id hold these verbatim, so a write path built on an +// empty read would detach every tracked entity from its table. +func TestVersionedEntityChangeSource_EntityGuidDecodesFromBinary(t *testing.T) { + raw, err := bson.Marshal(bson.D{ + {Key: "$Type", Value: "ChangeDataCapture$VersionedEntityChangeSource"}, + {Key: "EntityGuid", Value: mmpr.IDToBsonBinary(entityGUID)}, + {Key: "ExposedName", Value: "Order"}, + }) + if err != nil { + t.Fatal(err) + } + o := initVersionedEntityChangeSource() + o.InitFromRaw(bson.Raw(raw)) + + if got := o.EntityGuid(); got != entityGUID { + t.Errorf("EntityGuid = %q, want %q — a binary GUID read as a string comes back empty", got, entityGUID) + } + if o.ExposedName() != "Order" { + t.Errorf("ExposedName = %q; the rest of the element must still decode", o.ExposedName()) + } +} + +func TestVersionedProperty_GuidsDecodeFromBinary(t *testing.T) { + for _, tc := range []struct { + name string + assoc string + wantAssoc string + whatItMeans string + }{ + {"association property", assocGUID, assocGUID, "the GUID of the TO entity"}, + {"attribute property", zeroGUID, zeroGUID, "the all-zero GUID Studio Pro writes for an attribute"}, + } { + t.Run(tc.name, func(t *testing.T) { + raw, err := bson.Marshal(bson.D{ + {Key: "$Type", Value: "ChangeDataCapture$VersionedProperty"}, + {Key: "PropertyGuid", Value: mmpr.IDToBsonBinary(attrGUID)}, + {Key: "AssociatedEntityGuid", Value: mmpr.IDToBsonBinary(tc.assoc)}, + {Key: "ExposedName", Value: "OrderId"}, + {Key: "Type", Value: "Int64"}, + }) + if err != nil { + t.Fatal(err) + } + o := initVersionedProperty() + o.InitFromRaw(bson.Raw(raw)) + + if got := o.PropertyGuid(); got != attrGUID { + t.Errorf("PropertyGuid = %q, want %q", got, attrGUID) + } + if got := o.AssociatedEntityGuid(); got != tc.wantAssoc { + t.Errorf("AssociatedEntityGuid = %q, want %q (%s)", got, tc.wantAssoc, tc.whatItMeans) + } + if o.Type() != "Int64" { + t.Errorf("Type = %q, want Int64", o.Type()) + } + }) + } +} + +// The live half of the document points at the frozen half. mxbuild does not +// validate these pointers, so nothing downstream would report their loss. +func TestLiveSideKeepsItsBaselinePointers(t *testing.T) { + srcRaw, err := bson.Marshal(bson.D{ + {Key: "$Type", Value: "ChangeDataCapture$EntityChangeSource"}, + {Key: "Entity", Value: "CdcPublish.Order"}, + {Key: "ExposedName", Value: "Order"}, + {Key: "VersionedSourcePointer", Value: mmpr.IDToBsonBinary(pointerID)}, + }) + if err != nil { + t.Fatal(err) + } + src := initEntityChangeSource() + src.InitFromRaw(bson.Raw(srcRaw)) + if got := src.VersionedSourceRefID(); got != element.ID(pointerID) { + t.Errorf("VersionedSourcePointer = %q, want %q", got, pointerID) + } + + for _, kind := range []string{"AttributeProperty", "AssociationProperty"} { + t.Run(kind, func(t *testing.T) { + propRaw, err := bson.Marshal(bson.D{ + {Key: "$Type", Value: "ChangeDataCapture$" + kind}, + {Key: "ExposedName", Value: "OrderId"}, + {Key: "VersionedPropertyPointer", Value: mmpr.IDToBsonBinary(attrGUID)}, + }) + if err != nil { + t.Fatal(err) + } + var got element.ID + if kind == "AttributeProperty" { + o := initAttributeProperty() + o.InitFromRaw(bson.Raw(propRaw)) + got = o.VersionedPropertyRefID() + } else { + o := initAssociationProperty() + o.InitFromRaw(bson.Raw(propRaw)) + got = o.VersionedPropertyRefID() + } + if got != element.ID(attrGUID) { + t.Errorf("VersionedPropertyPointer = %q, want %q", got, attrGUID) + } + }) + } +} + +// AppName is the second segment of every topic the service publishes to +// (cdc....), and mxbuild reports CE1577 on +// an empty one — so losing it is both a build error and, if it ever got past +// one, a publisher shouting into a topic nobody reads. +func TestEntityChangeDataCaptureService_AppName(t *testing.T) { + raw, err := bson.Marshal(bson.D{ + {Key: "$Type", Value: "ChangeDataCapture$EntityChangeDataCaptureService"}, + {Key: "Name", Value: "OrdersCDCService"}, + {Key: "ServiceName", Value: "OrdersCDCService"}, + {Key: "AppName", Value: "TestApp"}, + }) + if err != nil { + t.Fatal(err) + } + o := initEntityChangeDataCaptureService() + o.InitFromRaw(bson.Raw(raw)) + + if got := o.AppName(); got != "TestApp" { + t.Errorf("AppName = %q, want TestApp", got) + } +} + +// A GUID written back has to reach storage as the same bytes it was read as: +// the runtime keys the database on it, so a re-encode that shifts a byte +// re-points an entity at nothing. codec.BinaryToUUID and mpr.IDToBsonBinary are +// the pair, and this pins them against the real values from the fixture. +func TestGuidBinaryRoundTripsExactly(t *testing.T) { + for _, guid := range []string{entityGUID, attrGUID, assocGUID, zeroGUID, pointerID} { + bin := mmpr.IDToBsonBinary(guid) + if got := codec.BinaryToUUID(bin.Data); got != guid { + t.Errorf("round trip changed the GUID: %q -> %q", guid, got) + } + if bin.Subtype != 0 { + t.Errorf("%s: subtype = %d, want 0 (what Studio Pro writes)", guid, bin.Subtype) + } + } +} diff --git a/modelsdk/gen/changedatacapture/types.go b/modelsdk/gen/changedatacapture/types.go index 182afd51b..4d86ecd72 100644 --- a/modelsdk/gen/changedatacapture/types.go +++ b/modelsdk/gen/changedatacapture/types.go @@ -51,6 +51,20 @@ type AssociationProperty struct { element.Base exposedName *property.Primitive[string] association *property.ByNameRef[element.Element] + // versionedProperty is hand-added (#2384 §4): the intra-unit pointer to this + // property's frozen counterpart in Baseline. + versionedProperty *property.ByIdRef[element.Element] +} + +// VersionedPropertyRefID is the $ID of this property's frozen counterpart in +// the service's Baseline. +func (o *AssociationProperty) VersionedPropertyRefID() element.ID { + return o.versionedProperty.RefID() +} + +// SetVersionedPropertyID sets the pointer to the baseline property. +func (o *AssociationProperty) SetVersionedPropertyID(id element.ID) { + o.versionedProperty.SetID(id) } // ExposedName returns the value of the exposedName property. @@ -75,6 +89,14 @@ func (o *AssociationProperty) SetAssociationQualifiedName(v string) { // InitFromRaw populates lazy-decoded property holders from raw BSON. func (o *AssociationProperty) InitFromRaw(raw bson.Raw) { + if val, err := raw.LookupErr("VersionedPropertyPointer"); err == nil { + if s, ok := val.StringValueOK(); ok { + o.versionedProperty.SetFromDecode(element.ID(s)) + } else if _, bdata, bok := val.BinaryOK(); bok { + o.versionedProperty.SetFromDecode(element.ID(codec.BinaryToUUID(bdata))) + } + } + o.exposedName.Init(raw) if val, err := raw.LookupErr("Association"); err == nil { if s, ok := val.StringValueOK(); ok { @@ -91,6 +113,20 @@ type AttributeProperty struct { element.Base exposedName *property.Primitive[string] attribute *property.ByNameRef[element.Element] + // versionedProperty is hand-added (#2384 §4): the intra-unit pointer to this + // property's frozen counterpart in Baseline. + versionedProperty *property.ByIdRef[element.Element] +} + +// VersionedPropertyRefID is the $ID of this property's frozen counterpart in +// the service's Baseline. +func (o *AttributeProperty) VersionedPropertyRefID() element.ID { + return o.versionedProperty.RefID() +} + +// SetVersionedPropertyID sets the pointer to the baseline property. +func (o *AttributeProperty) SetVersionedPropertyID(id element.ID) { + o.versionedProperty.SetID(id) } // ExposedName returns the value of the exposedName property. @@ -115,6 +151,14 @@ func (o *AttributeProperty) SetAttributeQualifiedName(v string) { // InitFromRaw populates lazy-decoded property holders from raw BSON. func (o *AttributeProperty) InitFromRaw(raw bson.Raw) { + if val, err := raw.LookupErr("VersionedPropertyPointer"); err == nil { + if s, ok := val.StringValueOK(); ok { + o.versionedProperty.SetFromDecode(element.ID(s)) + } else if _, bdata, bok := val.BinaryOK(); bok { + o.versionedProperty.SetFromDecode(element.ID(codec.BinaryToUUID(bdata))) + } + } + o.exposedName.Init(raw) if val, err := raw.LookupErr("Attribute"); err == nil { if s, ok := val.StringValueOK(); ok { @@ -129,6 +173,12 @@ func (o *AttributeProperty) InitFromRaw(raw bson.Raw) { type EntityChangeDataCaptureService struct { element.Base + // appName is hand-added (ako/mxcli#2384 §4): it is the second segment of + // every topic this service publishes to — + // cdc.... — so a service written without + // it publishes where nobody is listening, and mxbuild reports CE1577 on an + // empty one. + appName *property.Primitive[string] name *property.Primitive[string] documentation *property.Primitive[string] excluded *property.Primitive[bool] @@ -139,6 +189,16 @@ type EntityChangeDataCaptureService struct { } // Name returns the value of the name property. +// AppName returns the value of the appName property. +func (o *EntityChangeDataCaptureService) AppName() string { + return o.appName.Get() +} + +// SetAppName sets the value of the appName property. +func (o *EntityChangeDataCaptureService) SetAppName(v string) { + o.appName.Set(v) +} + func (o *EntityChangeDataCaptureService) Name() string { return o.name.Get() } @@ -215,6 +275,7 @@ func (o *EntityChangeDataCaptureService) SetBaseline(v element.Element) { // InitFromRaw populates lazy-decoded property holders from raw BSON. func (o *EntityChangeDataCaptureService) InitFromRaw(raw bson.Raw) { + o.appName.Init(raw) o.name.Init(raw) o.documentation.Init(raw) o.excluded.Init(raw) @@ -243,6 +304,11 @@ type EntityChangeSource struct { entity *property.ByNameRef[element.Element] exposedName *property.Primitive[string] properties *property.PartList[element.Element] + // versionedSource is hand-added (#2384 §4): the intra-unit pointer from the + // live source to its frozen counterpart in Baseline. mxbuild does not + // validate it, but Studio Pro writes it and the two halves are only + // relatable through it. + versionedSource *property.ByIdRef[element.Element] } // EntityQualifiedName returns the value of the entity property. @@ -255,6 +321,17 @@ func (o *EntityChangeSource) SetEntityQualifiedName(v string) { o.entity.SetQualifiedName(v) } +// VersionedSourceRefID is the $ID of this source's frozen counterpart in the +// service's Baseline. +func (o *EntityChangeSource) VersionedSourceRefID() element.ID { + return o.versionedSource.RefID() +} + +// SetVersionedSourceID sets the pointer to the baseline source. +func (o *EntityChangeSource) SetVersionedSourceID(id element.ID) { + o.versionedSource.SetID(id) +} + // ExposedName returns the value of the exposedName property. func (o *EntityChangeSource) ExposedName() string { return o.exposedName.Get() @@ -282,6 +359,14 @@ func (o *EntityChangeSource) RemoveProperties(index int) { // InitFromRaw populates lazy-decoded property holders from raw BSON. func (o *EntityChangeSource) InitFromRaw(raw bson.Raw) { + if val, err := raw.LookupErr("VersionedSourcePointer"); err == nil { + if s, ok := val.StringValueOK(); ok { + o.versionedSource.SetFromDecode(element.ID(s)) + } else if _, bdata, bok := val.BinaryOK(); bok { + o.versionedSource.SetFromDecode(element.ID(codec.BinaryToUUID(bdata))) + } + } + if val, err := raw.LookupErr("Entity"); err == nil { if s, ok := val.StringValueOK(); ok { o.entity.SetFromDecode(s) @@ -384,8 +469,12 @@ func (o *VersionedEntityChangeDataCaptureService) InitFromRaw(raw bson.Raw) { type VersionedEntityChangeSource struct { element.Base - version *property.Part[element.Element] - entityGuid *property.Primitive[string] + version *property.Part[element.Element] + // entityGuid is the DATABASE's identity for the exposed entity — + // mendixsystem$entity.id holds it verbatim — stored as BSON binary. Same + // defect as VersionedProperty.propertyGuid: as a string primitive it read + // back empty. + entityGuid *property.ByIdRef[element.Element] exposedName *property.Primitive[string] properties *property.PartList[element.Element] } @@ -402,12 +491,12 @@ func (o *VersionedEntityChangeSource) SetVersion(v element.Element) { // EntityGuid returns the value of the entityGuid property. func (o *VersionedEntityChangeSource) EntityGuid() string { - return o.entityGuid.Get() + return string(o.entityGuid.RefID()) } // SetEntityGuid sets the value of the entityGuid property. func (o *VersionedEntityChangeSource) SetEntityGuid(v string) { - o.entityGuid.Set(v) + o.entityGuid.SetID(element.ID(v)) } // ExposedName returns the value of the exposedName property. @@ -440,7 +529,13 @@ func (o *VersionedEntityChangeSource) InitFromRaw(raw bson.Raw) { if child, err := codec.DecodeChild(raw, "Version"); err == nil { o.version.SetFromDecode(child) } - o.entityGuid.Init(raw) + if val, err := raw.LookupErr("EntityGuid"); err == nil { + if s, ok := val.StringValueOK(); ok { + o.entityGuid.SetFromDecode(element.ID(s)) + } else if _, bdata, bok := val.BinaryOK(); bok { + o.entityGuid.SetFromDecode(element.ID(codec.BinaryToUUID(bdata))) + } + } o.exposedName.Init(raw) if children, err := codec.DecodeChildren(raw, "Properties"); err == nil { for _, child := range children { @@ -455,19 +550,38 @@ func (o *VersionedEntityChangeSource) InitFromRaw(raw bson.Raw) { type VersionedProperty struct { element.Base - propertyGuid *property.Primitive[string] - exposedName *property.Primitive[string] - propType *property.Enum[string] + // propertyGuid is the DATABASE's identity for the attribute or association + // this exposes — mendixsystem$attribute.id holds it verbatim. It is stored + // as BSON binary, so it is a ByIdRef and not a string primitive: read + // through DecodeString it came back EMPTY on every Studio Pro-authored + // document, silently (CLAUDE.md, "A GUID Is the Database's Identity"). + propertyGuid *property.ByIdRef[element.Element] + // associatedEntityGuid is hand-added (#2384 §4): for an association it is + // the GUID of the TO entity, and for an attribute the all-zero GUID. + associatedEntityGuid *property.ByIdRef[element.Element] + exposedName *property.Primitive[string] + propType *property.Enum[string] } // PropertyGuid returns the value of the propertyGuid property. func (o *VersionedProperty) PropertyGuid() string { - return o.propertyGuid.Get() + return string(o.propertyGuid.RefID()) } // SetPropertyGuid sets the value of the propertyGuid property. func (o *VersionedProperty) SetPropertyGuid(v string) { - o.propertyGuid.Set(v) + o.propertyGuid.SetID(element.ID(v)) +} + +// AssociatedEntityGuid is the GUID of the TO entity for an association +// property, or the all-zero GUID for an attribute. +func (o *VersionedProperty) AssociatedEntityGuid() string { + return string(o.associatedEntityGuid.RefID()) +} + +// SetAssociatedEntityGuid sets the associatedEntityGuid property. +func (o *VersionedProperty) SetAssociatedEntityGuid(v string) { + o.associatedEntityGuid.SetID(element.ID(v)) } // ExposedName returns the value of the exposedName property. @@ -492,7 +606,20 @@ func (o *VersionedProperty) SetType(v string) { // InitFromRaw populates lazy-decoded property holders from raw BSON. func (o *VersionedProperty) InitFromRaw(raw bson.Raw) { - o.propertyGuid.Init(raw) + if val, err := raw.LookupErr("PropertyGuid"); err == nil { + if s, ok := val.StringValueOK(); ok { + o.propertyGuid.SetFromDecode(element.ID(s)) + } else if _, bdata, bok := val.BinaryOK(); bok { + o.propertyGuid.SetFromDecode(element.ID(codec.BinaryToUUID(bdata))) + } + } + if val, err := raw.LookupErr("AssociatedEntityGuid"); err == nil { + if s, ok := val.StringValueOK(); ok { + o.associatedEntityGuid.SetFromDecode(element.ID(s)) + } else if _, bdata, bok := val.BinaryOK(); bok { + o.associatedEntityGuid.SetFromDecode(element.ID(codec.BinaryToUUID(bdata))) + } + } o.exposedName.Init(raw) if val, err := raw.LookupErr("Type"); err == nil { if s, ok := val.StringValueOK(); ok { @@ -515,7 +642,9 @@ func initAssociationProperty() *AssociationProperty { o.exposedName.Bind(&o.Base, 0) o.association = property.NewByNameRef[element.Element]("Association", "DomainModels$AssociationBase") o.association.Bind(&o.Base, 1) - o.SetProperties([]element.Property{o.exposedName, o.association}) + o.versionedProperty = property.NewByIdRef[element.Element]("VersionedPropertyPointer") + o.versionedProperty.Bind(&o.Base, 2) + o.SetProperties([]element.Property{o.exposedName, o.association, o.versionedProperty}) return o } @@ -538,7 +667,9 @@ func initAttributeProperty() *AttributeProperty { o.exposedName.Bind(&o.Base, 0) o.attribute = property.NewByNameRef[element.Element]("Attribute", "DomainModels$Attribute") o.attribute.Bind(&o.Base, 1) - o.SetProperties([]element.Property{o.exposedName, o.attribute}) + o.versionedProperty = property.NewByIdRef[element.Element]("VersionedPropertyPointer") + o.versionedProperty.Bind(&o.Base, 2) + o.SetProperties([]element.Property{o.exposedName, o.attribute, o.versionedProperty}) return o } @@ -571,7 +702,9 @@ func initEntityChangeDataCaptureService() *EntityChangeDataCaptureService { o.serviceName.Bind(&o.Base, 5) o.baseline = property.NewPart[element.Element]("Baseline") o.baseline.Bind(&o.Base, 6) - o.SetProperties([]element.Property{o.name, o.documentation, o.excluded, o.exportLevel, o.sources, o.serviceName, o.baseline}) + o.appName = property.NewPrimitive[string]("AppName", property.DecodeString) + o.appName.Bind(&o.Base, 7) + o.SetProperties([]element.Property{o.name, o.documentation, o.excluded, o.exportLevel, o.sources, o.serviceName, o.baseline, o.appName}) return o } @@ -596,7 +729,9 @@ func initEntityChangeSource() *EntityChangeSource { o.exposedName.Bind(&o.Base, 1) o.properties = property.NewPartList[element.Element]("Properties") o.properties.Bind(&o.Base, 2) - o.SetProperties([]element.Property{o.entity, o.exposedName, o.properties}) + o.versionedSource = property.NewByIdRef[element.Element]("VersionedSourcePointer") + o.versionedSource.Bind(&o.Base, 3) + o.SetProperties([]element.Property{o.entity, o.exposedName, o.properties, o.versionedSource}) return o } @@ -663,7 +798,7 @@ func initVersionedEntityChangeSource() *VersionedEntityChangeSource { o.SetTypeName("ChangeDataCapture$VersionedEntityChangeSource") o.version = property.NewPart[element.Element]("Version") o.version.Bind(&o.Base, 0) - o.entityGuid = property.NewPrimitive[string]("EntityGuid", property.DecodeString) + o.entityGuid = property.NewByIdRef[element.Element]("EntityGuid") o.entityGuid.Bind(&o.Base, 1) o.exposedName = property.NewPrimitive[string]("ExposedName", property.DecodeString) o.exposedName.Bind(&o.Base, 2) @@ -688,13 +823,15 @@ func NewVersionedEntityChangeSource() *VersionedEntityChangeSource { func initVersionedProperty() *VersionedProperty { o := &VersionedProperty{} o.SetTypeName("ChangeDataCapture$VersionedProperty") - o.propertyGuid = property.NewPrimitive[string]("PropertyGuid", property.DecodeString) + o.propertyGuid = property.NewByIdRef[element.Element]("PropertyGuid") o.propertyGuid.Bind(&o.Base, 0) o.exposedName = property.NewPrimitive[string]("ExposedName", property.DecodeString) o.exposedName.Bind(&o.Base, 1) o.propType = property.NewEnum[string]("Type") o.propType.Bind(&o.Base, 2) - o.SetProperties([]element.Property{o.propertyGuid, o.exposedName, o.propType}) + o.associatedEntityGuid = property.NewByIdRef[element.Element]("AssociatedEntityGuid") + o.associatedEntityGuid.Bind(&o.Base, 3) + o.SetProperties([]element.Property{o.propertyGuid, o.exposedName, o.propType, o.associatedEntityGuid}) return o } From a591848ff9a4fc7738a5ec5ba9669c69c800cd8f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 11:47:27 +0000 Subject: [PATCH 12/28] check: judge the values of an enumeration split (CE0079, CE1613, CE0771) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011oK82tETa46YQmMS58pQjL --- ...num-split-values-ce0079-ce1613-ce0771.json | 1 + CHANGELOG.md | 1 + cmd/mxcli/syntax/features_microflow.go | 3 + .../enum-split-duplicate-values.fail.mdl | 28 ++ mdl-examples/bug-tests/enum-split-values.mdl | 41 +++ mdl/executor/typecheck.go | 2 + mdl/executor/validate_enum_split_values.go | 261 ++++++++++++++++++ .../validate_enum_split_values_test.go | 184 ++++++++++++ mdl/executor/validate_microflow.go | 2 + mdl/executor/validate_nanoflow.go | 1 + 10 files changed, 524 insertions(+) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-11-18-enum-split-values-ce0079-ce1613-ce0771.json create mode 100644 mdl-examples/bug-tests/enum-split-duplicate-values.fail.mdl create mode 100644 mdl-examples/bug-tests/enum-split-values.mdl create mode 100644 mdl/executor/validate_enum_split_values.go create mode 100644 mdl/executor/validate_enum_split_values_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-enum-split-values-ce0079-ce1613-ce0771.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-enum-split-values-ce0079-ce1613-ce0771.json new file mode 100644 index 000000000..3f32bc6ae --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-enum-split-values-ce0079-ce1613-ce0771.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-18", "refs": ["ako/mxcli#1258"], "ce": ["CE0079", "CE1613", "CE0771"], "symptom": "An enumeration split `case $S/C when … end case` that left an enumeration value without a branch (CE0079), named a value the enumeration does not have, or the right value in another case (CE1613), or named a value in two branches (CE0771) passed `check` and `exec`; the build rejected each. Only the missing `(empty)` branch (MDL056) and an `else` were refused.", "file": "mdl/executor/validate_enum_split_values.go: checkEnumSplitDuplicateValues (MDL105, microflow and nanoflow validators) and enumSplitValueViolations (MDL106, called from TypeCheckProgram next to the expression hints). Tests: validate_enum_split_values_test.go; fixtures mdl-examples/bug-tests/enum-split-duplicate-values.fail.mdl and enum-split-values.mdl", "cause": "MDL056 was written knowing the other half of CE0079, a value with no branch, 'needs the enumeration's member list' and was left undone (its own comment says so); the member list is exactly what the catalog-backed tier already holds (exprcatalog.Reader.EnumCases / AttributeEnumQN, overlaid with the script's own enumerations by declareScriptTypes), so the deferral had outlived its reason.", "insight": "**A comment that defers a check 'until there is context' is a lead: look for the tier that now has the context before measuring from scratch.** CE0773, the code this started from, never appeared on 11.16.1 for any hand-written split (an `else` is a parse error with an explanatory hint, and the older records were the writer's own NoCase bug), so the useful result was the neighbouring gap. Measured one flow per project (a shared project hides CE0079/CE0771 behind CE1613): every value + (empty) and several values per branch clean; a value left out CE0079 on an attribute subject, a parameter subject and a nanoflow; an unknown value or the wrong case CE1613; a value in two branches CE0771 (one per branch, also when the repeat is inside a multi-value branch). An unknown value is reported alone because the build reports CE1613 first. Needs-the-project rules cannot have a `.fail.mdl` fixture: check-mdl runs `check` with no project and requires a fail fixture to fail, so MDL106's negatives live in Go tests (script enumeration, project enumeration through the fixture's Ticket.Status) plus the isolated measurements; MDL105 needs no enumeration and has the fixture. Sweep: 1,328 repo .mdl files, 0 MDL105 hits; 16 files with an enum split under `check -p`, one MDL106 hit - mdl-examples/bug-tests/1929-ended-branch-decision-one-merge.mdl, whose splits name R and G but not B; applied to a fresh project, mx check reports CE0079 for both flows, so the hit is true and the fixture (about describe round trips) was never built against the build. Control: stubbing both rules fails 11 sub-tests; `check -p` agrees with mx check on all 11 measured cases."} diff --git a/CHANGELOG.md b/CHANGELOG.md index 57c7b84e3..e2a8f8849 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`check` judges the values of an enumeration split** (ako/mxcli#1258) — a `case $S/C when … end case` that leaves an enumeration value without a branch, names a value the enumeration does not have, or names one value in two branches passed `check` and `exec`, and the build reported CE0079, CE1613 (values are case-sensitive) and CE0771. Two new rules: MDL105 (a value named twice, no project needed) and MDL106 (an unknown or missing value, under `check -p`, which reads the enumeration from the script or the project). An unknown value is reported alone, as the build does. A split whose subject does not resolve — a rule call, an association path, an enumeration the script alters or drops — is not judged. Measured on MxBuild 11.16.1, microflows and nanoflows. CE0773, which older records list beside CE0079, does not appear for any of these; it is the follow-on of a branch with no configured value, which the grammar already refuses for `else`. - **`check` rejects an import mapping's custom handler that does not name exactly its microflow's parameters** (ako/mxcli#1258) — `find … by M.MF ( P: member )` with a parameter the microflow does not declare (or the right name in another case) passed `check` and `exec` and the build reported CE1613 "The selected parameter … no longer exists"; leaving a parameter out was CE0277 "Parameter mapping of microflow for obtaining objects needs to be refreshed". New MDL104, which reads the microflow from the script or the project; a microflow it cannot find is not judged, and an unknown name is reported before a missing one, as the build does. A parameter fed a member of the wrong type (CE0282) is not covered. Measured on MxBuild 11.16.1, one mapping per project. - **`check` rejects `rest call … returns mapping M as list of E` over an object-rooted mapping** (ako/mxcli#1258) — an import mapping whose root is one object returns one object, and declaring the call's result as a list passed `check` and `exec` and the build reported CE0243 "The mapping used to return a value of type 'List of …', but now returns a value of type '…'". New MDL103, which reads the mapping's shape from the script or the project (the same resolver as MDL-MAP04); a mapping whose shape cannot be established is not judged. `as Module.E` is fine on either root — over an array it binds one object out of the list. Measured on MxBuild 11.16.1. - **`check` rejects an `alter entity … modify attribute` that leaves a published OData attribute published as the wrong type** (ako/mxcli#1258) — changing the type of an attribute a published service exposes (String to Integer, Decimal to Integer, Boolean or DateTime to String, String to Decimal) passed `check` and `exec` and the build reported CE5016 "Attribute '…' has type …, but is published as …", one per attribute. New MDL102, which reads the service from the script and from the project. A change inside one EDM type (String(50) to String(80); Integer to Long) is fine, and so is a `create or modify published odata service` AFTER the alter, which re-derives the types — before the alter it does not help. Measured on MxBuild 11.16.1. diff --git a/cmd/mxcli/syntax/features_microflow.go b/cmd/mxcli/syntax/features_microflow.go index 5f4db8fc7..6bcfbd8c5 100644 --- a/cmd/mxcli/syntax/features_microflow.go +++ b/cmd/mxcli/syntax/features_microflow.go @@ -253,6 +253,9 @@ func init() { " WHEN (empty) THEN -- required (MDL056 / CE0079)\n" + " ...\n" + "END CASE; -- no ELSE (MDL008)\n\n" + + "-- Every enumeration value needs a branch (CE0079; mxcli check -p: MDL106), a value\n" + + "-- the enumeration does not have is CE1613 (values are case-sensitive; MDL106), and\n" + + "-- one named in two branches is CE0771 (MDL105).\n\n" + "CASE Module.Rule(Param = $Value) -- a RULE returning an enumeration\n" + " WHEN Value1 THEN ... -- same call syntax as IF on a rule;\n" + " WHEN (empty) THEN ... -- with -p every value needs a branch\n" + diff --git a/mdl-examples/bug-tests/enum-split-duplicate-values.fail.mdl b/mdl-examples/bug-tests/enum-split-duplicate-values.fail.mdl new file mode 100644 index 000000000..6f6004e16 --- /dev/null +++ b/mdl-examples/bug-tests/enum-split-duplicate-values.fail.mdl @@ -0,0 +1,28 @@ +mdl 1; +-- NEGATIVE TEST — `mxcli check` must REFUSE this file (MDL105), with no project. +-- +-- An enumeration split that names the same value in two branches passed `check` +-- and `exec`, and the build reported +-- +-- [CE0771] (one error per branch that repeats the value) +-- +-- Measured on MxBuild 11.16.1, also when the repeat is inside a multi-value branch +-- (`when R, G` and `when G`). Values are case-sensitive. See +-- enum-split-values.mdl for the valid forms. The related errors — a value left +-- out (CE0079) and a value the enumeration does not have (CE1613) — need the +-- enumeration's members, so MDL106 reports them under `check -p`. +create enumeration MyFirstModule.EsdColor (R 'R', G 'G', B 'B'); +create persistent entity MyFirstModule.EsdShape (C: Enumeration(MyFirstModule.EsdColor)); + +create microflow MyFirstModule.EsdSameValueTwice ($S: MyFirstModule.EsdShape) returns Integer as $N +begin + declare $N Integer = 0; + case $S/C + when R then set $N = 1; + when R then set $N = 2; + when G then set $N = 3; + when B then set $N = 4; + when (empty) then set $N = 5; + end case; + return $N; +end; diff --git a/mdl-examples/bug-tests/enum-split-values.mdl b/mdl-examples/bug-tests/enum-split-values.mdl new file mode 100644 index 000000000..46cc81c6e --- /dev/null +++ b/mdl-examples/bug-tests/enum-split-values.mdl @@ -0,0 +1,41 @@ +mdl 1; +-- Control for enum-split-duplicate-values.fail.mdl: the valid forms of an enumeration +-- split — every value once plus (empty), several values in one branch, on an attribute +-- and on a parameter typed by the enumeration. Builds with 0 errors on MxBuild 11.16.1. +create enumeration MyFirstModule.EsvColor (R 'R', G 'G', B 'B'); +create persistent entity MyFirstModule.EsvShape (C: Enumeration(MyFirstModule.EsvColor)); + +create microflow MyFirstModule.EsvOnAttribute ($S: MyFirstModule.EsvShape) returns Integer as $N +begin + declare $N Integer = 0; + case $S/C + when R then set $N = 1; + when G then set $N = 2; + when B then set $N = 3; + when (empty) then set $N = 4; + end case; + return $N; +end; + +create microflow MyFirstModule.EsvSeveralValuesInOneBranch ($S: MyFirstModule.EsvShape) returns Integer as $N +begin + declare $N Integer = 0; + case $S/C + when R, G then set $N = 1; + when B then set $N = 2; + when (empty) then set $N = 3; + end case; + return $N; +end; + +create microflow MyFirstModule.EsvOnParameter ($P: MyFirstModule.EsvColor) returns Integer as $N +begin + declare $N Integer = 0; + case $P + when R then set $N = 1; + when G then set $N = 2; + when B then set $N = 3; + when (empty) then set $N = 4; + end case; + return $N; +end; diff --git a/mdl/executor/typecheck.go b/mdl/executor/typecheck.go index a98532b2e..6985a9e47 100644 --- a/mdl/executor/typecheck.go +++ b/mdl/executor/typecheck.go @@ -70,6 +70,8 @@ func (e *Executor) TypeCheckProgram(prog *ast.Program) []linter.Violation { out = append(out, adapter.CheckNanoflow(s).AsViolations()...) } } + // Enumeration splits against the members of the enumeration they split on. + out = append(out, enumSplitValueViolations(reader, prog)...) return out } diff --git a/mdl/executor/validate_enum_split_values.go b/mdl/executor/validate_enum_split_values.go new file mode 100644 index 000000000..9b41a3280 --- /dev/null +++ b/mdl/executor/validate_enum_split_values.go @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "fmt" + "sort" + "strings" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/exprcatalog" + "github.com/mendixlabs/mxcli/mdl/linter" +) + +// An enumeration split (`case $S/C when R … end case`) needs exactly one branch per +// enumeration value plus (empty), and every value written must exist. Measured on +// MxBuild 11.16.1, one flow per project (a shared project is wrong: CE1613 is an +// unresolved reference and hides every other error), with `check` and `exec` +// accepting every rejected case: +// +// every value + (empty), or several values in one branch clean +// a value left out (attribute subject, parameter subject) CE0079 +// the same on a nanoflow CE0079 +// a value that is not in the enumeration CE1613 +// the right value in another case CE1613 (values are case-sensitive) +// a value in two branches, also inside a multi-value one CE0771 (one per branch) +// +// The missing (empty) branch was already MDL056. CE0773, which older records list +// beside CE0079, does not appear for any of these on 11.16.1: it is the build's +// follow-on error for a branch with no configured value, which an `else` produces +// and the grammar already refuses. +const ( + // enumSplitDuplicateRule needs no enumeration: the same value twice is wrong + // whatever the enumeration holds, so it runs with the other flow rules. + enumSplitDuplicateRule = "MDL105" + // enumSplitValueRule needs the enumeration's members, so it runs where the + // script's and the project's enumerations are known (`check -p`). + enumSplitValueRule = "MDL106" +) + +// splitValueName is a branch value without any module or enumeration prefix. +func splitValueName(v string) string { + v = strings.TrimSpace(v) + if i := strings.LastIndex(v, "."); i >= 0 { + v = v[i+1:] + } + return v +} + +func isEmptyBranch(v string) bool { return strings.EqualFold(strings.TrimSpace(v), "(empty)") } + +// caseValues is every value a split's branches name, in order, (empty) left out. +func caseValues(stmt *ast.EnumSplitStmt) []string { + var out []string + for _, c := range stmt.Cases { + vals := c.Values + if len(vals) == 0 && c.Value != "" { + vals = []string{c.Value} + } + for _, v := range vals { + if !isEmptyBranch(v) { + out = append(out, splitValueName(v)) + } + } + } + return out +} + +// checkEnumSplitDuplicateValues (MDL105) reports a value that two branches of one +// enumeration split both name — CE0771, one error per branch that repeats it. +// Values are case-sensitive, so `r` and `R` are not the same value. +func (v *microflowValidator) checkEnumSplitDuplicateValues(body []ast.MicroflowStatement) { + forEachMicroflowStatement(body, func(s ast.MicroflowStatement) { + stmt, ok := s.(*ast.EnumSplitStmt) + if !ok { + return + } + seen := map[string]int{} + var dups []string + for _, val := range caseValues(stmt) { + seen[val]++ + if seen[val] == 2 { + dups = append(dups, val) + } + } + if len(dups) == 0 { + return + } + v.addViolation(enumSplitDuplicateRule, linter.SeverityError, + fmt.Sprintf("the case statement on '%s' names %s in more than one branch — mxbuild reports CE0771 "+ + "for each branch that repeats a value (values are case-sensitive)", + enumSplitSubject(stmt), strings.Join(dups, ", ")), + "Name each enumeration value in exactly one branch (a branch may list several: `when R, G then`).") + }) +} + +// enumSplitValueViolations (MDL106) judges every enumeration split in the script's +// flows against the members of the enumeration it splits on: a value that is not +// a member is CE1613, and — once every value written exists — a member with no +// branch is CE0079. The build reports CE1613 before CE0079, so an unknown value is +// reported alone. +// +// Only a split whose subject resolves is judged: a parameter typed by an +// enumeration, or `$Obj/Attr` where $Obj is a parameter or a retrieved/created +// object of an entity whose attribute is an enumeration. A rule-call subject, an +// association path, an enumeration the script alters or drops (its members are no +// longer the reader's), or anything the reader does not know stays silent. +func enumSplitValueViolations(reader *exprcatalog.Reader, prog *ast.Program) []linter.Violation { + changed := map[string]bool{} // lower-cased enumerations the script alters or drops + for _, stmt := range prog.Statements { + switch s := stmt.(type) { + case *ast.AlterEnumerationStmt: + changed[strings.ToLower(s.Name.String())] = true + case *ast.DropEnumerationStmt: + changed[strings.ToLower(s.Name.String())] = true + } + } + var out []linter.Violation + judge := func(kind string, name ast.QualifiedName, params []ast.MicroflowParam, body []ast.MicroflowStatement) { + vars := enumSplitScope(reader, params, body) + forEachMicroflowStatement(body, func(s ast.MicroflowStatement) { + stmt, ok := s.(*ast.EnumSplitStmt) + if !ok || stmt.Condition != nil || stmt.Variable == "" { + return + } + enumQN := resolveSplitEnumeration(reader, vars, stmt.Variable) + if enumQN == "" || changed[strings.ToLower(enumQN)] { + return + } + cases, ok := reader.EnumCases(enumQN) + if !ok { + return + } + if v, bad := judgeSplitValues(kind, name, stmt, enumQN, cases); bad { + out = append(out, v) + } + }) + } + for _, stmt := range prog.Statements { + switch s := stmt.(type) { + case *ast.CreateMicroflowStmt: + judge("microflow", s.Name, s.Parameters, s.Body) + case *ast.CreateNanoflowStmt: + judge("nanoflow", s.Name, s.Parameters, s.Body) + } + } + return out +} + +// splitVar is what a flow variable is known to be: an enumeration value, or an +// object of an entity. +type splitVar struct{ enumQN, entityQN string } + +// enumSplitScope types the flow's parameters and the objects its statements bind. +func enumSplitScope(reader *exprcatalog.Reader, params []ast.MicroflowParam, body []ast.MicroflowStatement) map[string]splitVar { + vars := map[string]splitVar{} + for _, p := range params { + dt := p.Type + switch { + case dt.Kind == ast.TypeEnumeration && (dt.EnumRef != nil || dt.EntityRef != nil): + qn := "" + if dt.EnumRef != nil { + qn = dt.EnumRef.String() + } else { + qn = dt.EntityRef.String() + } + vars[p.Name] = splitVar{enumQN: qn} + case dt.Kind == ast.TypeEntity && dt.EntityRef != nil: + // A bare `Module.Name` parses as an entity reference whatever it names; + // one the reader knows as an enumeration, and not as an entity, IS the + // enumeration (the same settlement scriptFlowParams makes). + qn := dt.EntityRef.String() + if _, isEnum := reader.EnumCases(qn); isEnum && !reader.KnowsEntity(qn) { + vars[p.Name] = splitVar{enumQN: qn} + } else { + vars[p.Name] = splitVar{entityQN: qn} + } + } + } + forEachMicroflowStatement(body, func(s ast.MicroflowStatement) { + switch st := s.(type) { + case *ast.DeclareStmt: + if st.Type.Kind == ast.TypeEnumeration && st.Type.EnumRef != nil { + vars[st.Variable] = splitVar{enumQN: st.Type.EnumRef.String()} + } + case *ast.RetrieveStmt: + if st.First && st.Variable != "" && st.StartVariable == "" && st.Source.Name != "" { + vars[st.Variable] = splitVar{entityQN: st.Source.String()} + } + case *ast.CreateObjectStmt: + if st.Variable != "" && st.EntityType.Name != "" { + vars[st.Variable] = splitVar{entityQN: st.EntityType.String()} + } + } + }) + return vars +} + +// resolveSplitEnumeration names the enumeration a split's subject holds, or "". +func resolveSplitEnumeration(reader *exprcatalog.Reader, vars map[string]splitVar, subject string) string { + obj, attr, isPath := strings.Cut(subject, "/") + v, ok := vars[obj] + if !ok { + return "" + } + if !isPath { + return v.enumQN + } + if v.entityQN == "" || strings.Contains(attr, "/") { + return "" + } + enumQN, _ := reader.AttributeEnumQN(v.entityQN, attr) + return enumQN +} + +func judgeSplitValues(kind string, name ast.QualifiedName, stmt *ast.EnumSplitStmt, enumQN string, members []string) (linter.Violation, bool) { + known := map[string]bool{} + for _, m := range members { + known[m] = true + } + covered := map[string]bool{} + var unknown []string + for _, val := range caseValues(stmt) { + if known[val] { + covered[val] = true + } else { + unknown = append(unknown, val) + } + } + loc := linter.Location{Module: name.Module, DocumentType: kind, DocumentName: name.Name} + subject := enumSplitSubject(stmt) + if len(unknown) > 0 { + sort.Strings(unknown) + return linter.Violation{ + RuleID: enumSplitValueRule, + Severity: linter.SeverityError, + Message: fmt.Sprintf("the case statement on '%s' names %s, which %s does not have — mxbuild reports CE1613 "+ + "(enumeration values are case-sensitive)", subject, strings.Join(unknown, ", "), enumQN), + Location: loc, + Suggestion: fmt.Sprintf("%s has: %s.", enumQN, strings.Join(members, ", ")), + }, true + } + var missing []string + for _, m := range members { + if !covered[m] { + missing = append(missing, m) + } + } + if len(missing) == 0 { + return linter.Violation{}, false + } + return linter.Violation{ + RuleID: enumSplitValueRule, + Severity: linter.SeverityError, + Message: fmt.Sprintf("the case statement on '%s' has no branch for %s of %s — an enumeration split needs one "+ + "for every value, and mxbuild reports CE0079 \"The '%s' condition value should be configured in properties "+ + "for an outgoing flow\"", subject, strings.Join(missing, ", "), enumQN, missing[0]), + Location: loc, + Suggestion: fmt.Sprintf("Add `when %s then …` for each missing value (a branch may list several: `when %s, … then`).", missing[0], missing[0]), + }, true +} diff --git a/mdl/executor/validate_enum_split_values_test.go b/mdl/executor/validate_enum_split_values_test.go new file mode 100644 index 000000000..e5f29cbfb --- /dev/null +++ b/mdl/executor/validate_enum_split_values_test.go @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/mdl/visitor" +) + +// An enumeration split (`case $S/C when R … end case`) needs exactly one branch per +// enum value plus (empty), and each value must exist. Measured on MxBuild 11.16.1, +// one flow per project (a shared project is wrong: CE1613 is an unresolved +// reference and hides every other error), `check` and `exec` accepting every +// rejected case: +// +// every value + (empty), or several values in one branch clean +// a value left out (attribute subject, parameter subject) CE0079 +// the same on a nanoflow CE0079 +// a value that is not in the enumeration (X) CE1613 +// the right value in another case (r for R) CE1613 (values are case-sensitive) +// a value in two branches, also inside a multi-value one CE0771 (one per branch) +// +// CE0773 does not appear for any of these on 11.16.1; it is the build's follow-on +// error for a branch with no configured value, which an `else` produces and the +// grammar already refuses. + +func enumSplitFlow(kind, subject, params, branches string) string { + return "mdl 1;\ncreate " + kind + " Es.F (" + params + ") returns Integer as $N\nbegin\n declare $N Integer = 0;\n case " + + subject + "\n" + branches + "\n end case;\n return $N;\nend;\n" +} + +func enumSplitDuplicateHits(t *testing.T, src string) []linter.Violation { + t.Helper() + prog, errs := visitor.Build(src) + if len(errs) > 0 { + t.Fatalf("parse: %v\n%s", errs, src) + } + var vs []linter.Violation + switch s := prog.Statements[0].(type) { + case *ast.CreateMicroflowStmt: + vs = ValidateMicroflow(s) + case *ast.CreateNanoflowStmt: + vs = ValidateNanoflow(s) + } + var out []linter.Violation + for _, v := range vs { + if v.RuleID == enumSplitDuplicateRule { + out = append(out, v) + } + } + return out +} + +func TestEnumSplitDuplicateValues(t *testing.T) { + cases := []struct { + name, branches string + want int + }{ + {"every value once (control)", " when R then set $N = 1;\n when G then set $N = 2;\n when (empty) then set $N = 4;", 0}, + {"the same value in two branches", " when R then set $N = 1;\n when R then set $N = 2;\n when (empty) then set $N = 4;", 1}, + {"a value repeated inside a multi-value branch's sibling", " when R, G then set $N = 1;\n when G then set $N = 2;\n when (empty) then set $N = 4;", 1}, + {"multi-value branches that do not overlap (control)", " when R, G then set $N = 1;\n when B then set $N = 2;\n when (empty) then set $N = 4;", 0}, + {"case differs, so not a duplicate (control)", " when r then set $N = 1;\n when R then set $N = 2;\n when (empty) then set $N = 4;", 0}, + } + for _, kind := range []string{"microflow", "nanoflow"} { + for _, c := range cases { + t.Run(kind+": "+c.name, func(t *testing.T) { + got := enumSplitDuplicateHits(t, enumSplitFlow(kind, "$S/C", "$S: Es.Shape", c.branches)) + if len(got) != c.want { + t.Fatalf("violations=%v, want %d", got, c.want) + } + if c.want > 0 && (!strings.Contains(got[0].Message, "CE0771") || !strings.Contains(got[0].Message, "G") && !strings.Contains(got[0].Message, "R")) { + t.Errorf("message should name CE0771 and the value: %s", got[0].Message) + } + }) + } + } +} + +const enumSplitScript = `mdl 1; +create enumeration MyFirstModule.Col (R 'R', G 'G', B 'B'); +create persistent entity MyFirstModule.Shape (C: Enumeration(MyFirstModule.Col)); +` + +func enumValueHits(t *testing.T, exec *Executor, src string) []linter.Violation { + t.Helper() + var out []linter.Violation + for _, v := range typeCheck(t, exec, src) { + if v.RuleID == enumSplitValueRule { + out = append(out, v) + } + } + return out +} + +func esFlow(kind, subject, params, branches string) string { + return strings.Replace(enumSplitFlow(kind, subject, params, branches), "mdl 1;\n", "", 1) +} + +func TestEnumSplitValues_ScriptEnumeration(t *testing.T) { + exec := typeCheckFixture(t) + const S = "$S: MyFirstModule.Shape" + const P = "$P: MyFirstModule.Col" + all := " when R then set $N = 1;\n when G then set $N = 2;\n when B then set $N = 3;\n when (empty) then set $N = 4;" + cases := []struct { + name, kind, subject, params, branches string + want int + code string + }{ + {"every value and (empty) (control)", "microflow", "$S/C", S, all, 0, ""}, + {"several values in one branch (control)", "microflow", "$S/C", S, + " when R, G then set $N = 1;\n when B then set $N = 3;\n when (empty) then set $N = 4;", 0, ""}, + {"a value left out", "microflow", "$S/C", S, + " when R then set $N = 1;\n when G then set $N = 2;\n when (empty) then set $N = 4;", 1, "CE0079"}, + {"two values left out: one violation naming both", "microflow", "$S/C", S, + " when R then set $N = 1;\n when (empty) then set $N = 4;", 1, "CE0079"}, + {"a value left out on a nanoflow", "nanoflow", "$S/C", S, + " when R then set $N = 1;\n when G then set $N = 2;\n when (empty) then set $N = 4;", 1, "CE0079"}, + {"a value that is not in the enumeration", "microflow", "$S/C", S, + all + "\n when X then set $N = 5;", 1, "CE1613"}, + {"the right value in another case", "microflow", "$S/C", S, + " when r then set $N = 1;\n when G then set $N = 2;\n when B then set $N = 3;\n when (empty) then set $N = 4;", 1, "CE1613"}, + {"an unknown value is reported alone, as the build does", "microflow", "$S/C", S, + " when R then set $N = 1;\n when X then set $N = 5;\n when (empty) then set $N = 4;", 1, "CE1613"}, + {"a parameter typed by the enumeration, all values (control)", "microflow", "$P", P, all, 0, ""}, + {"a parameter typed by the enumeration, a value left out", "microflow", "$P", P, + " when R then set $N = 1;\n when G then set $N = 2;\n when (empty) then set $N = 4;", 1, "CE0079"}, + {"a subject nothing resolves is not judged (control)", "microflow", "$Z/C", S, + " when R then set $N = 1;\n when (empty) then set $N = 4;", 0, ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + src := enumSplitScript + esFlow(c.kind, c.subject, c.params, c.branches) + got := enumValueHits(t, exec, src) + if len(got) != c.want { + t.Fatalf("violations=%v, want %d", got, c.want) + } + if c.want > 0 { + v := got[0] + if !strings.Contains(v.Message, c.code) || v.Severity != linter.SeverityError { + t.Errorf("want an error naming %s: %+v", c.code, v) + } + } + }) + } + // Both left-out values are named, in one message. + got := enumValueHits(t, exec, enumSplitScript+esFlow("microflow", "$S/C", S, + " when R then set $N = 1;\n when (empty) then set $N = 4;")) + if len(got) != 1 || !strings.Contains(got[0].Message, "G") || !strings.Contains(got[0].Message, "B") { + t.Errorf("both missing values should be named once: %v", got) + } +} + +// The members of an enumeration the script ALTERS or DROPS are not the ones the +// reader holds, so a split on it is not judged. +func TestEnumSplitValues_AlteredEnumerationIsNotJudged(t *testing.T) { + exec := typeCheckFixture(t) + src := enumSplitScript + "alter enumeration MyFirstModule.Col add value Y caption 'Y';\n" + + esFlow("microflow", "$S/C", "$S: MyFirstModule.Shape", + " when R then set $N = 1;\n when (empty) then set $N = 4;") + if got := enumValueHits(t, exec, src); len(got) != 0 { + t.Errorf("an altered enumeration must not be judged: %v", got) + } +} + +// The enumeration and entity live in the PROJECT (the fixture's Ticket.Status is +// OrderStatus(Open, Closed)). +func TestEnumSplitValues_ProjectEnumeration(t *testing.T) { + exec := typeCheckFixture(t) + src := func(branches string) string { + return esFlow("microflow", "$T/Status", "$T: MyFirstModule.Ticket", branches) + } + if got := enumValueHits(t, exec, src(" when Open then set $N = 1;\n when Closed then set $N = 2;\n when (empty) then set $N = 4;")); len(got) != 0 { + t.Errorf("control: every project value covered was reported: %v", got) + } + got := enumValueHits(t, exec, src(" when Open then set $N = 1;\n when (empty) then set $N = 4;")) + if len(got) != 1 || !strings.Contains(got[0].Message, "Closed") || !strings.Contains(got[0].Message, "CE0079") { + t.Errorf("a project enumeration value left out: want one CE0079 naming Closed, got %v", got) + } +} diff --git a/mdl/executor/validate_microflow.go b/mdl/executor/validate_microflow.go index a7a671a36..3e90967b3 100644 --- a/mdl/executor/validate_microflow.go +++ b/mdl/executor/validate_microflow.go @@ -191,6 +191,8 @@ func (v *microflowValidator) validate(body []ast.MicroflowStatement) { v.checkDuplicateVariableNames(v.params, body) // CE0023: a primitive where UNION/INTERSECT/SUBTRACT/EQUALS/CONTAINS need an object or list. v.checkListOperandKinds(body) + // CE0771: a value two branches of one enumeration split both name. + v.checkEnumSplitDuplicateValues(body) // ako/mxcli#962: the other half of #953's void-call finding — the name is // inert both ways, so reading it is CE0109. See validate_void_call_output.go. v.checkVoidCallOutputUse(v.params, body) diff --git a/mdl/executor/validate_nanoflow.go b/mdl/executor/validate_nanoflow.go index c3b7a1b6e..c1d1835bc 100644 --- a/mdl/executor/validate_nanoflow.go +++ b/mdl/executor/validate_nanoflow.go @@ -50,6 +50,7 @@ func validateNanoflowWith(stmt *ast.CreateNanoflowStmt, voids *voidCodeActions) v.seedPrimitiveKinds(stmt.Parameters, stmt.Body) v.checkDuplicateVariableNames(v.params, stmt.Body) v.checkListOperandKinds(stmt.Body) + v.checkEnumSplitDuplicateValues(stmt.Body) // A nanoflow's show message has the same ShowMessageType (mendixlabs/mxcli#1280). v.checkShowMessageType(stmt.Body) // MDL-TEMPLATE01: a nanoflow's log, show message and validation feedback From 59350fa61214202917982b636c777e48855ec3f2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 18:01:59 +0000 Subject: [PATCH 13/28] fix(lint): parse_xpath reads true()/false() and reports a partial parse as recovered In XPath mode the Boolean literal was returned before the call check, so the '()' after true/false ended the parse and dropped the rest of the constraint; parse_xpath discarded the E014 that said so. true()/false() now parse as Booleans, and an E014 parse returns a recovered root. Closes #2398 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011oK82qrr1WnXxq1eRn7qRJ --- ...11-18-parse-xpath-bool-call-truncates.json | 1 + CHANGELOG.md | 1 + mdl/exprcheck/parser.go | 19 ++++-- mdl/linter/parse_xpath_bool_call_2398_test.go | 66 +++++++++++++++++++ mdl/linter/starlark.go | 13 +++- 5 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-other/2026-11-18-parse-xpath-bool-call-truncates.json create mode 100644 mdl/linter/parse_xpath_bool_call_2398_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-18-parse-xpath-bool-call-truncates.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-18-parse-xpath-bool-call-truncates.json new file mode 100644 index 000000000..59ea147f8 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-18-parse-xpath-bool-call-truncates.json @@ -0,0 +1 @@ +{"area": "mdl-other", "date": "2026-11-18", "refs": ["ako/mxcli#2398", "mendixlabs/mxcli#1894"], "ce": [], "symptom": "Starlark parse_xpath('[IsConfidential = false() or Mod.Report_Owner = \\'[%CurrentUser%]\\']') returned only (IsConfidential = false), with no recovered node and no error; the same constraint with the terms swapped, or with bare false, parsed whole.", "file": "mdl/exprcheck/parser.go parseIdentLed (true()/false() in XPath mode); mdl/linter/starlark.go builtinParseXPath (E014 → recovered root). Test: mdl/linter/parse_xpath_bool_call_2398_test.go", "cause": "parseIdentLed returned a BoolLit for true/false before the call check, leaving `()` on the stream; Parse then reported E014 (trailing tokens), which builtinParseXPath discarded with `ast, _ :=`.", "insight": "**A parser that can stop early must say so to every consumer, not only the one that prints hints.** check printed E014, but the lint builtin threw the hints away, so rules got a partial tree that looked complete — and the truncation depended on term order, the worst kind of false negative for an access-rule rule. Fix both: accept the XPath spelling, and map an error-severity trailing-token hint to the documented `recovered` root."} diff --git a/CHANGELOG.md b/CHANGELOG.md index 23c56cdcd..7e09b4431 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **Lint `parse_xpath` parses `true()` / `false()` and no longer returns a partial tree** (ako/mxcli#2398, mendixlabs/mxcli#1894) — everything after an XPath `true()` or `false()` was dropped without a word, so `[IsConfidential = false() or Owner = '[%CurrentUser%]']` reached a rule as `IsConfidential = false`. Both forms now parse whole, and a constraint that does not parse to its end comes back as a `recovered` node, as the lint-rules skill documents. - **`marketplace diff` and `update` no longer report untouched pluggable-widget snippets as edits** (ako/mxcli#2367, mendixlabs/mxcli#1882) — right after an install, Workflow Commons showed 2 changed snippets on the first `diff` and 8 on every later one, so `update` needed `--force`, which also dropped the guard for real edits. The reference cache now keeps the widget definitions, and a reference gets the same widget and design-property repairs install runs on the app. Measured: 0 false edits on both runs, and a real edit is still reported. Cached references from earlier versions are rebuilt once. - **An imported navigation association that would clash is named `_`** (ako/mxcli#2354) — `create external entities` named Product's `Category` navigation `Category_2`, because the name collides with the entity Category. It is now `Product_Category`, with a number only if that is taken too. Names that do not collide are unchanged, and a re-import still recognises existing associations by their navigation property. - **`describe contract operation from openapi` shows query parameters as they are imported** (ako/mxcli#2353) — the preview showed an integer query parameter as `Integer` while the imported service described it as `String`. Mendix stores no type for a consumed REST query parameter (it is text), so the preview now shows `String`, matching the import; path parameters keep the spec's type. diff --git a/mdl/exprcheck/parser.go b/mdl/exprcheck/parser.go index 14d83f504..5c53f80e0 100644 --- a/mdl/exprcheck/parser.go +++ b/mdl/exprcheck/parser.go @@ -595,10 +595,21 @@ func parseIdentLed(s *Stream, ctx Context) (RobustExpr, []Hint) { t := s.Consume() name := t.Text switch strings.ToLower(name) { - case "true": - return &BoolLit{baseNode: baseNode{P: t.Pos}, Value: true}, nil - case "false": - return &BoolLit{baseNode: baseNode{P: t.Pos}, Value: false}, nil + case "true", "false": + // XPath writes a Boolean as `true()` / `false()`. Returning the literal + // before the call check left `()` on the stream, so the parse ended + // there and dropped everything after it — `[A = false() or B]` read as + // `A = false` (ako/mxcli#2398). + if ctx.XPath && s.Peek().Kind == TokLParen { + mark := s.Mark() + s.Consume() + if s.Peek().Kind == TokRParen { + s.Consume() + } else { + s.Reset(mark) + } + } + return &BoolLit{baseNode: baseNode{P: t.Pos}, Value: strings.EqualFold(name, "true")}, nil case "empty": return &EmptyExpr{baseNode: baseNode{P: t.Pos}}, nil case "null": diff --git a/mdl/linter/parse_xpath_bool_call_2398_test.go b/mdl/linter/parse_xpath_bool_call_2398_test.go new file mode 100644 index 000000000..f43a1e1c4 --- /dev/null +++ b/mdl/linter/parse_xpath_bool_call_2398_test.go @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: Apache-2.0 + +package linter + +import ( + "strings" + "testing" + + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" +) + +// ako/mxcli#2398 (mendixlabs/mxcli#1894): parse_xpath stopped at `false()`, +// left `() or …` on the stream, and returned `(IsConfidential = false)` as if +// it were the whole constraint — so a rule over an access rule's `or` branches +// judged a fragment, depending on the order of the terms. + +func parseXPathForTest(t *testing.T, src string) *starlarkstruct.Struct { + t.Helper() + r := &StarlarkRule{} + v, err := r.builtinParseXPath(nil, nil, starlark.Tuple{starlark.String(src)}, nil) + if err != nil { + t.Fatal(err) + } + return v.(*starlarkstruct.Struct) +} + +func attrString(t *testing.T, s *starlarkstruct.Struct, name string) string { + t.Helper() + v, err := s.Attr(name) + if err != nil { + return "" + } + if str, ok := v.(starlark.String); ok { + return string(str) + } + return "" +} + +func TestParseXPath_BooleanCallKeepsTheRest_Issue2398(t *testing.T) { + for _, src := range []string{ + "[IsConfidential = false() or Mod.Report_Owner = '[%CurrentUser%]']", + "[Mod.Report_Owner = '[%CurrentUser%]' or IsConfidential = true()]", + // CONTROL: the bare form always parsed whole. + "[IsConfidential = false or Mod.Report_Owner = '[%CurrentUser%]']", + } { + root := parseXPathForTest(t, src) + if k := attrString(t, root, "kind"); k != "bin" { + t.Fatalf("%s: root kind = %q, want bin", src, k) + } + if op := attrString(t, root, "op"); !strings.EqualFold(op, "or") { + t.Errorf("%s: root op = %q, want or — a branch was dropped", src, op) + } + } +} + +// A parse that stopped early is a recovered root, not a partial tree. +func TestParseXPath_TrailingTokensIsRecovered_Issue2398(t *testing.T) { + root := parseXPathForTest(t, "[A = 1 ) or B = 2]") + if k := attrString(t, root, "kind"); k != "recovered" { + t.Errorf("root kind = %q, want recovered: the rule must see the parse stopped early", k) + } + if src := attrString(t, root, "source"); src == "" { + t.Error("recovered root has no source") + } +} diff --git a/mdl/linter/starlark.go b/mdl/linter/starlark.go index 8abfe2c01..4a728e3e2 100644 --- a/mdl/linter/starlark.go +++ b/mdl/linter/starlark.go @@ -1641,7 +1641,18 @@ func (r *StarlarkRule) builtinParseXPath(_ *starlark.Thread, _ *starlark.Builtin parser := exprcheck.NewParser() ctx := exprcheck.NewSyntaxContext("", "") ctx.XPath = true - ast, _ := parser.Parse(inner, ctx) + ast, hs := parser.Parse(inner, ctx) + // A parse that stopped before the end (E014, trailing tokens) is a + // partial tree that looks whole; a rule over it judged a fragment of the + // constraint (ako/mxcli#2398). Hand the rule a recovered root instead. + for _, h := range hs { + if h.Code == "E014" { + return robustExprToStarlark(&exprcheck.RecoveredExpr{ + SourceFragment: inner, + Reason: "unexpected '" + h.YouWrote + "' after the expression: the constraint did not parse to its end", + }), nil + } + } return robustExprToStarlark(ast), nil } From 42425d2f89998d4ce6bcb06b59905810c6d4031f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 18:09:43 +0000 Subject: [PATCH 14/28] fix(translations): match a source text regardless of CRLF or LF line endings A describe translations file edited by a tool that normalises line endings lost the CR of every CRLF source, and create translations then matched none of them. Sources now match exactly first and then with CRLF read as LF, and the translation takes the source's line breaks. Closes #2383 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011oK82qrr1WnXxq1eRn7qRJ --- ...18-translations-crlf-source-unmatched.json | 1 + CHANGELOG.md | 1 + mdl/translations/apply.go | 3 +- mdl/translations/line_endings_2383_test.go | 54 +++++++++++++++++++ mdl/translations/outofscope.go | 5 +- mdl/translations/project.go | 43 ++++++++++++++- 6 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-other/2026-11-18-translations-crlf-source-unmatched.json create mode 100644 mdl/translations/line_endings_2383_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-18-translations-crlf-source-unmatched.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-18-translations-crlf-source-unmatched.json new file mode 100644 index 000000000..fb693facc --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-18-translations-crlf-source-unmatched.json @@ -0,0 +1 @@ +{"area": "mdl-other", "date": "2026-11-18", "refs": ["ako/mxcli#2383", "mendixlabs/mxcli#1890"], "ce": ["CE4899"], "symptom": "A describe translations file for Workflow Commons, filled in and run back with create or modify translations, reported every CRLF source (\"{1} days\\r\\n\", the confirmation questions) as 'No text has … as its source', and CE4899 stayed.", "file": "mdl/translations/project.go Dictionary.matcher / lookup (used by Apply, PatchUnit, OutOfScope). Test: mdl/translations/line_endings_2383_test.go", "cause": "Sources were matched byte for byte. describe writes the CR raw inside the quoted string (verified on current main: 8 CR lines for WorkflowCommons), but any editor, agent or git checkout that normalises line endings turns the file's sources into LF, which then matched nothing.", "insight": "**A key that survives only byte-exact transport will not survive an agent's edit.** The round trip worked when the file was untouched, so the defect was not in describe; the reporter's step 'fill in a translation' was. Match with line endings ignored (exact first), and write the translation with the source's CRLF so Studio Pro stores what it would. Measured on 11.12.6: a CR-stripped file sets {1} Tage in both documents, stored as {1} Tage\\r\\n."} diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e09b4431..cd70c5957 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`create translations` matches a source whatever its line endings** (ako/mxcli#2383, mendixlabs/mxcli#1890) — Marketplace texts carry CRLF, and a `describe translations` file whose CRs were lost when it was edited reported each of them as unknown, leaving confirmation questions without a default-language text (CE4899). A source now matches with CRLF and LF treated alike, and its translation is stored with the source's line breaks. - **Lint `parse_xpath` parses `true()` / `false()` and no longer returns a partial tree** (ako/mxcli#2398, mendixlabs/mxcli#1894) — everything after an XPath `true()` or `false()` was dropped without a word, so `[IsConfidential = false() or Owner = '[%CurrentUser%]']` reached a rule as `IsConfidential = false`. Both forms now parse whole, and a constraint that does not parse to its end comes back as a `recovered` node, as the lint-rules skill documents. - **`marketplace diff` and `update` no longer report untouched pluggable-widget snippets as edits** (ako/mxcli#2367, mendixlabs/mxcli#1882) — right after an install, Workflow Commons showed 2 changed snippets on the first `diff` and 8 on every later one, so `update` needed `--force`, which also dropped the guard for real edits. The reference cache now keeps the widget definitions, and a reference gets the same widget and design-property repairs install runs on the app. Measured: 0 false edits on both runs, and a real edit is still reported. Cached references from earlier versions are rebuilt once. - **An imported navigation association that would clash is named `_`** (ako/mxcli#2354) — `create external entities` named Product's `Category` navigation `Category_2`, because the name collides with the entity Category. It is now `Product_Category`, with a number only if that is taken too. Names that do not collide are unchanged, and a re-import still recognises existing associations by their navigation property. diff --git a/mdl/translations/apply.go b/mdl/translations/apply.go index cee80de88..de032cfa4 100644 --- a/mdl/translations/apply.go +++ b/mdl/translations/apply.go @@ -81,12 +81,13 @@ func PatchUnit(raw []byte, sourceLang, lang string, dict map[string]string, mode return raw, stats } + m := Dictionary(dict).matcher() for _, t := range findTexts(doc) { src := t.Source(sourceLang) if src == "" { continue } - want, named := dict[src] + _, want, named := m.lookup(src) if want != "" { if t.setTranslation(lang, want) { diff --git a/mdl/translations/line_endings_2383_test.go b/mdl/translations/line_endings_2383_test.go new file mode 100644 index 000000000..69b9c6f88 --- /dev/null +++ b/mdl/translations/line_endings_2383_test.go @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: Apache-2.0 + +package translations + +import ( + "testing" + + "github.com/mendixlabs/mxcli/model" +) + +// ako/mxcli#2383 (mendixlabs/mxcli#1890): Marketplace texts carry CRLF +// ("{1} days\r\n", Workflow Commons' confirmation questions). A file whose +// CRs were lost on the way — an editor, an agent, git — named every one of +// them as unknown ("No text has … as its source"), and the confirmation +// questions stayed CE4899. A source matches whatever its line endings. +func TestApply_MatchesASourceRegardlessOfLineEndings_Issue2383(t *testing.T) { + p := newProject(t, + unit(t, text(tr("en_US", "{1} days\r\n"))), + unit(t, text(tr("en_US", "Sure?\r\n\r\nThis withdraws it."))), + ) + dict := Dictionary{"{1} days\n": "{1} Tage\n", "Sure?\n\nThis withdraws it.": "Sicher?\n\nDies zieht es zurück."} + stats, err := Apply(p, "en_US", "de_CH", dict, ModeMerge, nil) + if err != nil { + t.Fatal(err) + } + if stats.Set != 2 || len(stats.Unmatched) != 0 { + t.Fatalf("stats = %+v, want 2 set and nothing unmatched", stats) + } + // The translation takes the source's line breaks, as Studio Pro stores them. + if got := langsOf(t, p.units["A"])[0]["de_CH"]; got != "{1} Tage\r\n" { + t.Errorf("translation = %q, want %q", got, "{1} Tage\r\n") + } + if got := langsOf(t, p.units["B"])[0]["de_CH"]; got != "Sicher?\r\n\r\nDies zieht es zurück." { + t.Errorf("translation = %q", got) + } + + // CONTROL: the exact spelling still matches, and a different text does not. + p = newProject(t, unit(t, text(tr("en_US", "{1} days\r\n")))) + stats, _ = Apply(p, "en_US", "de_CH", Dictionary{"{1} days\r\n": "{1} Tage\r\n", "{1} weeks\n": "x"}, ModeMerge, nil) + if stats.Set != 1 || len(stats.Unmatched) != 1 || stats.Unmatched[0] != "{1} weeks\n" { + t.Errorf("stats = %+v, want 1 set and {1} weeks unmatched", stats) + } +} + +func TestOutOfScope_MatchesRegardlessOfLineEndings_Issue2383(t *testing.T) { + p := newProject(t, unit(t, text(tr("en_US", "{1} days\r\n")))) + missed, err := OutOfScope(p, "en_US", func(model.ID) bool { return false }, Dictionary{"{1} days\n": "{1} Tage\n"}) + if err != nil { + t.Fatal(err) + } + if len(missed) != 1 || missed[0] != "{1} days\n" { + t.Errorf("missed = %q, want the file's own key", missed) + } +} diff --git a/mdl/translations/outofscope.go b/mdl/translations/outofscope.go index 041129cc3..cc2d45518 100644 --- a/mdl/translations/outofscope.go +++ b/mdl/translations/outofscope.go @@ -36,6 +36,7 @@ func OutOfScope(p Project, sourceLang string, scope Scope, dict Dictionary) ([]s return nil, fmt.Errorf("list units: %w", err) } missed := map[string]bool{} + m := dict.matcher() for _, u := range units { if u == nil || scope.includes(u.ID) { continue @@ -45,8 +46,8 @@ func OutOfScope(p Project, sourceLang string, scope Scope, dict Dictionary) ([]s continue // a unit that cannot be read contributes nothing } for _, e := range CollectFromUnit(raw, sourceLang) { - if _, named := dict[e.Source]; named { - missed[e.Source] = true + if key, _, named := m.lookup(e.Source); named { + missed[key] = true } } } diff --git a/mdl/translations/project.go b/mdl/translations/project.go index 458513086..ad0397a05 100644 --- a/mdl/translations/project.go +++ b/mdl/translations/project.go @@ -5,6 +5,7 @@ package translations import ( "fmt" "sort" + "strings" "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" @@ -34,6 +35,43 @@ func (s Scope) includes(id model.ID) bool { return s == nil || s(id) } // means "present but not translated yet". type Dictionary map[string]string +// matcher finds a source's entry in a Dictionary, exactly or with line +// endings ignored: Marketplace texts carry CRLF ("{1} days\r\n"), and a file +// whose CRs an editor, an agent or git normalised away named each of them as +// unknown (ako/mxcli#2383). +type matcher struct { + dict Dictionary + byLF map[string]string // source with CRLF as LF → the dictionary key +} + +func (d Dictionary) matcher() matcher { + m := matcher{dict: d, byLF: make(map[string]string, len(d))} + for k := range d { + m.byLF[lf(k)] = k + } + return m +} + +// lookup returns the dictionary key naming src, its translation, and whether +// there is one. A translation for a CRLF source takes CRLF line breaks, as +// Studio Pro stores them. +func (m matcher) lookup(src string) (key, translation string, ok bool) { + if t, ok := m.dict[src]; ok { + return src, t, true + } + key, ok = m.byLF[lf(src)] + if !ok { + return "", "", false + } + t := m.dict[key] + if strings.Contains(src, "\r\n") && !strings.Contains(t, "\r") { + t = strings.ReplaceAll(t, "\n", "\r\n") + } + return key, t, true +} + +func lf(s string) string { return strings.ReplaceAll(s, "\r\n", "\n") } + // Stats is what an Apply did across the project. type Stats struct { Set int @@ -148,6 +186,7 @@ func Apply(p Project, sourceLang, lang string, dict Dictionary, mode Mode, scope matched := map[string]bool{} removed := map[string]bool{} + m := dict.matcher() for _, u := range units { if u == nil || !scope.includes(u.ID) { continue @@ -157,8 +196,8 @@ func Apply(p Project, sourceLang, lang string, dict Dictionary, mode Mode, scope continue } for _, e := range CollectFromUnit(raw, sourceLang) { - if _, ok := dict[e.Source]; ok { - matched[e.Source] = true + if key, _, ok := m.lookup(e.Source); ok { + matched[key] = true } } out, us := PatchUnit(raw, sourceLang, lang, dict, mode) From 8d247c5eac4764f020844b38f8cde7cb37b3376a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 18:13:27 +0000 Subject: [PATCH 15/28] fix(enumeration): a value without a caption takes its name as the caption create enumeration and alter enumeration add value stored an empty caption for a value written without one, a blank option nothing reported. The omitted caption now defaults to the value's name, as Studio Pro fills it; an explicit '' is kept. Closes #2397 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011oK82qrr1WnXxq1eRn7qRJ --- ...2026-11-18-enum-value-without-caption.json | 1 + CHANGELOG.md | 1 + cmd/mxcli/syntax/features_domain_model.go | 2 +- mdl/visitor/enum_caption_default_2397_test.go | 36 +++++++++++++++++++ mdl/visitor/visitor_enumeration.go | 4 ++- mdl/visitor/visitor_helpers.go | 6 ++++ 6 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-visitor/2026-11-18-enum-value-without-caption.json create mode 100644 mdl/visitor/enum_caption_default_2397_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-visitor/2026-11-18-enum-value-without-caption.json b/.claude/skills/fix-issue/findings/mdl-visitor/2026-11-18-enum-value-without-caption.json new file mode 100644 index 000000000..da11c4ae3 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-visitor/2026-11-18-enum-value-without-caption.json @@ -0,0 +1 @@ +{"area": "mdl-visitor", "date": "2026-11-18", "refs": ["ako/mxcli#2397", "mendixlabs/mxcli#1893"], "ce": [], "symptom": "Enumeration values added without a caption (`create enumeration M.Color (Red)`, `alter enumeration M.Color add value Blue`) were stored with an empty caption: blank options in every drop-down, describe printed `Red ''`, and mx check reported 0 errors.", "file": "mdl/visitor/visitor_helpers.go buildEnumValues; mdl/visitor/visitor_enumeration.go ExitAlterEnumerationAction (add value). Test: mdl/visitor/enum_caption_default_2397_test.go", "cause": "The grammar makes the caption optional in both statements and the visitor defaulted an absent one to the empty string.", "insight": "**An optional property should default to what the authoring tool would fill, not to the zero value — especially when no checker flags the zero value.** mx check accepts an empty caption, so nothing downstream ever reported it. Distinguish 'omitted' (→ the name, as Studio Pro fills it) from an explicit '' (kept). Measured on 11.12.6: Red 'Red', Blue 'Blue', Green 'Green!' kept; mx check 0 errors."} diff --git a/CHANGELOG.md b/CHANGELOG.md index cd70c5957..6747e2319 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **An enumeration value written without a caption gets its name as the caption** (ako/mxcli#2397, mendixlabs/mxcli#1893) — `create enumeration … (Red)` and `alter enumeration … add value Blue` stored an empty caption, so the value showed as a blank option and nothing reported it. The caption now defaults to the value's name, as Studio Pro fills it; `''` written explicitly stays empty. - **`create translations` matches a source whatever its line endings** (ako/mxcli#2383, mendixlabs/mxcli#1890) — Marketplace texts carry CRLF, and a `describe translations` file whose CRs were lost when it was edited reported each of them as unknown, leaving confirmation questions without a default-language text (CE4899). A source now matches with CRLF and LF treated alike, and its translation is stored with the source's line breaks. - **Lint `parse_xpath` parses `true()` / `false()` and no longer returns a partial tree** (ako/mxcli#2398, mendixlabs/mxcli#1894) — everything after an XPath `true()` or `false()` was dropped without a word, so `[IsConfidential = false() or Owner = '[%CurrentUser%]']` reached a rule as `IsConfidential = false`. Both forms now parse whole, and a constraint that does not parse to its end comes back as a `recovered` node, as the lint-rules skill documents. - **`marketplace diff` and `update` no longer report untouched pluggable-widget snippets as edits** (ako/mxcli#2367, mendixlabs/mxcli#1882) — right after an install, Workflow Commons showed 2 changed snippets on the first `diff` and 8 on every later one, so `update` needed `--force`, which also dropped the guard for real edits. The reference cache now keeps the widget definitions, and a reference gets the same widget and design-property repairs install runs on the app. Measured: 0 false edits on both runs, and a real edit is still reported. Cached references from earlier versions are rebuilt once. diff --git a/cmd/mxcli/syntax/features_domain_model.go b/cmd/mxcli/syntax/features_domain_model.go index 1e4305614..2fbf7df04 100644 --- a/cmd/mxcli/syntax/features_domain_model.go +++ b/cmd/mxcli/syntax/features_domain_model.go @@ -414,7 +414,7 @@ func init() { "caption", "list enumerations", "describe enumeration", "drop enumeration", }, - Syntax: "CREATE ENUMERATION Module.Name (\n ValueName 'Display Caption',\n ...\n) [FOLDER 'path'] [EXPORT LEVEL API|HIDDEN];\n\nALTER ENUMERATION Module.Name SET EXPORT LEVEL API|HIDDEN; -- see domain-model.export-level\nALTER ENUMERATION Module.Name ADD VALUE [IF NOT EXISTS] NewValue [CAPTION 'Display Caption'];\nALTER ENUMERATION Module.Name RENAME VALUE OldName TO NewName;\nALTER ENUMERATION Module.Name MODIFY VALUE ValueName CAPTION 'New Caption';\nALTER ENUMERATION Module.Name DROP VALUE [IF EXISTS] ValueName;\n\nIF NOT EXISTS / IF EXISTS make a script RE-RUNNABLE. Without them the second\nrun errors and exec STOPS THERE, so one already-present value leaves every\nlater statement unapplied. A defensive drop-then-add is not a substitute: the\ndrop fails when the value is absent and the add when it is present.\n\nLIST ENUMERATIONS;\nLIST ENUMERATIONS IN ;\nDESCRIBE ENUMERATION Module.Name;\nDROP ENUMERATION Module.Name;\n\nUsing in entity:\n AttrName: Enumeration(Module.EnumName)\n\nThe System module's enumerations are platform built-ins with no stored\nunit. LIST ENUMERATIONS and DESCRIBE ENUMERATION report them anyway, so\ntheir values can be read instead of guessed at until the build rejects one\nwith CE1613. They are READ-ONLY: DESCRIBE prints them as -- comment lines,\nand CREATE / ALTER / DROP / MOVE naming System is refused.\n mxcli -p app.mpr describe enumeration System.WorkflowActivityType", + Syntax: "CREATE ENUMERATION Module.Name (\n ValueName 'Display Caption',\n ...\n) [FOLDER 'path'] [EXPORT LEVEL API|HIDDEN];\n\nALTER ENUMERATION Module.Name SET EXPORT LEVEL API|HIDDEN; -- see domain-model.export-level\nALTER ENUMERATION Module.Name ADD VALUE [IF NOT EXISTS] NewValue [CAPTION 'Display Caption'];\nALTER ENUMERATION Module.Name RENAME VALUE OldName TO NewName;\nALTER ENUMERATION Module.Name MODIFY VALUE ValueName CAPTION 'New Caption';\nALTER ENUMERATION Module.Name DROP VALUE [IF EXISTS] ValueName;\n\nA value written without a caption takes its name as the caption, as Studio\nPro fills it; write '' for an empty one.\n\nIF NOT EXISTS / IF EXISTS make a script RE-RUNNABLE. Without them the second\nrun errors and exec STOPS THERE, so one already-present value leaves every\nlater statement unapplied. A defensive drop-then-add is not a substitute: the\ndrop fails when the value is absent and the add when it is present.\n\nLIST ENUMERATIONS;\nLIST ENUMERATIONS IN ;\nDESCRIBE ENUMERATION Module.Name;\nDROP ENUMERATION Module.Name;\n\nUsing in entity:\n AttrName: Enumeration(Module.EnumName)\n\nThe System module's enumerations are platform built-ins with no stored\nunit. LIST ENUMERATIONS and DESCRIBE ENUMERATION report them anyway, so\ntheir values can be read instead of guessed at until the build rejects one\nwith CE1613. They are READ-ONLY: DESCRIBE prints them as -- comment lines,\nand CREATE / ALTER / DROP / MOVE naming System is refused.\n mxcli -p app.mpr describe enumeration System.WorkflowActivityType", Example: "mdl 1;\nCREATE ENUMERATION MyModule.OrderStatus (\n Pending 'Pending Approval',\n Processing 'Being Processed',\n Shipped 'Shipped to Customer'\n);\n\n-- Using in an entity\nCREATE PERSISTENT ENTITY MyModule.Order (\n OrderNumber: String(20) NOT NULL,\n Status: Enumeration(MyModule.OrderStatus)\n);", SeeAlso: []string{"domain-model.enumeration", "domain-model.entity.attributes", "domain-model.export-level"}, }) diff --git a/mdl/visitor/enum_caption_default_2397_test.go b/mdl/visitor/enum_caption_default_2397_test.go new file mode 100644 index 000000000..5ae42ad7e --- /dev/null +++ b/mdl/visitor/enum_caption_default_2397_test.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +// ako/mxcli#2397 (mendixlabs/mxcli#1893): an enumeration value written without +// a caption was stored with an empty one — `create enumeration M.Color (Red)` +// and `alter enumeration M.Color add value Blue` both gave a drop-down option +// with no text, and mx check said nothing. Studio Pro fills a new value's +// caption from its name; so does mxcli. A caption written as '' stays empty. +func TestEnumValueCaptionDefaultsToName_Issue2397(t *testing.T) { + prog, errs := Build(`create enumeration M.Color ( Red, Green 'Green!', Grey '' ); +alter enumeration M.Color add value Blue; +alter enumeration M.Color add value Pink caption 'Rose';`) + if len(errs) > 0 { + t.Fatal(errs) + } + ce := prog.Statements[0].(*ast.CreateEnumerationStmt) + want := map[string]string{"Red": "Red", "Green": "Green!", "Grey": ""} + for _, v := range ce.Values { + if v.Caption != want[v.Name] { + t.Errorf("create: %s caption = %q, want %q", v.Name, v.Caption, want[v.Name]) + } + } + if a := prog.Statements[1].(*ast.AlterEnumerationStmt); a.Caption != "Blue" { + t.Errorf("add value Blue: caption = %q, want Blue", a.Caption) + } + if a := prog.Statements[2].(*ast.AlterEnumerationStmt); a.Caption != "Rose" { + t.Errorf("add value Pink caption 'Rose': caption = %q, want Rose", a.Caption) + } +} diff --git a/mdl/visitor/visitor_enumeration.go b/mdl/visitor/visitor_enumeration.go index fe5332647..0b50ccf38 100644 --- a/mdl/visitor/visitor_enumeration.go +++ b/mdl/visitor/visitor_enumeration.go @@ -61,7 +61,9 @@ func (b *Builder) ExitAlterEnumerationAction(ctx *parser.AlterEnumerationActionC } if ctx.ADD() != nil && len(ids) >= 1 { - caption := "" + // No caption written: the value's name, as for `create + // enumeration` (ako/mxcli#2397). + caption := ids[0] if ctx.STRING_LITERAL() != nil { caption = unquoteStringLit(ctx.STRING_LITERAL()) } diff --git a/mdl/visitor/visitor_helpers.go b/mdl/visitor/visitor_helpers.go index d6bf79fcd..b63e21226 100644 --- a/mdl/visitor/visitor_helpers.go +++ b/mdl/visitor/visitor_helpers.go @@ -147,6 +147,12 @@ func buildEnumValues(ctx parser.IEnumerationValueListContext, b *Builder) []ast. } if ev.STRING_LITERAL() != nil { enumVal.Caption = unquoteStringLit(ev.STRING_LITERAL()) + } else { + // No caption written: Studio Pro fills a new value's caption from + // its name, and an empty one is a blank option in every drop-down + // that mx check does not report (ako/mxcli#2397). A caption + // written as '' is kept empty. + enumVal.Caption = enumVal.Name } values = append(values, enumVal) } From 52d6b91afd8aebc9ecb1258a52070385312e03ff Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 18:19:43 +0000 Subject: [PATCH 16/28] fix(pages): address a legacy grid's control-bar buttons and search fields under NewButtons Forms$GridControlBar.Items and Forms$SearchBar.Items are stored as NewButtons, but the widget finder, describe and the catalog read Items, so the buttons were 'not found', unlisted and absent from show widgets. All read the stored key now; describe also prints each button's caption and action. Closes #2382 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011oK82qrr1WnXxq1eRn7qRJ --- ...18-legacy-grid-newbuttons-storage-key.json | 1 + CHANGELOG.md | 1 + .../legacy_grid_newbuttons_2382_test.go | 90 +++++++++++++++++++ mdl/backend/pagemutator/legacygrid.go | 13 ++- mdl/backend/pagemutator/mutator.go | 16 ++-- mdl/catalog/builder_pages.go | 14 +++ mdl/catalog/legacy_grid_buttons_2382_test.go | 31 +++++++ .../cmd_pages_describe_legacy_grid.go | 27 +++++- .../legacy_grid_describe_buttons_2382_test.go | 33 +++++++ 9 files changed, 218 insertions(+), 8 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-backend/2026-11-18-legacy-grid-newbuttons-storage-key.json create mode 100644 mdl/backend/pagemutator/legacy_grid_newbuttons_2382_test.go create mode 100644 mdl/catalog/legacy_grid_buttons_2382_test.go create mode 100644 mdl/executor/legacy_grid_describe_buttons_2382_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-backend/2026-11-18-legacy-grid-newbuttons-storage-key.json b/.claude/skills/fix-issue/findings/mdl-backend/2026-11-18-legacy-grid-newbuttons-storage-key.json new file mode 100644 index 000000000..38b5055d1 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-backend/2026-11-18-legacy-grid-newbuttons-storage-key.json @@ -0,0 +1 @@ +{"area": "mdl-backend", "date": "2026-11-18", "refs": ["ako/mxcli#2382", "mendixlabs/mxcli#1889"], "ce": ["CE1613"], "symptom": "On OIDC SSO 2.3.2's OIDC.Snip_Configuration, `alter snippet … set (Caption: 'Edit X') on actionButton4` (a legacy grid control-bar button) failed with 'widget \"actionButton4\" not found'; describe listed the grid's columns but no buttons, though its comment said buttons were addressable; show widgets omitted them.", "file": "mdl/backend/pagemutator/legacygrid.go controlBarItemKeys (finder, nearest-DS, scope walks in mutator.go); mdl/executor/cmd_pages_describe_legacy_grid.go storedItems (+ caption from CaptionTemplate, action); mdl/catalog/builder_pages.go walkWidget. Tests: *_2382_test.go in pagemutator, executor, catalog", "cause": "Forms$GridControlBar.Items and Forms$SearchBar.Items are STORED as `NewButtons` (modelsdk/gen/keyaudit_test.go lists both); every walk read `Items`. #1459's search-field addressing was tested on a synthetic grid built with `Items`, so it never worked on a real project either.", "insight": "**Build a legacy-shape test fixture from a dumped real unit, not from the metamodel names.** `mxcli bson dump -t snippet -o OIDC.Snip_Configuration` showed ControlBar and SearchBar both holding `NewButtons`; the key audit already listed the mismatch but nothing consulted it when #1459 added the walk. Measured on the real snippet: set Caption and set Action on actionButton4 written; describe lists 6 buttons with caption and action; show widgets lists them (after a catalog rebuild — the catalog cache carries no builder version, so an old cache keeps the old index)."} diff --git a/CHANGELOG.md b/CHANGELOG.md index 6747e2319..e72d5b0e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **A legacy Data grid's control-bar buttons and search fields can be addressed** (ako/mxcli#2382, mendixlabs/mxcli#1889) — they are stored under `NewButtons`, and every lookup read `Items`, so `alter page … set (…) on actionButton4` and `drop widget` said "widget not found", describe listed none of them, and `show widgets` omitted them. All three now read the stored list: a button's caption and action can be changed (the repair for a CE1613 after a microflow rename), describe lists each button with its caption and action, and `show widgets` includes them. - **An enumeration value written without a caption gets its name as the caption** (ako/mxcli#2397, mendixlabs/mxcli#1893) — `create enumeration … (Red)` and `alter enumeration … add value Blue` stored an empty caption, so the value showed as a blank option and nothing reported it. The caption now defaults to the value's name, as Studio Pro fills it; `''` written explicitly stays empty. - **`create translations` matches a source whatever its line endings** (ako/mxcli#2383, mendixlabs/mxcli#1890) — Marketplace texts carry CRLF, and a `describe translations` file whose CRs were lost when it was edited reported each of them as unknown, leaving confirmation questions without a default-language text (CE4899). A source now matches with CRLF and LF treated alike, and its translation is stored with the source's line breaks. - **Lint `parse_xpath` parses `true()` / `false()` and no longer returns a partial tree** (ako/mxcli#2398, mendixlabs/mxcli#1894) — everything after an XPath `true()` or `false()` was dropped without a word, so `[IsConfidential = false() or Owner = '[%CurrentUser%]']` reached a rule as `IsConfidential = false`. Both forms now parse whole, and a constraint that does not parse to its end comes back as a `recovered` node, as the lint-rules skill documents. diff --git a/mdl/backend/pagemutator/legacy_grid_newbuttons_2382_test.go b/mdl/backend/pagemutator/legacy_grid_newbuttons_2382_test.go new file mode 100644 index 000000000..f5e48e08f --- /dev/null +++ b/mdl/backend/pagemutator/legacy_grid_newbuttons_2382_test.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pagemutator + +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/backend/bsonnav" + "github.com/mendixlabs/mxcli/mdl/bsonutil" +) + +// ako/mxcli#2382 (mendixlabs/mxcli#1889): a legacy grid's control-bar buttons +// and search fields are stored under `NewButtons`, the storage name of +// Forms$GridControlBar.Items and Forms$SearchBar.Items — read from OIDC SSO +// 2.3.2's OIDC.Snip_Configuration. The finder walked `Items`, so +// `set (Caption: …) on actionButton4` was "widget not found", and #1459's +// search-field addressing never reached a real search field either. + +func gridActionButton(name, caption string) bson.D { + return bson.D{ + {Key: "$ID", Value: bsonutil.NewIDBsonBinary()}, + {Key: "$Type", Value: "Forms$GridActionButton"}, + {Key: "Name", Value: name}, + {Key: "CaptionTemplate", Value: bson.D{ + {Key: "$ID", Value: bsonutil.NewIDBsonBinary()}, + {Key: "$Type", Value: "Forms$ClientTemplate"}, + {Key: "Template", Value: legacyGridText(caption)}, + }}, + } +} + +func makeStoredLegacyGridPage() bson.D { + return makeRawPage(bson.D{ + {Key: "$ID", Value: bsonutil.NewIDBsonBinary()}, + {Key: "$Type", Value: "Forms$DataGrid"}, + {Key: "Name", Value: "grid1"}, + {Key: "DataSource", Value: bson.D{ + {Key: "$ID", Value: bsonutil.NewIDBsonBinary()}, + {Key: "$Type", Value: "Forms$GridXPathSource"}, + {Key: "SearchBar", Value: bson.D{ + {Key: "$ID", Value: bsonutil.NewIDBsonBinary()}, + {Key: "$Type", Value: "Forms$SearchBar"}, + {Key: "NewButtons", Value: bson.A{int32(2), + legacyGridSearchField("Forms$ComparisonSearchField", "searchField1", "Alias", "OIDC.Client.Alias"), + }}, + }}, + }}, + {Key: "ControlBar", Value: bson.D{ + {Key: "$ID", Value: bsonutil.NewIDBsonBinary()}, + {Key: "$Type", Value: "Forms$GridControlBar"}, + {Key: "NewButtons", Value: bson.A{int32(2), + gridActionButton("actionButton1", "New"), + gridActionButton("actionButton4", "Edit"), + }}, + }}, + {Key: "Columns", Value: bson.A{int32(2), legacyGridColumn("column1", "Alias", "OIDC.Client.Alias")}}, + }) +} + +func TestLegacyGridStoredButtonsAndSearchFieldsResolve_Issue2382(t *testing.T) { + for _, name := range []string{"actionButton4", "searchField1", "column1"} { + if findBsonWidget(makeStoredLegacyGridPage(), name) == nil { + t.Errorf("%s not found in a grid stored as Studio Pro stores it", name) + } + } +} + +func TestLegacyGridButtonCaptionAndDrop_Issue2382(t *testing.T) { + raw := makeStoredLegacyGridPage() + m := &Mutator{rawData: raw, widgetFinder: findBsonWidget} + if err := m.SetWidgetProperty("actionButton4", "Caption", "Edit X"); err != nil { + t.Fatalf("set Caption: %v", err) + } + btn := findBsonWidget(raw, "actionButton4").widget + items := bsonnav.ToBsonA(bsonnav.DGet(bsonnav.DGetDoc(bsonnav.DGetDoc(btn, "CaptionTemplate"), "Template"), "Items")) + if got := bsonnav.DGetString(items[1].(bson.D), "Text"); got != "Edit X" { + t.Errorf("caption = %q, want Edit X", got) + } + + if err := m.DropWidget([]backend.WidgetRef{{Widget: "actionButton4"}}); err != nil { + t.Fatalf("drop: %v", err) + } + cb := bsonnav.DGetDoc(legacyGridDoc(t, raw), "ControlBar") + if got := itemNames(t, bsonnav.ToBsonA(bsonnav.DGet(cb, "NewButtons"))); len(got) != 1 || got[0] != "actionButton1" { + t.Errorf("control bar after drop = %v, want [actionButton1]", got) + } +} diff --git a/mdl/backend/pagemutator/legacygrid.go b/mdl/backend/pagemutator/legacygrid.go index 881dc229d..c9388a573 100644 --- a/mdl/backend/pagemutator/legacygrid.go +++ b/mdl/backend/pagemutator/legacygrid.go @@ -32,8 +32,19 @@ func legacyGridItemLists(wDoc bson.D) []namedItemList { lists := []namedItemList{{wDoc, "Columns"}} if ds := bsonnav.DGetDoc(wDoc, "DataSource"); ds != nil { if sb := bsonnav.DGetDoc(ds, "SearchBar"); sb != nil { - lists = append(lists, namedItemList{sb, "Items"}) + for _, key := range controlBarItemKeys { + lists = append(lists, namedItemList{sb, key}) + } } } return lists } + +// controlBarItemKeys are the keys a control bar or search bar holds its +// buttons and fields under. Forms$GridControlBar.Items and +// Forms$SearchBar.Items are STORED as `NewButtons` (the storage name, read from +// OIDC SSO 2.3.2's Snip_Configuration; modelsdk/gen/keyaudit_test.go lists +// both). Walking only `Items` left every real control-bar button and search +// field "not found" (ako/mxcli#2382); `Items` stays for any shape that uses the +// metamodel name. +var controlBarItemKeys = []string{"NewButtons", "Items"} diff --git a/mdl/backend/pagemutator/mutator.go b/mdl/backend/pagemutator/mutator.go index fb07c8f68..b6779a214 100644 --- a/mdl/backend/pagemutator/mutator.go +++ b/mdl/backend/pagemutator/mutator.go @@ -1686,8 +1686,10 @@ func findInWidgetChildren(wDoc bson.D, widgetName string) *bsonWidgetResult { // ControlBar if controlBar := bsonnav.DGetDoc(wDoc, "ControlBar"); controlBar != nil { - if result := findInWidgetArray(controlBar, "Items", widgetName); result != nil { - return result + for _, key := range controlBarItemKeys { + if result := findInWidgetArray(controlBar, key, widgetName); result != nil { + return result + } } } // Legacy DataGrid: Columns and the source's search bar (#1459). @@ -2514,8 +2516,10 @@ func findNearestDSInChildren(wDoc bson.D, widgetName string, curDS bson.D) (bson } } if controlBar := bsonnav.DGetDoc(wDoc, "ControlBar"); controlBar != nil { - if ds, found := findNearestDSInWidgets(controlBar, "Items", widgetName, curDS); found { - return ds, true + for _, key := range controlBarItemKeys { + if ds, found := findNearestDSInWidgets(controlBar, key, widgetName, curDS); found { + return ds, true + } } } if strings.Contains(typeName, "CustomWidget") { @@ -2716,7 +2720,9 @@ func collectWidgetScopeInChildren(wDoc bson.D, scope map[string]model.ID) { collectWidgetScope(wDoc, "Items", scope) } if controlBar := bsonnav.DGetDoc(wDoc, "ControlBar"); controlBar != nil { - collectWidgetScope(controlBar, "Items", scope) + for _, key := range controlBarItemKeys { + collectWidgetScope(controlBar, key, scope) + } } for _, l := range legacyGridItemLists(wDoc) { collectWidgetScope(l.doc, l.key, scope) diff --git a/mdl/catalog/builder_pages.go b/mdl/catalog/builder_pages.go index f3fdcfc9b..1c7acdbec 100644 --- a/mdl/catalog/builder_pages.go +++ b/mdl/catalog/builder_pages.go @@ -741,6 +741,20 @@ func walkWidget(w map[string]any, parentID string, depth int) []rawWidgetInfo { } } + // A legacy grid's control-bar buttons are named widgets `alter page` + // addresses; their list is STORED as `NewButtons` (the storage name of + // Forms$GridControlBar.Items), so `show widgets` listed none of them + // (ako/mxcli#2382). + if cb, ok := w["ControlBar"].(map[string]any); ok { + for _, key := range []string{"NewButtons", "Items"} { + for _, b := range getBsonArrayElements(cb[key]) { + if bMap, ok := b.(map[string]any); ok { + result = append(result, walkWidget(bMap, childParent, childDepth)...) + } + } + } + } + // Handle DataView/ListView footer widgets footerWidgets := getBsonArrayElements(w["FooterWidgets"]) for _, fw := range footerWidgets { diff --git a/mdl/catalog/legacy_grid_buttons_2382_test.go b/mdl/catalog/legacy_grid_buttons_2382_test.go new file mode 100644 index 000000000..b2bf77b41 --- /dev/null +++ b/mdl/catalog/legacy_grid_buttons_2382_test.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import "testing" + +// ako/mxcli#2382: a legacy grid's control-bar buttons are stored under +// ControlBar.NewButtons; `show widgets` listed none of them, so a tool could +// not find the button to re-point after a microflow was renamed. +func TestWalkWidget_LegacyGridControlBarButtons_Issue2382(t *testing.T) { + grid := map[string]any{ + "$Type": "Forms$DataGrid", + "Name": "grid1", + "ControlBar": map[string]any{ + "$Type": "Forms$GridControlBar", + "NewButtons": []any{int32(2), + map[string]any{"$Type": "Forms$GridActionButton", "Name": "actionButton4"}, + }, + }, + } + got := map[string]string{} + for _, w := range walkWidget(grid, "", 0) { + got[w.Name] = w.WidgetType + } + if got["actionButton4"] != "Forms$GridActionButton" { + t.Errorf("widgets = %v, want actionButton4 listed", got) + } + if got["grid1"] != "Forms$DataGrid" { + t.Errorf("widgets = %v, want grid1 listed", got) + } +} diff --git a/mdl/executor/cmd_pages_describe_legacy_grid.go b/mdl/executor/cmd_pages_describe_legacy_grid.go index 907dc2157..f696a109b 100644 --- a/mdl/executor/cmd_pages_describe_legacy_grid.go +++ b/mdl/executor/cmd_pages_describe_legacy_grid.go @@ -30,6 +30,7 @@ type rawLegacyGridItem struct { Caption string Attribute string Type string + Action string // a control-bar button's action, as MDL } // parseLegacyGrid reads the named parts of a Forms$DataGrid: ControlBar.Items, @@ -37,17 +38,28 @@ type rawLegacyGridItem struct { func parseLegacyGrid(ctx *ExecContext, w map[string]any) *rawLegacyGrid { g := &rawLegacyGrid{} if cb, ok := w["ControlBar"].(map[string]any); ok { - g.ControlBar = parseLegacyGridItems(ctx, cb["Items"]) + g.ControlBar = parseLegacyGridItems(ctx, storedItems(cb)) } g.Columns = parseLegacyGridItems(ctx, w["Columns"]) if ds, ok := w["DataSource"].(map[string]any); ok { if sb, ok := ds["SearchBar"].(map[string]any); ok { - g.SearchFields = parseLegacyGridItems(ctx, sb["Items"]) + g.SearchFields = parseLegacyGridItems(ctx, storedItems(sb)) } } return g } +// storedItems is a control bar's or search bar's list. The metamodel calls it +// Items; it is STORED as `NewButtons` (Forms$GridControlBar, Forms$SearchBar), +// so reading `Items` alone listed no button and no search field of a real grid +// (ako/mxcli#2382). +func storedItems(bar map[string]any) any { + if v, ok := bar["NewButtons"]; ok && v != nil { + return v + } + return bar["Items"] +} + func parseLegacyGridItems(ctx *ExecContext, v any) []rawLegacyGridItem { var out []rawLegacyGridItem for _, e := range getBsonArrayElements(v) { @@ -61,6 +73,14 @@ func parseLegacyGridItems(ctx *ExecContext, v any) []rawLegacyGridItem { Caption: extractTextCaption(ctx, m), Type: extractString(m["Type"]), } + if item.Caption == "" { + // A control-bar button keeps its caption as a client template, + // under CaptionTemplate, like an action button. + item.Caption = extractButtonCaption(ctx, m) + } + if _, ok := m["Action"].(map[string]any); ok { + item.Action = extractButtonAction(ctx, m) + } if ref, ok := m["AttributeRef"].(map[string]any); ok && ref != nil { item.Attribute = columnAttributeFromRef(ctx, ref) } @@ -120,6 +140,9 @@ func legacyItemProps(it rawLegacyGridItem, withType bool) string { if withType && it.Type != "" { props = append(props, "Type: "+it.Type) } + if it.Action != "" { + props = append(props, "Action: "+legacyCommentText(it.Action)) + } return strings.Join(props, ", ") } diff --git a/mdl/executor/legacy_grid_describe_buttons_2382_test.go b/mdl/executor/legacy_grid_describe_buttons_2382_test.go new file mode 100644 index 000000000..aaefe9efa --- /dev/null +++ b/mdl/executor/legacy_grid_describe_buttons_2382_test.go @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import "testing" + +// ako/mxcli#2382: describe printed a legacy grid's columns but none of its +// control-bar buttons or search fields: it read ControlBar.Items and +// SearchBar.Items, and both lists are stored as `NewButtons`. +func TestParseLegacyGrid_StoredNewButtons_Issue2382(t *testing.T) { + ctx, _ := newMockCtx(t) + text := func(s string) map[string]any { + return map[string]any{"$Type": "Texts$Text", "Items": []any{int32(3), + map[string]any{"$Type": "Texts$Translation", "LanguageCode": "en_US", "Text": s}}} + } + grid := map[string]any{ + "$Type": "Forms$DataGrid", + "ControlBar": map[string]any{"NewButtons": []any{int32(2), + map[string]any{"$Type": "Forms$GridActionButton", "Name": "actionButton4", + "CaptionTemplate": map[string]any{"$Type": "Forms$ClientTemplate", "Template": text("Edit")}}, + }}, + "DataSource": map[string]any{"SearchBar": map[string]any{"NewButtons": []any{int32(2), + map[string]any{"$Type": "Forms$ComparisonSearchField", "Name": "searchField1", "Caption": text("Alias")}, + }}}, + } + g := parseLegacyGrid(ctx, grid) + if len(g.ControlBar) != 1 || g.ControlBar[0].Name != "actionButton4" || g.ControlBar[0].Caption != "Edit" { + t.Errorf("control bar = %+v, want actionButton4 'Edit'", g.ControlBar) + } + if len(g.SearchFields) != 1 || g.SearchFields[0].Name != "searchField1" { + t.Errorf("search fields = %+v, want searchField1", g.SearchFields) + } +} From 2e6804e25301e24a251a145e52254bcb055e5b1c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 18:22:05 +0000 Subject: [PATCH 17/28] fix(release): publish each nightly under its dated tag, and give nightly-stamp advice that can be followed The nightly workflow kept only a rolling nightly release, so the dated version mxcli prints could not be downloaded and the bootstrap script's MXCLI_TAG advice gave a 404. Each nightly is now also published as a pre-release under its dated tag (the 30 most recent kept), and the bootstrap advice for a nightly stamp names both the dated tag and MXCLI_TAG=nightly. Closes #2359 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011oK82qrr1WnXxq1eRn7qRJ --- .../2026-11-18-nightly-not-pinnable.json | 1 + .github/workflows/nightly.yml | 39 ++++++++++++++++++- CHANGELOG.md | 1 + cmd/mxcli/init_hook.go | 13 ++++++- cmd/mxcli/init_hook_nightly_tag_2359_test.go | 38 ++++++++++++++++++ docs-site/src/ide/syncing.md | 6 ++- 6 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/other/2026-11-18-nightly-not-pinnable.json create mode 100644 cmd/mxcli/init_hook_nightly_tag_2359_test.go diff --git a/.claude/skills/fix-issue/findings/other/2026-11-18-nightly-not-pinnable.json b/.claude/skills/fix-issue/findings/other/2026-11-18-nightly-not-pinnable.json new file mode 100644 index 000000000..4925491eb --- /dev/null +++ b/.claude/skills/fix-issue/findings/other/2026-11-18-nightly-not-pinnable.json @@ -0,0 +1 @@ +{"area": "other", "date": "2026-11-18", "refs": ["ako/mxcli#2359"], "ce": [], "symptom": "mxcli --version reported nightly-20261117-b7dc2f863 and mxcli init stamped it, but only a rolling `nightly` release tag existed: no rollback to the build a project was made with, and the bootstrap script's 'Set MXCLI_TAG to nightly-… or newer' downloaded a 404.", "file": ".github/workflows/nightly.yml (Compute nightly version, Publish dated nightly with pruning to KEEP=30); cmd/mxcli/init_hook.go bootstrap advice for a nightly stamp and a failed dated download; docs-site/src/ide/syncing.md. Test: cmd/mxcli/init_hook_nightly_tag_2359_test.go", "cause": "The nightly job deleted and recreated one `nightly` release every night, while the version string baked into the binaries named a dated build that was never published; the bootstrap advice assumed every stamp version is a downloadable tag, true only for vX.Y.Z.", "insight": "**A version a binary prints should be a name something can fetch.** The fix publishes the dated tag beside the rolling one (computed once so binary and tag agree) and bounds the count, and the advice no longer promises a download that may be gone. The workflow is not runnable here: actionlint is absent and the release steps need GitHub; the YAML parses, and the shell is plain gh CLI. The bootstrap script downloads from mendixlabs/mxcli, so dated pins work there once the workflow change is upstream."} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2cf9dbf79..8888b7553 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -434,9 +434,17 @@ jobs: env: GH_TOKEN: ${{ github.token }} + # One version string for the binaries and the dated release that carries + # them: `mxcli --version` prints it, `mxcli init` stamps it into a + # project, and the bootstrap script can then download exactly that build. + - name: Compute nightly version + if: steps.check.outputs.skip != 'true' + id: version + run: echo "version=nightly-$(date -u +%Y%m%d)-$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" + - name: Build release binaries if: steps.check.outputs.skip != 'true' - run: VERSION=nightly-$(date -u +%Y%m%d)-$(git rev-parse --short HEAD) make release + run: VERSION=${{ steps.version.outputs.version }} make release - name: Generate changelog if: steps.check.outputs.skip != 'true' @@ -478,3 +486,32 @@ jobs: env: GH_TOKEN: ${{ github.token }} NOTES: ${{ steps.changelog.outputs.body }} + + # The rolling `nightly` tag moves every night, so the build a project was + # made with could not be fetched again: no rollback, and the bootstrap + # script's "set MXCLI_TAG to " downloaded a 404 (ako/mxcli#2359). + # Each nightly is also published under its dated tag — the version + # `mxcli --version` prints — and the newest KEEP of those are kept. + - name: Publish dated nightly + if: steps.check.outputs.skip != 'true' + run: | + if ! gh release view "$VERSION" >/dev/null 2>&1; then + gh release create "$VERSION" \ + --prerelease \ + --latest=false \ + --target "$(git rev-parse HEAD)" \ + --title "Nightly $(date -u +%Y-%m-%d) ($VERSION)" \ + --notes "$NOTES" \ + bin/mxcli-* + fi + gh release list --limit 200 --json tagName,createdAt \ + -q '.[] | select(.tagName | test("^nightly-[0-9]{8}-")) | [.createdAt, .tagName] | @tsv' | + sort -r | tail -n +$((KEEP + 1)) | cut -f2 | + while read -r old; do + gh release delete "$old" --cleanup-tag -y || true + done + env: + GH_TOKEN: ${{ github.token }} + NOTES: ${{ steps.changelog.outputs.body }} + VERSION: ${{ steps.version.outputs.version }} + KEEP: '30' diff --git a/CHANGELOG.md b/CHANGELOG.md index e72d5b0e4..697a4d151 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **Nightly builds can be pinned and rolled back** (ako/mxcli#2359) — only a rolling `nightly` release existed, so the dated version `mxcli --version` prints could not be downloaded, and the bootstrap script's advice to set `MXCLI_TAG` to it gave a 404. Each nightly is now also published under its dated tag, the 30 most recent are kept, and the bootstrap script's advice for a nightly stamp names that tag and `MXCLI_TAG=nightly` as the way to the current build. - **A legacy Data grid's control-bar buttons and search fields can be addressed** (ako/mxcli#2382, mendixlabs/mxcli#1889) — they are stored under `NewButtons`, and every lookup read `Items`, so `alter page … set (…) on actionButton4` and `drop widget` said "widget not found", describe listed none of them, and `show widgets` omitted them. All three now read the stored list: a button's caption and action can be changed (the repair for a CE1613 after a microflow rename), describe lists each button with its caption and action, and `show widgets` includes them. - **An enumeration value written without a caption gets its name as the caption** (ako/mxcli#2397, mendixlabs/mxcli#1893) — `create enumeration … (Red)` and `alter enumeration … add value Blue` stored an empty caption, so the value showed as a blank option and nothing reported it. The caption now defaults to the value's name, as Studio Pro fills it; `''` written explicitly stays empty. - **`create translations` matches a source whatever its line endings** (ako/mxcli#2383, mendixlabs/mxcli#1890) — Marketplace texts carry CRLF, and a `describe translations` file whose CRs were lost when it was edited reported each of them as unknown, leaving confirmation questions without a default-language text (CE4899). A source now matches with CRLF and LF treated alike, and its translation is stored with the source's line breaks. diff --git a/cmd/mxcli/init_hook.go b/cmd/mxcli/init_hook.go index 81abfdf47..a509abc1b 100644 --- a/cmd/mxcli/init_hook.go +++ b/cmd/mxcli/init_hook.go @@ -193,6 +193,9 @@ if [ ! -x "$MXCLI" ] || [ -n "$stale" ]; then else rm -f ./mxcli.download echo "Could not download mxcli from ${url}." >&2 + case "$TAG" in + nightly-*) echo " Only the most recent dated nightlies are kept; MXCLI_TAG=nightly fetches the current one." >&2 ;; + esac if [ ! -x "$MXCLI" ]; then echo "Fetch it manually, or set MXCLI_TAG to a released version." >&2 exit 1 @@ -203,7 +206,15 @@ fi if older_than_stamp "$MXCLI"; then echo "WARNING: $MXCLI ($(bin_version "$MXCLI")) is still older than this project's tooling ($(stamp_field version))." >&2 - echo " Set MXCLI_TAG to $(stamp_field version) or newer (or 'nightly') and re-run this script." >&2 + case "$(stamp_field version)" in + # A dated nightly is published under its own tag, but only the most recent + # ones are kept; 'nightly' always names the current build (ako/mxcli#2359). + nightly-*) + echo " Re-run this script with MXCLI_TAG=$(stamp_field version) (dated nightlies are kept for the most recent builds only)," >&2 + echo " or with MXCLI_TAG=nightly for the current nightly." >&2 ;; + *) + echo " Set MXCLI_TAG to $(stamp_field version) or newer (or 'nightly') and re-run this script." >&2 ;; + esac fi # Keep the project's tooling (skills, bundled lint rules, the mxcli section of diff --git a/cmd/mxcli/init_hook_nightly_tag_2359_test.go b/cmd/mxcli/init_hook_nightly_tag_2359_test.go new file mode 100644 index 000000000..5e59b3662 --- /dev/null +++ b/cmd/mxcli/init_hook_nightly_tag_2359_test.go @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "strings" + "testing" +) + +// ako/mxcli#2359: with a nightly stamp, the bootstrap script told the user to +// "Set MXCLI_TAG to nightly-20261117-b7dc2f863 or newer", which downloaded a +// 404: only the rolling `nightly` tag existed. Dated nightlies are now +// published, but only the most recent ones are kept, so the advice says so +// and names `nightly` as the way to the current build. +func TestBootstrapScript_NightlyStampAdviceIsDownloadable_Issue2359(t *testing.T) { + requireSh(t) + e := newBootEnv(t, "nightly-21000101-deadbee") // newer than the fake download + out := e.run(t) + if !strings.Contains(out, "is still older than this project's tooling") { + t.Fatalf("no staleness warning; output:\n%s", out) + } + for _, want := range []string{"MXCLI_TAG=nightly-21000101-deadbee", "MXCLI_TAG=nightly", "most recent"} { + if !strings.Contains(out, want) { + t.Errorf("advice lacks %q:\n%s", want, out) + } + } + if strings.Contains(out, "or newer (or 'nightly')") { + t.Errorf("still the release-version advice for a nightly stamp:\n%s", out) + } +} + +// CONTROL: a release stamp keeps the release advice. (A nightly stand-in is +// not provably older than a release stamp, so this reads the script itself.) +func TestBootstrapScript_ReleaseStampAdviceUnchanged_Issue2359(t *testing.T) { + if !strings.Contains(renderBootstrapScript("App.mpr"), "or newer (or 'nightly') and re-run this script.") { + t.Error("the release-stamp advice is gone from the bootstrap script") + } +} diff --git a/docs-site/src/ide/syncing.md b/docs-site/src/ide/syncing.md index 4e08cfbee..e4295fb01 100644 --- a/docs-site/src/ide/syncing.md +++ b/docs-site/src/ide/syncing.md @@ -95,7 +95,11 @@ guard is the bootstrap script, `.claude/bootstrap-mxcli.sh`, which a newer project it compares that binary's version with the stamp, and it does not link an older one — it downloads `MXCLI_TAG` instead. `MXCLI_TAG` defaults to the release of the mxcli that generated the script (`vX.Y.Z`), or `nightly` when that -was a nightly or development build. A +was a nightly or development build. Each nightly is also published under its +dated tag (`nightly-YYYYMMDD-`, the version `mxcli --version` prints), and +the most recent dated nightlies are kept, so `MXCLI_TAG` can pin or roll back +to one of those; an older one is gone, and `MXCLI_TAG=nightly` fetches the +current build. A `./mxcli` older than the stamp is replaced the same way, through a temporary file, so a `./mxcli` that is a symlink to the `PATH` binary never has the download written through it. If the download fails, the older `./mxcli` is From 1b22ee48a72f483d007ac01b211e498327b683ce Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 18:23:26 +0000 Subject: [PATCH 18/28] test: keep the #2397 doc comment gofmt-stable Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011oK82qrr1WnXxq1eRn7qRJ --- mdl/visitor/enum_caption_default_2397_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mdl/visitor/enum_caption_default_2397_test.go b/mdl/visitor/enum_caption_default_2397_test.go index 5ae42ad7e..56db38031 100644 --- a/mdl/visitor/enum_caption_default_2397_test.go +++ b/mdl/visitor/enum_caption_default_2397_test.go @@ -12,7 +12,7 @@ import ( // a caption was stored with an empty one — `create enumeration M.Color (Red)` // and `alter enumeration M.Color add value Blue` both gave a drop-down option // with no text, and mx check said nothing. Studio Pro fills a new value's -// caption from its name; so does mxcli. A caption written as '' stays empty. +// caption from its name; so does mxcli. An explicitly empty caption stays empty. func TestEnumValueCaptionDefaultsToName_Issue2397(t *testing.T) { prog, errs := Build(`create enumeration M.Color ( Red, Green 'Green!', Grey '' ); alter enumeration M.Color add value Blue; From 6c22c9c83ab7e5ddfdd7e5dd78f5b76816d44f7f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 21:02:03 +0000 Subject: [PATCH 19/28] chore: ignore generated widget skill docs in testdata fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running `mxcli lint` or `refresh catalog full` against a fixture writes per-widget skill docs into that project's .claude/skills/widgets/ (43 files for testdata/pedapp), which showed up as untracked noise in the work tree. testdata/expr-checker/.gitignore already ignores its own copy with this rationale; testdata/pedapp/.gitignore did not. Fixed at the root instead of per-fixture, scoped to testdata/, so any fixture a lint or catalog run opens is covered — the same reasoning as the **/.mxcli/unit-types.idx entry above it. Verified: the fixture's generated files are ignored, the repo's own .claude/ is unaffected, and no tracked file becomes ignored. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011oK82sY9fcHtMuZcMNRm2B --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 0ae724663..a396706dd 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,9 @@ go-test-output.txt # MPR v2 unit-type index, written next to any project mxcli opens — including # the testdata/ fixtures tests open in place. **/.mxcli/unit-types.idx + +# Per-widget skill docs, derived from a project's widgets/*.mpk and rewritten by +# `refresh catalog full` (executor.RegenerateWidgetDocs), not authored here. +# Written next to any fixture a lint or catalog run opens, the same way the +# unit-type index above is; testdata/expr-checker also ignores its own copy. +**/testdata/**/.claude/skills/widgets/ From fead2d453cfc56728b1ff323b31298a874be583b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 21:25:38 +0000 Subject: [PATCH 20/28] =?UTF-8?q?docs(proposal):=20verify=20gate=20?= =?UTF-8?q?=E2=80=94=20lint=20baseline=20from=20day=20one,=20discoverabili?= =?UTF-8?q?ty,=20benchmark=20evidence=20(#2333)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011oK82stfDEB1TGfJLVwUxc --- docs/11-proposals/PROPOSAL_verify_gate.md | 67 +++++++++++++++++++---- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/docs/11-proposals/PROPOSAL_verify_gate.md b/docs/11-proposals/PROPOSAL_verify_gate.md index 9e0eefcef..76f8a5e18 100644 --- a/docs/11-proposals/PROPOSAL_verify_gate.md +++ b/docs/11-proposals/PROPOSAL_verify_gate.md @@ -11,7 +11,7 @@ related: # Proposal: `mxcli verify` — a staged gate with baseline comparison **Status:** Draft -**Date:** 2026-11-16 +**Date:** 2026-11-16 (updated 2026-11-18) **Issue:** ako/mxcli#2333 **Relates to:** ako/mxcli#1313 (`mx check` baseline), #1312 (`diff --mx-check`), #1258 (`check --full`), #2308 (`replace()` passes `check`, fails `mx check`) @@ -35,7 +35,35 @@ gate must include `mx check`. heuristic each time `check` passes what `mx check` rejects) and `PROPOSAL_check_diagnostics_catalog.md`. `verify` fixes none of them. It exists so that the gap, whatever its size at any moment, cannot pass unnoticed: it makes the -~8 s authoritative stage part of the loop. Closing the gap stays with `check`. +~8 s authoritative stage part of the loop. Closing the gap stays with `check`, and +that work is **step one and keeps going** regardless of this proposal: each `mx +check` validation moved into `mxcli check` shrinks what only the slow stage can +catch. It never reaches zero, which is why the gate keeps `mxcheck` as a stage. + +## Evidence that steps get skipped + +An external benchmark (mxbench.mecowhy.dev, 2026-11-17; summarised in a comment on +#2333) built the same 111-story app nine times (three tools by three models). +Its maintainability critique (microflows with overlapping actions, saves in a +loop, no business key, unreachable template modules, over-wide entities, missing +access rules) maps almost entirely onto **existing `lint` rules** (MPR008, +CONV011, ARCH003, QUAL004, DESIGN001, SEC001 and others). None of it appears as a +caught problem because no agent ran `mxcli lint`; three agents on three tools and +three models each skipped a gate that was installed and correct. Two consequences +for the design: + +- **A stage nothing invokes is indistinguishable from one that does not exist.** + Discoverability is part of the deliverable, not a follow-up (see Phasing). +- **`lint` needs its baseline from the first release.** Run against an AI-built + app with no baseline, lint fires in the hundreds; a gate whose first run + reports 400 findings gets switched off. + +The same evidence bounds what the gate can promise. The benchmark's non-lint +findings (a guest who can write a price, a look-up limit that resets with a new +browser session, paying one order emptying the cart, a home page that overflows +on a phone) need a model reader plus probing a running app. No static stage +reaches them; only `smoke` and beyond can, and `verify` should say so rather than +imply a clean result means "correct". ## Proposed behaviour @@ -70,7 +98,7 @@ mxcli verify -p app.mpr [--base | --baseline ] no runtime for a selected stage), so "gate failed" and "gate could not run" are never confused. When stages fail, the code is a **bitmask** (1 check, 2 lint, 4 mxcheck, 8 build, 16 test, 32 smoke) offset so it cannot collide with 0 or 2 - — exact encoding to be fixed in phase 2 — because with run-all as the default + — exact encoding to be fixed in step 2 — because with run-all as the default several stages can fail at once. The per-stage detail is always in the output; the code is only a summary. **Relation to existing codes:** `check` exits with a variable code (`cmd_check.go`) and `lint` exits 1. `verify`'s table is its own @@ -152,7 +180,7 @@ the output**. Phase 1 must therefore classify each code the parser sees: delta, never a list, and `--format json` says so. The CE0463 stale/persisting labelling from the re-check must survive unchanged as -a label on class 3 findings. The first question for phase 1 is the classification +a label on class 3 findings. The first question for step 1 is the classification table (code → class), built from real `mx check` output; stability under rename or move (formerly the only question) applies only to class 1. @@ -179,19 +207,34 @@ type Stage interface { ## Out of scope -- Fixing findings; running stages in parallel (cost order + early stop is the point); +- Fixing findings; running stages in parallel (stages run in cost order; `--fail-fast` is the CI mode); replacing `docker check`, `lint`, `test` or `playwright check`. - `check --full` (#1258): `mxcheck` stage calls `docker check` until it exists, then may be swapped or made optional. ## Phasing -1. **Structured `mx check` findings** — parser + `docker.CheckFindings()`, tests - against recorded `mx` output fixtures. Standalone PR; unblocks #1313. -2. **`mxcli verify`** with `check`, `lint`, `mxcheck`; `--baseline`, - `--update-baseline`, `--format`, exit-code table, `--fail-fast`. -3. **`--base `** via scratch export. -4. **Opt-in `build`, `test`, `smoke` stages.** +0. **Standing: close the `check` / `mx check` gap** per + `PROPOSAL_check_mxbuild_gap_heuristics.md`. Already under way (each validation + moved into `mxcli check` removes a class of late failures); independent of, and + not blocked by, the steps below. +1. **Structured `mx check` findings**: parser + `docker.CheckFindings()`, tests + against recorded `mx` output fixtures. Standalone PR; unblocks the `mxcheck` + baseline, #1313 and #1312. +2. **`mxcli verify`, first release**: stages `check`, `lint`, `mxcheck`; + `--format`, exit-code table, `--fail-fast`; and a **`lint` baseline from day + one** (`--baseline`, `--update-baseline`, reusing `mdl/linter/baseline.go` + through the adapter described above), so the first run on an existing project + reports only what is new. The `mxcheck` stage baselines once step 1 lands; until + then it reports all findings and says it is unbaselined. +3. **Make `verify` the thing everything points at**: skills, `mxcli init` output + and the project `CLAUDE.md` guidance name `verify` as the single "did my change + break anything?" command, instead of listing six commands. Ships with step 2, + not after it. A guard test (in the style of `init_claudemd_gates_test.go`) + keeps the pointer from drifting. +4. **`--base `** via scratch export. +5. **Opt-in `build`, `test`, `smoke` stages.** `smoke` is also the only stage that + can reach the runtime-only findings listed under Evidence. ## Testing @@ -212,7 +255,7 @@ type Stage interface { `verify`, under a namespace.) 2. **One baseline file or one per stage?** One file is simpler for agents; per-stage keeps `lint --baseline` files valid unchanged. -3. **Code classification** (phase 1): which `mx check` codes are located, +3. **Code classification** (step 1): which `mx check` codes are located, document-only, or unlocated, and is count-based matching acceptable for the unlocated ones? 4. **`--base` for MPR v1** — extract via `git show` of the single `.mpr`, or require v2? From 6bb96d16663b805360e2cacb6ef3d06ce20bf52b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 21:25:44 +0000 Subject: [PATCH 21/28] =?UTF-8?q?docs(proposal):=20verify=20gate=20?= =?UTF-8?q?=E2=80=94=20consistent=20step=20numbering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011oK82stfDEB1TGfJLVwUxc --- docs/11-proposals/PROPOSAL_verify_gate.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/11-proposals/PROPOSAL_verify_gate.md b/docs/11-proposals/PROPOSAL_verify_gate.md index 76f8a5e18..59db50b33 100644 --- a/docs/11-proposals/PROPOSAL_verify_gate.md +++ b/docs/11-proposals/PROPOSAL_verify_gate.md @@ -147,7 +147,7 @@ mxcli verify -p app.mpr [--base | --baseline ] **count**, not a list. The lint stage therefore either keeps lint's own file and `Compare` and adapts the result, or `Finding` grows an `ID` and the matching rules are restated. - Phase 2 decides; it is not "reuse as is". + Step 2 decides; it is not "reuse as is". - `docker.Check(opts) error` (`cmd/mxcli/docker/check.go`) runs `mx` on a temporary copy and **writes free text** to `opts.Stdout`; it returns only an error. (`appErrorCount` is an unexported helper.) **There are no structured `mx check` @@ -169,7 +169,7 @@ The key `(code, document, element path)` is only as good as `mx check`'s text. I `recheckCE0463` (`docker/check.go`): *"mx check's text output does not name a CE0463's page, so two same-named widgets on different pages print the same line; the lines are compared as counts."* For CE0463 the element identity is **not in -the output**. Phase 1 must therefore classify each code the parser sees: +the output**. Step 1 must therefore classify each code the parser sees: 1. **Located** — the line names document and element (e.g. CE0109 on a microflow activity). Key = `(code, document, path)`; `resolved` is a list. From d9b154a23f6bc36bb8587f45397a5e72afdd551d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 21:40:51 +0000 Subject: [PATCH 22/28] external entities: import a key the service does not allow filtering on as not filterable (CE6630) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011oK82tETa46YQmMS58pQjL --- ...nal-entity-key-part-filterable-ce6630.json | 1 + CHANGELOG.md | 1 + .../contracts/key-filterable-metadata.xml | 35 +++++++++ .../key-filterable-from-contract.mdl | 34 ++++++++ mdl/backend/modelsdk/domainmodel.go | 19 ++++- mdl/backend/modelsdk/domainmodel_write.go | 2 +- .../modelsdk/odata_key_filterable_test.go | 70 +++++++++++++++++ mdl/executor/cmd_contract.go | 3 + .../cmd_contract_key_filterable_test.go | 78 +++++++++++++++++++ sdk/domainmodel/domainmodel.go | 6 ++ 10 files changed, 245 insertions(+), 4 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-executor/2026-11-18-external-entity-key-part-filterable-ce6630.json create mode 100644 mdl-examples/bug-tests/contracts/key-filterable-metadata.xml create mode 100644 mdl-examples/bug-tests/key-filterable-from-contract.mdl create mode 100644 mdl/backend/modelsdk/odata_key_filterable_test.go create mode 100644 mdl/executor/cmd_contract_key_filterable_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-external-entity-key-part-filterable-ce6630.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-external-entity-key-part-filterable-ce6630.json new file mode 100644 index 000000000..972e14645 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-18-external-entity-key-part-filterable-ce6630.json @@ -0,0 +1 @@ +{"area": "mdl/executor", "date": "2026-11-18", "refs": ["ako/mxcli#1258"], "ce": ["CE6630"], "symptom": "`create external entities from` over a contract whose entity set has FilterRestrictions Filterable=false, or NonFilterableProperties naming the key, built an app mx check rejected with one CE6630 \"'Id' is marked Filterable=False in the OData service, but True in the app\" per such entity. The key attribute's own Filterable was already false; every other property was right.", "file": "sdk/domainmodel RemoteKeyPart.NotFilterable; mdl/executor/cmd_contract.go (key parts get !entitySet.AttrFilterable(keyName)); mdl/backend/modelsdk/domainmodel_write.go odataKeyPartToGen (was SetFilterable(true)) and domainmodel.go odataKeyFromGen/storedKeyPartFilterable. Tests: cmd_contract_key_filterable_test.go, odata_key_filterable_test.go; fixture mdl-examples/bug-tests/key-filterable-from-contract.mdl + contracts/key-filterable-metadata.xml", "cause": "A Rest$ODataKeyPart carries its own Filterable beside the attribute's Rest$ODataMappedValue.Filterable, and mxbuild compares both with the service. The writer hard-coded the key part's to true, so the four earlier CE6630 fixes (set-level and property-list shapes of Filterable, Sortable, Top, Skip, Updatable) all set the attribute and none could reach the key part. The reader never read it, so a rewrite of a Studio Pro entity would also have flipped a stored false to true.", "insight": "**Five CE6630 records were the importer getting a capability wrong one term at a time; the efficient way to close the class was one contract per capability shape against the real build, not another report.** 13 shapes, one contract and one project each (whole set Sortable/Filterable=false, NonSortable/NonFilterableProperties naming the label or the key, TopSupported, SkipSupported, CountRestrictions, Insert/Update/DeleteRestrictions, no annotation): 11 clean, 2 CE6630 - both on the KEY and both Filterable. Sortable on the key was already fine, which is what pointed away from the contract parser (AttrFilterable and AttrSortable are the same shape) and at a field only the key has. A model field with a historic default needs a zero value that keeps the default: `NotFilterable` rather than `Filterable`, so every other builder of a key part is unchanged. The reader must not read an absent property as false (storedKeyPartFilterable defaults true), the same trap storedLocalizeDate documents. A hand-written `create external entity` is NOT a CE6630 gap worth a check: it produces CE6010 (no key), CE6612 on every attribute and Countable/Top/Skip mismatches whatever the contract says, so it is not a form that builds on its own. My first matrix also forgot ODataVersion: OData4 (CE6631 on everything) and used the reserved attribute name `Id` in a hand-written entity (CE7247); both were mine. Control: reverting the writer, the importer or the reader each fails its own tests (2, 3 and 1); the fixture gives 2 CE6630 with the pre-fix binary and 0 errors with the fixed one."} diff --git a/CHANGELOG.md b/CHANGELOG.md index 96749e3de..fc4844a1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`create external entities from` no longer imports a key the service says is not filterable as filterable** (ako/mxcli#1258) — a service whose entity set is not filterable, or whose `NonFilterableProperties` names the key, imported into an app that failed to build with CE6630 "'Code' is marked Filterable=False in the OData service, but True in the app", once per such entity. The attribute's own flag was right; the entity's remote key (`Rest$ODataKey`) holds a key part with a second `Filterable`, which was written `true` every time. It now follows the same contract restrictions, a re-import refreshes it, and reading an entity keeps a stored `false` so a rewrite of a Studio Pro-authored entity no longer flips it. Measured on MxBuild 11.16.1 over 13 capability shapes, one contract and one project each: 11 were already clean, the two above are fixed. - **Nightly builds can be pinned and rolled back** (ako/mxcli#2359) — only a rolling `nightly` release existed, so the dated version `mxcli --version` prints could not be downloaded, and the bootstrap script's advice to set `MXCLI_TAG` to it gave a 404. Each nightly is now also published under its dated tag, the 30 most recent are kept, and the bootstrap script's advice for a nightly stamp names that tag and `MXCLI_TAG=nightly` as the way to the current build. - **A legacy Data grid's control-bar buttons and search fields can be addressed** (ako/mxcli#2382, mendixlabs/mxcli#1889) — they are stored under `NewButtons`, and every lookup read `Items`, so `alter page … set (…) on actionButton4` and `drop widget` said "widget not found", describe listed none of them, and `show widgets` omitted them. All three now read the stored list: a button's caption and action can be changed (the repair for a CE1613 after a microflow rename), describe lists each button with its caption and action, and `show widgets` includes them. - **An enumeration value written without a caption gets its name as the caption** (ako/mxcli#2397, mendixlabs/mxcli#1893) — `create enumeration … (Red)` and `alter enumeration … add value Blue` stored an empty caption, so the value showed as a blank option and nothing reported it. The caption now defaults to the value's name, as Studio Pro fills it; `''` written explicitly stays empty. diff --git a/mdl-examples/bug-tests/contracts/key-filterable-metadata.xml b/mdl-examples/bug-tests/contracts/key-filterable-metadata.xml new file mode 100644 index 000000000..18dd251e4 --- /dev/null +++ b/mdl-examples/bug-tests/contracts/key-filterable-metadata.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + Code + + + + + + + + diff --git a/mdl-examples/bug-tests/key-filterable-from-contract.mdl b/mdl-examples/bug-tests/key-filterable-from-contract.mdl new file mode 100644 index 000000000..07961072e --- /dev/null +++ b/mdl-examples/bug-tests/key-filterable-from-contract.mdl @@ -0,0 +1,34 @@ +mdl 1; +-- CE6630 on an external entity's KEY: "'Code' is marked Filterable=False in the OData +-- service, but True in the app." +-- +-- An external entity's remote key (Rest$ODataKey) holds a key part that carries its +-- own Filterable, which mxbuild compares with the service's, besides the attribute's +-- own flag. `create external entities from` set the attribute's flag from the contract +-- but wrote the key part true every time, so a service that does not allow filtering +-- on its key -- the whole set not filterable, or NonFilterableProperties naming the key +-- -- imported into an app that failed to build, one CE6630 per such entity. +-- +-- Measured on MxBuild 11.16.1, one contract and one project per case: 13 capability +-- shapes (no annotation, whole set not sortable / not filterable, the label or the key +-- in NonSortableProperties / NonFilterableProperties, TopSupported, SkipSupported, +-- CountRestrictions, Insert/Update/DeleteRestrictions). Eleven built clean before the +-- fix; the two that did not were the two above. All thirteen build with 0 errors now. +-- +-- MetadataUrl is relative to the APP's directory, so copy the contract there first: +-- mkdir -p /path/to/app/mdl-examples/bug-tests/contracts +-- cp mdl-examples/bug-tests/contracts/key-filterable-metadata.xml \ +-- /path/to/app/mdl-examples/bug-tests/contracts/ +-- mxcli exec mdl-examples/bug-tests/key-filterable-from-contract.mdl -p /path/to/app/app.mpr + +create module KeyFilt; + +create constant KeyFilt.SvcUrl ( Type: string, DefaultValue: 'https://example.invalid/odata/' ); + +create consumed odata service KeyFilt.Client ( + ODataVersion: OData4, + MetadataUrl: './mdl-examples/bug-tests/contracts/key-filterable-metadata.xml', + ServiceUrl: '@KeyFilt.SvcUrl' +); + +create external entities from KeyFilt.Client into KeyFilt; diff --git a/mdl/backend/modelsdk/domainmodel.go b/mdl/backend/modelsdk/domainmodel.go index a469b100f..eea987165 100644 --- a/mdl/backend/modelsdk/domainmodel.go +++ b/mdl/backend/modelsdk/domainmodel.go @@ -708,9 +708,10 @@ func odataKeyFromGen(key element.Element) []*domainmodel.RemoteKeyPart { continue } kp := &domainmodel.RemoteKeyPart{ - Name: p.EntityKeyPartName(), - RemoteName: p.Name(), - RemoteType: p.RemoteType(), + Name: p.EntityKeyPartName(), + RemoteName: p.Name(), + RemoteType: p.RemoteType(), + NotFilterable: !storedKeyPartFilterable(p), } if t := p.Type(); t != nil { kp.Type = attributeTypeFromGen(t) @@ -720,6 +721,18 @@ func odataKeyFromGen(key element.Element) []*domainmodel.RemoteKeyPart { return parts } +// storedKeyPartFilterable reads a key part's Filterable. A document that does +// not store it is read as filterable, the value every earlier mxcli wrote: reading +// the absent property as false would mark the key not filterable on the next write. +func storedKeyPartFilterable(p *genRest.ODataKeyPart) bool { + if raw := p.Raw(); raw != nil { + if _, err := raw.LookupErr("Filterable"); err != nil { + return true + } + } + return p.Filterable() +} + // storedLocalizeDate reads DateTimeAttributeType.LocalizeDate. An absent property // is Mendix's default, true: mxcli wrote DateTime attributes without it until // #743, and reading those as false would unlocalize them on the next write of diff --git a/mdl/backend/modelsdk/domainmodel_write.go b/mdl/backend/modelsdk/domainmodel_write.go index 0c672e88b..25cea56a7 100644 --- a/mdl/backend/modelsdk/domainmodel_write.go +++ b/mdl/backend/modelsdk/domainmodel_write.go @@ -829,7 +829,7 @@ func odataKeyPartToGen(kp *domainmodel.RemoteKeyPart) element.Element { part := genRest.NewODataKeyPart() part.SetEntityKeyPartName(kp.Name) part.SetName(kp.RemoteName) - part.SetFilterable(true) + part.SetFilterable(!kp.NotFilterable) part.SetRemoteType(kp.RemoteType) t := attributeTypeToGen(kp.Type) assignID(t) diff --git a/mdl/backend/modelsdk/odata_key_filterable_test.go b/mdl/backend/modelsdk/odata_key_filterable_test.go new file mode 100644 index 000000000..53ef04e14 --- /dev/null +++ b/mdl/backend/modelsdk/odata_key_filterable_test.go @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/modelsdk/codec" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// A Rest$ODataKeyPart carries its own Filterable, which mxbuild compares with the +// service's: a key the contract says cannot be filtered on, stored true, is CE6630 +// "'Id' is marked Filterable=False in the OData service, but True in the app". +// It used to be written true whatever the contract said. +func encodedKeyPartFilterable(t *testing.T, kp *domainmodel.RemoteKeyPart) bool { + t.Helper() + b, err := (&codec.Encoder{}).Encode(odataKeyToGen([]*domainmodel.RemoteKeyPart{kp})) + if err != nil { + t.Fatal(err) + } + var doc bson.D + if err := bson.Unmarshal(b, &doc); err != nil { + t.Fatal(err) + } + for _, e := range doc { + if e.Key != "Parts" { + continue + } + arr, _ := e.Value.(bson.A) + part, _ := arr[len(arr)-1].(bson.D) + for _, f := range part { + if f.Key == "Filterable" { + v, _ := f.Value.(bool) + return v + } + } + } + t.Fatalf("no Filterable stored on the key part: %v", doc) + return false +} + +func TestODataKeyPart_FilterableIsWrittenFromTheModel(t *testing.T) { + base := domainmodel.RemoteKeyPart{Name: "Id", RemoteName: "Id", RemoteType: "Edm.String", Type: &domainmodel.StringAttributeType{}} + + filterable := base + if !encodedKeyPartFilterable(t, &filterable) { + t.Error("a key part with nothing said stays filterable (the historic value)") + } + notFilterable := base + notFilterable.NotFilterable = true + if encodedKeyPartFilterable(t, ¬Filterable) { + t.Error("a key part the contract says is not filterable was written Filterable=true") + } +} + +// Reading must keep what the document says, or a rewrite of a Studio Pro entity +// whose key is not filterable would flip it back to true. +func TestODataKeyPart_FilterableSurvivesARoundTrip(t *testing.T) { + for _, notFilterable := range []bool{false, true} { + kp := &domainmodel.RemoteKeyPart{Name: "Id", RemoteName: "Id", RemoteType: "Edm.String", + Type: &domainmodel.StringAttributeType{}, NotFilterable: notFilterable} + back := odataKeyFromGen(odataKeyToGen([]*domainmodel.RemoteKeyPart{kp})) + if len(back) != 1 || back[0].NotFilterable != notFilterable { + t.Errorf("NotFilterable %v did not survive: %+v", notFilterable, back) + } + } +} diff --git a/mdl/executor/cmd_contract.go b/mdl/executor/cmd_contract.go index 85ada7964..0f4fa9501 100644 --- a/mdl/executor/cmd_contract.go +++ b/mdl/executor/cmd_contract.go @@ -618,6 +618,9 @@ func createExternalEntities(ctx *ExecContext, s *ast.CreateExternalEntitiesStmt) RemoteName: keyName, RemoteType: keyProp.Type, Type: edmToDomainModelAttrType(keyProp, true), + // The key part has its own Filterable, which mxbuild compares with + // the service's — the same restrictions the attribute follows. + NotFilterable: !entitySet.AttrFilterable(keyName), }) } diff --git a/mdl/executor/cmd_contract_key_filterable_test.go b/mdl/executor/cmd_contract_key_filterable_test.go new file mode 100644 index 000000000..ce6bc7477 --- /dev/null +++ b/mdl/executor/cmd_contract_key_filterable_test.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" +) + +// CE6630 "'Id' is marked Filterable=False in the OData service, but True in the +// app." Measured on MxBuild 11.16.1, one contract and one project per case, the +// importer (`create external entities from`) against: no annotation, a whole set +// that is not sortable/filterable, NonSortableProperties / NonFilterableProperties +// naming the label or the key, TopSupported, SkipSupported, CountRestrictions, +// Insert/Update/DeleteRestrictions. Eleven of thirteen were clean; the two that +// were not both restricted FILTERING of the KEY: the whole set Filterable=false, +// and NonFilterableProperties naming the key. The attribute's own flag was right — +// it is the Rest$ODataKeyPart, which carries a second Filterable, that was always +// written true. +func keyFilterableContract(annotation string) string { + return ` + + + + + + + + + + ` + annotation + ` + + + +` +} + +const ( + filterTerm = `` + sortTerm = `` + wholeSetOff = `` +) + +func nonFilterable(name string) string { + return filterTerm + `` + name + + `` +} + +func TestImportedKeyPart_FilterableFollowsTheContract(t *testing.T) { + cases := []struct { + name, annotation string + wantNotFilter bool + }{ + {"no annotation (control)", "", false}, + {"the whole set is not filterable", filterTerm + wholeSetOff, true}, + {"the key is listed as not filterable", nonFilterable("Id"), true}, + {"another property is listed (control)", nonFilterable("Label"), false}, + {"the whole set is not SORTABLE (control)", sortTerm + strings.Replace(wholeSetOff, "Filterable", "Sortable", 1), false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + entities, _ := importComplexTypeContract(t, keyFilterableContract(c.annotation)) + e := entities["Thing"] + if e == nil || len(e.RemoteKeyParts) != 1 { + t.Fatalf("want an imported Thing with one key part, got %+v", e) + } + if got := e.RemoteKeyParts[0].NotFilterable; got != c.wantNotFilter { + t.Errorf("key part NotFilterable = %v, want %v", got, c.wantNotFilter) + } + // The attribute's own flag was already right and must stay in step. + for _, a := range e.Attributes { + if a.Name == "Id" && a.Filterable == c.wantNotFilter { + t.Errorf("attribute Id Filterable = %v disagrees with the key part (NotFilterable %v)", a.Filterable, c.wantNotFilter) + } + } + }) + } +} diff --git a/sdk/domainmodel/domainmodel.go b/sdk/domainmodel/domainmodel.go index 51c3a09af..e97752221 100644 --- a/sdk/domainmodel/domainmodel.go +++ b/sdk/domainmodel/domainmodel.go @@ -129,6 +129,12 @@ type RemoteKeyPart struct { RemoteName string // OData property name (often same as Name) RemoteType string // OData EDM type, e.g. "Edm.String" Type AttributeType + // NotFilterable is true when the service says the key cannot be filtered on. + // A key part carries its own Filterable, which mxbuild compares with the + // service's (CE6630), so it is a model fact like the attribute's. The zero + // value keeps the historic filterable key, so code that builds a key part + // without a contract in hand is unchanged. + NotFilterable bool } // GetName returns the entity's name. From 0d6b395a787009f46ee3451e83841c79ee841c11 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 21:46:21 +0000 Subject: [PATCH 23/28] chore: ignore mxcli's working directory in top-level testdata fixtures Any command that opens a fixture writes .mxcli/ beside it (catalog.db, the unit index, the unit-contents cache). testdata/pedapp/.gitignore ignores its own; testdata/testapp-views and testdata/expr-checker have no entry, so running lint against them leaves an untracked directory behind. The root ignored only **/.mxcli/unit-types.idx, one file inside it. Anchored at /testdata/ deliberately, NOT **/testdata/**: modelsdk/mpr/testdata/v1-project/.mxcli/catalog.db is tracked as a fixture, and a recursive pattern would shadow it. Verified: the byproduct is ignored, that tracked catalog.db is still visible to git, and `git ls-files | git check-ignore --stdin` is empty, so no tracked file became ignored. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011oK82sY9fcHtMuZcMNRm2B --- .gitignore | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitignore b/.gitignore index a396706dd..8cd8572ad 100644 --- a/.gitignore +++ b/.gitignore @@ -80,3 +80,11 @@ go-test-output.txt # Written next to any fixture a lint or catalog run opens, the same way the # unit-type index above is; testdata/expr-checker also ignores its own copy. **/testdata/**/.claude/skills/widgets/ + +# mxcli's own working directory inside a top-level testdata fixture: catalog, +# unit index and unit-contents cache, all written by any command that opens the +# fixture. testdata/pedapp ignores its own; the others have no .gitignore. +# Anchored at /testdata/ ON PURPOSE: modelsdk/mpr/testdata/v1-project/.mxcli/ +# holds a catalog.db that IS tracked, as a fixture, and a /**/ pattern would +# shadow it. +/testdata/*/.mxcli/ From 8d7d1f6ec1372a163add3ae830781572f92c4267 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 21:46:39 +0000 Subject: [PATCH 24/28] feat(lint): MPR016 reports entities whose boxes overlap on the domain model Closes ako/mxcli#2406. The sibling of MPR008, which does this for microflow activities. The diagram is the first thing someone opening an unfamiliar app reads, and a box hidden behind another is a member list they never see. Neither Studio Pro's check nor mxbuild reports it, because the model is perfectly valid. A pair is named as a pair; entities that collide transitively are ONE finding however large the cluster, so a model with every entity on one spot reports once rather than as 325 pairs. The finding names the repair that already exists: `mxcli layout --only-overlapping`. Geometry is dmlayout's own estimator (an entity stores only its top-left Location; Studio Pro derives the box when it draws, so the size is estimated from the name and member list), so the rule and the placer agree by construction instead of being kept in step by hand. The predicate is new and deliberately NOT the placer's. Rect.overlaps kept boxes a 20px clearance apart, which is what placing needs and wrong for a check: two entities 10px apart are badly laid out and do not overlap, and a finding has to be true. It is now Rect.TooClose (unchanged behaviour, same single call site in clear()), beside a new Rect.Intersects with no margin. A test pins the difference and fails if the rule is ever switched over. Unlike MPR008 this does not skip the origin: an unpositioned activity defaults to (0,0) so there it means "no position recorded", while entities are placed on every create path, so entities sharing a position IS the defect. Platform modules are skipped via the new LintContext.OwnedModuleNamesByID. A rule that reads the MODEL does not inherit the notPlatformModule WHERE clause that protects catalog-backed rules, so before that existed this reported 41 System entities, naming FileDocument and HttpRequest exactly as QUAL002 once did. Source cannot distinguish System (empty for it as for the user's own modules), hence the module-ID test. Recorded as a finding, because the rules proposed in #2403, #2405 and #2407 all read the model and start out the same way. Measured - 0 findings on testdata/pedapp and testdata/testapp-views. - 0 findings on 24 entities of varied name length and 0-21 attributes created with no @position, so dmlayout's auto-placement is clean; full `layout` (Plan) and `layout --only-overlapping` are clean too, and idempotent. - Planted overlaps -- a pair, a tall entity reaching the box below it, and a pile of four -- are each reported, the pile as one finding; then `layout --only-overlapping` moves 5 of 10 entities and lint returns to 0, so detector and repair agree. - Correction to the issue's hypothesis: dmlayout is NOT the source of the overlaps that motivated the rule. The unguarded path is an explicit @position written by a script, which nothing warns about. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011oK82sY9fcHtMuZcMNRm2B --- ...ading-lint-rule-reports-system-module.json | 1 + CHANGELOG.md | 8 + cmd/mxcli/cmd_lint.go | 1 + mdl/dmlayout/place.go | 26 +- mdl/dmlayout/place_test.go | 4 +- mdl/linter/context.go | 66 ++++ .../rules/mpr016_overlapping_entities.go | 196 +++++++++++ .../rules/mpr016_overlapping_entities_test.go | 310 ++++++++++++++++++ 8 files changed, 607 insertions(+), 5 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/mdl-other/2026-11-18-model-reading-lint-rule-reports-system-module.json create mode 100644 mdl/linter/rules/mpr016_overlapping_entities.go create mode 100644 mdl/linter/rules/mpr016_overlapping_entities_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-18-model-reading-lint-rule-reports-system-module.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-18-model-reading-lint-rule-reports-system-module.json new file mode 100644 index 000000000..0ae80b69b --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-18-model-reading-lint-rule-reports-system-module.json @@ -0,0 +1 @@ +{"area": "mdl/linter", "date": "2026-11-18", "symptom": "A new lint rule that reads the MODEL rather than the catalog reports Mendix's own platform content. MPR016's first run on testdata/pedapp reported \"41 entities overlap each other on the System domain model diagram: ConsumedODataConfiguration, Error, FileDocument, HttpHeader, HttpMessage, HttpRequest and 35 more\" — a model no user can edit (`mxcli layout` refuses System outright). The project's own modules were clean.", "cause": "Catalog-backed rules get the platform filter for free: every ctx.Entities()/Pages()/... query ends in notPlatformModule(alias) (mdl/linter/context.go), which is `COALESCE(Source,'')='' AND COALESCE(Id,'')<>'00000000-0000-0000-0000-000000000001'`. A rule reading the model through the reader (here ListDomainModels, for entity Location, which the catalog does not hold) never passes through that WHERE clause, so it starts out seeing System and every Marketplace module. Fixed by adding LintContext.OwnedModuleNamesByID, the Go counterpart of notPlatformModule, and keying the rule's module resolution on it.", "file": "`mdl/linter/context.go` (OwnedModuleNamesByID, notPlatformModule, systemModuleID); `mdl/linter/rules/mpr016_overlapping_entities.go`; test `mdl/linter/rules/mpr016_overlapping_entities_test.go` (TestOverlappingEntitiesRule_SkipsPlatformModules)", "insight": "The platform filter lives in the SQL, so it protects only rules that ask the catalog. Any rule needing geometry, an expression or anything else the catalog does not carry must re-apply it in Go, and every such rule starts out broken in exactly this way — QUAL002 shipped it once on these same entities, and MPR016 reproduced it three months later. Two cheap discriminators: run the new rule next to a catalog-based rule with a similar threshold (MPR003 caps a module at 15 persistent entities and reported NOTHING on the same 41-entity System module, which localises the difference to the data source in one step), and read the finding's own entity names — FileDocument and HttpRequest are Mendix's, not any app's. Do NOT reach for modules.Source to fix it: Source is empty for System exactly as for the user's own modules, which is the trap the ID test exists for. Rules reading the model are about to become common (an attribute read-but-never-written check, a model-name-in-a-string-literal check and an index check all need the model), so the helper is the fix rather than a filter per rule."} diff --git a/CHANGELOG.md b/CHANGELOG.md index 96749e3de..da3fa736b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Added + +- **Lint `MPR016`: entities whose boxes cover each other on the domain model diagram** (ako/mxcli#2406) — the sibling of MPR008, which does this for microflow activities. The diagram is the first thing someone opening an unfamiliar app reads, and a box behind another box is a member list they never see; neither Studio Pro's check nor mxbuild reports it, because the model is valid. A pair is named as a pair, and a cluster that collides transitively is ONE finding however large — a model with every entity on one spot is 1 finding, not 325 pairs. The repair is the command the finding names, `mxcli layout --only-overlapping`. + + Box geometry is `dmlayout`'s own estimator (an entity stores only its top-left `Location`; Studio Pro derives the box when it draws, so the size is estimated from the name and member list), so the rule and the placer agree by construction rather than by being kept in step. The PREDICATE is new and deliberately not the placer's: `Rect.TooClose` keeps boxes a 20px clearance apart, which is what placing needs, while a finding must be true — two entities 10px apart are badly laid out and do not overlap. `Rect.Intersects` has no margin, and a test pins the difference. + + Measured: 0 findings on `testdata/pedapp` and `testdata/testapp-views`, and 0 on 24 entities of varied name length and attribute count (0 to 21) created with no `@position`, so `dmlayout`'s auto-placement, full `Plan` and `--only-overlapping` repair all satisfy the rule. Planted overlaps — a pair, a tall entity reaching the box below it, and a pile of four — are each reported, and `mxcli layout --only-overlapping` takes the project back to 0 by moving 5 of 10 entities. Platform modules are skipped: a rule that reads the model does not inherit the catalog's `notPlatformModule` filter, and before that was added the rule reported 41 System entities (`LintContext.OwnedModuleNamesByID` is the Go counterpart of that SQL). + ### Fixed - **Nightly builds can be pinned and rolled back** (ako/mxcli#2359) — only a rolling `nightly` release existed, so the dated version `mxcli --version` prints could not be downloaded, and the bootstrap script's advice to set `MXCLI_TAG` to it gave a 404. Each nightly is now also published under its dated tag, the 30 most recent are kept, and the bootstrap script's advice for a nightly stamp names that tag and `MXCLI_TAG=nightly` as the way to the current build. diff --git a/cmd/mxcli/cmd_lint.go b/cmd/mxcli/cmd_lint.go index a5eefc2fe..be6024654 100644 --- a/cmd/mxcli/cmd_lint.go +++ b/cmd/mxcli/cmd_lint.go @@ -518,6 +518,7 @@ func builtinLintRules() []linter.Rule { rules.NewSecurityLevelIgnoresRulesRule(), // SEC013 - security level Off/Prototype ignores the project's rules rules.NewOverlappingActivitiesRule(), // MPR008 - requires BSON inspection rules.NewLoopChildContainmentRule(), // MPR011 - requires BSON inspection + rules.NewOverlappingEntitiesRule(), // MPR016 - reads entity geometry from the domain model rules.NewNoCommitInLoopRule(), // CONV011-CONV014 - require BSON inspection rules.NewExclusiveSplitCaptionRule(), rules.NewErrorHandlingOnCallsRule(), diff --git a/mdl/dmlayout/place.go b/mdl/dmlayout/place.go index 7a1cb4261..9522b9909 100644 --- a/mdl/dmlayout/place.go +++ b/mdl/dmlayout/place.go @@ -20,12 +20,32 @@ type Rect struct { X, Y, W, H int } -// overlaps reports whether two boxes come closer than clearance. -func (r Rect) overlaps(o Rect) bool { +// TooClose reports whether two boxes come closer than clearance. This is the +// predicate for PLACING: touching boxes are as unreadable as overlapping ones, +// so a placer separates them. +// +// It is deliberately NOT the predicate for checking a stored model. Two boxes +// 10px apart satisfy TooClose and do not overlap, so a lint rule that used this +// would report an overlap that is not there — see Intersects. +func (r Rect) TooClose(o Rect) bool { return r.X < o.X+o.W+clearance && o.X < r.X+r.W+clearance && r.Y < o.Y+o.H+clearance && o.Y < r.Y+r.H+clearance } +// Intersects reports whether two boxes actually cover each other, with no +// clearance margin. This is the predicate for CHECKING a model that already +// exists (linter rule MPR016): a finding has to be true, and "closer than I +// would have placed them" is not an overlap. +// +// Both boxes are still estimates (see the package comment), so a finding of a +// few pixels in one axis may not be visible in Studio Pro. The estimates are +// generous rather than tight, which errs towards reporting; MPR016 therefore +// reports a cluster rather than a pixel count, and the repair is idempotent. +func (r Rect) Intersects(o Rect) bool { + return r.X < o.X+o.W && o.X < r.X+r.W && + r.Y < o.Y+o.H && o.Y < r.Y+r.H +} + // EntityBox is the estimated box an entity occupies at its stored Location. func EntityBox(e *domainmodel.Entity) Rect { w, h := boxSize(e) @@ -79,7 +99,7 @@ func obstacles(dm *domainmodel.DomainModel, skip *domainmodel.Entity) []Rect { func clear(r Rect, obs []Rect) bool { for _, o := range obs { - if r.overlaps(o) { + if r.TooClose(o) { return false } } diff --git a/mdl/dmlayout/place_test.go b/mdl/dmlayout/place_test.go index 2485aa99c..5c6b750c4 100644 --- a/mdl/dmlayout/place_test.go +++ b/mdl/dmlayout/place_test.go @@ -27,12 +27,12 @@ func overlapping(dm *domainmodel.DomainModel) []string { var hits []string for i, a := range dm.Entities { for _, b := range dm.Entities[i+1:] { - if EntityBox(a).overlaps(EntityBox(b)) { + if EntityBox(a).TooClose(EntityBox(b)) { hits = append(hits, a.Name+"/"+b.Name) } } for _, an := range dm.Annotations { - if EntityBox(a).overlaps(AnnotationBox(an)) { + if EntityBox(a).TooClose(AnnotationBox(an)) { hits = append(hits, a.Name+"/annotation") } } diff --git a/mdl/linter/context.go b/mdl/linter/context.go index d2b7259a7..b49958196 100644 --- a/mdl/linter/context.go +++ b/mdl/linter/context.go @@ -13,6 +13,7 @@ import ( "github.com/mendixlabs/mxcli/mdl/catalog" "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" "github.com/mendixlabs/mxcli/sdk/microflows" "github.com/mendixlabs/mxcli/sdk/pages" "github.com/mendixlabs/mxcli/sdk/security" @@ -94,6 +95,71 @@ func (ctx *LintContext) FullMicroflow(id model.ID) (*microflows.Microflow, error return ctx.fullMFCache[id], nil } +// DomainModels returns the domain model of every module, or nil when the +// reader cannot supply them. +// +// Entity geometry (an entity's Location, and the members its drawn box is sized +// from) is not in the catalog, so a rule that checks the diagram has to read +// the model. This is an optional reader capability rather than a method on +// LintReader: eight readers in the linter test packages implement that +// interface and would each need a stub, while in production the reader is +// always an mdl/backend Backend, whose interface declares ListDomainModels. So +// the assertion holds wherever lint actually runs, and the same arrangement is +// how JarFiles reaches Path(). +func (ctx *LintContext) DomainModels() []*domainmodel.DomainModel { + if ctx == nil || ctx.reader == nil { + return nil + } + r, ok := ctx.reader.(interface { + ListDomainModels() ([]*domainmodel.DomainModel, error) + }) + if !ok { + return nil + } + dms, err := r.ListDomainModels() + if err != nil { + return nil + } + return dms +} + +// OwnedModuleNamesByID maps the ID of every module the user actually owns to +// its name — not System, and not downloaded from the Marketplace or installed +// as a package. A domain model records only its containing module's ID, so a +// rule that reads the model needs this both to name the module and to stay off +// modules the user cannot edit. +// +// This is the Go counterpart of the notPlatformModule SQL fragment, and it has +// to make the same two tests for the same reason. A rule that reads the MODEL +// rather than the catalog does not get that WHERE clause for free, so it starts +// out seeing every platform module — which is how MPR016, in its first run, +// reported 41 System entities as overlapping, naming FileDocument and +// HttpRequest just as QUAL002 once did. Filtering on Source alone does not +// help: it is empty for System exactly as it is for the user's own modules, +// which is why the ID test is here too. +func (ctx *LintContext) OwnedModuleNamesByID() map[model.ID]string { + out := map[model.ID]string{} + if ctx == nil || ctx.reader == nil { + return out + } + mods, err := ctx.reader.ListModules() + if err != nil { + return out + } + for _, m := range mods { + if m == nil || string(m.ID) == systemModuleID { + continue + } + // The catalog's modules.Source is non-empty for exactly these two, and + // notPlatformModule requires it empty. + if m.FromAppStore || strings.TrimSpace(m.AppStoreGuid) != "" || m.PackageFile != "" { + continue + } + out[m.ID] = m.Name + } + return out +} + // Reader returns the LintReader, or nil if not set. func (ctx *LintContext) Reader() LintReader { return ctx.reader diff --git a/mdl/linter/rules/mpr016_overlapping_entities.go b/mdl/linter/rules/mpr016_overlapping_entities.go new file mode 100644 index 000000000..c0dd2466a --- /dev/null +++ b/mdl/linter/rules/mpr016_overlapping_entities.go @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rules + +import ( + "fmt" + "sort" + "strings" + + "github.com/mendixlabs/mxcli/mdl/dmlayout" + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// OverlappingEntitiesRule flags entities whose boxes cover each other on a +// domain model diagram. +// +// This is the sibling of MPR008, which does the same for microflow activities. +// The defect is the same class and so is the consequence: the diagram is the +// first thing a person opens to understand an app they are taking over, and a +// box hidden behind another is a member list they never read. Neither Studio +// Pro's own check nor mxbuild reports it, because the model is valid. +// +// mxcli has a particular stake in it. The positions are usually OURS: dmlayout +// places every entity a script does not position, so an overlap is as likely to +// be the placer's output as the author's doing. That makes this rule a check on +// mdl/dmlayout as much as on the project. +// +// # Geometry +// +// An entity stores only its Location (its box's TOP-LEFT corner). There is no +// Size in the model — Studio Pro derives the box from the name and member list +// when it draws, and mxcli never sees the result. So the box is an estimate, +// and the estimate used here is dmlayout's own (dmlayout.EntityBox): the rule +// and the placer have to agree about how big a box is, or `mxcli layout +// --only-overlapping` would leave behind overlaps this rule still reports, or +// chase ones it does not. Sharing the estimator makes that agreement structural +// rather than a thing to keep in step by hand. +// +// The predicate, though, is NOT the placer's. dmlayout.Rect.TooClose keeps +// boxes a clearance apart, which is right when placing and wrong when checking: +// two entities 10px apart are badly laid out but they do not overlap, and this +// rule may only report what is true. Hence Intersects, with no margin. +type OverlappingEntitiesRule struct{} + +func NewOverlappingEntitiesRule() *OverlappingEntitiesRule { + return &OverlappingEntitiesRule{} +} + +func (r *OverlappingEntitiesRule) ID() string { return "MPR016" } +func (r *OverlappingEntitiesRule) Name() string { return "OverlappingEntities" } +func (r *OverlappingEntitiesRule) Category() string { return "correctness" } +func (r *OverlappingEntitiesRule) DefaultSeverity() linter.Severity { return linter.SeverityWarning } +func (r *OverlappingEntitiesRule) Description() string { + return "Entities whose boxes cover each other on the domain model diagram, hiding their members from anyone reading it" +} + +func (r *OverlappingEntitiesRule) Check(ctx *linter.LintContext) []linter.Violation { + dms := ctx.DomainModels() + if len(dms) == 0 { + return nil + } + moduleNames := ctx.OwnedModuleNamesByID() + + var violations []linter.Violation + for _, dm := range dms { + if dm == nil || len(dm.Entities) < 2 { + continue + } + module := moduleNames[dm.ContainerID] + if module == "" || ctx.IsExcluded(module) { + continue + } + for _, cluster := range overlappingEntityClusters(dm.Entities) { + if ctx.IsDocumentExcluded(module + "." + cluster[0].Name) { + continue + } + violations = append(violations, linter.Violation{ + RuleID: r.ID(), + Severity: r.DefaultSeverity(), + Message: overlappingEntitiesMessage(module, cluster), + Location: linter.Location{ + Module: module, + DocumentType: "entity", + DocumentName: cluster[0].Name, + DocumentID: string(cluster[0].ID), + }, + Suggestion: "Run 'mxcli layout --only-overlapping' (with --dry-run first) to move just the entities that collide and leave every other position alone, or give them explicit positions.", + }) + } + } + return violations +} + +// maxNamedInCluster bounds how many entity names a message spells out. A pile of +// 26 named in full is a message nobody reads to the end, and the count carries +// the point by itself. +const maxNamedInCluster = 6 + +// overlappingEntitiesMessage words one finding. A pair reads as a pair; a larger +// cluster leads with the count, because "these 26 are stacked up" is the fact +// worth acting on and the individual names are not. +func overlappingEntitiesMessage(module string, cluster []*domainmodel.Entity) string { + names := make([]string, 0, len(cluster)) + for _, e := range cluster { + names = append(names, e.Name) + } + if len(cluster) == 2 { + return fmt.Sprintf("Entities '%s' and '%s' overlap on the %s domain model diagram, so one hides part of the other.", + names[0], names[1], module) + } + shown := names + suffix := "" + if len(shown) > maxNamedInCluster { + shown = shown[:maxNamedInCluster] + suffix = fmt.Sprintf(" and %d more", len(names)-maxNamedInCluster) + } + return fmt.Sprintf("%d entities overlap each other on the %s domain model diagram: %s%s.", + len(cluster), module, strings.Join(shown, ", "), suffix) +} + +// overlappingEntityClusters groups entities whose estimated boxes intersect into +// connected clusters, and returns only the clusters with something in them — +// one per group of entities that collide, never one per pair. +// +// Pairwise would be wrong at scale, not just noisy. The benchmark that motivated +// this rule found a model with "all 26 entities at the same position, one on top +// of another", which is 325 pairs and one fact. Grouping transitively reports +// that as a single finding whatever the pile's size, and needs no threshold to +// decide when a pile is a pile. +// +// Unlike MPR008 this does NOT skip the origin. An unpositioned microflow +// activity defaults to (0,0), so there the origin means "no position recorded"; +// entities get a position from dmlayout on every create path, so entities at one +// position IS the defect, and skipping it would miss the worst case there is. +// +// The result is deterministic: entities are sorted by name within a cluster and +// clusters by their first name, so a finding does not move between runs. +func overlappingEntityClusters(ents []*domainmodel.Entity) [][]*domainmodel.Entity { + live := make([]*domainmodel.Entity, 0, len(ents)) + for _, e := range ents { + if e != nil { + live = append(live, e) + } + } + if len(live) < 2 { + return nil + } + + boxes := make([]dmlayout.Rect, len(live)) + for i, e := range live { + boxes[i] = dmlayout.EntityBox(e) + } + + // Union-find over the intersection graph: a cluster is a connected + // component, so A over B over C is one finding even when A and C do not + // touch each other. + parent := make([]int, len(live)) + for i := range parent { + parent[i] = i + } + var find func(int) int + find = func(i int) int { + for parent[i] != i { + parent[i] = parent[parent[i]] + i = parent[i] + } + return i + } + for i := 0; i < len(live); i++ { + for j := i + 1; j < len(live); j++ { + if boxes[i].Intersects(boxes[j]) { + if a, b := find(i), find(j); a != b { + parent[a] = b + } + } + } + } + + groups := map[int][]*domainmodel.Entity{} + for i, e := range live { + root := find(i) + groups[root] = append(groups[root], e) + } + + var out [][]*domainmodel.Entity + for _, g := range groups { + if len(g) < 2 { + continue + } + sort.SliceStable(g, func(i, j int) bool { return g[i].Name < g[j].Name }) + out = append(out, g) + } + sort.SliceStable(out, func(i, j int) bool { return out[i][0].Name < out[j][0].Name }) + return out +} diff --git a/mdl/linter/rules/mpr016_overlapping_entities_test.go b/mdl/linter/rules/mpr016_overlapping_entities_test.go new file mode 100644 index 000000000..6ac33fcd2 --- /dev/null +++ b/mdl/linter/rules/mpr016_overlapping_entities_test.go @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rules + +import ( + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/catalog" + "github.com/mendixlabs/mxcli/mdl/dmlayout" + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" +) + +// ent builds an entity at a position with n attributes, so a test can state a +// box's height through the thing that actually drives it. +func ent(name string, x, y, attrs int) *domainmodel.Entity { + e := &domainmodel.Entity{Name: name, Location: model.Point{X: x, Y: y}} + for i := 0; i < attrs; i++ { + e.Attributes = append(e.Attributes, &domainmodel.Attribute{Name: "A"}) + } + return e +} + +// names flattens clusters to compare them without depending on pointer order. +func names(clusters [][]*domainmodel.Entity) []string { + var out []string + for _, c := range clusters { + var n []string + for _, e := range c { + n = append(n, e.Name) + } + out = append(out, strings.Join(n, "+")) + } + return out +} + +func TestOverlappingEntitiesRule_Metadata(t *testing.T) { + r := NewOverlappingEntitiesRule() + if r.ID() != "MPR016" { + t.Errorf("ID = %q, want MPR016", r.ID()) + } + if r.Name() != "OverlappingEntities" { + t.Errorf("Name = %q, want OverlappingEntities", r.Name()) + } + // Same category and severity as MPR008, its microflow-side sibling. + if r.Category() != "correctness" { + t.Errorf("Category = %q, want correctness", r.Category()) + } + if r.DefaultSeverity() != linter.SeverityWarning { + t.Errorf("Severity = %v, want warning", r.DefaultSeverity()) + } +} + +// Two boxes that genuinely cover each other: the report's "the payment attempt +// covers the bottom of the cart". +func TestOverlappingEntityClusters_ReportsARealOverlap(t *testing.T) { + got := overlappingEntityClusters([]*domainmodel.Entity{ + ent("Cart", 0, 0, 0), + ent("PaymentAttempt", 100, 30, 0), + }) + if want := []string{"Cart+PaymentAttempt"}; len(got) != 1 || names(got)[0] != want[0] { + t.Fatalf("clusters = %v, want %v", names(got), want) + } +} + +// The discriminating case for this rule's whole design. +// +// dmlayout.Rect.overlaps keeps boxes `clearance` (20px) apart, because that is +// what PLACING needs: boxes 10px apart are unreadable and the placer should +// separate them. A CHECK may not use that predicate. Two entities 10px apart +// are badly laid out, but they do not overlap, and a finding that said they did +// would be false — so the rule uses Intersects, which has no margin. +// +// If this test fails because the rule started reporting, the rule has been +// switched to the placer's predicate and every tight-but-legal diagram in every +// project now produces findings. +func TestOverlappingEntityClusters_IgnoresBoxesThatMerelySitClose(t *testing.T) { + a, b := ent("Cart", 0, 0, 0), ent("Coupon", 170, 0, 0) + + // The control: the placer's predicate DOES fire on this pair, which is why + // the rule needs its own. + if !dmlayout.EntityBox(a).TooClose(dmlayout.EntityBox(b)) { + t.Fatal("precondition: the placer's predicate should consider a 10px gap too close") + } + if dmlayout.EntityBox(a).Intersects(dmlayout.EntityBox(b)) { + t.Fatal("precondition: a 10px gap is not an intersection") + } + + if got := overlappingEntityClusters([]*domainmodel.Entity{a, b}); len(got) != 0 { + t.Fatalf("clusters = %v, want none for boxes that only sit close", names(got)) + } +} + +// The report's worst case: "All 26 entities sit at the same position on the +// diagram, one on top of another." Reporting that as 26*25/2 = 325 pairwise +// findings would bury the one fact worth knowing, so a connected cluster is one +// finding however large it is. +func TestOverlappingEntityClusters_CollapsesAPileIntoOneCluster(t *testing.T) { + var ents []*domainmodel.Entity + for i := 0; i < 26; i++ { + ents = append(ents, ent(string(rune('A'+i)), 0, 0, 0)) + } + got := overlappingEntityClusters(ents) + if len(got) != 1 { + t.Fatalf("got %d clusters, want 1 for a pile of 26", len(got)) + } + if len(got[0]) != 26 { + t.Fatalf("cluster holds %d entities, want 26", len(got[0])) + } +} + +// Two separate collisions are two findings, not one cluster of four. +func TestOverlappingEntityClusters_SeparatesIndependentOverlaps(t *testing.T) { + got := overlappingEntityClusters([]*domainmodel.Entity{ + ent("Cart", 0, 0, 0), + ent("Coupon", 50, 10, 0), + ent("Order", 1000, 1000, 0), + ent("OrderLine", 1050, 1010, 0), + }) + if len(got) != 2 { + t.Fatalf("clusters = %v, want 2", names(got)) + } +} + +// A box's height comes from its attribute count, so a tall entity reaches a +// neighbour that a default-sized one would clear. This is the mendixlabs/mxcli#1479 +// lesson — a row pitch alone cannot place a 20-attribute entity — and it is the +// test that proves the rule measures boxes through dmlayout's estimator rather +// than assuming one size for every entity. +func TestOverlappingEntityClusters_HeightFollowsTheAttributeCount(t *testing.T) { + // Control: with no attributes the upper box is 60px tall and clears y=100. + if got := overlappingEntityClusters([]*domainmodel.Entity{ + ent("Order", 0, 0, 0), + ent("OrderLine", 0, 100, 0), + }); len(got) != 0 { + t.Fatalf("control: clusters = %v, want none", names(got)) + } + + // The same pair, with 20 attributes on the upper entity: 34 + 20*16 = 354px + // tall, so it now covers the box at y=100. + if got := overlappingEntityClusters([]*domainmodel.Entity{ + ent("Order", 0, 0, 20), + ent("OrderLine", 0, 100, 0), + }); len(got) != 1 { + t.Fatalf("clusters = %v, want the tall entity to reach its neighbour", names(got)) + } +} + +// Unlike MPR008, this rule does NOT skip the origin. An unpositioned microflow +// activity defaults to (0,0), so MPR008 treats that as "no position recorded"; +// entities are placed by dmlayout on every create path, so several entities at +// one position is the pile defect itself, and skipping it would miss the worst +// case in the benchmark that motivated the rule. +func TestOverlappingEntityClusters_DoesNotSkipTheOrigin(t *testing.T) { + got := overlappingEntityClusters([]*domainmodel.Entity{ + ent("Cart", 0, 0, 0), + ent("Coupon", 0, 0, 0), + }) + if len(got) != 1 { + t.Fatalf("clusters = %v, want the pair at the origin reported", names(got)) + } +} + +func TestOverlappingEntityClusters_IgnoresNilAndSingletons(t *testing.T) { + if got := overlappingEntityClusters(nil); len(got) != 0 { + t.Fatalf("clusters = %v, want none", names(got)) + } + if got := overlappingEntityClusters([]*domainmodel.Entity{nil, ent("Cart", 0, 0, 0), nil}); len(got) != 0 { + t.Fatalf("clusters = %v, want none", names(got)) + } +} + +// dmReader supplies domain models the way the production backend does. The +// capability is not in LintReader (see LintContext.DomainModels), so the rule +// reaches it by assertion and this reader is what satisfies it. +type dmReader struct { + rawUnitReader + dms []*domainmodel.DomainModel + modules []*model.Module +} + +func (r dmReader) ListDomainModels() ([]*domainmodel.DomainModel, error) { return r.dms, nil } +func (r dmReader) ListModules() ([]*model.Module, error) { return r.modules, nil } + +func newDMContext(t *testing.T, r linter.LintReader) *linter.LintContext { + t.Helper() + cat, err := catalog.New() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cat.Close() }) + return linter.NewLintContext(cat, r) +} + +// End to end: the rule reads the domain model through the context, names both +// entities, and points at the command that repairs it. +func TestOverlappingEntitiesRule_ReportsThroughTheContext(t *testing.T) { + dm := &domainmodel.DomainModel{ + ContainerID: model.ID("m1"), + Entities: []*domainmodel.Entity{ent("Cart", 0, 0, 0), ent("PaymentAttempt", 100, 30, 0)}, + } + ctx := newDMContext(t, dmReader{ + dms: []*domainmodel.DomainModel{dm}, + modules: []*model.Module{{Name: "Shop", BaseElement: model.BaseElement{ID: model.ID("m1")}}}, + }) + + got := NewOverlappingEntitiesRule().Check(ctx) + if len(got) != 1 { + t.Fatalf("got %d violations, want 1: %+v", len(got), got) + } + v := got[0] + if v.RuleID != "MPR016" { + t.Errorf("RuleID = %q", v.RuleID) + } + if v.Location.Module != "Shop" { + t.Errorf("Location.Module = %q, want Shop", v.Location.Module) + } + for _, want := range []string{"Cart", "PaymentAttempt"} { + if !strings.Contains(v.Message, want) { + t.Errorf("message %q does not name %q", v.Message, want) + } + } + if !strings.Contains(v.Suggestion, "--only-overlapping") { + t.Errorf("suggestion %q does not name the repair command", v.Suggestion) + } +} + +// An excluded module produces nothing, like every other rule. +func TestOverlappingEntitiesRule_HonoursModuleExclusion(t *testing.T) { + dm := &domainmodel.DomainModel{ + ContainerID: model.ID("m1"), + Entities: []*domainmodel.Entity{ent("Cart", 0, 0, 0), ent("PaymentAttempt", 100, 30, 0)}, + } + ctx := newDMContext(t, dmReader{ + dms: []*domainmodel.DomainModel{dm}, + modules: []*model.Module{{Name: "Shop", BaseElement: model.BaseElement{ID: model.ID("m1")}}}, + }) + ctx.SetExcludedModules([]string{"Shop"}) + + if got := NewOverlappingEntitiesRule().Check(ctx); len(got) != 0 { + t.Fatalf("got %d violations for an excluded module, want 0", len(got)) + } +} + +// A reader that cannot supply domain models yields no findings rather than a +// crash or a rule failure — the same way MPR008 returns nil on a nil reader. +func TestOverlappingEntitiesRule_ReaderWithoutDomainModels(t *testing.T) { + ctx := newDMContext(t, rawUnitReader{}) + if got := NewOverlappingEntitiesRule().Check(ctx); len(got) != 0 { + t.Fatalf("got %d violations, want 0", len(got)) + } + if got := NewOverlappingEntitiesRule().Check(linter.NewLintContextFromDB(nil)); len(got) != 0 { + t.Fatalf("got %d violations with a nil reader, want 0", len(got)) + } +} + +// The System module is skipped, and so is anything from the Marketplace. +// +// This is a regression guard, not a hypothetical. MPR016's first run against +// testdata/pedapp reported "41 entities overlap on the System domain model +// diagram: ConsumedODataConfiguration, Error, FileDocument, HttpHeader, +// HttpMessage, HttpRequest and 35 more" — Mendix's own model, which no user can +// lay out (`mxcli layout` refuses System outright). A rule that reads the MODEL +// does not inherit the notPlatformModule WHERE clause that keeps catalog-based +// rules off platform modules, so it sees them all until it filters. +// +// Note System is NOT identifiable by its Source, which is empty for it exactly +// as for the user's own modules; it has a fixed ID. QUAL002 shipped this same +// bug once, on the same entities. +func TestOverlappingEntitiesRule_SkipsPlatformModules(t *testing.T) { + const systemModuleID = "00000000-0000-0000-0000-000000000001" + + overlapping := func() []*domainmodel.Entity { + return []*domainmodel.Entity{ent("Cart", 0, 0, 0), ent("PaymentAttempt", 100, 30, 0)} + } + for _, tc := range []struct { + name string + module *model.Module + }{ + {"System", &model.Module{Name: "System", BaseElement: model.BaseElement{ID: model.ID(systemModuleID)}}}, + {"Marketplace", &model.Module{Name: "Administration", FromAppStore: true, AppStoreVersion: "4.3.2", + BaseElement: model.BaseElement{ID: model.ID("m1")}}}, + {"MarketplaceByGuidAlone", &model.Module{Name: "Atlas_Core", AppStoreGuid: "abc", + BaseElement: model.BaseElement{ID: model.ID("m1")}}}, + } { + t.Run(tc.name, func(t *testing.T) { + dm := &domainmodel.DomainModel{ContainerID: tc.module.ID, Entities: overlapping()} + ctx := newDMContext(t, dmReader{ + dms: []*domainmodel.DomainModel{dm}, + modules: []*model.Module{tc.module}, + }) + if got := NewOverlappingEntitiesRule().Check(ctx); len(got) != 0 { + t.Fatalf("got %d violations for %s, want 0: %+v", len(got), tc.name, got) + } + }) + } + + // The control: the identical model in a module the user owns IS reported, + // so the filter above is selective and not just switching the rule off. + dm := &domainmodel.DomainModel{ContainerID: model.ID("m1"), Entities: overlapping()} + ctx := newDMContext(t, dmReader{ + dms: []*domainmodel.DomainModel{dm}, + modules: []*model.Module{{Name: "Shop", BaseElement: model.BaseElement{ID: model.ID("m1")}}}, + }) + if got := NewOverlappingEntitiesRule().Check(ctx); len(got) != 1 { + t.Fatalf("control: got %d violations for an owned module, want 1", len(got)) + } +} From 78a3cc5e7fabc9fd7502f9fa9b16902530980faa Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 22:13:23 +0000 Subject: [PATCH 25/28] =?UTF-8?q?docs(proposal):=20verify=20gate=20?= =?UTF-8?q?=E2=80=94=20MxBuild=20staging=20(unmasked/hidden),=20optional?= =?UTF-8?q?=20mendix=20check=20pre-stage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011oK82stfDEB1TGfJLVwUxc --- docs/11-proposals/PROPOSAL_verify_gate.md | 112 ++++++++++++++++++++-- 1 file changed, 104 insertions(+), 8 deletions(-) diff --git a/docs/11-proposals/PROPOSAL_verify_gate.md b/docs/11-proposals/PROPOSAL_verify_gate.md index 59db50b33..fdd6255a5 100644 --- a/docs/11-proposals/PROPOSAL_verify_gate.md +++ b/docs/11-proposals/PROPOSAL_verify_gate.md @@ -11,7 +11,7 @@ related: # Proposal: `mxcli verify` — a staged gate with baseline comparison **Status:** Draft -**Date:** 2026-11-16 (updated 2026-11-18) +**Date:** 2026-11-16 (updated 2026-11-19) **Issue:** ako/mxcli#2333 **Relates to:** ako/mxcli#1313 (`mx check` baseline), #1312 (`diff --mx-check`), #1258 (`check --full`), #2308 (`replace()` passes `check`, fails `mx check`) @@ -91,7 +91,7 @@ mxcli verify -p app.mpr [--base | --baseline ] reruns. `--fail-fast` stops at the first stage with new findings, for CI. Opt-in stages (`build`, `test`, `smoke`) are still skipped unless selected, and are skipped when an earlier stage failed, since they need a model that builds. -- Findings are classified **new / resolved / pre-existing** against the baseline, +- Findings are classified **new / resolved / pre-existing** against the baseline (`mxcheck` adds **unmasked** and **hidden**, see the staging section), keyed by `(code, document, element path)` — never by message text or line number, which drift between runs (the problem #1313 describes). - **Exit code:** 0 when nothing is new; 2 when the gate could not run (no `mx`, @@ -132,7 +132,16 @@ mxcli verify -p app.mpr [--base | --baseline ] 2. `--base ` — run the same stages on a scratch export of that ref, then compare. Roughly doubles `mxcheck` cost; results are cacheable per commit. -## Existing pieces and gaps (checked against `main` at `ecdd9b7c`) +## Existing pieces and gaps (checked against `main` at `ecdd9b7c`; see also the update below) + +**Update 2026-11-19.** Since this section was written, `main` gained about twenty +`check` validations (step 0: MDL097/MDL098, the `replace()` fix, DECLARE/SET/RETURN +type checks, and others), the `mdl-examples/check-gap/` parity harness, +`scripts/diagcoverage` and `scripts/maskmatrix`. None of it parses `mx check` +output into findings: the harness compares codes in shell and `diagcoverage` reads +only the code catalog, so **step 1 is still needed**. What the work adds is +fixtures to build the parser's golden files from, and two measurements that change +the design (next two sections). - `mdl/linter/baseline.go` has `WriteBaseline`/`LoadBaseline`/`Compare`, but it is **not reusable as is** against the `Finding` below: @@ -163,6 +172,74 @@ mxcli verify -p app.mpr [--base | --baseline ] Playwright test scripts). `mxcli verify` next to `mxcli playwright verify` is the ordinary noun-under-a-namespace case, not a collision of two top-level commands. +## MxBuild reports in stages: new, resolved and unmasked + +Measured in `PROPOSAL_check_diagnostics_catalog.md` (MxBuild 11.16.1, six defects +and every ordered pair, `make mask-matrix`): `mx check` does not report every +error, it reports the earliest stage that has any. + +| stage | reported | hides | +|---|---|---| +| 0 | an unresolved reference (CE1613) | every other error in every document | +| 1 | a duplicate variable (CE0111) | every other error in the same microflow | +| 2 | an undefined variable (CE0109) | the stage 3 group, in the same microflow | +| 3 | CE0117, CE0068, CE0038, CE0067 | nothing | + +For a baseline diff this is a correctness problem, in both directions: + +- **False "new".** The baseline has a stage 1 error in microflow `M.F`. The change + fixes it, and a stage 3 error that was always there appears. A plain diff blames + the change for it. +- **False "resolved".** The change introduces a stage 0 error. Every other finding + in the project disappears from the report and a plain diff calls them all + resolved. + +So `verify` classifies the `mxcheck` stage's findings into four states, not three: + +| state | meaning | +|---|---| +| **new** | in the report, not in the baseline, and the baseline was not hiding it | +| **unmasked** | new in the report, but the baseline had an earlier-stage finding in a scope that hides it (same document for stages 1 and 2, anywhere for stage 0). It may be old; the report cannot tell | +| **resolved** | in the baseline, absent from the report, and nothing in the report hides it | +| **hidden** | in the baseline, absent from the report, and the report has a finding that hides it. Not resolved | + +Rules that follow: + +- `unmasked` is reported with the finding that hid it, and **does not fail the gate + by default** (it is most likely pre-existing). `--strict` makes it fail, for + teams that would rather pay a second pass. The text output states how many + findings the staging may still be holding back. +- A new stage 0 or earlier-stage finding always fails the gate, and the `hidden` + count is printed next to it so a reader does not read "400 resolved". +- Each finding carries its **stage** (the classification table of step 1 gets a + stage column). A code not in the measured set is treated as stage 3 (hides + nothing, hidden by earlier stages): the conservative reading for `unmasked`, and + recorded as a limit until `maskmatrix` covers more codes. The measurement covers + microflows and pages only. +- The `check` stage (`mxcli check`) and `lint` do not stage: `mxcli check` + deliberately reports every real error (the catalog proposal's "report every real + error" rule), so new, resolved and unchanged are exact there. +- Staging is the main reason to run `check` first and `mxcheck` second: the more + `mxcli check` reports, the less the slow stage can hide. + +## An optional fast `mxcheck` pre-stage: Mendix CLI 2 + +The same spike measured Mendix CLI 2's `mendix check` at 1.3 to 1.7 s against 6 to +8.5 s for `mx check`, with no staging. It is proprietary (never committed, path +supplied by the user) and gave false errors on some real apps, so it cannot be the +authoritative stage. Design consequences: + +- `mxcheck` keeps `mx check` as the authority. `--mxcheck-with mendix` (name open) + runs `mendix check` as an **advisory pre-stage**: its findings are reported and + never gate on their own; anything that would gate must be confirmed by `mx check`. +- Codes differ between the checkers for one finding (CE1613 in `mx` is CE7710 in + `mendix check`), and it reports more errors in a project whose references do not + resolve. A finding therefore records its **checker**, and a baseline is only + compared with findings from the same checker. A baseline file records the + checker and its version, and a mismatch is an error, not a diff. +- Off by default; revisit once there is evidence about its false-positive rate on + real projects. + ## Finding identity: for which codes can the key be built at all? The key `(code, document, element path)` is only as good as `mx check`'s text. In @@ -189,7 +266,10 @@ or move (formerly the only question) applies only to class 1. New package `cmd/mxcli/verify/` (orchestrator, no business logic): ```go -type Finding struct { Stage, Code, Document, DocumentID, Path, Message string } +type Finding struct { + Stage, Checker, Code, Document, DocumentID, Path, Message string + MxStage int // MxBuild report stage 0-3 for `mxcheck` findings, else -1 +} type Stage interface { Name() string Run(ctx context.Context, p Project) ([]Finding, error) @@ -219,13 +299,15 @@ type Stage interface { moved into `mxcli check` removes a class of late failures); independent of, and not blocked by, the steps below. 1. **Structured `mx check` findings**: parser + `docker.CheckFindings()`, tests - against recorded `mx` output fixtures. Standalone PR; unblocks the `mxcheck` - baseline, #1313 and #1312. + against recorded `mx` output, built from the `mdl-examples/check-gap/` + fixtures. Includes the code classification table with its **stage** column. + Standalone PR; unblocks the `mxcheck` baseline, #1313 and #1312. 2. **`mxcli verify`, first release**: stages `check`, `lint`, `mxcheck`; `--format`, exit-code table, `--fail-fast`; and a **`lint` baseline from day one** (`--baseline`, `--update-baseline`, reusing `mdl/linter/baseline.go` through the adapter described above), so the first run on an existing project - reports only what is new. The `mxcheck` stage baselines once step 1 lands; until + reports only what is new. The `mxcheck` stage baselines once step 1 lands, with the four-state + classification above; until then it reports all findings and says it is unbaselined. 3. **Make `verify` the thing everything points at**: skills, `mxcli init` output and the project `CLAUDE.md` guidance name `verify` as the single "did my change @@ -240,6 +322,12 @@ type Stage interface { - Unit: finding key normalisation per class (located / document-only / unlocated) (same finding, different line/message → same key); classification new/resolved/pre-existing; default run-all vs `--fail-fast`. +- Staging: for each stage pair from `maskmatrix`, a fixture diff classified + unmasked (not new) when the baseline hid it and hidden (not resolved) when the + change introduces the hiding finding; **control:** the same finding with no + hiding baseline finding must be new and must fail the gate. +- Checker identity: a baseline from one checker compared against the other is an + error. - Parser: golden `mx check` outputs (CE0109, CE0844, CE7811, CE0463 labelled). - Control (per CLAUDE.md): a "nothing new" test paired with one that injects a finding and must fail. @@ -262,5 +350,13 @@ type Stage interface { 5. **Should `check` stage fail on warnings,** or only errors? 6. **Exit-code encoding** — bitmask offsets, and whether to align with `check`'s and `lint`'s existing codes. -7. **Default stage set** — with run-all as the default, `mxcheck` costs ~8 s once +7. **Unmasked policy.** Non-failing by default with `--strict`, or the other way + round? Failing wastes a pass on findings that are usually old; not failing can + let a change that introduced a hidden error through. +8. **Staging coverage.** The measurement covers six codes in microflows and + pages. How far must `maskmatrix` be extended before `unmasked` is trusted + beyond them? +9. **`mendix check` pre-stage:** worth a flag now, or defer until its + false-positive rate is measured on real projects? +10. **Default stage set** — with run-all as the default, `mxcheck` costs ~8 s once per iteration; is on-by-default accepted? From 3a1e6c99a5898529023eaec0dcf7b45dcb593edb Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 22:44:42 +0000 Subject: [PATCH 26/28] feat(lint): MPR017 reports a Boolean guard that can only hold one value Refs ako/mxcli#2403. Blocked on ako/mxcli#2417 for a project's first run. A decision tests a flag whose every reachable value is the same, so the branch it guards is unreachable and the flow reads as though it has a safety interlock when it has none. Studio Pro and mxbuild both pass it: the model is valid. The test is the attribute's REACHABLE VALUE SET -- its default plus every literal any Create/Change activity assigns -- not "is it ever written". #2403 described it as "read but never written", but the defect it cites is a reset flag that IS written, only ever to false, so a "never written" rule would have missed the case the issue was raised for. The inverse control matters as much: a default of true with the only write false is TWO values, so that guard can change state once and is correctly not reported. A write the rule cannot see reads as no write, so four exclusions, each a measured false-positive source: bound to a page widget (a user may type into it, and #2403 called this the decisive one), assigned from an expression rather than a literal, calculated, or not read in any condition. Writes are collected from every module including Marketplace ones while only the project's own attributes are reported -- a Marketplace flow writing an app's attribute is rare, but counting it as no write would be a false positive. Boolean only for now: enumerating "the literals I could see" is a far weaker claim for other types. The mirror verdict, written but never read, is not in this rule. Two silent failures found on the way, both recorded as a finding: 1. linter.CatalogRequirer is OPTIONAL, so a rule that does not declare RequiredCatalogMode gets a fast catalog, where activities / widgets / xpath_expressions are never written -- the rule then finds no conditions and reports a clean project. Its own doc warns of this ("silently returning empty results"). Now declares CatalogFull, with a test. 2. Under that, ctx.AttributesFor fails with "no such table: attributes" whenever lint BUILDS the catalog rather than loading it (#2417), and once that error is recorded the context stops answering later reads. So the rule refuses rather than passing: it separates "no entities, nothing to check" from "entities but no readable attributes" and reports the latter as a linter.RuleFailure naming #2417. That guard is to be DELETED when #2417 is fixed; it is a workaround, and it only protects the one rule that knows. Measured - 13 tests, including the inverse control (a flag written both ways is not reported) and each of the four exclusions. - Positive control, which is what caught (1) and (2): a fixture of one entity and three microflows -- a guard reading the flag, a flow writing only false, and a control flag written both ways. The rule reports the first and ignores the control. Before the fixes it reported NOTHING on that fixture, while two real projects showed zero false positives -- which looked like success and was a rule that could never fire. - 0 findings on testdata/pedapp and testdata/testapp-views. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011oK82sY9fcHtMuZcMNRm2B --- ...e-silently-finds-nothing-fast-catalog.json | 1 + CHANGELOG.md | 7 + cmd/mxcli/cmd_lint.go | 1 + .../rules/mpr017_fixed_guard_attribute.go | 428 ++++++++++++++++++ .../mpr017_fixed_guard_attribute_test.go | 232 ++++++++++ 5 files changed, 669 insertions(+) create mode 100644 .claude/skills/fix-issue/findings/mdl-other/2026-11-18-new-lint-rule-silently-finds-nothing-fast-catalog.json create mode 100644 mdl/linter/rules/mpr017_fixed_guard_attribute.go create mode 100644 mdl/linter/rules/mpr017_fixed_guard_attribute_test.go diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-18-new-lint-rule-silently-finds-nothing-fast-catalog.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-18-new-lint-rule-silently-finds-nothing-fast-catalog.json new file mode 100644 index 000000000..a2c54c0f5 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-18-new-lint-rule-silently-finds-nothing-fast-catalog.json @@ -0,0 +1 @@ +{"area": "mdl/linter", "date": "2026-11-18", "symptom": "A new Go lint rule that reads the catalog reported NOTHING on a project that provably had the defect it detects, then reported correctly on the very next identical run. MPR017 found no decision expressions on a freshly built catalog and the real finding once the catalog was cached.", "cause": "Two separate causes, stacked, each silent. (1) linter.CatalogRequirer is an OPTIONAL interface: a rule that does not implement RequiredCatalogMode gets CatalogFast, so activities / widgets / xpath_expressions are never written and every expression-reading rule sees an empty project. Its own doc warns of exactly this ('silently returning empty results'). (2) Under it, ctx.AttributesFor fails with 'no such table: attributes' whenever lint BUILDS the catalog rather than loading it (ako/mxcli#2417), and once that query error is recorded the context stops answering later reads — a second `for e := range ctx.Entities()` loop did not execute its body once. Fixed (1) by declaring CatalogFull; worked around (2) by refusing with a linter.RuleFailure when entities read but attributes did not.", "file": "`mdl/linter/rules/mpr017_fixed_guard_attribute.go` (RequiredCatalogMode, collectAttributeFacts); `mdl/linter/catalog_mode.go`; `mdl/linter/context.go` (AttributesFor, recordQueryError); test `mdl/linter/rules/mpr017_fixed_guard_attribute_test.go` (TestFixedGuardAttributeRule_RequiresFullCatalog)", "insight": "A new rule's first measurement must be a POSITIVE control: build the defect and watch the rule fire. Two projects' worth of 'no false positives' looked like success and was a rule that could never fire at all — the same shape as #808. The discriminator that localised both causes in one step was running the new rule beside an established one in the same invocation: SEC001 reported its entities on the fresh catalog while the new rule reported nothing, which placed the fault in the new rule's inputs rather than in the catalog. Second: run every new rule twice on a brand-new project, because fresh-build and cached-catalog are different code paths and only the cached one works today. Third: anything a rule reads that the catalog writes only in full mode needs RequiredCatalogMode, and nothing fails loudly when it is missing — grep a new rule for activities/widgets/xpath_expressions/permissions before trusting a clean run. And a rule that cannot read its input must report linter.RuleFailure, never an empty result: `No issues found.` printed above an `Error:` line is what a CI gate reads as a pass."} diff --git a/CHANGELOG.md b/CHANGELOG.md index da3fa736b..5940d8a48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- **Lint `MPR017`: a Boolean flag a decision tests but which can only ever hold one value** (ako/mxcli#2403) — the branch it guards is unreachable, and the flow reads as though it has a safety interlock when it has none. Nothing else reports it: the model is valid and the build is clean. + + The test is the attribute's **reachable value set** — its default plus every literal any Create/Change activity assigns — not "is it ever written". The defect this comes from is a reset flag that *is* written, only ever to `false`, so a "never written" rule would have missed the case it was raised for; and a default of `true` with the only write `false` is two values, so that guard can change state once and is correctly not reported. Four exclusions keep a write the rule cannot see from reading as no write: bound to a page widget (a user may type into it), assigned from an expression rather than a literal, calculated, or not read in any condition. Writes are collected from every module including Marketplace ones, while only the project's own attributes are reported. Boolean only for now — enumerating "the literals I could see" is a far weaker claim for other types. + + **Blocked by ako/mxcli#2417 on a project's first lint run.** `AttributesFor` fails with "no such table: attributes" when lint builds the catalog rather than loading it, so the rule cannot see attributes until a second run. Rather than report a clean project, it refuses: "MPR017 could not run: read N entities but no attributes". The guard distinguishes "no entities, nothing to check" from "entities but no readable attributes" and is to be deleted when #2417 is fixed. + + - **Lint `MPR016`: entities whose boxes cover each other on the domain model diagram** (ako/mxcli#2406) — the sibling of MPR008, which does this for microflow activities. The diagram is the first thing someone opening an unfamiliar app reads, and a box behind another box is a member list they never see; neither Studio Pro's check nor mxbuild reports it, because the model is valid. A pair is named as a pair, and a cluster that collides transitively is ONE finding however large — a model with every entity on one spot is 1 finding, not 325 pairs. The repair is the command the finding names, `mxcli layout --only-overlapping`. Box geometry is `dmlayout`'s own estimator (an entity stores only its top-left `Location`; Studio Pro derives the box when it draws, so the size is estimated from the name and member list), so the rule and the placer agree by construction rather than by being kept in step. The PREDICATE is new and deliberately not the placer's: `Rect.TooClose` keeps boxes a 20px clearance apart, which is what placing needs, while a finding must be true — two entities 10px apart are badly laid out and do not overlap. `Rect.Intersects` has no margin, and a test pins the difference. diff --git a/cmd/mxcli/cmd_lint.go b/cmd/mxcli/cmd_lint.go index be6024654..bcbb15510 100644 --- a/cmd/mxcli/cmd_lint.go +++ b/cmd/mxcli/cmd_lint.go @@ -519,6 +519,7 @@ func builtinLintRules() []linter.Rule { rules.NewOverlappingActivitiesRule(), // MPR008 - requires BSON inspection rules.NewLoopChildContainmentRule(), // MPR011 - requires BSON inspection rules.NewOverlappingEntitiesRule(), // MPR016 - reads entity geometry from the domain model + rules.NewFixedGuardAttributeRule(), // MPR017 - a Boolean guard with one reachable value rules.NewNoCommitInLoopRule(), // CONV011-CONV014 - require BSON inspection rules.NewExclusiveSplitCaptionRule(), rules.NewErrorHandlingOnCallsRule(), diff --git a/mdl/linter/rules/mpr017_fixed_guard_attribute.go b/mdl/linter/rules/mpr017_fixed_guard_attribute.go new file mode 100644 index 000000000..4dca70c7d --- /dev/null +++ b/mdl/linter/rules/mpr017_fixed_guard_attribute.go @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rules + +import ( + "fmt" + "sort" + "strings" + + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/microflows" +) + +// FixedGuardAttributeRule flags a Boolean attribute that a decision tests but +// that can only ever hold one value, so the decision's outcome is fixed. +// +// From the benchmark that motivated ako/mxcli#2403, found in all three apps read: +// +// "The tick, the housekeeping and the scheduled reset check a flag so that +// they do not work during the data reset, but no flow ever sets it to true; +// the reset only sets it to false. So the tick can work on data while the +// reset deletes and reloads it." +// +// Nothing in the toolchain reports this. The model is valid, the build is +// clean, and the flow reads as though it has a safety interlock. It does not: +// the branch it guards is unreachable. +// +// # Why "one reachable value" and not "never written" +// +// #2403 described this as "read in conditions but never written", which is the +// shape it takes when a flag is forgotten entirely. But the case above is NOT +// that — the flag IS written, to false, by the reset. A "never written" rule +// would miss the very defect it was raised for. +// +// So the test is the attribute's set of reachable values: the default, plus +// every literal any Change activity assigns. One value means the condition's +// outcome cannot vary. That subsumes "never written" (a value set of one, or of +// none) and catches the real case. The control the other way matters just as +// much: a default of true with the only write false is TWO values, so the guard +// can change state once and is not dead. +// +// # What makes a finding safe to report +// +// A write the rule cannot see reads as "no write", so every source of writes +// has to be accounted for or the rule reports working flags as dead. Four +// exclusions, each measured as a real false-positive source: +// +// - bound to a page widget (linter.Widget.AttributeRef): a user can type into +// it and the model has no idea what they will type. #2403 called this the +// decisive one, and it is — without it the rule fires across a normal app. +// - written from an expression rather than a literal: the value set is +// unknown, so nothing follows. +// - calculated: Mendix computes it, the model never assigns it. +// - not read in any condition: then there is no guard, dead or otherwise. +// (The mirror case, written but never read, is a separate verdict and is +// deliberately not in this rule.) +// +// Writes are collected from EVERY module, including Marketplace ones, while +// only the project's own attributes are reported. A Marketplace flow writing an +// app's attribute is rare, but counting it as no write would be a false +// positive, and the asymmetry costs nothing. +// +// Boolean only for now. A String or Integer read in a condition and assigned +// one literal is the same shape, but enumerating "the literals I could see" is +// a much weaker claim for those types, so they are left out rather than +// reported at low confidence. +type FixedGuardAttributeRule struct{} + +func NewFixedGuardAttributeRule() *FixedGuardAttributeRule { + return &FixedGuardAttributeRule{} +} + +func (r *FixedGuardAttributeRule) ID() string { return "MPR017" } +func (r *FixedGuardAttributeRule) Name() string { return "FixedGuardAttribute" } +func (r *FixedGuardAttributeRule) Category() string { return "correctness" } +func (r *FixedGuardAttributeRule) DefaultSeverity() linter.Severity { return linter.SeverityWarning } + +// RequiredCatalogMode declares that this rule needs a FULL catalog. +// +// It reads activities (decision expressions), widgets (AttributeRef) and +// xpath_expressions, all of which REFRESH CATALOG FULL writes and a fast +// catalog does not. Without this declaration the lint entry point builds a +// fast catalog and the rule finds no conditions at all — so it reports +// NOTHING on a project that has the defect, which is worse than a wrong +// finding because it looks like a pass. +// +// Measured: on a freshly built catalog the rule found nothing, while a second +// run against the cached catalog (built in full mode by an earlier command) +// reported correctly. That difference is this declaration. +func (r *FixedGuardAttributeRule) RequiredCatalogMode() linter.CatalogMode { + return linter.CatalogFull +} + +func (r *FixedGuardAttributeRule) Description() string { + return "A Boolean attribute a decision tests but which can only hold one value, so the branch it guards is unreachable" +} + +// attrFacts is what the project says about one attribute. +type attrFacts struct { + // readInCondition: some decision or XPath consults it. + readInCondition bool + // widgetBound: a page widget is bound to it, so a user may write it. + widgetBound bool + // unknownWrite: assigned from something other than a literal, so the + // reachable set cannot be enumerated. + unknownWrite bool + // literals are the distinct literal values assigned to it. + literals map[string]bool +} + +// guardFinding is one reportable attribute and why. +type guardFinding struct { + attr linter.Attribute + values int // size of the reachable value set + only string // the one value, for the message +} + +func (r *FixedGuardAttributeRule) Check(ctx *linter.LintContext) []linter.Violation { + if ctx == nil || ctx.Reader() == nil { + return nil + } + facts, why := collectAttributeFacts(ctx) + if why != "" { + // The writes could not be read. "I could not see the writes" must never + // be reported as "there are no writes", so the rule reports its own + // failure rather than a silently empty result (linter.RuleFailure is + // kept out of the score and listed separately). + return []linter.Violation{{ + RuleID: r.ID(), + Severity: r.DefaultSeverity(), + Message: "MPR017 could not run: " + why, + RuleFailure: true, + }} + } + + var owned []linter.Attribute + for e := range ctx.Entities() { + if ctx.IsExcluded(e.ModuleName) { + continue + } + for a := range ctx.AttributesFor(e.QualifiedName) { + owned = append(owned, a) + } + } + + var violations []linter.Violation + for _, f := range fixedGuards(owned, facts) { + qn := f.attr.EntityQualifiedName + "." + f.attr.Name + msg := fmt.Sprintf("Attribute '%s' is tested by a decision but can only ever be %s, "+ + "so the branch it guards is unreachable.", qn, f.only) + if f.values == 0 { + msg = fmt.Sprintf("Attribute '%s' is tested by a decision but nothing ever assigns it "+ + "and it has no default, so the branch it guards is unreachable.", qn) + } + violations = append(violations, linter.Violation{ + RuleID: r.ID(), + Severity: r.DefaultSeverity(), + Message: msg, + Location: linter.Location{ + Module: f.attr.ModuleName, + DocumentType: "entity", + DocumentName: entityName(f.attr.EntityQualifiedName), + DocumentID: f.attr.EntityID, + }, + Suggestion: "Either assign the other value where the guarded work should be skipped, " + + "or remove the flag and the decision that reads it. A flag with one reachable " + + "value reads as an interlock and is not one.", + }) + } + return violations +} + +func entityName(qualified string) string { + if i := strings.LastIndex(qualified, "."); i >= 0 { + return qualified[i+1:] + } + return qualified +} + +// fixedGuards is the decision, kept pure so it can be tested without a project. +// Deterministic: findings are ordered by qualified name. +func fixedGuards(attrs []linter.Attribute, facts map[string]*attrFacts) []guardFinding { + var out []guardFinding + for _, a := range attrs { + if !strings.EqualFold(a.DataType, "Boolean") || a.IsCalculated { + continue + } + f := facts[a.EntityQualifiedName+"."+a.Name] + if f == nil || !f.readInCondition || f.widgetBound || f.unknownWrite { + continue + } + reachable := map[string]bool{} + for v := range f.literals { + reachable[v] = true + } + if a.DefaultValue != "" { + reachable[strings.ToLower(strings.TrimSpace(a.DefaultValue))] = true + } + if len(reachable) > 1 { + continue + } + only := "" + for v := range reachable { + only = v + } + out = append(out, guardFinding{attr: a, values: len(reachable), only: only}) + } + sort.SliceStable(out, func(i, j int) bool { + return out[i].attr.EntityQualifiedName+"."+out[i].attr.Name < + out[j].attr.EntityQualifiedName+"."+out[j].attr.Name + }) + return out +} + +// attributeReadInExpression reports whether expr reads the member `name`. +// +// Matched as `/name` followed by a non-identifier character, which is how a +// member read is spelled in a Mendix expression ($Var/Attr, and across an +// association $Var/Mod.Assoc/Attr). It cannot resolve WHICH entity $Var holds, +// so the match is by name alone and therefore over-broad: a read of another +// entity's same-named attribute marks this one read too. That direction is +// deliberate — it costs findings, not correctness. +func attributeReadInExpression(expr, name string) bool { + if expr == "" || name == "" { + return false + } + needle := "/" + name + for i := 0; ; { + j := strings.Index(expr[i:], needle) + if j < 0 { + return false + } + end := i + j + len(needle) + if end >= len(expr) || !isIdentRune(expr[end]) { + return true + } + i = end + } +} + +func isIdentRune(b byte) bool { + return b == '_' || (b >= '0' && b <= '9') || (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') +} + +// collectAttributeFacts sweeps the project once. The returned string is empty +// on success and otherwise says why the sweep could not be trusted — the rule +// then reports that instead of findings, because "I could not see the writes" +// must never be reported as "there are no writes". +func collectAttributeFacts(ctx *linter.LintContext) (map[string]*attrFacts, string) { + facts := map[string]*attrFacts{} + get := func(qn string) *attrFacts { + if f := facts[qn]; f != nil { + return f + } + f := &attrFacts{literals: map[string]bool{}} + facts[qn] = f + return f + } + + // Attribute names of interest, and the qualified names each maps to. Reads + // are matched by name (see attributeReadInExpression), so one name can mark + // several attributes read. + byName := map[string][]string{} + entities := 0 + for e := range ctx.Entities() { + entities++ + for a := range ctx.AttributesFor(e.QualifiedName) { + qn := a.EntityQualifiedName + "." + a.Name + get(qn) + byName[a.Name] = append(byName[a.Name], qn) + } + } + // No attributes readable. If the project genuinely has no entities that is + // simply nothing to check; but entities WITH no readable attributes means + // the attribute read failed, and this rule must then say so rather than + // report a clean run. + // + // It is not hypothetical: on a freshly built catalog ctx.AttributesFor + // fails with "no such table: attributes" (ako/mxcli#2417) while entities + // and activities resolve, and the context stops answering afterwards. Left + // silent, this rule reported NOTHING on a project that provably has the + // defect, which is indistinguishable from a pass. + if len(byName) == 0 { + if entities > 0 { + return nil, fmt.Sprintf("read %d entities but no attributes "+ + "(ako/mxcli#2417: AttributesFor fails on a freshly built catalog); "+ + "re-run to use the cached catalog", entities) + } + return facts, "" + } + + markRead := func(expr string) { + if expr == "" { + return + } + for name, qns := range byName { + if attributeReadInExpression(expr, name) { + for _, qn := range qns { + get(qn).readInCondition = true + } + } + } + } + + // Reads: decision and other activity condition expressions, from the + // catalog, and every XPath constraint. Neither is platform-filtered, which + // is what we want for reads. + // The `activities` VIEW, not activities_data: the _data tables are the + // builders' storage and hold rows for every snapshot, while the view joins + // the current one. Reading the table directly made this rule find no + // conditions at all on the run right after a write, when the catalog was + // rebuilt in-process — so it reported nothing on a project that did have + // the defect. + rows, err := ctx.Query(`SELECT COALESCE(ConditionExpression, '') FROM activities + WHERE COALESCE(ConditionExpression, '') <> ''`) + if err != nil { + return nil, fmt.Sprintf("reading decision expressions: %v", err) + } + for rows.Next() { + var expr string + if rows.Scan(&expr) == nil { + markRead(expr) + } + } + rows.Close() + for x := range ctx.XPathExpressions() { + markRead(x.XPathExpression) + } + + // A page widget bound to the attribute: a user may write it. + for w := range ctx.Widgets() { + if w.AttributeRef == "" { + continue + } + if f := facts[w.AttributeRef]; f != nil { + f.widgetBound = true + continue + } + // Bound by bare name or an unexpected spelling: mark every attribute of + // that name, erring towards not reporting. + for _, qn := range byName[entityName(w.AttributeRef)] { + get(qn).widgetBound = true + } + } + + // Writes: every Change/Create member assignment in every module. Uses the + // reader rather than ctx.Microflows(), which is platform-filtered — a write + // from a Marketplace flow is still a write. + reader := ctx.Reader() + flows, err := reader.ListMicroflows() + if err != nil { + return nil, fmt.Sprintf("reading microflows for member assignments: %v", err) + } + for _, mf := range flows { + if mf == nil { + continue + } + full := mf + if mf.ObjectCollection == nil && mf.ID != "" { + if loaded, err := ctx.FullMicroflow(model.ID(mf.ID)); err == nil && loaded != nil { + full = loaded + } + } + if full.ObjectCollection == nil { + continue + } + walkMemberChanges(full.ObjectCollection.Objects, func(mc *microflows.MemberChange) { + qn := mc.AttributeQualifiedName + targets := []string{} + if f := facts[qn]; f != nil { + targets = append(targets, qn) + } else if qn != "" { + targets = append(targets, byName[entityName(qn)]...) + } + for _, t := range targets { + f := get(t) + if lit, ok := booleanLiteral(mc.Value); ok { + f.literals[lit] = true + } else { + f.unknownWrite = true + } + } + // The assigned value is itself an expression, so it reads whatever + // it mentions. + markRead(mc.Value) + }) + } + return facts, "" +} + +// booleanLiteral normalises a value that is exactly a boolean literal. +// Anything else is an expression whose result is not known here. +func booleanLiteral(v string) (string, bool) { + s := strings.ToLower(strings.TrimSpace(v)) + if s == "true" || s == "false" { + return s, true + } + return "", false +} + +// walkMemberChanges visits every member assignment, descending into loops the +// way CONV020 does — a change inside a loop is still a change. +func walkMemberChanges(objs []microflows.MicroflowObject, visit func(*microflows.MemberChange)) { + for _, o := range objs { + switch a := o.(type) { + case *microflows.LoopedActivity: + if a.ObjectCollection != nil { + walkMemberChanges(a.ObjectCollection.Objects, visit) + } + case *microflows.ActionActivity: + switch act := a.Action.(type) { + case *microflows.ChangeObjectAction: + for _, mc := range act.Changes { + if mc != nil { + visit(mc) + } + } + case *microflows.CreateObjectAction: + for _, mc := range act.InitialMembers { + if mc != nil { + visit(mc) + } + } + } + } + } +} diff --git a/mdl/linter/rules/mpr017_fixed_guard_attribute_test.go b/mdl/linter/rules/mpr017_fixed_guard_attribute_test.go new file mode 100644 index 000000000..fc9ef9b90 --- /dev/null +++ b/mdl/linter/rules/mpr017_fixed_guard_attribute_test.go @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rules + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/linter" +) + +func attr(entity, name, dataType, def string) linter.Attribute { + return linter.Attribute{ + Name: name, + EntityQualifiedName: entity, + ModuleName: "Shop", + DataType: dataType, + DefaultValue: def, + } +} + +func TestFixedGuardAttributeRule_Metadata(t *testing.T) { + r := NewFixedGuardAttributeRule() + if r.ID() != "MPR017" { + t.Errorf("ID = %q, want MPR017", r.ID()) + } + if r.Name() != "FixedGuardAttribute" { + t.Errorf("Name = %q, want FixedGuardAttribute", r.Name()) + } + if r.Category() != "correctness" { + t.Errorf("Category = %q, want correctness", r.Category()) + } +} + +// The defect this rule exists for, as the benchmark's MxScout stage found it in +// all three apps it read: +// +// "The tick, the housekeeping and the scheduled reset check a flag so that +// they do not work during the data reset, but no flow ever sets it to true; +// the reset only sets it to false." +// +// The flag IS written. A rule looking for "read but never written" would not +// see this at all, which is why the condition is a value set of one: every +// reachable value is false, so every decision on it has a fixed outcome. +func TestFixedGuards_FlagWrittenOnlyToOneConstant(t *testing.T) { + facts := map[string]*attrFacts{ + "Simulation.SimulationSettings.IsResetInProgress": { + readInCondition: true, + literals: map[string]bool{"false": true}, + }, + } + got := fixedGuards([]linter.Attribute{ + attr("Simulation.SimulationSettings", "IsResetInProgress", "Boolean", "false"), + }, facts) + if len(got) != 1 { + t.Fatalf("got %d findings, want 1", len(got)) + } + if got[0].values != 1 { + t.Errorf("values = %d, want 1", got[0].values) + } +} + +// The control that makes the rule mean something: a flag written to BOTH values +// is a working guard and must not be reported. +func TestFixedGuards_FlagWrittenToBothValuesIsFine(t *testing.T) { + facts := map[string]*attrFacts{ + "Shop.Settings.IsOpen": { + readInCondition: true, + literals: map[string]bool{"true": true, "false": true}, + }, + } + if got := fixedGuards([]linter.Attribute{ + attr("Shop.Settings", "IsOpen", "Boolean", "false"), + }, facts); len(got) != 0 { + t.Fatalf("got %d findings, want 0: %+v", len(got), got) + } +} + +// A default of true with the only write false still has two reachable values, +// so the guard can change state once and is not dead. Getting this wrong is the +// easy mistake: counting writes alone would report it. +func TestFixedGuards_DefaultCountsAsAReachableValue(t *testing.T) { + facts := map[string]*attrFacts{ + "Shop.Settings.IsFirstRun": { + readInCondition: true, + literals: map[string]bool{"false": true}, + }, + } + if got := fixedGuards([]linter.Attribute{ + attr("Shop.Settings", "IsFirstRun", "Boolean", "true"), + }, facts); len(got) != 0 { + t.Fatalf("got %d findings, want 0 (default true + write false = two values)", len(got)) + } +} + +// Never written at all, and no default: still one value (empty), still a fixed +// guard. This is the "read but never written" case the issue led with. +func TestFixedGuards_NeverWrittenAtAll(t *testing.T) { + facts := map[string]*attrFacts{ + "Shop.Settings.IsPaused": {readInCondition: true, literals: map[string]bool{}}, + } + if got := fixedGuards([]linter.Attribute{ + attr("Shop.Settings", "IsPaused", "Boolean", ""), + }, facts); len(got) != 1 { + t.Fatalf("got %d findings, want 1", len(got)) + } +} + +// Four reasons not to report, each a false positive the rule would otherwise +// produce on an ordinary app. +func TestFixedGuards_Exclusions(t *testing.T) { + for _, tc := range []struct { + name string + a linter.Attribute + facts *attrFacts + }{ + { + // A user can type into it, so its value is not the model's to know. + // This is the exclusion the issue called decisive: without it the + // rule fires on half of a normal app. + "bound to a page widget", + attr("Shop.Settings", "IsOpen", "Boolean", "false"), + &attrFacts{readInCondition: true, widgetBound: true, literals: map[string]bool{"false": true}}, + }, + { + // Written from an expression, not a literal: the value set is + // unknown, so nothing can be concluded. + "written from an expression", + attr("Shop.Settings", "IsOpen", "Boolean", "false"), + &attrFacts{readInCondition: true, unknownWrite: true, literals: map[string]bool{"false": true}}, + }, + { + // Never consulted in a condition, so there is no guard to be dead. + // The written-but-never-read case is a separate verdict, not this one. + "not read in any condition", + attr("Shop.Settings", "IsOpen", "Boolean", "false"), + &attrFacts{literals: map[string]bool{"false": true}}, + }, + { + // Mendix computes it; the model does not assign it. + "calculated", + func() linter.Attribute { + a := attr("Shop.Settings", "IsOpen", "Boolean", "") + a.IsCalculated = true + return a + }(), + &attrFacts{readInCondition: true, literals: map[string]bool{}}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + qn := tc.a.EntityQualifiedName + "." + tc.a.Name + got := fixedGuards([]linter.Attribute{tc.a}, map[string]*attrFacts{qn: tc.facts}) + if len(got) != 0 { + t.Fatalf("got %d findings, want 0: %+v", len(got), got) + } + }) + } +} + +// Only Boolean attributes in v1. A String or Integer read in a condition and +// written to one literal is the same shape of defect, but "the literals I could +// see" is a far weaker claim for those types, so they are left out rather than +// reported at low confidence. +func TestFixedGuards_BooleanOnlyForNow(t *testing.T) { + for _, dt := range []string{"String", "Integer", "Decimal", "DateTime", "Enum"} { + facts := map[string]*attrFacts{ + "Shop.Settings.Mode": {readInCondition: true, literals: map[string]bool{"'a'": true}}, + } + if got := fixedGuards([]linter.Attribute{ + attr("Shop.Settings", "Mode", dt, ""), + }, facts); len(got) != 0 { + t.Errorf("%s: got %d findings, want 0", dt, len(got)) + } + } +} + +// Reads are matched by attribute name against expression text, which cannot +// resolve which entity a variable holds. That makes the match deliberately +// over-broad: a read of SomeOther.IsOpen marks every IsOpen as read, so the +// rule under-reports rather than reporting something false. +func TestAttributeReadInExpression(t *testing.T) { + for _, tc := range []struct { + expr, name string + want bool + }{ + {"$Settings/IsResetInProgress = true", "IsResetInProgress", true}, + {"$Settings/Shop.Settings_Order/IsOpen", "IsOpen", true}, + {"not $S/IsOpen", "IsOpen", true}, + {"$S/IsOpenToday", "IsOpen", false}, // prefix, not the member + {"$S/WasIsOpen", "IsOpen", false}, // suffix, not the member + {"'IsOpen'", "IsOpen", false}, // a string literal, not a member read + {"$IsOpen", "IsOpen", false}, // a variable of that name, not a member + {"", "IsOpen", false}, + } { + if got := attributeReadInExpression(tc.expr, tc.name); got != tc.want { + t.Errorf("attributeReadInExpression(%q, %q) = %v, want %v", tc.expr, tc.name, got, tc.want) + } + } +} + +// A reader that cannot supply microflows yields no findings rather than +// reporting every attribute as unwritten — the failure mode that would make +// this rule dangerous, since "I could not see the writes" must never read as +// "there are no writes". +func TestFixedGuardAttributeRule_NoReaderReportsNothing(t *testing.T) { + if got := NewFixedGuardAttributeRule().Check(linter.NewLintContextFromDB(nil)); len(got) != 0 { + t.Fatalf("got %d findings with no reader, want 0", len(got)) + } +} + +// The rule must declare that it needs a full catalog. +// +// Regression guard, not a hypothetical: without it the lint entry point builds +// a FAST catalog, the `activities` table is not written, and the rule finds no +// decision expressions — so it reports nothing on a project that does have the +// defect. Measured before the declaration: a fresh catalog gave "No issues +// found" while a second run against the cached (full) catalog reported the +// defect correctly. linter.CatalogRequirer's own doc warns about exactly this +// ("silently returning empty results"). +func TestFixedGuardAttributeRule_RequiresFullCatalog(t *testing.T) { + r := NewFixedGuardAttributeRule() + req, ok := any(r).(linter.CatalogRequirer) + if !ok { + t.Fatal("rule does not implement linter.CatalogRequirer, so lint will build a fast catalog and the rule will silently find nothing") + } + if got := req.RequiredCatalogMode(); got != linter.CatalogFull { + t.Errorf("RequiredCatalogMode = %v, want CatalogFull (it reads activities, widgets and xpath_expressions)", got) + } + // And the computed mode for a run including this rule must be full. + if got := linter.RequiredCatalogMode([]linter.Rule{r}); got != linter.CatalogFull { + t.Errorf("RequiredCatalogMode([rule]) = %v, want CatalogFull", got) + } +} From dab667df16612ecffc1a9a145291d8ccaef71f05 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 22:45:27 +0000 Subject: [PATCH 27/28] feat(docker): parse mx check output into structured findings First step of PROPOSAL_verify_gate.md (#2333): ParseCheckOutput, a location class per finding, the measured MxBuild report stage, a message-free Key, and a Verify cross-check against mx's own error count. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011oK82stfDEB1TGfJLVwUxc --- cmd/mxcli/docker/check_findings.go | 230 ++++++++++++++++++++++++ cmd/mxcli/docker/check_findings_test.go | 112 ++++++++++++ 2 files changed, 342 insertions(+) create mode 100644 cmd/mxcli/docker/check_findings.go create mode 100644 cmd/mxcli/docker/check_findings_test.go diff --git a/cmd/mxcli/docker/check_findings.go b/cmd/mxcli/docker/check_findings.go new file mode 100644 index 000000000..97f127ce3 --- /dev/null +++ b/cmd/mxcli/docker/check_findings.go @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "bytes" + "fmt" + "regexp" + "strconv" + "strings" +) + +// A CheckFinding is one diagnostic line of `mx check`, parsed. It is the unit a +// baseline compares (PROPOSAL_verify_gate.md): the text of `mx check` is not +// stable between runs, so a caller compares Keys and counts, never lines. +// +// mx check prints one line per finding: +// +// [error] [CE0117] "Error(s) in expression." at Microflow 'ACT_X' +type CheckFinding struct { + Severity string // "error", "warning" or "deprecation", as mx prints it + Code string // "CE0117" + Message string // the quoted text; display only, never part of the Key + Location string // text after `" at `, or "" when the line names none + Kind string // "Microflow", "Data grid 2", ... (the Location before its quoted name) + Name string // the quoted name in Location; "" when empty or absent + Class LocationClass + Stage int // MxBuild report stage 0..3, see StageOf + Line string +} + +// LocationClass says how much of "which element" a finding's text carries, +// which decides how a baseline can match it (PROPOSAL_verify_gate.md, +// "Finding identity"). +type LocationClass int + +const ( + // Located: the line names the kind and name of the element. + Located LocationClass = iota + // DocumentOnly: the line names a document, not the element in it. + DocumentOnly + // Unlocated: the line names nothing that identifies it. Matched as a count + // per distinct line, as recheckCE0463 already does. + Unlocated +) + +func (c LocationClass) String() string { + switch c { + case Located: + return "located" + case DocumentOnly: + return "document-only" + default: + return "unlocated" + } +} + +// Key is what a baseline matches on: severity, code and location, never the +// message. For an Unlocated finding the location is the whole line's text, so +// the same line repeated is the same key and the caller counts it. +func (f CheckFinding) Key() string { + return f.Severity + "\x00" + f.Code + "\x00" + f.Class.String() + "\x00" + f.Location +} + +var findingRe = regexp.MustCompile(`^\[(error|warning|deprecation)\]\s+\[(C[EWD]\d+)\]\s+"`) + +// locationRe splits "Microflow 'ACT_X'" into kind and name. +var locationRe = regexp.MustCompile(`^(.*?)\s*'(.*)'$`) + +// ParseCheckOutput reads the diagnostics out of `mx check` output. Lines that +// are not findings (banners, the closing "The app contains: N errors.") are +// skipped. A finding the parser cannot split still counts: its Message is the +// rest of the line. +func ParseCheckOutput(output string) []CheckFinding { + var out []CheckFinding + for _, raw := range strings.Split(output, "\n") { + line := strings.TrimSpace(strings.TrimRight(raw, "\r")) + m := findingRe.FindStringSubmatchIndex(line) + if m == nil { + continue + } + f := CheckFinding{Severity: line[m[2]:m[3]], Code: line[m[4]:m[5]], Line: line} + rest := line[m[1]:] // after the opening quote + if i := strings.LastIndex(rest, `" at `); i >= 0 { + f.Message, f.Location = rest[:i], strings.TrimSpace(rest[i+len(`" at `):]) + } else { + f.Message = strings.TrimSuffix(rest, `"`) + } + if lm := locationRe.FindStringSubmatch(f.Location); lm != nil { + f.Kind, f.Name = strings.TrimSpace(lm[1]), lm[2] + } else { + f.Kind = f.Location + } + f.Class = classify(f) + f.Stage = StageOf(f.Code) + out = append(out, f) + } + return out +} + +// unlocatedCodes are codes whose text does not name the element, so two +// same-named elements on different documents print the same line. CE0463 does +// not name its page (docker/check.go, recheckCE0463). +var unlocatedCodes = map[string]bool{"CE0463": true} + +// documentKinds are the kinds whose name is a document, so the line says which +// document but not what in it. A kind not listed names an element (a widget, +// an activity) and counts as Located. The list is the measured set, not a +// claim of completeness: an unlisted document kind errs towards Located. +var documentKinds = map[string]bool{ + "Microflow": true, + "Nanoflow": true, + "Page": true, + "Snippet": true, + "Layout": true, + "Domain model of module": true, + "Enumeration": true, + "Workflow": true, + "Java action": true, + "Rule": true, + "Constant": true, + "Import mapping": true, + "Export mapping": true, + "Published REST service": true, + "Consumed REST service": true, +} + +func classify(f CheckFinding) LocationClass { + switch { + case unlocatedCodes[f.Code], f.Location == "", f.Name == "": + return Unlocated + case documentKinds[f.Kind]: + return DocumentOnly + default: + return Located + } +} + +// mxStages is MxBuild's report stage per code, as measured in +// PROPOSAL_check_diagnostics_catalog.md ("MxBuild reports errors in stages", +// MxBuild 11.16.1, microflows and pages only). An earlier stage hides the later +// ones: 0 everything, 1 and 2 the rest of the same microflow. +var mxStages = map[string]int{ + "CE1613": 0, + "CE0111": 1, + "CE0109": 2, + "CE0117": 3, + "CE0068": 3, + "CE0038": 3, + "CE0067": 3, +} + +// StageOf is the MxBuild stage of code. A code outside the measured set is stage +// 3: it hides nothing and anything earlier may hide it, which is the cautious +// reading for a baseline diff. +func StageOf(code string) int { + if s, ok := mxStages[code]; ok { + return s + } + return 3 +} + +var appErrorsRe = regexp.MustCompile(`The app contains: (\d+) errors?\.`) + +// CheckReport is the parsed result of one `mx check` run. +type CheckReport struct { + Findings []CheckFinding + // Declared is the error count mx printed on its closing line; HasDeclared + // is false when that line is absent (mx did not complete). + Declared int + HasDeclared bool +} + +// Errors counts the findings of severity error. +func (r CheckReport) Errors() int { + n := 0 + for _, f := range r.Findings { + if f.Severity == "error" { + n++ + } + } + return n +} + +// Verify cross-checks the parse against the count mx printed. A mismatch means +// the parser lost lines (or mx changed its format), and a baseline built from it +// would silently hide findings, so a caller must not treat the report as +// complete. +func (r CheckReport) Verify() error { + if !r.HasDeclared { + return fmt.Errorf("mx check did not print its closing error count; the report may be incomplete") + } + if got := r.Errors(); got != r.Declared { + return fmt.Errorf("parsed %d errors but mx check reported %d; the output format may have changed", got, r.Declared) + } + return nil +} + +// ParseCheckReport parses output and reads mx's own error count. +func ParseCheckReport(output string) CheckReport { + r := CheckReport{Findings: ParseCheckOutput(output)} + if m := appErrorsRe.FindStringSubmatch(output); m != nil { + if n, err := strconv.Atoi(m[1]); err == nil { + r.Declared, r.HasDeclared = n, true + } + } + return r +} + +// CheckFindings runs `mx check` on a temporary copy of the project and returns +// its parsed report. It checks the project as stored (no update-widgets), never +// modifies it, and returns the report together with mx's own exit error, which +// is non-zero whenever the project has errors: a non-nil error is not a failure +// to run unless the report is empty and Verify fails. +func CheckFindings(opts CheckOptions) (CheckReport, error) { + projectVersion := "" + if opts.ProjectPath != "" { + if reader, err := openReadOnly(opts.ProjectPath); err == nil { + projectVersion = reader.ProjectVersion().ProductVersion + reader.Disconnect() + } + } + mxPath, err := resolveMxForCheck(opts.MxBuildPath, projectVersion) + if err != nil { + return CheckReport{}, err + } + var out, errOut bytes.Buffer + runErr := MxCheckOnCopy(mxPath, opts.ProjectPath, nil, &out, &errOut) + return ParseCheckReport(out.String() + "\n" + errOut.String()), runErr +} diff --git a/cmd/mxcli/docker/check_findings_test.go b/cmd/mxcli/docker/check_findings_test.go new file mode 100644 index 000000000..23b17b5af --- /dev/null +++ b/cmd/mxcli/docker/check_findings_test.go @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "strings" + "testing" +) + +// The finding lines are the ones mx check printed in the repository's own +// records (docker/check.go's CE0463 comment, the fix-issue findings); the +// closing line is mx's. +const sampleCheck = `Checking project... +[error] [CE0463] "The definition of this widget has changed. ..." at Data grid 2 'dgRequests' +[error] [CE0463] "The definition of this widget has changed. ..." at Data grid 2 'dgRequests' +[error] [CE0117] "Error(s) in expression." at Microflow 'ACT_X' +[error] [CE0066] "Entity access is out of date…" at Domain model of module 'Administration' +[error] [CE0090] "The 'I913.Animal' value should be configured for an outgoing flow." at Object type decision '' +[error] [CE0109] "Variable has no value." +The app contains: 6 errors. +` + +func TestParseCheckOutput(t *testing.T) { + fs := ParseCheckOutput(sampleCheck) + if len(fs) != 6 { + t.Fatalf("parsed %d findings, want 6: %+v", len(fs), fs) + } + for _, c := range []struct { + i int + code, kind string + name string + class LocationClass + stage int + messagePart string + }{ + {0, "CE0463", "Data grid 2", "dgRequests", Unlocated, 3, "widget has changed"}, + {2, "CE0117", "Microflow", "ACT_X", DocumentOnly, 3, "Error(s) in expression."}, + {3, "CE0066", "Domain model of module", "Administration", DocumentOnly, 3, "Entity access"}, + {4, "CE0090", "Object type decision", "", Unlocated, 3, "'I913.Animal'"}, + {5, "CE0109", "", "", Unlocated, 2, "Variable has no value."}, + } { + f := fs[c.i] + if f.Code != c.code || f.Kind != c.kind || f.Name != c.name || f.Class != c.class || f.Stage != c.stage || + !strings.Contains(f.Message, c.messagePart) { + t.Errorf("finding %d = %+v, want code %s kind %q name %q class %v stage %d message ~%q", + c.i, f, c.code, c.kind, c.name, c.class, c.stage, c.messagePart) + } + } +} + +// A finding on a widget with a code that does name its element is Located; the +// CE0463 override is what makes the same text Unlocated. This is the control for +// the table: without it every widget finding would pass as Unlocated. +func TestClassifyLocatedNeedsAnElementName(t *testing.T) { + f := ParseCheckOutput(`[error] [CE0001] "msg" at Data grid 2 'dgRequests'`)[0] + if f.Class != Located { + t.Errorf("class = %v, want located", f.Class) + } + f = ParseCheckOutput(`[error] [CE0463] "msg" at Data grid 2 'dgRequests'`)[0] + if f.Class != Unlocated { + t.Errorf("CE0463 class = %v, want unlocated", f.Class) + } +} + +// The key ignores the message: the same finding reworded or on another line is +// one finding, which is the point of keying (the control: a different location +// is a different key). +func TestKeyIgnoresMessage(t *testing.T) { + a := ParseCheckOutput(`[error] [CE0117] "one wording" at Microflow 'ACT_X'`)[0] + b := ParseCheckOutput(`[error] [CE0117] "another wording" at Microflow 'ACT_X'`)[0] + c := ParseCheckOutput(`[error] [CE0117] "one wording" at Microflow 'ACT_Y'`)[0] + if a.Key() != b.Key() { + t.Error("the same finding with a different message must have the same key") + } + if a.Key() == c.Key() { + t.Error("a different location must have a different key") + } +} + +func TestStageOf(t *testing.T) { + for code, want := range map[string]int{"CE1613": 0, "CE0111": 1, "CE0109": 2, "CE0067": 3, "CE9999": 3} { + if got := StageOf(code); got != want { + t.Errorf("StageOf(%s) = %d, want %d", code, got, want) + } + } +} + +func TestParseCheckReportVerify(t *testing.T) { + r := ParseCheckReport(sampleCheck) + if !r.HasDeclared || r.Declared != 6 { + t.Fatalf("declared = %d (%v), want 6", r.Declared, r.HasDeclared) + } + if err := r.Verify(); err != nil { + t.Errorf("Verify() = %v on a complete report", err) + } + + // Control: a format change that loses lines must be reported, not absorbed. + lost := strings.Replace(sampleCheck, "[error] [CE0117]", "[err] [CE0117]", 1) + if err := ParseCheckReport(lost).Verify(); err == nil { + t.Error("Verify() = nil although a finding line was not parsed") + } + if err := ParseCheckReport("[error] [CE0117] \"x\" at Microflow 'M'\n").Verify(); err == nil { + t.Error("Verify() = nil without mx's closing count") + } +} + +func TestParseCheckOutputKeepsSeverityAndCRLF(t *testing.T) { + fs := ParseCheckOutput("[warning] [CW0001] \"w\" at Page 'P'\r\n[deprecation] [CD0001] \"d\"\r\n") + if len(fs) != 2 || fs[0].Severity != "warning" || fs[1].Severity != "deprecation" { + t.Fatalf("got %+v", fs) + } +} From f64749ed6e15f2a57c932d0afd9a81352f043290 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 23:01:28 +0000 Subject: [PATCH 28/28] fix(sql): address a generated connector's tables by their schema `SQL GENERATE CONNECTOR INTO ` emitted bare table names, so none of its queries could run against the database they were generated from: SELECT id, CustomerNumber, Name, Email, Active FROM customer ERROR: Table with name customer does not exist! Did you mean "cdcpublish.customer"? SHOW TABLES and SHOW VIEWS return (table_schema, table_name) and both readers kept only the name, so TableSchema.Schema -- a field that already existed -- was never populated and the emitter had nothing to qualify with. The quieter half of the same defect: a name two schemas share was refused as ambiguous by the describe step, and the reader's `continue // skip tables that fail` swallowed it, so the object vanished from the output with no warning. Two apps publishing CDC for an entity of the same name is the ordinary case. Both readers now go through one `tableAddresses`, `ReadTableSchema` splits a qualified name into Schema and Name so the Mendix entity is still called after the bare name, and the emitted query addresses `ts.QualifiedName()`. The whole pipeline is tested without PostgreSQL -- modernc SQLite, DriverPostgres so the real query templates run, and an attached information_schema, the harness the IMPORT ... LINK fix established. The repo's other DB-backed sql tests are gated on MXCLI_TEST_PG_DSN and skip in CI, which is how ako/mxcli#808 shipped green. Controls, each on the end-to-end test: emitter back to ts.Name leaves every FROM unqualified; reader back to row[1] does the same and additionally reports `1 entities, want 3`. Verified end to end against a DuckGraphLake lake serving Mendix entity CDC, where each publishing app gets a schema of its own: 6 of 6 generated queries run, where 5 of 6 failed before. The one that passed before is a table in `main`, in the search path -- which is why a single-schema database never showed the defect. Fixes #2414 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011oK82qQLKUyoXXGmbmX68t --- ...enerate-drops-schema-from-every-query.json | 1 + cmd/mxcli/syntax/features_integration.go | 13 +- sql/generate.go | 16 +- sql/generate_schema_qualified_2414_test.go | 209 ++++++++++++++++++ sql/schema.go | 66 ++++-- 5 files changed, 287 insertions(+), 18 deletions(-) create mode 100644 .claude/skills/fix-issue/findings/other/2026-11-18-sql-generate-drops-schema-from-every-query.json create mode 100644 sql/generate_schema_qualified_2414_test.go diff --git a/.claude/skills/fix-issue/findings/other/2026-11-18-sql-generate-drops-schema-from-every-query.json b/.claude/skills/fix-issue/findings/other/2026-11-18-sql-generate-drops-schema-from-every-query.json new file mode 100644 index 000000000..bb9cb7820 --- /dev/null +++ b/.claude/skills/fix-issue/findings/other/2026-11-18-sql-generate-drops-schema-from-every-query.json @@ -0,0 +1 @@ +{"area":"sql","date":"2026-11-18","symptom":"SQL GENERATE CONNECTOR INTO emitted queries that cannot run against the database they were generated from: `SELECT id, CustomerNumber, Name, Email, Active FROM customer` -> `ERROR: Table with name customer does not exist! Did you mean \"cdcpublish.customer\"?`. Worse and quieter: a table name that two schemas share was refused as ambiguous by the describe step and the object was dropped from the output entirely, so three views in gave one entity out with no warning (ako/mxcli#2414).","cause":"ShowTables/ShowViews return (table_schema, table_name) and both readers kept only row[1] (sql/schema.go ReadAllTableSchemas, ReadAllViewSchemas), so TableSchema.Schema -- a field that already existed -- was never populated and the emitter had nothing to qualify with; generate.go then addressed the table as ts.Name. The drop is the second-order effect: ReadTableSchema on a bare name hits refuseAmbiguousTable, which correctly refuses a name in two schemas, and the reader's `continue // skip tables that fail` swallows it.","file":"sql/schema.go (ReadAllTableSchemas, ReadAllViewSchemas, ReadTableSchema, new tableAddresses/qualifiedTableName/TableSchema.QualifiedName); sql/generate.go (renderConnector, extracted from GenerateConnector); test sql/generate_schema_qualified_2414_test.go","insight":"The whole sql/ metadata pipeline runs end to end with no PostgreSQL: modernc SQLite, DriverPostgres so the real query templates execute, and `ATTACH ':memory:' AS information_schema` with tables/columns fixtures (the harness the IMPORT ... LINK fix established in sql/import_assoc_test.go). Use it -- the repo's other DB-backed sql tests are gated on MXCLI_TEST_PG_DSN and therefore skip in CI, which is how ako/mxcli#808 shipped green, and the lake this was found against is DuckDB, which rejects `serial` so it cannot host them either. Two controls, each on the end-to-end test: emitter back to ts.Name -> every FROM unqualified; reader back to row[1] -> same, plus `1 entities, want 3`. The reason the defect survived is that it is invisible on exactly the database most people try: a single schema in the connection's search path works, and in the live reproduction 1 of 6 generated queries passed for that reason (main.ro_probe). Reserved words are a red herring -- bare `FROM order` is a syntax error, but `FROM cdcpublish.order` parses, so qualifying subsumes it; whether quoting is still needed is unmeasured on strict PostgreSQL. Found by pointing mxcli at a DuckGraphLake lake serving Mendix entity CDC, where every publishing app gets a schema of its own, so nothing is in the search path.","refs":["ako/mxcli#2414","ako/mxcli#2415","ako/mxcli#2416"]} diff --git a/cmd/mxcli/syntax/features_integration.go b/cmd/mxcli/syntax/features_integration.go index 9b9925910..b4b10b47e 100644 --- a/cmd/mxcli/syntax/features_integration.go +++ b/cmd/mxcli/syntax/features_integration.go @@ -491,10 +491,17 @@ func init() { Summary: "Auto-generate Database Connector MDL from external database schema", Keywords: []string{ "generate connector", "database connector", - "constants", "non-persistent entity", "jdbc", + "constants", "non-persistent entity", "jdbc", "schema", }, - Syntax: "SQL GENERATE CONNECTOR INTO Module\n [TABLES (t1, t2)]\n [VIEWS (v1, v2)]\n [EXEC];", - Example: "SQL source GENERATE CONNECTOR INTO HRModule;\nSQL source GENERATE CONNECTOR INTO HRModule TABLES (employees, departments) EXEC;", + Syntax: "SQL GENERATE CONNECTOR INTO Module\n" + + " [TABLES (t1, schema.t2)]\n" + + " [VIEWS (v1, schema.v2)]\n" + + " [EXEC];\n" + + " -- A name may be schema-qualified, and must be where two\n" + + " -- schemas hold the same one. Each generated QUERY selects\n" + + " -- FROM the schema the object was discovered in, so it runs\n" + + " -- wherever the connection's search path points.", + Example: "SQL source GENERATE CONNECTOR INTO HRModule;\nSQL source GENERATE CONNECTOR INTO HRModule TABLES (employees, hr.departments) EXEC;", SeeAlso: []string{"sql", "sql.connect"}, }) diff --git a/sql/generate.go b/sql/generate.go index 0d1821d8e..4f4809f3f 100644 --- a/sql/generate.go +++ b/sql/generate.go @@ -70,6 +70,15 @@ func GenerateConnector(ctx context.Context, cfg *GenerateConfig) (*GenerateResul return nil, fmt.Errorf("no tables or views found") } + renderConnector(cfg, schemas, result) + return result, nil +} + +// renderConnector writes the MDL for schemas that have already been discovered. +// It is separate from discovery so the emitted text — in particular how a table +// is addressed in a generated query (ako/mxcli#2414) — can be tested without a +// database. +func renderConnector(cfg *GenerateConfig, schemas []*TableSchema, result *GenerateResult) *GenerateResult { // Generate MDL in two parts: executable (constants + entities) and connection reference var exec strings.Builder // constants + entities (parseable by mxcli) var conn strings.Builder // DATABASE CONNECTION definition (reference) @@ -144,7 +153,10 @@ func GenerateConnector(ctx context.Context, cfg *GenerateConfig) (*GenerateResul entityName: entityName, columns: ts.Columns, mappings: mappings, - tableName: ts.Name, + // The address, not the Mendix name: a query that selects FROM the + // bare name fails on any database that keeps the object outside the + // connection's search path (ako/mxcli#2414). + tableName: ts.QualifiedName(), }) } @@ -197,7 +209,7 @@ func GenerateConnector(ctx context.Context, cfg *GenerateConfig) (*GenerateResul result.ExecutableMDL = exec.String() result.ConnectionMDL = conn.String() result.MDL = exec.String() + "-- Database Connection (create in Studio Pro or configure via Database Connector module)\n" + conn.String() - return result, nil + return result } type columnMapping struct { diff --git a/sql/generate_schema_qualified_2414_test.go b/sql/generate_schema_qualified_2414_test.go new file mode 100644 index 000000000..1e93f8d3b --- /dev/null +++ b/sql/generate_schema_qualified_2414_test.go @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sql + +import ( + "context" + "strings" + "testing" +) + +// ako/mxcli#2414: the generated connector addressed every table by its bare +// name, so none of its queries could be run against the database they were +// generated from: +// +// SELECT id, CustomerNumber, Name, Email, Active FROM customer +// ERROR: Table with name customer does not exist! +// Did you mean "cdcpublish.customer"? +// +// SHOW TABLES returns (table_schema, table_name) and the reader kept only the +// name, so the schema never reached TableSchema and the FROM clause could not +// be qualified. Measured against a DuckGraphLake lake serving Mendix entity +// CDC, where each publishing app's views live in a schema of their own. +// +// This is tested on the emitted text rather than against a live database: the +// symptom is the query string, and a DSN-gated test would skip in CI, which is +// how ako/mxcli#808 shipped green. + +func cdcSchemas() []*TableSchema { + return []*TableSchema{ + {Schema: "cdcpublish", Name: "customer", Columns: []ColumnSchema{ + {Name: "id", DataType: "bigint"}, + {Name: "Name", DataType: "varchar"}, + }}, + {Schema: "cdcpublish", Name: "order", IsView: true, Columns: []ColumnSchema{ + {Name: "id", DataType: "bigint"}, + {Name: "OrderNumber", DataType: "bigint"}, + }}, + } +} + +func TestRenderConnector_QualifiesTheTableWithItsSchema_Issue2414(t *testing.T) { + cfg := &GenerateConfig{ + Conn: &Connection{Driver: DriverPostgres, Alias: "lake"}, + Module: "LakeSql", + Alias: "lake", + } + got := renderConnector(cfg, cdcSchemas(), &GenerateResult{}).MDL + + for _, want := range []string{"FROM cdcpublish.customer", "FROM cdcpublish.order"} { + if !strings.Contains(got, want) { + t.Errorf("generated SQL does not address the table by schema: want %q in\n%s", want, got) + } + } + // The bare form is the defect, not merely a missing improvement: a query + // that reads FROM customer fails outright on the database it came from. + for _, bare := range []string{"FROM customer", "FROM order"} { + if strings.Contains(got, bare+"'") { + t.Errorf("generated SQL still addresses a table unqualified: %q", bare) + } + } + // The schema belongs in the address only. It must not leak into the Mendix + // names, which are what a modeller and every expression refer to. + for _, want := range []string{"ENTITY LakeSql.Customer (", "ENTITY LakeSql.Order (", + "QUERY GetAllCustomers", "QUERY GetAllOrders"} { + if !strings.Contains(got, want) { + t.Errorf("want %q in\n%s", want, got) + } + } + if strings.Contains(got, "LakeSql.CdcpublishCustomer") || strings.Contains(got, "Cdcpublish_") { + t.Error("the schema reached an entity name") + } +} + +// CONTROL: a driver or database that reports no schema must still produce the +// unqualified name it did before, not a leading dot. +func TestRenderConnector_NoSchemaStaysUnqualified_Issue2414(t *testing.T) { + cfg := &GenerateConfig{ + Conn: &Connection{Driver: DriverPostgres, Alias: "hr"}, + Module: "HR", + Alias: "hr", + } + schemas := []*TableSchema{{Name: "employees", Columns: []ColumnSchema{{Name: "id", DataType: "bigint"}}}} + got := renderConnector(cfg, schemas, &GenerateResult{}).MDL + + if !strings.Contains(got, "FROM employees'") { + t.Errorf("want FROM employees in\n%s", got) + } + if strings.Contains(got, "FROM .employees") { + t.Error("an empty schema produced a leading dot") + } +} + +// The reader's half of the same defect: the schema column of SHOW TABLES has to +// survive into the name the describe step is given, or TableSchema.Schema stays +// empty however the emitter is written. +func TestQualifiedTableName_KeepsTheSchemaColumn_Issue2414(t *testing.T) { + for _, tc := range []struct { + name string + row []any + want string + }{ + {"schema and name", []any{"cdcpublish", "customer"}, "cdcpublish.customer"}, + {"reserved word as a name", []any{"cdcpublish", "order"}, "cdcpublish.order"}, + {"no schema reported", []any{"", "employees"}, "employees"}, + {"name only", []any{"employees"}, "employees"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := qualifiedTableName(tc.row); got != tc.want { + t.Errorf("qualifiedTableName(%v) = %q, want %q", tc.row, got, tc.want) + } + }) + } +} + +// The readers must go through that helper, not read the rows themselves: the +// defect was one column dropped in a loop body, and a test on the helper alone +// passes while the loop ignores it. Stubbing this function is what reproduces +// the bug end to end. +func TestTableAddresses_EveryRowKeepsItsSchema_Issue2414(t *testing.T) { + rows := [][]any{ + {"cdcpublish", "customer"}, + {"cdcpublish", "order"}, + {"main", "ro_probe"}, + {"", "unqualified"}, + {"too_short"}, + {}, + } + got := tableAddresses(rows) + want := []string{"cdcpublish.customer", "cdcpublish.order", "main.ro_probe", "unqualified", "too_short"} + if len(got) != len(want) { + t.Fatalf("tableAddresses = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("address %d = %q, want %q", i, got[i], want[i]) + } + } +} + +// ReadTableSchema is also reached with a name the user qualified themselves +// (TABLES (cdcpublish.customer)). The schema has to land in Schema and the bare +// name in Name, or the entity is called CdcpublishCustomer. +func TestTableSchemaQualifiedName_Issue2414(t *testing.T) { + for _, tc := range []struct{ schema, name, want string }{ + {"cdcpublish", "customer", "cdcpublish.customer"}, + {"", "customer", "customer"}, + } { + ts := &TableSchema{Schema: tc.schema, Name: tc.name} + if got := ts.QualifiedName(); got != tc.want { + t.Errorf("QualifiedName(%q, %q) = %q, want %q", tc.schema, tc.name, got, tc.want) + } + } +} + +// End to end over the whole pipeline — ShowTables, DescribeTable, the readers +// and the emitter — with no PostgreSQL. The driver is DriverPostgres so the +// real query templates run; the database is modernc SQLite with an attached +// information_schema, the harness the IMPORT ... LINK fix established. +// +// This is the test that fails if either half of the fix is reverted: a reader +// that drops the schema column, or an emitter that addresses ts.Name. +func TestGenerateConnector_EndToEnd_QualifiesEverySchema_Issue2414(t *testing.T) { + conn := openSQLite(t, "lake", + `ATTACH DATABASE ':memory:' AS information_schema`, + `CREATE TABLE information_schema.tables (table_schema TEXT, table_name TEXT, table_type TEXT)`, + `CREATE TABLE information_schema.columns (table_schema TEXT, table_name TEXT, + column_name TEXT, data_type TEXT, is_nullable TEXT, column_default TEXT, ordinal_position INTEGER)`, + // Two apps' CDC views, as DuckGraphLake exposes them: a schema per + // publishing app, and a name that repeats across them. + `INSERT INTO information_schema.tables VALUES + ('cdcpublish', 'customer', 'VIEW'), + ('cdcpublish', 'order', 'VIEW'), + ('otherapp', 'customer', 'VIEW')`, + `INSERT INTO information_schema.columns VALUES + ('cdcpublish', 'customer', 'id', 'bigint', 'NO', NULL, 1), + ('cdcpublish', 'customer', 'Name', 'character varying', 'YES', NULL, 2), + ('cdcpublish', 'order', 'id', 'bigint', 'NO', NULL, 1), + ('cdcpublish', 'order', 'OrderNumber', 'bigint', 'YES', NULL, 2), + ('otherapp', 'customer', 'id', 'bigint', 'NO', NULL, 1), + ('otherapp', 'customer', 'Email', 'text', 'YES', NULL, 2)`) + conn.Driver = DriverPostgres + + res, err := GenerateConnector(context.Background(), &GenerateConfig{ + Conn: conn, Module: "LakeSql", Alias: "lake", + }) + if err != nil { + t.Fatal(err) + } + + for _, want := range []string{ + "FROM cdcpublish.customer", "FROM cdcpublish.order", "FROM otherapp.customer", + } { + if !strings.Contains(res.MDL, want) { + t.Errorf("want %q in the generated connector:\n%s", want, res.MDL) + } + } + + // Each schema's columns must be read as that schema's. Before the fix the + // describe step was given a bare name, so a name two schemas share was + // refused as ambiguous and the object was dropped from the output entirely + // — three views in, fewer out, silently (ReadAllTableSchemas skips a table + // whose describe fails). + if n := strings.Count(res.MDL, "CREATE NON-PERSISTENT ENTITY"); n != 3 { + t.Errorf("%d entities, want 3 — a view shared between schemas was dropped:\n%s", n, res.MDL) + } + if !strings.Contains(res.MDL, "Email") || !strings.Contains(res.MDL, "OrderNumber") { + t.Errorf("a schema's own columns did not reach its entity:\n%s", res.MDL) + } +} diff --git a/sql/schema.go b/sql/schema.go index ac0aaae91..68732c0a1 100644 --- a/sql/schema.go +++ b/sql/schema.go @@ -24,6 +24,48 @@ type TableSchema struct { IsView bool } +// QualifiedName is how the table must be addressed in a generated query: the +// schema it was discovered in, when one is known. A database reports no schema +// for some drivers, and there the bare name is the address (ako/mxcli#2414). +func (t *TableSchema) QualifiedName() string { + if t.Schema == "" { + return t.Name + } + return t.Schema + "." + t.Name +} + +// tableAddresses is the address of every object a SHOW TABLES / SHOW VIEWS +// result names. The readers go through it rather than reading the rows +// themselves, so a schema cannot be dropped again without changing a function +// a test covers (ako/mxcli#2414). +func tableAddresses(rows [][]any) []string { + var out []string + for _, row := range rows { + if addr := qualifiedTableName(row); addr != "" { + out = append(out, addr) + } + } + return out +} + +// qualifiedTableName is the address in a SHOW TABLES / SHOW VIEWS row, whose +// columns are (table_schema, table_name). The schema is part of the address, +// so dropping it leaves a name that cannot be selected from (ako/mxcli#2414). +func qualifiedTableName(row []any) string { + if len(row) < 2 { + if len(row) == 1 { + return fmt.Sprintf("%v", row[0]) + } + return "" + } + name := fmt.Sprintf("%v", row[1]) + schema := fmt.Sprintf("%v", row[0]) + if schema == "" { + return name + } + return schema + "." + name +} + // pkQuery returns the query to find primary key columns for a given table. var pkQuery = map[DriverName]string{ DriverPostgres: `SELECT a.attname @@ -65,8 +107,14 @@ func ReadTableSchema(ctx context.Context, conn *Connection, tableName string, is } } + // tableName may be qualified, either because discovery qualified it or + // because the user wrote TABLES (sales.customers). The schema belongs in + // Schema and the bare name in Name: the name is what every Mendix entity + // and query is called after (ako/mxcli#2414). + schemaName, bareName := splitTableName(tableName) schema := &TableSchema{ - Name: tableName, + Schema: schemaName, + Name: bareName, IsView: isView, } @@ -98,12 +146,8 @@ func ReadAllTableSchemas(ctx context.Context, conn *Connection) ([]*TableSchema, } var schemas []*TableSchema - for _, row := range result.Rows { - if len(row) < 2 { - continue - } - tableName := fmt.Sprintf("%v", row[1]) - ts, err := ReadTableSchema(ctx, conn, tableName, false) + for _, addr := range tableAddresses(result.Rows) { + ts, err := ReadTableSchema(ctx, conn, addr, false) if err != nil { continue // skip tables that fail } @@ -120,12 +164,8 @@ func ReadAllViewSchemas(ctx context.Context, conn *Connection) ([]*TableSchema, } var schemas []*TableSchema - for _, row := range result.Rows { - if len(row) < 2 { - continue - } - viewName := fmt.Sprintf("%v", row[1]) - ts, err := ReadTableSchema(ctx, conn, viewName, true) + for _, addr := range tableAddresses(result.Rows) { + ts, err := ReadTableSchema(ctx, conn, addr, true) if err != nil { continue // skip views that fail }