diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-17-no-baseline-drops-dependent-associations.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-17-no-baseline-drops-dependent-associations.json new file mode 100644 index 0000000000..1657845870 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-11-17-no-baseline-drops-dependent-associations.json @@ -0,0 +1 @@ +{"area": "cmd-mxcli", "date": "2026-11-17", "refs": ["ako/mxcli#2368", "mendixlabs/mxcli#1883"], "ce": [], "symptom": "marketplace update --no-baseline deleted an association another module owns into the updated module (MyFirstModule.Request_Configuration → WorkflowCommons.Configuration) without naming it; mx check then reported 0 errors, so nothing pointed at the loss.", "file": "cmd/mxcli/cmd_marketplace_update.go noBaselineGate; cmd/mxcli/marketplace/snapshot.go SnapshotDependents. Tests: cmd/mxcli/cmd_marketplace_update_no_baseline_2368_test.go, cmd/mxcli/marketplace/no_baseline_dependents_2368_test.go", "cause": "--no-baseline skipped the whole drift computation, and the dependent-association search (#1502) lived inside it, although it reads only the project and needs no baseline.", "insight": "**An opt-out flag should skip only the part it cannot do.** --no-baseline exists because the baseline package cannot be built; the cross-module association search does not use the baseline, so bundling it into the same branch made the flag silently drop a check it never had a reason to drop. Split each safety check by its inputs before deciding which flag disables it."} diff --git a/.claude/skills/fix-issue/findings/mdl-backend/2026-11-17-grant-xpath-whitespace-duplicates-rule.json b/.claude/skills/fix-issue/findings/mdl-backend/2026-11-17-grant-xpath-whitespace-duplicates-rule.json new file mode 100644 index 0000000000..d1011adb3b --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-backend/2026-11-17-grant-xpath-whitespace-duplicates-rule.json @@ -0,0 +1 @@ +{"area": "mdl/backend", "date": "2026-11-17", "refs": ["ako/mxcli#2355"], "ce": [], "symptom": "`grant read * on entity Shop.Item to Shop.User where [Name = 'a' or Name = 'b'];` followed by the same grant with the XPath wrapped over two lines left two identical rules; diff reported a change, and on a real project upgraded from nightly-20261020 to nightly-20261117 every re-run duplicated two grants stored on one line by the older build.", "file": "mdl/types/xpath_same.go SameXPathConstraint; mdl/backend/modelsdk/domainmodel_security_write.go AddEntityAccessRule (match + keep stored spelling); mdl/executor/cmd_entities_access.go formatAccessRuleResult. Test: mdl/backend/modelsdk/access_rule_xpath_spacing_2355_test.go", "cause": "The upsert key (role set + constraint, #936) compared the constraint byte for byte, and FormatXPathConstraint keeps a constraint that already fits as written, so a script's line break reached both the comparison and the stored value.", "insight": "**A string that is part of an identity key must be compared in its canonical form, and a match must keep the stored spelling.** Comparing whitespace-insensitively outside literals (the same rule canon's CarryUnexpressed already applies to XPath) merges the grants; writing back the stored constraint on a match is what makes the re-run elide instead of rewriting the rule in the new spelling. Control: whitespace inside a string literal still distinguishes rules. Live on 11.12.6: the reporter's a.mdl then b.mdl gives one rule, diff 'exec would write nothing', exec 'Unchanged entity access'."} diff --git a/.claude/skills/fix-issue/findings/mdl-executor/2026-11-17-retrieve-list-returned-as-object.json b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-17-retrieve-list-returned-as-object.json new file mode 100644 index 0000000000..f516022d33 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-executor/2026-11-17-retrieve-list-returned-as-object.json @@ -0,0 +1 @@ +{"area": "mdl-executor", "date": "2026-11-17", "refs": ["ako/mxcli#2364"], "ce": ["CE0117"], "symptom": "Under `mdl 1;`, `retrieve $Item from Lm.Item where … limit 1; return $Item;` in a microflow returning Lm.Item passed check (\"Expression types OK\") and exec; mx check reported CE0117 \"Error(s) in expression.\" at End event.", "file": "mdl/executor/validate_microflow_retrieve_single.go checkRetrieveListAsObject (return case) / returnTypeName. Test: mdl/executor/retrieve_list_returned_2364_test.go", "cause": "MDL-RETRIEVE03 knew which variables a database retrieve bound to a list, but only judged `change $L` and `$L/Attr` as object uses; `return $L` against a non-list return type was not one of them.", "insight": "**A list-as-object rule must enumerate every object slot, and `return` is one.** Measured on 11.12.6: `limit 1` returned from a flow returning the entity → CE0117 at End event; `first` → 0 errors. A bare `Module.Name` return type is parsed as an enum-or-entity reference, so formatDataTypeForMDL printed it as \"Unknown\" — name it from EntityRef/EnumRef."} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-11-17-sec012-ignores-role-visibility.json b/.claude/skills/fix-issue/findings/mdl-other/2026-11-17-sec012-ignores-role-visibility.json new file mode 100644 index 0000000000..561d34b5fb --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-11-17-sec012-ignores-role-visibility.json @@ -0,0 +1 @@ +{"area": "mdl-other", "date": "2026-11-17", "refs": ["ako/mxcli#2356"], "ce": ["CE2729"], "symptom": "mxcli lint SEC012 (ReachablePageNotReadable) reported 19 CE2729 errors on an 11.15 project where mx check reported 0: every one was data inside a container with `Visible: roles (…)` limited to module roles that may read it.", "file": "mdl/linter/rules/reachable_page_read_access.go roleVisibility.hidden / restrictsAway / refersTo (unit ids from loadReachModel). Test: mdl/linter/rules/reachable_page_role_visibility_2356_test.go", "cause": "The rule works from catalog refs, which say a page uses an entity or attribute but not in which widget, so a widget's ConditionalVisibilitySettings.ModuleRoles never reached it; mxbuild does not check read access for a widget the user role cannot see.", "insight": "**When a lint rule mirrors an mxbuild check, measure both directions on one page.** 11.12.6: a data grid over an attribute the role may not read, inside a container `visible: roles (Admin)` — mx check 0 errors, SEC012 now 0 (was 1); the same container `visible: roles (Admin, User)` — CE2729 and SEC012 both 1. The catalog cannot answer per-widget questions; the stored widget tree (LintReader.GetRawUnit) can, and the finding is suppressed only when every occurrence is under a restriction that excludes all of the user role's module roles — an unreadable unit or no occurrence keeps the finding."} diff --git a/.claude/skills/fix-issue/findings/mdl-visitor/2026-11-17-alter-page-visible-roles-expression.json b/.claude/skills/fix-issue/findings/mdl-visitor/2026-11-17-alter-page-visible-roles-expression.json new file mode 100644 index 0000000000..491d7fe6c7 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-visitor/2026-11-17-alter-page-visible-roles-expression.json @@ -0,0 +1 @@ +{"area": "mdl-visitor", "date": "2026-11-17", "refs": ["ako/mxcli#2371"], "ce": ["CE0117"], "symptom": "`alter page … { set ( visible: roles (A, B) ) on cntAdmin }` printed \"Altered page\", describe showed `Visible: [roles (A, B)]`, and mx check reported CE0117 \"Error(s) in expression.\" at the container.", "file": "mdl/visitor/visitor_alter_page.go buildAlterSet; mdl/visitor/visible_roles_split.go splitVisibleRoles; mdl/backend/pagemutator/mutator.go setRawWidgetPropertyMut case visibleroles. Tests: mdl/visitor/alter_visible_roles_2371_test.go, mdl/backend/pagemutator/visible_roles_2371_test.go", "cause": "CREATE PAGE has a grammar alternative for `Visible: roles (…)` and #2319 split `roles (…) and `; the alter `set` path takes `visible:` as a widget expression, so the roles reached the mutator as VisibleIf text and were written as the visibility expression. The mutator had no case for module roles at all.", "insight": "**A fix to one statement's reading of a property is not a fix to the property.** #2319 fixed `Visible: roles` in CREATE; ALTER reads the same words through a different rule (alterPageAssignment → widget expression) and kept the old defect. When fixing a property spelling, grep every grammar rule that can carry it. Measured on 11.12.6: after the fix both `roles (A, B)` and `roles (A) and 2 = 2` describe back as roles (plus condition) and mx check reports 0 errors."} diff --git a/.claude/skills/fix-issue/findings/other/2026-11-17-placeholder-pattern-not-rerunnable.json b/.claude/skills/fix-issue/findings/other/2026-11-17-placeholder-pattern-not-rerunnable.json new file mode 100644 index 0000000000..3cfeb019f7 --- /dev/null +++ b/.claude/skills/fix-issue/findings/other/2026-11-17-placeholder-pattern-not-rerunnable.json @@ -0,0 +1 @@ +{"area": "other", "date": "2026-11-17", "refs": ["ako/mxcli#2358"], "ce": [], "symptom": "A script following the resolve-forward-references Placeholder Pattern (placeholder snippet, pages calling it, then create or modify of the real snippet) never ran clean twice: diff reported 'Snippet Admin.Snippet_AdminMenu: rewritten with only element $IDs changed (canonically equal)' on every run.", "file": ".claude/skills/mendix/resolve-forward-references/SKILL.md (placeholder step and both examples)", "cause": "Each run shrank the finished snippet to the placeholder (create or modify), then rebuilt it; the rebuild is reconciled against the placeholder that is now stored, so the original element $IDs cannot be carried back. The skill's own spelling, a plain create, fails outright on a second run ('snippet already exists').", "insight": "**A step that only exists for the first run must be a no-op on every later one: `create snippet|page … if not exists`, which the grammar and executor already supported.** Measured on 11.12.6: with if not exists, run 2 skips the placeholder, '3 documents already in sync', and diff says exec would write nothing; the reporter's create-or-modify placeholder reproduces the ID churn (the control); the skill's plain create fails on run 2. No code change: the fix is the documented pattern."} diff --git a/.claude/skills/fix-issue/findings/other/2026-11-17-sql-describe-schema-qualified.json b/.claude/skills/fix-issue/findings/other/2026-11-17-sql-describe-schema-qualified.json new file mode 100644 index 0000000000..e1b3dd8955 --- /dev/null +++ b/.claude/skills/fix-issue/findings/other/2026-11-17-sql-describe-schema-qualified.json @@ -0,0 +1 @@ +{"area": "other", "date": "2026-11-17", "refs": ["ako/mxcli#2361"], "ce": [], "symptom": "`mxcli sql --alias shop --describe sales.customers` printed an empty column list and (0 rows), no error; `--describe customers` worked. --tables prints schema and table apart, so the qualified form is the natural next step.", "file": "sql/meta.go DescribeTable / columnsFilter / splitTableName / refuseAmbiguousTable. Test: sql/describe_qualified_2361_test.go (needs MXCLI_TEST_PG_DSN)", "cause": "The PostgreSQL and SQL Server templates compared table_name to the whole argument, so `sales.customers` matched no row; only the Oracle path split the schema off. An empty result was printed as a table with no columns.", "insight": "**Test the unverified case the reporter lists.** The unqualified form was the 'control', but on PostgreSQL 16 with `customers` in two schemas it returned both tables' columns as one (6 rows) — a second, silent defect. Measured: qualified now 3 columns; a missing table and an ambiguous name are errors that name the accepted form or the candidates; the revert gives 0 columns and no errors."} diff --git a/.claude/skills/fix-issue/findings/other/2026-11-17-sql-writes-without-prompt.json b/.claude/skills/fix-issue/findings/other/2026-11-17-sql-writes-without-prompt.json new file mode 100644 index 0000000000..6e6e920008 --- /dev/null +++ b/.claude/skills/fix-issue/findings/other/2026-11-17-sql-writes-without-prompt.json @@ -0,0 +1 @@ +{"area": "other", "date": "2026-11-17", "refs": ["ako/mxcli#2362"], "ce": [], "symptom": "`mxcli sql --alias shop \"delete from sales.orders\"` deleted every row and printed \"(0 rows)\", which reads as an empty select; there was no read-only mode and no affected-row count.", "file": "sql/readonly.go ReadStatement / Run / executeReadOnly; cmd/mxcli/cmd_sql.go (--write). Test: sql/readonly_2362_test.go (needs MXCLI_TEST_PG_DSN)", "cause": "The CLI passed every statement to db.QueryContext; a DELETE returns no rows, so its effect looked like an empty result.", "insight": "**A keyword check is not a read-only mode.** `with d as (delete from t returning id) select * from d` starts with WITH and deletes on PostgreSQL; only the transaction stops it. Measured on PostgreSQL 16: with the read-only, rolled-back transaction the CTE fails and both rows stay; replacing it with a plain query (the revert) deletes them. Keep both layers: the keyword refusal gives the clear message, the transaction holds for what the keyword misses."} diff --git a/.claude/skills/fix-issue/findings/sdk/2026-11-17-js-action-stub-not-mxbuild-shape.json b/.claude/skills/fix-issue/findings/sdk/2026-11-17-js-action-stub-not-mxbuild-shape.json new file mode 100644 index 0000000000..0e9d60ccae --- /dev/null +++ b/.claude/skills/fix-issue/findings/sdk/2026-11-17-js-action-stub-not-mxbuild-shape.json @@ -0,0 +1 @@ +{"area": "sdk", "date": "2026-11-17", "refs": ["ako/mxcli#2357"], "ce": [], "symptom": "After upgrading to nightly-20261117, exec/diff of an unchanged script with JavaScript actions rewrote every .js stub: the action's documentation vanished from the JSDoc, the banner lost '- the import list' and gained a 'Special characters' line, optional parameters lost their [brackets], and CRLF became LF.", "file": "sdk/javaactions/jssource.go JSSource.Generate / completeJSImports / jsDocType / jsDocReturn / enumUnion; mdl/backend/modelsdk/javascript_write.go WriteJavaScriptSourceFile / enumerationValueNames. Goldens: sdk/javaactions/testdata/mxbuild/JS_*_11.12.6.js.golden. Tests: sdk/javaactions/jssource_mxbuild_2357_test.go", "cause": "mxcli's JavaScript generator never matched mxbuild's file; the files on disk were mxbuild's, and the #2131 indentation fix made each re-run's user code differ from them, so every run regenerated the file in mxcli's shape.", "insight": "**Measure the generator against the tool that owns the file, for every type, in one build.** mxbuild 11.12.6 --target=deploy on actions mxcli created shows: banner with '- the import list' and no 'Special characters' line, `import { Big } from \"big.js\";` always, the action's documentation first in the JSDoc, `[name]` for an optional parameter, Integer/Long/Decimal as Big (not number), objects MxObject, lists MxObject[], enumerations `undefined|\"Red\"|\"Blue\"`, CRLF. The old generator also mistyped Integer/Long/Decimal as number. Control after the fix: mxcli writes 10 actions, mxbuild regenerates them, all 10 files are byte-identical, and a re-run reports Unchanged."} diff --git a/.claude/skills/mendix/database-connections/SKILL.md b/.claude/skills/mendix/database-connections/SKILL.md index e8c9c4210c..3edff1c220 100644 --- a/.claude/skills/mendix/database-connections/SKILL.md +++ b/.claude/skills/mendix/database-connections/SKILL.md @@ -11,18 +11,18 @@ description: "Connect a Mendix app to an external database over JDBC with the Ex | lines | section | |---|---| | 30-32 | Purpose | -| 33-49 | When to Use This Skill | -| 50-84 | Prerequisites | -| 85-199 | Database Connection Syntax | -| 200-261 | Query Definition Syntax | -| 262-359 | Complete Examples | -| 360-375 | Viewing Connections | -| 376-396 | Best Practices | -| 397-410 | Troubleshooting | -| 411-422 | Related Commands | -| 423-548 | Executing Queries from Microflows | -| 549-563 | Importing Data from External Databases | -| 564-567 | References | +| 33-52 | When to Use This Skill | +| 53-87 | Prerequisites | +| 88-202 | Database Connection Syntax | +| 203-264 | Query Definition Syntax | +| 265-362 | Complete Examples | +| 363-378 | Viewing Connections | +| 379-399 | Best Practices | +| 400-413 | Troubleshooting | +| 414-425 | Related Commands | +| 426-551 | Executing Queries from Microflows | +| 552-566 | Importing Data from External Databases | +| 567-570 | References | # Skill: Create External Database Connections @@ -46,6 +46,9 @@ Create and manage external database connections in Mendix using the External Dat > SQL source GENERATE CONNECTOR INTO MyModule TABLES (employees, departments) EXEC; > ``` > For manual exploration, use `sql source show tables;` and `sql source describe tablename;`. +> From the shell, `mxcli sql --alias ""` is **read-only**: a write is refused +> unless `--write` is passed. Ask the user before passing `--write` to a database you were +> pointed at to explore. ## Prerequisites diff --git a/.claude/skills/mendix/download-marketplace-content/SKILL.md b/.claude/skills/mendix/download-marketplace-content/SKILL.md index 87da024648..b3bc13f51b 100644 --- a/.claude/skills/mendix/download-marketplace-content/SKILL.md +++ b/.claude/skills/mendix/download-marketplace-content/SKILL.md @@ -14,16 +14,16 @@ description: "The full lifecycle of Mendix Marketplace modules and widgets from | 45-61 | Prerequisites: Authenticate | | 62-95 | Step 1 — Discover | | 96-105 | Step 2 — Download a `.mpk` to disk (optional) | -| 106-225 | Step 3 — Install into a project | -| 226-290 | Step 4 — Repair the model after the install (required, headless) | -| 291-347 | Step 5 — Before updating: has the module been edited? | -| 348-406 | Step 6 — Update an installed module | -| 407-460 | Worked example: the agent-editor stack on a vanilla app | -| 461-482 | Package a module of the project (`marketplace package`) | -| 483-490 | Notes | -| 491-494 | Platform authentication (`mxcli auth login/logout/status/list`) with PAT scheme for marketplace-api | -| 495-498 | Marketplace download/install (`mxcli marketplace download/install`) — the content API now exposes a per-version downloadUrl (303→public CDN); install is type-aware (widget→widgets/, new module→`mx module-import`); existing-module updates are reported, not applied (entity-ID/local-edit safety — see PROPOSAL_marketplace_modules | -| 499-503 | Marketplace drift detection (`mxcli marketplace diff -p app.mpr [--to VERSION] [--json]`) | +| 106-227 | Step 3 — Install into a project | +| 228-292 | Step 4 — Repair the model after the install (required, headless) | +| 293-349 | Step 5 — Before updating: has the module been edited? | +| 350-408 | Step 6 — Update an installed module | +| 409-462 | Worked example: the agent-editor stack on a vanilla app | +| 463-484 | Package a module of the project (`marketplace package`) | +| 485-492 | Notes | +| 493-496 | Platform authentication (`mxcli auth login/logout/status/list`) with PAT scheme for marketplace-api | +| 497-500 | Marketplace download/install (`mxcli marketplace download/install`) — the content API now exposes a per-version downloadUrl (303→public CDN); install is type-aware (widget→widgets/, new module→`mx module-import`); existing-module updates are reported, not applied (entity-ID/local-edit safety — see PROPOSAL_marketplace_modules | +| 501-505 | Marketplace drift detection (`mxcli marketplace diff -p app.mpr [--to VERSION] [--json]`) | # Download, Install and Update Marketplace Content @@ -221,7 +221,9 @@ version "6.0.0" not found; run 'mxcli marketplace versions ' to list availab `--force` does not help — it overrides a *finding*, and here there is no finding. `--no-baseline` accepts that the question cannot be answered and updates anyway. -Commit first: local edits to that module go without being named. +Commit first: local edits to that module go without being named. Associations +other modules own into it are still found: the update refuses without `--force` +and `--save-edits ` saves them for replay. ## Step 4 — Repair the model after the install (required, headless) diff --git a/.claude/skills/mendix/migrate-k2-nintex/SKILL.md b/.claude/skills/mendix/migrate-k2-nintex/SKILL.md index 8919a7971c..df612b2d29 100644 --- a/.claude/skills/mendix/migrate-k2-nintex/SKILL.md +++ b/.claude/skills/mendix/migrate-k2-nintex/SKILL.md @@ -449,7 +449,7 @@ create microflow CRM.SUB_GetManager ($Employee: HR.Employee) returns HR.Employee as $Manager begin retrieve $Manager from HR.Employee - where [HR.Employee_Reports = $Employee]; + where [HR.Employee_Reports = $Employee] first; return $Manager; end; ``` diff --git a/.claude/skills/mendix/resolve-forward-references/SKILL.md b/.claude/skills/mendix/resolve-forward-references/SKILL.md index 6ae234105d..5a04eeaeb8 100644 --- a/.claude/skills/mendix/resolve-forward-references/SKILL.md +++ b/.claude/skills/mendix/resolve-forward-references/SKILL.md @@ -11,14 +11,14 @@ description: "Order MDL statements so that every reference resolves — executio | lines | section | |---|---| | 26-52 | Why Forward References Fail | -| 53-70 | The Placeholder Pattern | -| 71-137 | Pattern 1 — Shared Navigation Snippet (most common) | -| 138-185 | Pattern 2 — Page References Another Page (new/edit from overview) | -| 186-218 | Pattern 3 — Microflow References a Page Not Yet Created | -| 219-237 | Ordering Rules for Dependency-Free Scripts | -| 238-261 | Choosing Between CREATE OR MODIFY and ALTER SNIPPET | -| 262-336 | Script Template for a Full CRUD Module | -| 337-341 | Related Skills | +| 53-79 | The Placeholder Pattern | +| 80-146 | Pattern 1 — Shared Navigation Snippet (most common) | +| 147-194 | Pattern 2 — Page References Another Page (new/edit from overview) | +| 195-227 | Pattern 3 — Microflow References a Page Not Yet Created | +| 228-246 | Ordering Rules for Dependency-Free Scripts | +| 247-270 | Choosing Between CREATE OR MODIFY and ALTER SNIPPET | +| 271-346 | Script Template for a Full CRUD Module | +| 347-351 | Related Skills | # Resolving Forward References in MDL Scripts @@ -54,11 +54,20 @@ This applies to the following reference types: The standard workaround is a three-step sequence: -1. **Create a minimal placeholder** for the document that will be referenced. +1. **Create a minimal placeholder** for the document that will be referenced — + with `create snippet|page … if not exists`, so the step does nothing once the + document exists. 2. **Create all documents that reference it.** They bind to the placeholder's ID. 3. **Fill in the placeholder** using `CREATE OR MODIFY` or `ALTER` — both preserve the original ID so existing bindings remain valid. +> **The placeholder step must be `if not exists`** (ako/mxcli#2358), or the script never +> runs clean twice. A plain `create` fails on the second run ("snippet already exists"). +> `create or modify` shrinks the finished document back to the placeholder, and step 3 +> then rebuilds it with new element IDs on every run: `diff` reports "rewritten with only +> element $IDs changed" and `mprcontents/` churns. With `if not exists` the second run +> skips step 1, step 3 finds the document as it left it, and nothing is written. + > **Write `CREATE OR MODIFY`, not `CREATE OR REPLACE`.** `OR REPLACE` is the deprecated > spelling of `OR MODIFY` (`MDL-DEPR001`, refused from `mdl 2`): it means the same and > also keeps the ID, so it does not break the bindings — but it is not the spelling to @@ -75,8 +84,8 @@ the snippet via `snippetcall` (references the snippet). Both sides reference eac ```sql mdl 1; --- Step 1: placeholder snippet (minimal valid content) -create snippet MyModule.NavMenu +-- Step 1: placeholder snippet (minimal valid content); a no-op once it exists +create snippet if not exists MyModule.NavMenu { layoutgrid g { row { column (desktopwidth: 12) { dynamictext loading (content: 'Loading...') @@ -84,7 +93,7 @@ create snippet MyModule.NavMenu }; -- Step 2: pages that embed the snippet (snippet already exists → resolves OK) -create page MyModule.Customer_Overview +create or modify page MyModule.Customer_Overview ( title: 'Customers', layout: Atlas_Core.Atlas_Default @@ -100,7 +109,7 @@ create page MyModule.Customer_Overview }} }; -create page MyModule.Order_Overview +create or modify page MyModule.Order_Overview ( title: 'Orders', layout: Atlas_Core.Atlas_Default @@ -144,7 +153,7 @@ page must exist before the overview can reference it. mdl 1; -- Solution: declare the target page first (even if empty), then the referencing page -create page MyModule.Customer_NewEdit +create or modify page MyModule.Customer_NewEdit ( params: ( $Customer: MyModule.Customer ), title: 'Edit Customer', @@ -161,7 +170,7 @@ create page MyModule.Customer_NewEdit }; -- Now the overview can safely reference the NewEdit page -create page MyModule.Customer_Overview +create or modify page MyModule.Customer_Overview ( title: 'Customers', layout: Atlas_Core.Atlas_Default @@ -268,8 +277,9 @@ mdl 1; -- Correct declaration order: snippets → pages → microflows → nav -- ============================================================ --- 1. Placeholder for shared navigation (will reference pages created below) -create snippet MyModule.AppNav +-- 1. Placeholder for shared navigation (will reference pages created below); +-- if not exists keeps a re-run from shrinking the finished snippet +create snippet if not exists MyModule.AppNav { layoutgrid g { row { column (desktopwidth: 12) { dynamictext placeholder (content: '...') @@ -277,7 +287,7 @@ create snippet MyModule.AppNav }; -- 2. NewEdit page (referenced by Overview's New button) -create page MyModule.Customer_NewEdit +create or modify page MyModule.Customer_NewEdit ( params: ( $Customer: MyModule.Customer ), title: 'Edit Customer', @@ -288,7 +298,7 @@ create page MyModule.Customer_NewEdit }; -- 3. Overview page (references NewEdit + NavMenu) -create page MyModule.Customer_Overview +create or modify page MyModule.Customer_Overview ( title: 'Customers', layout: Atlas_Core.Atlas_Default @@ -316,7 +326,7 @@ create or modify snippet MyModule.AppNav }; -- 5. Microflows (pages already exist) -create microflow MyModule.ACT_Customer_New () +create or modify microflow MyModule.ACT_Customer_New () begin $c = create MyModule.Customer (); show page MyModule.Customer_NewEdit (Customer = $c); diff --git a/CHANGELOG.md b/CHANGELOG.md index 273e364033..88b10eca2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **`alter page … set (visible: roles (…))` sets the module roles** (ako/mxcli#2371) — the alter form stored `roles (…)` as the visibility expression, reported success, and left a page `mx check` rejected with CE0117. It now writes the "Visible for module roles" setting as `create page` does, keeping the widget's condition, and `roles (…) and ` sets both. +- **`check` reports a `limit 1` list returned as a single object** (ako/mxcli#2364) — under `mdl 1;` `retrieve … limit 1` binds a list of one, and returning it from a microflow that returns the entity passed check and exec, then failed `mx check` with CE0117 at the End event. MDL-RETRIEVE03 now reports it and suggests `first`, `head($Item)` or a list return type. +- **`sql --describe schema.table` describes the table** (ako/mxcli#2361) — the qualified name `--tables` suggests returned an empty column list and no error on PostgreSQL and SQL Server. It now matches schema and table, a table that is not found is an error naming the accepted form, and an unqualified name held by several schemas is refused with the candidates instead of listing all their columns as one table. +- **`mxcli sql` is read-only unless `--write` is given** (ako/mxcli#2362) — any statement ran, and `delete from sales.orders` printed `(0 rows)` while emptying the table. A statement that is not a read is now refused, a read runs in a read-only transaction that is rolled back (so a deleting `WITH` fails too), and `--write` allows changes and reports the rows affected. +- **`marketplace update --no-baseline` names and gates on associations other modules own into the module** (ako/mxcli#2368, mendixlabs/mxcli#1883) — the flag skipped the local-edit check, and with it the search for cross-module associations the update deletes, so an app's own association disappeared without a word and `mx check` stayed clean. Finding them needs only the project, so `--no-baseline` now refuses without `--force` when any exist, saves them with `--save-edits`, and lists them under "Removed from other modules". +- **`lint` SEC012 skips data hidden from the user role** (ako/mxcli#2356) — data inside a container with `Visible: roles (…)` was reported as CE2729 for user roles that cannot see the container, though mxbuild does not check it: 19 errors on a project where `mx check` reported none. The rule now reads the page's widget tree and drops a finding when every place showing that data is hidden from all of the user role's module roles; the same container visible to the role is still reported, as mxbuild reports it. +- **A JavaScript action's `.js` file is written as mxbuild writes it** (ako/mxcli#2357) — a re-run of an unchanged script rewrote every JavaScript action file in mxcli's own shape: no documentation in the JSDoc, another banner, no `[optional]` parameters, LF, and Integer/Decimal typed as `number`. The generator now writes mxbuild's bytes (measured on 11.12.6 for every parameter and return type): the action's documentation, `Big` for numbers, enumerations as their values and CRLF. After an mxbuild build, all 10 measured files are unchanged, and a re-run writes nothing. +- **A grant whose `where` is wrapped differently merges into the stored rule** (ako/mxcli#2355) — the rule was matched by its XPath compared byte for byte, so the same constraint over two lines, or stored on one line by an older mxcli or Studio Pro, added a second, identical rule on every run. Constraints that differ only in whitespace outside string literals now match, and the stored spelling is kept, so a re-run writes nothing. - **`check` rejects `Visible: roles (…)` naming a role of another module** (ako/mxcli#1258) — a widget can only be shown to the module roles of its own page, snippet or layout. A role of another module (`Administration.Administrator` on a page in `Wd`, or a role of any other module) passed `check --references` and `exec`, and the build reported CE0149 "Visibility needs to be updated, reselect roles in properties". New MDL-GRANT04, the sibling of MDL-GRANT01 (CE0148 for document grants): it needs no project, and covers a container, dynamic text, layout-grid row, navigation-list item, tab page, snippet, layout, and a widget inserted by `alter page`. Whether the role may view the page does not matter, only its module. Measured on MxBuild 11.16.1 over 17 documents. - **`mxcli oql` against a Mendix 9 runtime says why it cannot work** (ako/mxcli#2341, mendixlabs/mxcli#1879) — a 9.x runtime has no `preview_execute_oql` action, and the error sent the user to restart with live-preview flags, which cannot help. When the action is missing, the runtime's version is now read, and below Mendix 10 the error says OQL over the admin API needs 10 or later and names that version. - **`report traceability --junit` shows each requirement's test results** (ako/mxcli#2340, mendixlabs/mxcli#1877) — the results were read and then dropped, so the report was byte-identical with and without `--junit`. Each requirement now has `testResults` (name and passed, failed, error, skipped or not run), and the table marks a failing test, so "built, test failing" no longer reads as "built and proven". diff --git a/Makefile b/Makefile index 198d44bdca..aea504e6af 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ GO_BUILD_FLAGS = -trimpath # Clean version for VS Code extension (must be valid semver: major.minor.patch) VSCE_VERSION = $(shell echo "$(VERSION)" | sed 's/^v//; s/-.*//' | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$$' || echo "0.0.0") -.PHONY: build build-debug size release clean test ci-local test-mdl check-mdl check-skill-mdl check-conformance conformance-shrink gen-migration-reference check-migration-reference check-skill-pack-js check-findings check-wiki-pages digest-status diag-coverage check-tunnel-deps check-test-timeouts check-widget-versions test-integration test-integration-executor test-integration-roundtrip test-integration-parity test-integration-upgrade test-integration-other grammar completions sync-skills sync-skill-packs sync-commands sync-mods sync-lint-rules sync-changelog sync-all docs documentation docs-site docs-serve vscode-ext vscode-install source-tree sbom sbom-report lint lint-go lint-ts fmt fmt-check vet skill-index check-skill-index bench bench-trend +.PHONY: build build-debug size release clean test ci-local test-mdl check-mdl check-skill-mdl check-conformance conformance-shrink gen-migration-reference check-migration-reference check-skill-pack-js check-findings check-wiki-pages digest-status diag-coverage mask-matrix check-tunnel-deps check-test-timeouts check-widget-versions test-integration test-integration-executor test-integration-roundtrip test-integration-parity test-integration-upgrade test-integration-other grammar completions sync-skills sync-skill-packs sync-commands sync-mods sync-lint-rules sync-changelog sync-all docs documentation docs-site docs-serve vscode-ext vscode-install source-tree sbom sbom-report lint lint-go lint-ts fmt fmt-check vet skill-index check-skill-index bench bench-trend # Helper: copy file only if content differs (avoids mtime updates that invalidate go build cache) # Usage: $(call copy-if-changed,src,dst) @@ -363,6 +363,14 @@ check-skill-pack-js: diag-coverage: @go run ./scripts/diagcoverage $(ARGS) +# Which errors a Mendix checker hides until others are fixed: every defect and every +# ordered pair of them in one project, checked once. Needs a healthy project and a +# MxBuild. Advisory. See scripts/maskmatrix and the staging section of +# docs/11-proposals/PROPOSAL_check_diagnostics_catalog.md. +# make mask-matrix ARGS="--project /path/to/App.mpr --mx ~/.mxcli/mxbuild/11.16.1/modeler/mx" +mask-matrix: + @go run ./scripts/maskmatrix $(ARGS) + # How far docs-wiki/bug-patterns/ has fallen behind the findings it digests. # Advisory, always exits 0 — see the header in the script for why. digest-status: diff --git a/cmd/mxcli/cmd_marketplace_update.go b/cmd/mxcli/cmd_marketplace_update.go index f0b7fab9d3..53ee220697 100644 --- a/cmd/mxcli/cmd_marketplace_update.go +++ b/cmd/mxcli/cmd_marketplace_update.go @@ -128,6 +128,13 @@ func runMarketplaceUpdate(cmd *cobra.Command, args []string) error { if noBaseline { fmt.Fprintf(out, "\n--no-baseline: skipping the local-edit check for %s.\n", moduleName) fmt.Fprintln(out, " Any local edits to this module will be discarded without being named.") + deps, derr := marketplace.SnapshotDependents(mprPath, moduleName, newBackendFactory()) + if derr != nil { + return fmt.Errorf("find associations other modules own into %s: %w", moduleName, derr) + } + if drift, err = noBaselineGate(out, moduleName, deps, force, saveEdits); err != nil { + return err + } } else { base, berr := pickVersion(versions.Items, installedVersionID, installedVersion) if berr != nil { @@ -197,6 +204,29 @@ func runMarketplaceUpdate(cmd *cobra.Command, args []string) error { return nil } +// noBaselineGate is the part of the local-edit check --no-baseline can still +// make: the associations other modules own that reach into the module. Finding +// them needs no baseline, and the update deletes them all the same, so they +// are saved, refused without --force and named in the report +// (ako/mxcli#2368). Edits to the module itself stay unknown. +func noBaselineGate(out io.Writer, module string, deps []marketplace.Element, force bool, saveEdits string) (*marketplace.Report, error) { + drift := &marketplace.Report{Module: module, Dependents: deps} + if len(deps) == 0 { + return drift, nil + } + if saveEdits != "" { + written, unsaved, err := marketplace.SaveEdits(saveEdits, drift) + if err != nil { + return nil, err + } + reportSavedEdits(out, saveEdits, written, unsaved) + } + if err := gateOnLocalEdits(out, drift, force, saveEdits); err != nil { + return nil, err + } + return drift, nil +} + // gateOnLocalEdits refuses an update that would destroy local work, unless the // user has said to proceed. The refusal names the elements rather than just // their count: "3 elements were modified" is not enough to decide with. @@ -343,7 +373,7 @@ func init() { marketplaceUpdateCmd.Flags().String("save-edits", "", "write locally changed elements to this directory as re-executable MDL") marketplaceUpdateCmd.Flags().Bool("force", false, "update even though local edits will be discarded") marketplaceUpdateCmd.Flags().Bool("no-baseline", false, - "update without checking for local edits, when the installed version is no longer published") + "update without checking for local edits to the module, when the installed version is no longer published (associations other modules own into it are still checked)") marketplaceUpdateCmd.Flags().String("profile", auth.ProfileDefault, "credential profile") marketplaceCmd.AddCommand(marketplaceUpdateCmd) diff --git a/cmd/mxcli/cmd_marketplace_update_no_baseline_2368_test.go b/cmd/mxcli/cmd_marketplace_update_no_baseline_2368_test.go new file mode 100644 index 0000000000..a7be817650 --- /dev/null +++ b/cmd/mxcli/cmd_marketplace_update_no_baseline_2368_test.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/cmd/mxcli/marketplace" +) + +// ako/mxcli#2368: under --no-baseline the update deleted an association another +// module owns without naming it. The edits to the module itself cannot be known +// without the baseline, but the associations reaching into it can, and they are +// gated like any other loss: refused without --force, named with it. +func TestNoBaselineGate_RefusesWhenAnAssociationReachesIn(t *testing.T) { + deps := []marketplace.Element{{ + Key: marketplace.ElementKey{Type: "ASSOCIATION", Name: "MyFirstModule.Request_Configuration"}, + MDL: "create or modify association MyFirstModule.Request_Configuration\nfrom MyFirstModule.Request to WorkflowCommons.Configuration;", + }} + var out bytes.Buffer + drift, err := noBaselineGate(&out, "WorkflowCommons", deps, false, "") + if err == nil { + t.Fatal("update allowed without --force although it removes MyFirstModule.Request_Configuration") + } + if !strings.Contains(err.Error(), "MyFirstModule.Request_Configuration") { + t.Errorf("refusal does not name the association: %v", err) + } + + out.Reset() + drift, err = noBaselineGate(&out, "WorkflowCommons", deps, true, "") + if err != nil { + t.Fatalf("--force must proceed: %v", err) + } + if len(drift.DependentNames()) != 1 { + t.Errorf("the report after the update would not name the removal: %v", drift.DependentNames()) + } + + // Control: with nothing reaching in, --no-baseline proceeds as before. + if _, err := noBaselineGate(&out, "WorkflowCommons", nil, false, ""); err != nil { + t.Fatalf("control: no dependents must pass: %v", err) + } +} + +func TestNoBaselineGate_SavesTheAssociation(t *testing.T) { + deps := []marketplace.Element{{ + Key: marketplace.ElementKey{Type: "ASSOCIATION", Name: "MyFirstModule.Request_Configuration"}, + MDL: "create or modify association MyFirstModule.Request_Configuration\nfrom MyFirstModule.Request to WorkflowCommons.Configuration;", + }} + var out bytes.Buffer + dir := t.TempDir() + if _, err := noBaselineGate(&out, "WorkflowCommons", deps, false, dir); err == nil { + t.Fatal("saving does not lift the refusal; --force does") + } + if !strings.Contains(out.String(), "Saved 1") { + t.Errorf("--save-edits did not save the association:\n%s", out.String()) + } +} diff --git a/cmd/mxcli/cmd_sql.go b/cmd/mxcli/cmd_sql.go index c324660006..c29f08bd4b 100644 --- a/cmd/mxcli/cmd_sql.go +++ b/cmd/mxcli/cmd_sql.go @@ -17,6 +17,11 @@ var sqlCmd = &cobra.Command{ Short: "Execute a SQL query against an external database", Long: `Execute a SQL query against an external database (PostgreSQL, Oracle, SQL Server). +A query runs read-only: a statement that is not a read (SELECT, WITH, +EXPLAIN, SHOW, VALUES) is refused, and a read runs in a read-only transaction +that is rolled back. Pass --write to change the database; the output then +says how many rows the statement affected. + Connection credentials are resolved in order: 1. --dsn flag 2. Environment variable MXCLI_SQL__DSN @@ -59,6 +64,9 @@ Examples: # Describe a table mxcli sql --alias mydb --describe users + # Change data (refused without --write) + mxcli sql --alias mydb --write "DELETE FROM orders WHERE id = 42" + # JSON output mxcli sql --alias mydb --json "SELECT * FROM orders" `, @@ -72,6 +80,7 @@ Examples: views, _ := cmd.Flags().GetBool("views") functions, _ := cmd.Flags().GetBool("functions") describe, _ := cmd.Flags().GetString("describe") + allowWrite, _ := cmd.Flags().GetBool("write") // Default alias if alias == "" { @@ -134,7 +143,7 @@ Examples: fmt.Fprintln(os.Stderr, "Error: query argument required (or use --tables / --views / --functions / --describe)") os.Exit(1) } - result, err = sqllib.Execute(ctx, conn, args[0]) + result, err = sqllib.Run(ctx, conn, args[0], allowWrite) } if err != nil { @@ -142,6 +151,11 @@ Examples: os.Exit(1) } + if result.RowsAffected != nil { + fmt.Fprintf(os.Stdout, "%d row(s) affected\n", *result.RowsAffected) + return + } + if jsonOutput { if err := sqllib.FormatJSON(os.Stdout, result); err != nil { fmt.Fprintf(os.Stderr, "Error formatting JSON: %v\n", err) @@ -164,6 +178,7 @@ func init() { sqlCmd.Flags().Bool("views", false, "List views") sqlCmd.Flags().Bool("functions", false, "List functions and procedures") sqlCmd.Flags().String("describe", "", "Describe the named table's or view's columns") + sqlCmd.Flags().Bool("write", false, "Allow statements that change the database (INSERT, UPDATE, DELETE, DDL); without it a query runs read-only") rootCmd.AddCommand(sqlCmd) } diff --git a/cmd/mxcli/marketplace/no_baseline_dependents_2368_test.go b/cmd/mxcli/marketplace/no_baseline_dependents_2368_test.go new file mode 100644 index 0000000000..58c185870e --- /dev/null +++ b/cmd/mxcli/marketplace/no_baseline_dependents_2368_test.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package marketplace + +import "testing" + +// ako/mxcli#2368 (mendixlabs/mxcli#1883): `update --no-baseline` skipped the +// whole local-edit check, and with it the search for associations other modules +// own into the module — which needs only the project, not the baseline. The +// update deleted MyFirstModule's association without naming it. +func TestSnapshotDependents_FindsAssociationsWithoutABaseline(t *testing.T) { + target := copyFixture(t) + + // Control: the fixture has none, so a hit below is the association's doing. + deps, err := SnapshotDependents(target, "Administration", testBackend) + if err != nil { + t.Fatal(err) + } + if len(deps) != 0 { + t.Fatalf("control: fixture already has dependents %v", deps) + } + + execMDL(t, target, crossModuleAssocMDL) + deps, err = SnapshotDependents(target, "Administration", testBackend) + if err != nil { + t.Fatal(err) + } + if len(deps) != 1 || deps[0].Key.Name != crossModuleAssoc || deps[0].MDL == "" { + t.Fatalf("dependents = %+v; want %s with its MDL", deps, crossModuleAssoc) + } +} diff --git a/cmd/mxcli/marketplace/snapshot.go b/cmd/mxcli/marketplace/snapshot.go index bc24ae207d..6f6e8ee0c1 100644 --- a/cmd/mxcli/marketplace/snapshot.go +++ b/cmd/mxcli/marketplace/snapshot.go @@ -168,24 +168,11 @@ func (s *Snapshot) Keys() []ElementKey { // with the *same* engine is what makes the result meaningful. func SnapshotModule(mprPath, moduleName string, newBackend func() backend.FullBackend) (*Snapshot, error) { var sink bytes.Buffer - exec := executor.New(&sink) - defer exec.Close() - if newBackend != nil { - exec.SetBackendFactory(newBackend) - } - - if err := exec.Execute(&ast.ConnectStmt{Path: mprPath}); err != nil { - return nil, fmt.Errorf("connect %s: %w", mprPath, err) - } - - // The catalog is built lazily by the statements that need it, and the - // enumeration below reads it directly rather than through a statement, so - // build it explicitly. Fast mode is enough: the `objects` index it populates - // is exactly what this needs, and full mode additionally parses every - // activity and widget for cross-references nothing here uses. - if err := exec.Execute(&ast.RefreshCatalogStmt{}); err != nil { - return nil, fmt.Errorf("build catalog for %s: %w", mprPath, err) + exec, err := openForSnapshot(mprPath, &sink, newBackend) + if err != nil { + return nil, err } + defer exec.Close() rows, err := queryModuleObjects(exec, moduleName) if err != nil { @@ -206,14 +193,59 @@ func SnapshotModule(mprPath, moduleName string, newBackend func() backend.FullBa snap.Elements[key] = describeElement(exec, &sink, row) } - deps, err := queryDependentAssociations(exec, moduleName) + if snap.Dependents, err = dependents(exec, &sink, moduleName); err != nil { + return nil, err + } + return snap, nil +} + +// SnapshotDependents captures only the associations other modules own that +// reach into moduleName. It needs nothing but the project, so `update +// --no-baseline` — which cannot build the baseline to find edits to the module +// — still names, saves and gates on what the update deletes outside it +// (ako/mxcli#2368). +func SnapshotDependents(mprPath, moduleName string, newBackend func() backend.FullBackend) ([]Element, error) { + var sink bytes.Buffer + exec, err := openForSnapshot(mprPath, &sink, newBackend) + if err != nil { + return nil, err + } + defer exec.Close() + return dependents(exec, &sink, moduleName) +} + +func dependents(exec *executor.Executor, sink *bytes.Buffer, moduleName string) ([]Element, error) { + rows, err := queryDependentAssociations(exec, moduleName) if err != nil { return nil, err } - for _, row := range deps { - snap.Dependents = append(snap.Dependents, describeElement(exec, &sink, row)) + var out []Element + for _, row := range rows { + out = append(out, describeElement(exec, sink, row)) } - return snap, nil + return out, nil +} + +// openForSnapshot connects an executor writing to sink and builds its catalog. +func openForSnapshot(mprPath string, sink *bytes.Buffer, newBackend func() backend.FullBackend) (*executor.Executor, error) { + exec := executor.New(sink) + if newBackend != nil { + exec.SetBackendFactory(newBackend) + } + if err := exec.Execute(&ast.ConnectStmt{Path: mprPath}); err != nil { + exec.Close() + return nil, fmt.Errorf("connect %s: %w", mprPath, err) + } + // The catalog is built lazily by the statements that need it, and the + // enumeration reads it directly rather than through a statement, so build + // it explicitly. Fast mode is enough: the `objects` index it populates is + // exactly what this needs, and full mode additionally parses every + // activity and widget for cross-references nothing here uses. + if err := exec.Execute(&ast.RefreshCatalogStmt{}); err != nil { + exec.Close() + return nil, fmt.Errorf("build catalog for %s: %w", mprPath, err) + } + return exec, nil } // moduleObject is one row of the enumeration. diff --git a/docs-site/src/guides/marketplace.md b/docs-site/src/guides/marketplace.md index f941d7f800..fac0b2993d 100644 --- a/docs-site/src/guides/marketplace.md +++ b/docs-site/src/guides/marketplace.md @@ -295,7 +295,7 @@ Without this, module install order silently decided which widget versions the pr ## When the installed version is no longer published -`diff` and `update` both download the installed version to establish the local-edit baseline, so both fail when it has been unpublished — as NanoflowCommons 6.0.0 has, while a blank 11.13 app still ships it. `--force` does not help: it overrides a finding, and there is no finding to override. `mxcli marketplace update … --no-baseline` accepts that the question cannot be answered and updates anyway, discarding any local edits to that module without naming them. +`diff` and `update` both download the installed version to establish the local-edit baseline, so both fail when it has been unpublished — as NanoflowCommons 6.0.0 has, while a blank 11.13 app still ships it. `--force` does not help: it overrides a finding, and there is no finding to override. `mxcli marketplace update … --no-baseline` accepts that the question cannot be answered and updates anyway, discarding any local edits to that module without naming them. Associations that *other* modules own into the module are still found, because that needs only the project: they are refused without `--force`, saved by `--save-edits`, and listed under "Removed from other modules" after the update. ## Why `diff` and `update` are slow, and what is cached diff --git a/docs-site/src/tools/external-sql.md b/docs-site/src/tools/external-sql.md index 7167674a5d..15477c29a3 100644 --- a/docs-site/src/tools/external-sql.md +++ b/docs-site/src/tools/external-sql.md @@ -59,6 +59,16 @@ For one-off queries without the REPL: mxcli sql --driver postgres --dsn 'postgres://user:pass@localhost:5432/mydb' "SELECT * FROM users" ``` +The subcommand is **read-only by default**. A statement that is not a read +(`INSERT`, `UPDATE`, `DELETE`, `MERGE`, DDL, procedure calls) is refused, and a +read runs in a read-only transaction that is rolled back, so a write inside a +read — a data-modifying `WITH` — fails instead of changing data. Pass `--write` +to change the database; the output then reports the rows affected: + +```bash +mxcli sql --alias mydb --write "DELETE FROM orders WHERE id = 42" +``` + ## Related Pages - [SQL CONNECT](sql-connect.md) -- Connection syntax and driver details diff --git a/docs/11-proposals/PROPOSAL_check_diagnostics_catalog.md b/docs/11-proposals/PROPOSAL_check_diagnostics_catalog.md index 81c6bbabb3..0b3281461f 100644 --- a/docs/11-proposals/PROPOSAL_check_diagnostics_catalog.md +++ b/docs/11-proposals/PROPOSAL_check_diagnostics_catalog.md @@ -112,14 +112,60 @@ MxBuild, Mendix CLI 2 (11.16 build), Linux x64, one clean project from `mxcli ne - **Codes differ for the same finding.** On a 28-error project (`testdata/testapp-views`), all 28 "... no longer exists" findings are CE1613 in `mx check` (11.14.0 and 11.16.1 alike) and CE7710 in `mendix check`. The harness must compare messages, not only codes. -- **Unconfirmed extras.** On that project `mendix check` reports 34 errors to `mx - check`'s 28: 3x CE0136 (retrieve without entity), 2x CE0535 (grid column weights), - 1x CE6083 (design property unsupported by theme). MxBuild 11.16.1 reports none of - them, so they stay unconfirmed. The CE0136 ones sit on retrieves whose entity no - longer exists, which `mx check` already reports as CE1613, so they look like - cascades rather than independent findings. No extras on the clean project. +- **The 6 extras are artifacts of a stripped project.** On `testdata/testapp-views/TestApp.mpr` + (a Studio Pro 11.14.0 project holding only `MyFirstModule` and `System`: no Atlas_Core, no + theme, no Administration module) `mendix check` reports 34 errors to MxBuild 11.16.1's 28. + The 28 "selected … no longer exists" findings are CE7710 in CLI 2 and CE1613 in MxBuild, + the same count. The 6 left over, classified by a control each: + - **3x CE0136** (retrieve without an entity) are a cascade of the unresolved entity + `Administration.Account`: they sit on the same three retrieve activities MxBuild reports + as CE1613. Defining the entity removes all three, and the retrieve errors with them. + - **2x CE0535** (grid column weights "add up to -1") are a cascade of the unresolved layout + `Atlas_Core.Atlas_TopBar`. The stored columns are identical to what mxcli writes for + `DesktopWidth: AutoFill` (Weight, TabletWeight, PhoneWeight and PreviewWidth all -1), and + that shape is clean in both checkers on a healthy project, in a theme-less project when + the layout resolves, and for a lone 6-wide column. Repointing the page at a layout that + exists clears both; CE6858 ("update Atlas UI to 2.4 to use Layout Grid on Native pages") + appears instead, for every page with a layout grid in a project without Atlas. + - **1x CE6083** (design property "Row gap" unsupported by the theme) survives the repoint, + and `mendix check` also raises it for a new page written into this project. MxBuild is + silent on all of them. It is clean in both checkers on a healthy project, where the Atlas + theme defines the property. The likely reason is that this project has no theme to + compare against (inferred, not measured). MxBuild does raise CE6083 for real projects: + 10 bug-fix records name it. + None is a check to port, and none is a CLI 2 false positive on a healthy project. Defining + the missing entity also made `mendix check` raise CE8412 (the demo users' entity must be + `System.User`): a result of the control, not of the project. +- **MxBuild reports errors in stages; `mendix check` does not.** Measured on MxBuild 11.16.1 + with six single defects and every ordered pair of them (30 documents, both orders) in one + project, plus a reference defect: + + | stage | what is reported | what it hides | + |---|---|---| + | 0 | a reference that does not resolve (CE1613) | **every other error in every document**, a page's CE0149 included | + | 1 | a duplicate variable name (CE0111) | every other error in the same microflow | + | 2 | an undefined variable (CE0109) | the stage 3 group, in the same microflow | + | 3 | CE0117 (wrong type), CE0068 (return in a loop), CE0038 (declare without a value), CE0067 (no final return) | nothing: all four are reported together | + + With one wrong-parameter call (CE1613) in 13 documents, MxBuild reported those 13 errors and + nothing from the other 36 documents; `mendix check` reported 115 errors in 50. Dropping the + reference defect made the undefined variable, the wrong type and the page's CE0149 appear + together. No pair differed by order, and no two defects share a code. `mendix check` reported + both defects in all 30 pair documents (judged by its own codes: CE0844 is its name for + MxBuild's CE0117). Not measured: Studio Pro itself (the same consistency checker, so likely + the same), documents other than microflows and pages, and the order among codes outside these + six. Reproduce it with `make mask-matrix` (`scripts/maskmatrix`). What follows from it: + - **Report every real error.** `mxcli check` reporting more than MxBuild is not a mismatch: + the rest are real, only hidden until the earlier stage is fixed. Copying the staging would + send an author through one fix-and-rebuild loop per stage. + - **One defect per fixture**, and a control with it removed. A fixture holding two defects + can look as if one is absent (the `captrack-19` rule stood down for exactly this reason). + - **The bug records skew early.** A later-stage code shows up in a report only after the + earlier ones were fixed, so the field ranking in `make diag-coverage` under-counts it. + - **Run the oracle on a project whose references resolve**, or stage 0 hides the rest. - **Conclusion.** `mendix check` works as a first-pass oracle: run it, then confirm any - code it reports against `mx check` before porting. It is a proprietary binary, so the + code it reports against `mx check` before porting. Run it on a project whose references + resolve (no CE7710 / CE1613): on one that does not, it reports cascades MxBuild suppresses. It is a proprietary binary, so the harness takes its path as an option and nothing is committed. ## Extraction methodology @@ -236,8 +282,8 @@ button, or a URL."* be reverse-engineered per rule. The catalog gives the *target + message*, not the predicate — sizing must account for that. 4. **Fast oracle.** The spike above supports `mendix check` as a first-pass oracle with - `mx check` confirming. Still open: the 6 unconfirmed extras on a broken project, a - corpus larger than 8 fixtures, and cold/warm timings. + `mx check` confirming. Still open: a corpus larger than 8 fixtures, and cold/warm + timings. The 6 extras on a broken project are classified above. 5. **Overlap with `PROPOSAL_check_mxbuild_gap_heuristics.md`.** Fold that proposal's 6 cases in as the first harvested batch, or keep it separate and cross-link? (Recommend: keep it as the tactical pilot; this proposal owns the methodology + backlog.) diff --git a/mdl-examples/bug-tests/ledger-17-slash-division.fail.mdl b/mdl-examples/bug-tests/ledger-17-slash-division.fail.mdl index 0cd1c9573f..47d931f60e 100644 --- a/mdl-examples/bug-tests/ledger-17-slash-division.fail.mdl +++ b/mdl-examples/bug-tests/ledger-17-slash-division.fail.mdl @@ -12,6 +12,9 @@ -- operator `/`), and the variable/variable form `$Dec / $Dec2` (which parses as a -- member-access path — the visitor preserves its `/ $` source so MDL045 sees it). -- +-- Measured on MxBuild 11.16.1 with every variable declared: each of the six statements +-- below is CE0117 on its own activity, and the same microflow written with `div` is clean. +-- -- Usage (expected to FAIL check): -- mxcli check mdl-examples/bug-tests/ledger-17-slash-division.fail.mdl -- ============================================================================ @@ -22,6 +25,14 @@ create microflow MyModule.SlashDivision ( ) returns decimal begin + -- Every variable is declared on purpose: an undeclared one is CE0109, which MxBuild + -- reports first and which hides the CE0117 these statements exist to show. + declare $Bad decimal = 0; + declare $AlsoBad decimal = 0; + declare $VarVar decimal = 0; + declare $Embed1 decimal = 0; + declare $Embed2 decimal = 0; + declare $Embed3 decimal = 0; set $Bad = $Dec / 2; -- MDL045: use `div` set $AlsoBad = ($Dec + 1) / $Dec2; -- MDL045: use `div` set $VarVar = $Dec / $Dec2; -- MDL045: variable/variable form (finding #17) diff --git a/mdl-examples/bug-tests/ledger-21-datetime-literals.fail.mdl b/mdl-examples/bug-tests/ledger-21-datetime-literals.fail.mdl index d60d904356..d174b78424 100644 --- a/mdl-examples/bug-tests/ledger-21-datetime-literals.fail.mdl +++ b/mdl-examples/bug-tests/ledger-21-datetime-literals.fail.mdl @@ -11,6 +11,9 @@ -- check time and points to the addDays()/addMonths() workaround (those DO take -- variables — step off a literal anchor date). -- +-- Measured on MxBuild 11.16.1 with the variable declared: the statement below is CE0117, +-- and `dateTime(2026, 1, 15)` with literal arguments is clean. +-- -- Usage (expected to FAIL check): -- mxcli check mdl-examples/bug-tests/ledger-21-datetime-literals.fail.mdl -- ============================================================================ @@ -21,6 +24,9 @@ create microflow MyModule.DateTimeLiterals ( ) returns dateTime begin + -- Declared on purpose: an undeclared variable is CE0109, which MxBuild reports first + -- and which hides the CE0117 this statement exists to show. + declare $Bad dateTime = [%CurrentDateTime%]; set $Bad = dateTime(2026, $Month, $Day); -- MDL046: variable args return $Bad; end diff --git a/mdl-examples/check-gap/README.md b/mdl-examples/check-gap/README.md index e6cb017a6e..d19cb11096 100644 --- a/mdl-examples/check-gap/README.md +++ b/mdl-examples/check-gap/README.md @@ -20,3 +20,13 @@ committed; pass its path). Gated: needs mxbuild, so it is not part of `make test A fixture that `mxcli check -p` passes while `mx check` reports a code is a backlog item. Codes can differ between checkers for the same finding (CE1613 in mx is CE7710 in `mendix check`), so compare messages when a code differs. + +`run-parity.sh` compares what each checker reports for one defect at a time. To see what a +checker HIDES when two defects share a document (MxBuild reports in stages; Mendix CLI 2 does +not), use `scripts/maskmatrix`: + + make mask-matrix ARGS="--project /tmp/clean/Spike.mpr \ + --mx ~/.mxcli/mxbuild/11.16.1/modeler/mx [--mendix /path/to/mendix]" + +It writes every defect and every ordered pair into a copy of the project, checks once, and +prints who hides whom, the stages that gives, and what an unresolved reference hides. diff --git a/mdl/backend/modelsdk/access_rule_xpath_spacing_2355_test.go b/mdl/backend/modelsdk/access_rule_xpath_spacing_2355_test.go new file mode 100644 index 0000000000..061121a878 --- /dev/null +++ b/mdl/backend/modelsdk/access_rule_xpath_spacing_2355_test.go @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 + +package modelsdkbackend + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/types" +) + +// ako/mxcli#2355: a grant matched the stored rule by role set and XPath +// constraint compared byte for byte, so the same constraint written over two +// lines — or stored on one line by an older mxcli or by Studio Pro — added a +// second, identical rule instead of merging, and a re-run of the script +// changed the project. Constraints that differ only in whitespace outside +// string literals are one constraint. +func TestGrant_XPathDifferingOnlyInWhitespaceMerges_Issue2355(t *testing.T) { + b, mod, dm := grantFixture(t) + oneLine := "[Name = 'a' or Name = 'b']" + if err := b.AddEntityAccessRule(grantParams(dm.ID, "ReadOnly", oneLine, nil)); err != nil { + t.Fatal(err) + } + wrapped := "[Name = 'a'\n or Name = 'b']" + if err := b.AddEntityAccessRule(grantParams(dm.ID, "ReadOnly", wrapped, nil)); err != nil { + t.Fatal(err) + } + rule := onlyRule(t, b, mod.ID) + // The stored spelling is kept, so a re-run in the other spelling writes + // nothing. + if rule.XPathConstraint != oneLine { + t.Errorf("constraint = %q, want the stored %q kept", rule.XPathConstraint, oneLine) + } +} + +// CONTROL: whitespace inside a string literal is part of the constraint, so a +// different literal is a different rule. +func TestGrant_XPathLiteralWhitespaceIsDifferent_Issue2355(t *testing.T) { + b, mod, dm := grantFixture(t) + if err := b.AddEntityAccessRule(grantParams(dm.ID, "ReadOnly", "[Name = 'a b']", nil)); err != nil { + t.Fatal(err) + } + if err := b.AddEntityAccessRule(grantParams(dm.ID, "ReadOnly", "[Name = 'ab']", nil)); err != nil { + t.Fatal(err) + } + if n := len(rulesOf(t, b, mod.ID)); n != 2 { + t.Errorf("rules = %d, want 2 (the literals differ)", n) + } +} + +func TestSameXPathConstraint_Issue2355(t *testing.T) { + for _, c := range []struct { + a, b string + same bool + }{ + {"[a = 'x' or b = 'y']", "[a = 'x'\n or b = 'y']", true}, + {"", "", true}, + {"[a = 'x y']", "[a = 'xy']", false}, + {"", "[a = 1]", false}, + } { + if got := types.SameXPathConstraint(c.a, c.b); got != c.same { + t.Errorf("SameXPathConstraint(%q, %q) = %v, want %v", c.a, c.b, got, c.same) + } + } +} diff --git a/mdl/backend/modelsdk/domainmodel_security_write.go b/mdl/backend/modelsdk/domainmodel_security_write.go index 21b9c8f6d0..c2e61e82b6 100644 --- a/mdl/backend/modelsdk/domainmodel_security_write.go +++ b/mdl/backend/modelsdk/domainmodel_security_write.go @@ -52,12 +52,21 @@ func (b *Backend) AddEntityAccessRule(p backend.EntityAccessRuleParams) error { var rule *genDm.AccessRule for _, el := range ent.AccessRulesItems() { r, ok := el.(*genDm.AccessRule) + // The same constraint written with other whitespace is the same rule + // (ako/mxcli#2355): matched byte for byte, a grant wrapped over two + // lines added a second, identical rule beside one stored on one line. if ok && sameStringSet(r.ModuleRolesQualifiedNames(), p.RoleNames) && - r.XPathConstraint() == p.XPathConstraint { + types.SameXPathConstraint(r.XPathConstraint(), p.XPathConstraint) { rule = r break } } + // A merged rule keeps its stored spelling, so a re-run that only formats + // the constraint differently writes nothing. + xpath := p.XPathConstraint + if rule != nil { + xpath = rule.XPathConstraint() + } // Everything the matched rule already grants, so the write below widens it // rather than replacing it. GRANT is additive by contract; narrowing is what @@ -98,7 +107,7 @@ func (b *Backend) AddEntityAccessRule(p backend.EntityAccessRuleParams) error { if def := types.HigherAccessRights(p.DefaultMemberAccess, priorDefault); def != "" { rule.SetDefaultMemberAccessRights(def) } - rule.SetXPathConstraint(p.XPathConstraint) + rule.SetXPathConstraint(xpath) for _, ma := range p.MemberAccesses { m := genDm.NewMemberAccess() assignID(m) diff --git a/mdl/backend/modelsdk/javascript_write.go b/mdl/backend/modelsdk/javascript_write.go index 4fcc632559..d4f58b7d40 100644 --- a/mdl/backend/modelsdk/javascript_write.go +++ b/mdl/backend/modelsdk/javascript_write.go @@ -178,7 +178,20 @@ func (b *Backend) WriteJavaScriptSourceFile(moduleName, actionName string, jsCod // calls (mendixlabs/mxcli#1429; the Java twin is ako/mxcli#705). existing, _ := os.ReadFile(path) imports, extraCode := javaactions.JavaScriptRetainedSections(string(existing)) - source := javaactions.GenerateJavaScriptSource(actionName, jsCode, params, returnType, imports, extraCode) + // The file as mxbuild writes it, the action's documentation and its + // enumerations' values included, so a build leaves it alone and a re-run + // writes nothing (ako/mxcli#2357). The action's unit is written before its + // file, so its documentation is read back from it. + doc := "" + if b.reader != nil { + if a, err := b.ReadJavaScriptActionByName(moduleName + "." + actionName); err == nil && a != nil { + doc = a.Documentation + } + } + source := javaactions.JSSource{ + Action: actionName, Documentation: doc, UserCode: jsCode, Params: params, ReturnType: returnType, + Imports: imports, ExtraCode: extraCode, EnumValues: b.enumerationValueNames, + }.Generate() changed, err := javaactions.WriteSourceIfChanged(path, source) b.noteFileWrite(changed) if err != nil { @@ -233,3 +246,40 @@ func (b *Backend) RenameJavaScriptSourceFile(moduleName, oldName, newName string } return nil } + +// enumerationValueNames is the value names of the enumeration qualifiedName, +// in order, or nil when it cannot be read. +func (b *Backend) enumerationValueNames(qualifiedName string) []string { + if b.reader == nil { + return nil + } + info, err := b.reader.GetRawUnitByName("enumeration", qualifiedName) + if err != nil || info == nil { + return nil + } + var doc bson.D + if bson.Unmarshal(info.Contents, &doc) != nil { + return nil + } + var names []string + for _, e := range doc { + if e.Key != "Values" { + continue + } + list, _ := e.Value.(bson.A) + for _, v := range list { + vd, ok := v.(bson.D) + if !ok { + continue + } + for _, f := range vd { + if f.Key == "Name" { + if n, ok := f.Value.(string); ok { + names = append(names, n) + } + } + } + } + } + return names +} diff --git a/mdl/backend/modelsdk/javascript_write_retain_test.go b/mdl/backend/modelsdk/javascript_write_retain_test.go index d823f230fb..b441ed27c0 100644 --- a/mdl/backend/modelsdk/javascript_write_retain_test.go +++ b/mdl/backend/modelsdk/javascript_write_retain_test.go @@ -64,7 +64,7 @@ func TestWriteJavaScriptSourceFile_EditedBodyKeepsImportsAndExtraCode(t *testing if err := b.WriteJavaScriptSourceFile("WebMod", "GetDistance", "return deg2rad(latitude).times(2);", params, &types.DecimalType{}); err != nil { t.Fatal(err) } - got, err := os.ReadFile(path) + got, err := readLF(path) if err != nil { t.Fatal(err) } @@ -89,7 +89,7 @@ func TestWriteJavaScriptSourceFile_EditedBodyKeepsImportsAndExtraCode(t *testing if err := b.WriteJavaScriptSourceFile("WebMod", "GetDistance", "return deg2rad(latitude).times(2);", params, &types.DecimalType{}); err != nil { t.Fatal(err) } - again, _ := os.ReadFile(path) + again, _ := readLF(path) if string(again) != src { t.Errorf("second regeneration changed the file:\n%s", again) } @@ -103,11 +103,13 @@ func TestWriteJavaScriptSourceFile_NewFileHasEmptySections(t *testing.T) { if err := b.WriteJavaScriptSourceFile("WebMod", "Fresh", "return 1;", nil, nil); err != nil { t.Fatal(err) } - got, err := os.ReadFile(filepath.Join(b.jsActionSourceDir("WebMod"), "Fresh.js")) + got, err := readLF(filepath.Join(b.jsActionSourceDir("WebMod"), "Fresh.js")) if err != nil { t.Fatal(err) } - if strings.Contains(string(got), "\nimport ") || !strings.Contains(string(got), "// BEGIN EXTRA CODE\n// END EXTRA CODE") { + // mxbuild writes the Big import in every file, and nothing else is there. + if strings.Count(string(got), "\nimport ") != 1 || !strings.Contains(string(got), "\nimport { Big } from \"big.js\";\n") || + !strings.Contains(string(got), "// BEGIN EXTRA CODE\n// END EXTRA CODE") { t.Errorf("new stub has unexpected sections:\n%s", got) } } @@ -126,13 +128,14 @@ func TestWriteJavaScriptSourceFile_ParametersAreLowerFirst(t *testing.T) { if err := b.WriteJavaScriptSourceFile("Demo", "JS_Open", `return url + (readOnly ? " (read only)" : "");`, params, &types.StringType{}); err != nil { t.Fatal(err) } - got, err := os.ReadFile(filepath.Join(b.jsActionSourceDir("Demo"), "JS_Open.js")) + got, err := readLF(filepath.Join(b.jsActionSourceDir("Demo"), "JS_Open.js")) if err != nil { t.Fatal(err) } for _, want := range []string{ - " * @param {string} url\n", - " * @param {boolean} readOnly\n", + // Not marked required, so optional, as mxbuild writes it. + " * @param {string} [url]\n", + " * @param {boolean} [readOnly]\n", "export async function JS_Open(url, readOnly) {\n", } { if !strings.Contains(string(got), want) { @@ -182,14 +185,22 @@ export async function Locate(address, geocodingProvider, providerApiKey) { if err := b.WriteJavaScriptSourceFile("Demo", "Locate", "return address + geocodingProvider + providerApiKey;", params, &types.StringType{}); err != nil { t.Fatal(err) } - got, _ := os.ReadFile(path) + got, _ := readLF(path) const sig = "export async function Locate(address, geocodingProvider, providerApiKey) {\n" if !strings.Contains(string(got), sig) { t.Errorf("rewrite changed the Studio Pro signature:\n%s", got) } for _, name := range []string{"address", "geocodingProvider", "providerApiKey"} { - if !strings.Contains(string(got), " "+name+"\n") { + if !strings.Contains(string(got), " ["+name+"]\n") { t.Errorf("JSDoc lost @param %s:\n%s", name, got) } } } + +// readLF reads a file with CRLF turned into LF: mxbuild, and so mxcli, write a +// JavaScript action's file in CRLF (ako/mxcli#2357), and these tests compare +// content, not line endings. +func readLF(path string) ([]byte, error) { + b, err := os.ReadFile(path) + return []byte(strings.ReplaceAll(string(b), "\r\n", "\n")), err +} diff --git a/mdl/backend/pagemutator/mutator.go b/mdl/backend/pagemutator/mutator.go index 04d0baa654..fb07c8f687 100644 --- a/mdl/backend/pagemutator/mutator.go +++ b/mdl/backend/pagemutator/mutator.go @@ -3313,6 +3313,27 @@ func setRawWidgetPropertyMut(widget bson.D, propName string, value any) error { return fmt.Errorf("widget does not support conditional visibility") } return nil + case "visibleroles": + // `visible: roles (…)`: "Visible for module roles", stored on the + // visibility node beside the condition, which it leaves as it is + // (ako/mxcli#2371). A by-name list: Studio Pro writes marker 1. + roles, ok := value.([]string) + if !ok || len(roles) == 0 { + return fmt.Errorf("VisibleRoles takes a list of module roles, got %T", value) + } + if _, has := bsonnav.DGet(widget, "ConditionalVisibilitySettings").(bson.D); !has { + if !setWidgetConditionalSettingMut(widget, "ConditionalVisibilitySettings", + "Forms$ConditionalVisibilitySettings", "", true) { + return fmt.Errorf("widget does not support conditional visibility") + } + } + list := bson.A{int32(1)} + for _, r := range roles { + list = append(list, r) + } + cvs := bsonnav.DGet(widget, "ConditionalVisibilitySettings").(bson.D) + bsonnav.DSet(cvs, "ModuleRoles", list) + return nil case "editableif": expr, _ := value.(string) if !setWidgetConditionalSettingMut(widget, "ConditionalEditabilitySettings", diff --git a/mdl/backend/pagemutator/visible_roles_2371_test.go b/mdl/backend/pagemutator/visible_roles_2371_test.go new file mode 100644 index 0000000000..dea1f9a1d8 --- /dev/null +++ b/mdl/backend/pagemutator/visible_roles_2371_test.go @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pagemutator + +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "github.com/mendixlabs/mxcli/mdl/backend/bsonnav" +) + +// ako/mxcli#2371: `alter page … set (visible: roles (A, B)) on c` stored the +// roles as the visibility expression; mx check reported CE0117. The roles are +// "Visible for module roles": ConditionalVisibilitySettings.ModuleRoles, a +// by-name list Studio Pro writes with marker 1. +func TestSetWidgetProperty_VisibleRoles_Issue2371(t *testing.T) { + for name, stored := range map[string]any{ + "no settings yet": nil, + "condition kept": bson.D{ + {Key: "$ID", Value: "x"}, + {Key: "$Type", Value: "Forms$ConditionalVisibilitySettings"}, + {Key: "Attribute", Value: ""}, + {Key: "Conditions", Value: bson.A{int32(2)}}, + {Key: "Expression", Value: "$currentObject/Active"}, + {Key: "IgnoreSecurity", Value: false}, + {Key: "ModuleRoles", Value: bson.A{int32(1), "M.Old"}}, + {Key: "SourceVariable", Value: nil}, + }, + } { + w := bson.D{ + {Key: "$Type", Value: "Forms$DivContainer"}, + {Key: "Name", Value: "ctn"}, + {Key: "ConditionalVisibilitySettings", Value: stored}, + } + rawData := makeRawPage(w) + m := &Mutator{rawData: rawData, widgetFinder: findBsonWidget} + if err := m.SetWidgetProperty("ctn", "VisibleRoles", []string{"M.Admin", "M.User"}); err != nil { + t.Fatalf("%s: %v", name, err) + } + cvs, ok := bsonnav.DGet(findBsonWidget(rawData, "ctn").widget, "ConditionalVisibilitySettings").(bson.D) + if !ok { + t.Fatalf("%s: no ConditionalVisibilitySettings written", name) + } + roles, _ := bsonnav.DGet(cvs, "ModuleRoles").(bson.A) + if len(roles) != 3 || roles[0] != int32(1) || roles[1] != "M.Admin" || roles[2] != "M.User" { + t.Errorf("%s: ModuleRoles = %v, want [1 M.Admin M.User]", name, roles) + } + wantExpr := "" + if stored != nil { + wantExpr = "$currentObject/Active" + } + if got := bsonnav.DGetString(cvs, "Expression"); got != wantExpr { + t.Errorf("%s: Expression = %q, want %q (the roles are not an expression)", name, got, wantExpr) + } + } +} diff --git a/mdl/executor/cmd_entities_access.go b/mdl/executor/cmd_entities_access.go index ef87abb2fa..59959525ae 100644 --- a/mdl/executor/cmd_entities_access.go +++ b/mdl/executor/cmd_entities_access.go @@ -8,6 +8,7 @@ import ( "slices" "strings" + "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/mdl/visitor" "github.com/mendixlabs/mxcli/sdk/domainmodel" ) @@ -255,7 +256,7 @@ func formatAccessRuleResult(ctx *ExecContext, moduleName, entityName string, rol // A role may hold one rule per XPath constraint (#936), and the backend // keys the rule by role set plus constraint — echoing any other match // would report a different rule's rights back to the user. - if !anyXPath && (rule.XPathConstraint != xpath || !sameRoleSet(rule.ModuleRoleNames, roleNames)) { + if !anyXPath && (!types.SameXPathConstraint(rule.XPathConstraint, xpath) || !sameRoleSet(rule.ModuleRoleNames, roleNames)) { continue } // Found a matching rule diff --git a/mdl/executor/retrieve_list_returned_2364_test.go b/mdl/executor/retrieve_list_returned_2364_test.go new file mode 100644 index 0000000000..cba1e940a2 --- /dev/null +++ b/mdl/executor/retrieve_list_returned_2364_test.go @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "strings" + "testing" +) + +// ako/mxcli#2364: under `mdl 1;` `retrieve … limit 1` binds a list of one, and +// `return $Item` from a microflow that returns `Lm.Item` passed check ("Expression +// types OK") while mx check reported CE0117 at the End event (measured by the +// reporter on 11.16.1). + +func returnFlow(rng, returns string) string { + return `mdl 1; +create microflow Lm.GetItem ($Name: String) returns ` + returns + ` begin + retrieve $Item from Lm.Item where Name = $Name ` + rng + `; + return $Item; +end;` +} + +func TestRetrieveListReturnedAsObject_Issue2364(t *testing.T) { + v := findViolation(violationsForMicroflowSource(t, returnFlow("limit 1", "Lm.Item")), retrieveListAsObjectRule) + if v == nil { + t.Fatal("returning a `limit 1` list from a microflow that returns Lm.Item is not reported") + } + for _, want := range []string{"$Item", "list", "first", "CE0117"} { + if !strings.Contains(v.Message+v.Suggestion, want) { + t.Errorf("violation lacks %q: %s / %s", want, v.Message, v.Suggestion) + } + } +} + +// CONTROLS: the object range returned as the object, and the list returned +// from a microflow that returns a list, are what mxbuild accepts. +func TestRetrieveListReturnedAsObject_Controls_Issue2364(t *testing.T) { + for name, src := range map[string]string{ + "first as object": returnFlow("first", "Lm.Item"), + "list as list": returnFlow("limit 1", "list of Lm.Item"), + } { + if v := findViolation(violationsForMicroflowSource(t, src), retrieveListAsObjectRule); v != nil { + t.Errorf("%s: unexpected %s: %s", name, v.RuleID, v.Message) + } + } +} diff --git a/mdl/executor/validate_microflow_retrieve_single.go b/mdl/executor/validate_microflow_retrieve_single.go index 9a95da85d4..7a1d5e6243 100644 --- a/mdl/executor/validate_microflow_retrieve_single.go +++ b/mdl/executor/validate_microflow_retrieve_single.go @@ -103,7 +103,25 @@ func (v *microflowValidator) checkRetrieveListAsObject(body []ast.MicroflowState "of the list first (`$item = head($%s);`).", name, name)) } }, - nil, + func(s ast.MicroflowStatement) { + // `return $L` from a flow whose return type is not a list + // (ako/mxcli#2364): mxbuild reports CE0117 "Error(s) in + // expression." at the End event. + ret, ok := s.(*ast.ReturnStmt) + if !ok || v.returnType == nil || v.returnType.Type.Kind == ast.TypeListOf || v.returnType.Type.Kind == ast.TypeVoid { + return + } + ve, ok := ret.Value.(*ast.VariableExpr) + if !ok || !list[ve.Name] { + return + } + v.addViolation(retrieveListAsObjectRule, linter.SeverityError, + fmt.Sprintf("$%s was retrieved as a list (under `mdl 1;` `limit 1` is a list of one), "+ + "but the %s returns %s, a single value — mxbuild rejects the return with CE0117 "+ + "\"Error(s) in expression.\" at the End event.", ve.Name, v.docType, returnTypeName(v.returnType.Type)), + fmt.Sprintf("Retrieve the object with `retrieve $%s from … first;`, return `head($%s)`, "+ + "or declare the return type as `list of …`.", ve.Name, ve.Name)) + }, func(s ast.MicroflowStatement) { for _, p := range statementProducedVars(s) { delete(list, p.name) @@ -237,3 +255,16 @@ func listUseOf(s ast.MicroflowStatement) (string, string) { } return "", "" } + +// returnTypeName names a declared return type for a message. A bare +// `Module.Name` is parsed as an enumeration-or-entity reference, which +// formatDataTypeForMDL does not name. +func returnTypeName(dt ast.DataType) string { + switch { + case dt.EntityRef != nil: + return dt.EntityRef.String() + case dt.EnumRef != nil: + return dt.EnumRef.String() + } + return formatDataTypeForMDL(dt) +} diff --git a/mdl/linter/rules/reachable_page_read_access.go b/mdl/linter/rules/reachable_page_read_access.go index 8c1c3f2b53..8a7ff4b4a8 100644 --- a/mdl/linter/rules/reachable_page_read_access.go +++ b/mdl/linter/rules/reachable_page_read_access.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/model" "github.com/mendixlabs/mxcli/sdk/security" ) @@ -90,6 +91,7 @@ type reachModel struct { entries []string // menu pages and microflows profileHomes map[string]string // navigation profile -> its home page roleHomes map[string]map[string][]string // profile -> user role -> role-based home pages/microflows + ids map[string]string // page/snippet -> unit id } // Check reports, per page or snippet a user role reaches, each attribute or @@ -111,7 +113,15 @@ func (r *ReachablePageReadAccessRule) Check(ctx *linter.LintContext) []linter.Vi } found := map[string]*finding{} var order []string - note := func(doc, entity, member, userRole string) { + vis := newRoleVisibility(reader, m.ids) + note := func(doc, entity, member, userRole string, roles map[string]bool) { + target := member + if target == "" { + target = entity + } + if vis.hidden(doc, target, roles) { + return + } k := doc + "\x00" + entity + "\x00" + member f, ok := found[k] if !ok { @@ -141,7 +151,7 @@ func (r *ReachablePageReadAccessRule) Check(ctx *linter.LintContext) []linter.Vi for _, e := range entities { if !strings.HasPrefix(e, "System.") && !m.readsEntity(e, roles) && !blindEntity[e] { blindEntity[e] = true - note(doc, e, "", ur.Name) + note(doc, e, "", ur.Name, roles) } } for _, e := range m.edges[doc] { @@ -162,7 +172,7 @@ func (r *ReachablePageReadAccessRule) Check(ctx *linter.LintContext) []linter.Vi } } if !readable { - note(doc, candidates[0], e.target, ur.Name) + note(doc, candidates[0], e.target, ur.Name, roles) } } } @@ -388,6 +398,7 @@ func loadReachModel(ctx *linter.LintContext) *reachModel { market: map[string]bool{}, views: map[string]map[string]bool{}, executes: map[string]map[string]bool{}, gen: map[string]string{}, reads: map[string]map[string]bool{}, members: map[string]map[string]bool{}, listed: map[string]map[string]bool{}, defaultRO: map[string]map[string]bool{}, roleHomes: map[string]map[string][]string{}, profileHomes: map[string]string{}, + ids: map[string]string{}, } query := func(q string, scan func(cols []string)) { rows, err := ctx.Query(q) @@ -415,14 +426,16 @@ func loadReachModel(ctx *linter.LintContext) *reachModel { } } query(`SELECT Name, COALESCE(Source, '') FROM modules`, func(c []string) { m.market[c[0]] = c[1] != "" }) - query(`SELECT QualifiedName, ModuleName FROM pages`, func(c []string) { + query(`SELECT QualifiedName, ModuleName, Id FROM pages`, func(c []string) { m.module[c[0]] = c[1] + m.ids[c[0]] = c[2] if m.views[c[0]] == nil { m.views[c[0]] = map[string]bool{} } }) - query(`SELECT QualifiedName, ModuleName FROM snippets`, func(c []string) { + query(`SELECT QualifiedName, ModuleName, Id FROM snippets`, func(c []string) { m.module[c[0]] = c[1] + m.ids[c[0]] = c[2] m.snippets[c[0]] = true }) query(`SELECT QualifiedName FROM nanoflows`, func(c []string) { @@ -494,3 +507,107 @@ func reachAny(granted map[string]bool, roles []string) bool { } return false } + +// roleVisibility answers whether everything a page or snippet shows of an +// entity or attribute sits inside widgets a user role cannot see. +// +// ako/mxcli#2356: mxbuild does not check read access for a widget hidden from +// the role by `Visible: roles (…)` (ConditionalVisibilitySettings.ModuleRoles), +// so data inside a container visible only to administrators is no CE2729 for a +// user role without that module role. The catalog's refs do not say which +// widget a reference sits in, so the stored widget tree is read. +type roleVisibility struct { + reader linter.LintReader + ids map[string]string + units map[string]map[string]any +} + +func newRoleVisibility(reader linter.LintReader, ids map[string]string) *roleVisibility { + return &roleVisibility{reader: reader, ids: ids, units: map[string]map[string]any{}} +} + +func (v *roleVisibility) unit(doc string) map[string]any { + if u, ok := v.units[doc]; ok { + return u + } + var u map[string]any + if id := v.ids[doc]; id != "" && v.reader != nil { + u, _ = v.reader.GetRawUnit(model.ID(id)) + } + v.units[doc] = u + return u +} + +// hidden reports whether every occurrence of target (an entity or attribute +// qualified name) in doc is inside a widget whose role restriction excludes +// all of roles. No occurrence found, or an unreadable unit, is not hidden: the +// finding stands. +func (v *roleVisibility) hidden(doc, target string, roles map[string]bool) bool { + u := v.unit(doc) + if u == nil { + return false + } + seen, visible := 0, false + var walk func(node any, hiddenAbove bool) + walk = func(node any, hiddenAbove bool) { + if visible { + return + } + switch n := node.(type) { + case map[string]any: + h := hiddenAbove || restrictsAway(n["ConditionalVisibilitySettings"], roles) + for _, val := range n { + walk(val, h) + } + case []any: + for _, val := range n { + walk(val, hiddenAbove) + } + case string: + if refersTo(n, target) { + seen++ + if !hiddenAbove { + visible = true + } + } + } + } + walk(u, false) + return seen > 0 && !visible +} + +// restrictsAway reports whether a ConditionalVisibilitySettings value limits +// the widget to module roles of which roles holds none. +func restrictsAway(settings any, roles map[string]bool) bool { + s, ok := settings.(map[string]any) + if !ok { + return false + } + list, _ := s["ModuleRoles"].([]any) + restricted := false + for _, r := range list { + name, ok := r.(string) + if !ok { + continue // the array's leading type marker + } + restricted = true + if roles[name] { + return false + } + } + return restricted +} + +// refersTo reports whether a stored string names target, alone or as a step +// of an association path ("Mod.Assoc/Mod.Entity"). +func refersTo(s, target string) bool { + if s == target { + return true + } + for _, step := range strings.Split(s, "/") { + if step == target { + return true + } + } + return false +} diff --git a/mdl/linter/rules/reachable_page_role_visibility_2356_test.go b/mdl/linter/rules/reachable_page_role_visibility_2356_test.go new file mode 100644 index 0000000000..b949d04e9d --- /dev/null +++ b/mdl/linter/rules/reachable_page_role_visibility_2356_test.go @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: Apache-2.0 + +package rules + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/linter" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/security" +) + +// ako/mxcli#2356: SEC012 reported data a user role may not read on a page it +// reaches, saying "mxbuild reports CE2729" — 19 times on a real 11.15 project +// where mx check reported 0. Every one was inside a container visible only to +// module roles that may read it (`Visible: roles (…)`): mxbuild does not check +// a widget hidden from the role, and neither does the rule now. + +// rawPageReader serves one page's stored widget tree for any id. +type rawPageReader struct { + navSecurityReader + raw map[string]any +} + +func (r rawPageReader) GetRawUnit(model.ID) (map[string]any, error) { return r.raw, nil } + +// pageWithGridIn is a page whose data grid over Probe.Ticket shows Title, inside +// a container visible only to the given module roles (none: no restriction). +func pageWithGridIn(visibleTo ...string) map[string]any { + grid := map[string]any{ + "$Type": "Forms$DataGrid", + "DataSource": map[string]any{"$Type": "Forms$DataViewSource", "EntityRef": map[string]any{"Entity": "Probe.Ticket"}}, + "Columns": []any{int32(3), map[string]any{"$Type": "Forms$DataGridColumn", + "AttributeRef": map[string]any{"$Type": "DomainModels$AttributeRef", "Attribute": "Probe.Ticket.Title"}}}, + } + container := map[string]any{"$Type": "Forms$DivContainer", "Widgets": []any{int32(3), grid}} + if len(visibleTo) > 0 { + roles := []any{int32(1)} + for _, r := range visibleTo { + roles = append(roles, r) + } + container["ConditionalVisibilitySettings"] = map[string]any{ + "$Type": "Forms$ConditionalVisibilitySettings", "ModuleRoles": roles} + } + return map[string]any{"$Type": "Forms$Page", "Widgets": []any{int32(3), container}} +} + +func (f *readAccessFixture) checkReachWithRaw(raw map[string]any) int { + f.t.Helper() + ctx := linter.NewLintContext(f.cat, rawPageReader{navSecurityReader{sec: f.sec}, raw}) + return len(NewReachablePageReadAccessRule().Check(ctx)) +} + +func roleVisibilityFixture(t *testing.T) *readAccessFixture { + f := reachFixture(t, security.SecurityLevelProduction) + f.grid("Probe.P_Menu", "Probe.Ticket", "Probe.Ticket.Title", "Probe.User") + f.menuItem("Probe.P_Menu") + return f +} + +func TestReachablePageRead_HiddenFromTheRoleIsSilent_Issue2356(t *testing.T) { + f := roleVisibilityFixture(t) + if n := f.checkReachWithRaw(pageWithGridIn("Probe.Admin")); n != 0 { + t.Errorf("violations = %d, want none: the grid is visible only to Probe.Admin, which ProbeUser does not hold", n) + } +} + +// CONTROLS: visible to a role the user role holds, or not restricted, is +// reported as before; so is a page whose stored tree cannot be read. +func TestReachablePageRead_VisibleToTheRoleIsReported_Issue2356(t *testing.T) { + f := roleVisibilityFixture(t) + if n := f.checkReachWithRaw(pageWithGridIn("Probe.User", "Probe.Admin")); n != 1 { + t.Errorf("visible to Probe.User: violations = %d, want 1", n) + } + if n := f.checkReachWithRaw(pageWithGridIn()); n != 1 { + t.Errorf("no restriction: violations = %d, want 1", n) + } + if n := f.checkReachWithRaw(nil); n != 1 { + t.Errorf("unreadable page: violations = %d, want 1", n) + } +} diff --git a/mdl/roundtrip/testapp_allowlist_test.go b/mdl/roundtrip/testapp_allowlist_test.go index a6053c89d7..d94b7e6152 100644 --- a/mdl/roundtrip/testapp_allowlist_test.go +++ b/mdl/roundtrip/testapp_allowlist_test.go @@ -107,8 +107,6 @@ var testAppKnownFailures = map[string]knownFailure{ "entity WorkflowCommons.ConflictedWorkflowView": {laws: []law{lawExec}, issue: "#721", why: "entity: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, "entity WorkflowCommons.MyInitiatedWorkflowView": {laws: []law{lawExec}, issue: "#721", why: "entity: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, "entity WorkflowCommons.UserTaskView": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "entity: executes since a Studio Pro index keeps its GUID on CREATE OR MODIFY (mendixlabs/mxcli#1470); re-executing adds a fourth access rule (the XPath-scoped grant is not matched to its stored rule) and the documentation's trailing whitespace drifts — not yet triaged into #721's classes"}, - "entity WorkflowCommons.WorkflowAttachment": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "entity: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, - "entity WorkflowCommons.WorkflowComment": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "entity: breaks getput and putget on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, "entity WorkflowCommons.WorkflowGroupView": {laws: []law{lawParse}, issue: "#721", why: "entity: breaks parse on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, "entity WorkflowCommons.WorkflowUserTaskView": {laws: []law{lawExec}, issue: "#721", why: "entity: breaks exec on TestApp, measured when it joined the harness (#743); not yet triaged into #721's classes"}, "entity WorkflowCommons.WorkflowView": {laws: []law{lawGetPut, lawPutGet}, issue: "#721", why: "entity: executes since a Studio Pro index keeps its GUID on CREATE OR MODIFY (mendixlabs/mxcli#1470); re-executing adds a fourth access rule (the XPath-scoped grant is not matched to its stored rule) and the documentation's trailing whitespace drifts — not yet triaged into #721's classes"}, diff --git a/mdl/types/xpath_same.go b/mdl/types/xpath_same.go new file mode 100644 index 0000000000..653ab68ff5 --- /dev/null +++ b/mdl/types/xpath_same.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import "strings" + +// SameXPathConstraint reports whether two XPath constraints are the same once +// whitespace outside string literals is ignored: the same constraint wrapped +// over lines, indented, or stored on one line by another writer +// (ako/mxcli#2355). Whitespace inside a literal is part of the value. +func SameXPathConstraint(a, b string) bool { + return xpathKey(a) == xpathKey(b) +} + +// xpathKey is s without whitespace outside its single-quoted literals. +func xpathKey(s string) string { + var sb strings.Builder + inLiteral := false + for _, r := range s { + if r == '\'' { + inLiteral = !inLiteral + } + if !inLiteral && (r == ' ' || r == '\t' || r == '\n' || r == '\r') { + continue + } + sb.WriteRune(r) + } + return sb.String() +} diff --git a/mdl/visitor/alter_visible_roles_2371_test.go b/mdl/visitor/alter_visible_roles_2371_test.go new file mode 100644 index 0000000000..24c73272e6 --- /dev/null +++ b/mdl/visitor/alter_visible_roles_2371_test.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 + +package visitor + +import ( + "reflect" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" +) + +// ako/mxcli#2371: the alter form of `visible: roles (…)` arrived at the +// executor as the expression text "roles (…)". +func TestAlterSetVisibleRoles_Issue2371(t *testing.T) { + for src, want := range map[string]map[string]any{ + "alter page M.P { set ( visible: roles (M.Admin, M.User) ) on c };": { + "VisibleRoles": []string{"M.Admin", "M.User"}, + }, + "alter page M.P { set ( visible: roles (M.Admin) and $currentObject/A = 1 ) on c };": { + "VisibleRoles": []string{"M.Admin"}, + "VisibleIf": "$currentObject/A = 1", + }, + // CONTROL: a plain condition stays a condition. + "alter page M.P { set ( visible: $currentObject/A = 1 ) on c };": { + "VisibleIf": "$currentObject/A = 1", + }, + } { + prog, errs := Build(src) + if len(errs) > 0 { + t.Fatalf("%s: %v", src, errs) + } + op := prog.Statements[0].(*ast.AlterPageStmt).Operations[0].(*ast.SetPropertyOp) + if !reflect.DeepEqual(op.Properties, want) { + t.Errorf("%s:\n got %#v\nwant %#v", src, op.Properties, want) + } + } +} diff --git a/mdl/visitor/visible_roles_split.go b/mdl/visitor/visible_roles_split.go index 59714c37f2..4b5ab3c80d 100644 --- a/mdl/visitor/visible_roles_split.go +++ b/mdl/visitor/visible_roles_split.go @@ -12,6 +12,23 @@ import ( var leadingVisibleRolesRe = regexp.MustCompile( `(?is)^\s*roles\s*\(\s*([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+(?:\s*,\s*[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+)*)\s*\)\s+and\s+(.+?)\s*$`) +// visibleRolesOnlyRe is `roles (Mod.A, Mod.B)` and nothing else. +var visibleRolesOnlyRe = regexp.MustCompile( + `(?is)^\s*roles\s*\(\s*([A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+(?:\s*,\s*[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+)*)\s*\)\s*$`) + +// splitVisibleRoles is splitLeadingVisibleRoles that also takes a bare +// `roles (…)`, whose condition is empty. `alter page … set (visible: …)` +// receives both as expression text (ako/mxcli#2371). +func splitVisibleRoles(expr string) (roles []string, cond string, ok bool) { + if m := visibleRolesOnlyRe.FindStringSubmatch(expr); m != nil { + for _, r := range strings.Split(m[1], ",") { + roles = append(roles, strings.TrimSpace(r)) + } + return roles, "", true + } + return splitLeadingVisibleRoles(expr) +} + // splitLeadingVisibleRoles splits `roles (…) and ` into the role // names and the condition (ako/mxcli#2319). It declines when the condition // has a top-level `or`: `roles (A) and x or y` reads (roles and x) or y, diff --git a/mdl/visitor/visitor_alter_page.go b/mdl/visitor/visitor_alter_page.go index 52a273b6b0..3153aaf45b 100644 --- a/mdl/visitor/visitor_alter_page.go +++ b/mdl/visitor/visitor_alter_page.go @@ -104,6 +104,20 @@ func (b *Builder) buildAlterSet(ctx *parser.AlterSetContext) ast.AlterPageOperat op.Properties[name] = value } } + // `visible: roles (…) [and ]` is Studio Pro's "Visible for + // module roles", a setting of its own beside the condition, as CREATE PAGE + // reads it (ako/mxcli#2319). It reached here as the expression text, and + // was stored as an invalid expression (ako/mxcli#2371). + if text, ok := op.Properties["VisibleIf"].(string); ok { + if roles, cond, ok := splitVisibleRoles(text); ok { + op.Properties["VisibleRoles"] = roles + if cond == "" { + delete(op.Properties, "VisibleIf") + } else { + op.Properties["VisibleIf"] = cond + } + } + } b.recordAlterPageSet(ctx) return op diff --git a/scripts/maskmatrix/main.go b/scripts/maskmatrix/main.go new file mode 100644 index 0000000000..5071b9dc95 --- /dev/null +++ b/scripts/maskmatrix/main.go @@ -0,0 +1,411 @@ +// SPDX-License-Identifier: Apache-2.0 + +// mask-matrix measures which errors a Mendix checker hides until others are fixed. +// +// It writes one microflow per defect and one per ordered pair of defects into a copy +// of a healthy project, runs `mx check` (and, optionally, Mendix CLI 2's `mendix +// check`) once over all of them, and reports, from what each checker says about the +// pairs, which defect hides which, the stages that gives, and whether the order inside +// a document matters. A second, smaller run shows what a reference that does not +// resolve hides across the whole app. +// +// go run ./scripts/maskmatrix --project /path/to/healthy/App.mpr \ +// --mx ~/.mxcli/mxbuild/11.16.1/modeler/mx [--mendix /path/to/mendix] +// +// The project must be healthy (it must check clean) and built with the Mendix version +// of the --mx you pass. See docs/11-proposals/PROPOSAL_check_diagnostics_catalog.md +// for what this found on MxBuild 11.16.1. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" +) + +func main() { + if err := run(os.Args[1:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "mask-matrix:", err) + os.Exit(1) + } +} + +type config struct { + project, mx, mxcli, mendix string + defects []Defect + probe, keep bool +} + +func run(args []string, stdout, stderr io.Writer) error { + fs := flag.NewFlagSet("mask-matrix", flag.ContinueOnError) + fs.SetOutput(stderr) + project := fs.String("project", "", "a healthy Mendix project (.mpr) to copy; required") + mx := fs.String("mx", "", "path to MxBuild's modeler/mx; required") + mxcli := fs.String("mxcli", "./bin/mxcli", "mxcli binary used to write the documents") + mendix := fs.String("mendix", "", "path to Mendix CLI 2's `mendix`, to compare (optional)") + only := fs.String("defects", "", "comma-separated subset of "+strings.Join(defectNames(Defects), ",")+" (default: all)") + noProbe := fs.Bool("no-reference-probe", false, "skip the run that shows what an unresolved reference hides") + keep := fs.Bool("keep", false, "keep the working copies and print where they are") + if err := fs.Parse(args); err != nil { + return err + } + if *project == "" || *mx == "" { + return fmt.Errorf("--project and --mx are required") + } + defects, err := selectDefects(*only) + if err != nil { + return err + } + c := config{project: *project, mx: *mx, mxcli: *mxcli, mendix: *mendix, defects: defects, probe: !*noProbe, keep: *keep} + return c.execute(stdout) +} + +func defectNames(ds []Defect) []string { + out := make([]string, len(ds)) + for i, d := range ds { + out[i] = d.Name + } + return out +} + +func selectDefects(only string) ([]Defect, error) { + if only == "" { + return Defects, nil + } + var out []Defect + for _, n := range strings.Split(only, ",") { + n = strings.TrimSpace(n) + found := false + for _, d := range Defects { + if d.Name == n { + out = append(out, d) + found = true + } + } + if !found { + return nil, fmt.Errorf("unknown defect %q (have %s)", n, strings.Join(defectNames(Defects), ", ")) + } + } + if len(out) < 2 { + return nil, fmt.Errorf("need at least two defects to compare") + } + return out, nil +} + +func (c config) execute(w io.Writer) error { + tmp, err := os.MkdirTemp("", "maskmatrix-") + if err != nil { + return err + } + if c.keep { + fmt.Fprintf(w, "working copies kept in %s\n\n", tmp) + } else { + defer os.RemoveAll(tmp) + } + mpr := filepath.Base(c.project) + base := filepath.Join(tmp, "base") + if err := copyProject(filepath.Dir(c.project), base); err != nil { + return fmt.Errorf("copy the project: %w", err) + } + + defects, gen, refused, err := c.write(tmp, base, mpr) + if err != nil { + return err + } + for _, r := range refused { + fmt.Fprintf(w, "left out %s: mxcli refuses to write it even under --no-check\n", r) + } + fmt.Fprintf(w, "%d documents (%d defects, every ordered pair, and a baseline) in a copy of %s\n\n", len(gen.Docs)+1, len(defects), mpr) + + checkers := []struct{ name, bin string }{{"MxBuild mx check", c.mx}} + if c.mendix != "" { + checkers = append(checkers, struct{ name, bin string }{"mendix check (CLI 2)", c.mendix}) + } + for _, ch := range checkers { + f, err := runChecker(ch.bin, base, mpr, filepath.Join(tmp, "run-"+strings.Fields(ch.name)[0])) + if err != nil { + return fmt.Errorf("%s: %w", ch.name, err) + } + if len(f["Z_baseline"]) > 0 { + fmt.Fprintf(w, "WARNING: %s reports errors on the clean baseline document; the project is not healthy\n", ch.name) + } + fmt.Fprintf(w, "=== %s\n", ch.name) + Report(w, defects, Analyze(defects, gen.Docs, f)) + } + if c.probe { + if err := c.referenceProbe(w, tmp, mpr); err != nil { + return fmt.Errorf("reference probe: %w", err) + } + } + return nil +} + +// write generates the documents and writes them into base with mxcli, leaving out any +// defect exec refuses (a rule it enforces even under --no-check rolls the whole script +// back, so the script is generated again without it). +func (c config) write(tmp, base, mpr string) ([]Defect, Generated, []string, error) { + defects := append([]Defect(nil), c.defects...) + var refused []string + for range c.defects { + gen := Generate(defects) + script := filepath.Join(tmp, "m.mdl") + if err := os.WriteFile(script, []byte(gen.Script), 0o644); err != nil { + return nil, gen, nil, err + } + out, err := exec.Command(c.mxcli, "exec", script, "-p", filepath.Join(base, mpr), "--no-check").CombinedOutput() + if err == nil { + return defects, gen, refused, nil + } + d := Refused(string(out)) + if d == "" { + return nil, gen, nil, fmt.Errorf("mxcli exec failed: %v\n%s", err, tail(string(out), 6)) + } + refused = append(refused, d) + var keep []Defect + for _, x := range defects { + if x.Name != d { + keep = append(keep, x) + } + } + defects = keep + if len(defects) < 2 { + return nil, gen, nil, fmt.Errorf("fewer than two defects can be written") + } + } + return nil, Generated{}, nil, fmt.Errorf("could not write the documents") +} + +// runChecker runs a checker on a fresh copy of dir (a checker may touch the project) +// and returns its findings per microflow. +func runChecker(bin, dir, mpr, work string) (Findings, error) { + if err := copyProject(dir, work); err != nil { + return nil, err + } + out := filepath.Join(filepath.Dir(work), filepath.Base(work)+".json") + cmd := exec.Command(bin, "check", mpr, "-j", out) + cmd.Dir = work + cmd.Env = checkerEnv() + // A checker exits non-zero when it finds errors; the JSON is the result. + if b, err := cmd.CombinedOutput(); err != nil { + if _, statErr := os.Stat(out); statErr != nil { + return nil, fmt.Errorf("%v\n%s", err, tail(string(b), 6)) + } + } + raw, err := os.ReadFile(out) + if err != nil { + return nil, err + } + return ParseFindings(raw) +} + +// checkerEnv preloads the system FreeType on Linux, as `mxcli docker check` does: +// the JVM's own copy lacks a symbol Skia needs, so a bundled mx can abort without it. +func checkerEnv() []string { + env := os.Environ() + if runtime.GOOS != "linux" || os.Getenv("LD_PRELOAD") != "" { + return env + } + arch := map[string]string{"amd64": "x86_64", "arm64": "aarch64"}[runtime.GOARCH] + if arch == "" { + return env + } + lib := "/usr/lib/" + arch + "-linux-gnu/libfreetype.so.6" + if _, err := os.Stat(lib); err == nil { + env = append(env, "LD_PRELOAD="+lib) + } + return env +} + +var docRE = regexp.MustCompile(`^Microflow '(\w+)'`) + +// ParseFindings reads the JSON a checker writes with -j: errors[].locations[] with the +// document and element each error is on. Errors on anything but a microflow are kept +// under their own document name. +func ParseFindings(raw []byte) (Findings, error) { + var doc struct { + Errors []struct { + Code string `json:"code"` + Locations []struct { + Document string `json:"document-name"` + Element string `json:"element-name"` + } `json:"locations"` + } `json:"errors"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + return nil, fmt.Errorf("not a checker report: %w", err) + } + f := Findings{} + for _, e := range doc.Errors { + for _, l := range e.Locations { + name := l.Document + if m := docRE.FindStringSubmatch(name); m != nil { + name = m[1] + } + f[name] = append(f[name], Finding{Code: e.Code, Element: l.Element}) + } + } + return f, nil +} + +// Report prints what Analyze found. +func Report(w io.Writer, defects []Defect, r Result) { + fmt.Fprintln(w, "each defect alone:") + for _, d := range r.Defects { + fmt.Fprintf(w, " %-9s %s\n", d, strings.Join(r.Signature[d], ", ")+silentNote(r, d)) + } + if len(r.SharedCodes) > 0 { + fmt.Fprintf(w, " note: %s are raised by more than one defect, so those defects cannot be told apart\n", strings.Join(r.SharedCodes, ", ")) + } + fmt.Fprintln(w, "\nwho hides whom (the same in both orders):") + any := false + for _, d := range r.Defects { + if hs := r.Hides[d]; len(hs) > 0 { + fmt.Fprintf(w, " %-9s hides %s\n", d, strings.Join(hs, ", ")) + any = true + } + } + if !any { + fmt.Fprintln(w, " nothing: every defect is reported alongside every other") + } + if len(r.OrderDependent) > 0 { + fmt.Fprint(w, "order-dependent pairs:") + for _, p := range r.OrderDependent { + fmt.Fprintf(w, " %s/%s", p[0], p[1]) + } + fmt.Fprintln(w) + } else { + fmt.Fprintln(w, "order inside a document: no pair differs when swapped") + } + fmt.Fprintln(w, "\nstages (a defect in an earlier stage hides the later ones; one stage is reported together):") + for i, st := range r.Stages { + fmt.Fprintf(w, " %d %s\n", i+1, strings.Join(st, ", ")) + } + fmt.Fprintln(w) +} + +func silentNote(r Result, d string) string { + if contains(r.Silent, d) { + return "(clean: the checker does not report this defect, so it is left out of the comparison)" + } + return "" +} + +// referenceProbe shows what a reference that does not resolve hides across the app: +// one document with such a reference, one with an undefined variable and one with a +// type error, checked before and after the reference defect is dropped. +func (c config) referenceProbe(w io.Writer, tmp, mpr string) error { + probe := filepath.Join(tmp, "probe") + if err := copyProject(filepath.Dir(c.project), probe); err != nil { + return err + } + script := filepath.Join(tmp, "probe.mdl") + body := `mdl 1; +create module Pr; +create microflow Pr.Helper ($Text: String) returns String begin return $Text; end; +create microflow Pr.M_ref () returns Integer begin $p = call microflow Pr.Helper(Wrong = 'a'); return 1; end; +create microflow Pr.M_undef () returns Integer begin declare $u Integer = $Nope; return 1; end; +create microflow Pr.M_type () returns Integer begin declare $t Integer = 'abc'; return 1; end; +` + if err := os.WriteFile(script, []byte(body), 0o644); err != nil { + return err + } + if out, err := exec.Command(c.mxcli, "exec", script, "-p", filepath.Join(probe, mpr), "--no-check").CombinedOutput(); err != nil { + return fmt.Errorf("mxcli exec: %v\n%s", err, tail(string(out), 6)) + } + show := func(label, work string) error { + f, err := runChecker(c.mx, probe, mpr, work) + if err != nil { + return err + } + var parts []string + for _, doc := range []string{"M_ref", "M_undef", "M_type"} { + for _, x := range f[doc] { + parts = append(parts, fmt.Sprintf("%s@%s", x.Code, doc)) + } + } + sort.Strings(parts) + if len(parts) == 0 { + parts = []string{"clean"} + } + fmt.Fprintf(w, " %-34s %s\n", label, strings.Join(parts, ", ")) + return nil + } + fmt.Fprintln(w, "=== what an unresolved reference hides (MxBuild mx check)") + if err := show("with the reference defect (M_ref):", filepath.Join(tmp, "probe-with")); err != nil { + return err + } + drop := filepath.Join(tmp, "drop.mdl") + if err := os.WriteFile(drop, []byte("mdl 1;\ndrop microflow Pr.M_ref;\n"), 0o644); err != nil { + return err + } + if out, err := exec.Command(c.mxcli, "exec", drop, "-p", filepath.Join(probe, mpr)).CombinedOutput(); err != nil { + return fmt.Errorf("mxcli exec (drop): %v\n%s", err, tail(string(out), 6)) + } + return show("after dropping it:", filepath.Join(tmp, "probe-without")) +} + +func tail(s string, n int) string { + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return strings.Join(lines, "\n") +} + +// skipAtTop are entries of a project's root that a checker never reads and that are +// large: `mxcli new` puts a copy of the mxcli binary there (110 MB), and every working +// copy would repeat it. +var skipAtTop = map[string]bool{"mxcli": true, "mxcli.exe": true, ".git": true} + +// copyProject copies a Mendix project directory to dst, leaving out skipAtTop and +// anything that is not a plain file or directory. +func copyProject(src, dst string) error { + return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + if rel == "." { + return os.MkdirAll(dst, 0o755) + } + if skipAtTop[strings.Split(filepath.ToSlash(rel), "/")[0]] { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + target := filepath.Join(dst, rel) + switch { + case d.IsDir(): + return os.MkdirAll(target, 0o755) + case !d.Type().IsRegular(): + return nil + } + in, err := os.Open(path) + if err != nil { + return err + } + defer in.Close() + out, err := os.Create(target) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + out.Close() + return err + } + return out.Close() + }) +} diff --git a/scripts/maskmatrix/matrix.go b/scripts/maskmatrix/matrix.go new file mode 100644 index 0000000000..7ecd2ae6b8 --- /dev/null +++ b/scripts/maskmatrix/matrix.go @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// Defect is one way a microflow can be wrong, written so that any two can share a +// microflow: each fragment owns its variable names and its erroneous statement carries +// a caption, so a finding can be traced back to it. +type Defect struct { + Name string + Fragment string + NoReturn bool // the flow has no final return (the defect is the omission) +} + +// Defects are the document-level defects the matrix combines. A reference defect (a +// call that names a parameter that does not exist) is not among them: MxBuild +// reports it first and it hides every other error in every document, so one such +// document would blank the whole matrix. ReferenceProbe covers it instead. +var Defects = []Defect{ + {Name: "UNDEF", Fragment: "@caption 'T_UNDEF' declare $u1 Integer = $Nope;"}, + {Name: "DUP", Fragment: "declare $d1 Integer = 1; @caption 'T_DUP' declare $d1 Integer = 2;"}, + {Name: "TYPE", Fragment: "@caption 'T_TYPE' declare $t1 Integer = 'abc';"}, + {Name: "RETLOOP", Fragment: "loop $i1 in $L begin return 1; end loop;"}, + {Name: "NOVALUE", Fragment: "@caption 'T_NOVALUE' declare $n1 Integer;"}, + {Name: "NORETURN", NoReturn: true}, +} + +// Module is the module the generated documents live in. +const Module = "Mk" + +// Generated is the script and the defects each document holds. +type Generated struct { + Script string + Docs map[string][]string // document name -> the defects in it, in order +} + +// Generate writes a script with one document per defect (S_) and one per ordered +// pair (X___), so the order within a document can be compared too, and a clean +// baseline (Z_baseline) as the control. +func Generate(defects []Defect) Generated { + g := Generated{Docs: map[string][]string{}} + var b strings.Builder + fmt.Fprintf(&b, "mdl 1;\ncreate module %s;\n", Module) + fmt.Fprintf(&b, "create persistent entity %s.Item ( Name: String(50) );\n", Module) + fmt.Fprintf(&b, "create microflow %s.Z_baseline ($L: list of %s.Item) returns Integer begin return 1; end;\n", Module, Module) + emit := func(name string, ds ...Defect) { + var body strings.Builder + ret := " return 1;" + var names []string + for _, d := range ds { + names = append(names, d.Name) + if d.Fragment != "" { + body.WriteString(d.Fragment + " ") + } + if d.NoReturn { + ret = "" + } + } + fmt.Fprintf(&b, "create microflow %s.%s ($L: list of %s.Item) returns Integer begin %s%s end;\n", + Module, name, Module, body.String(), strings.TrimSpace(ret)) + g.Docs[name] = names + } + for _, d := range defects { + emit("S_"+d.Name, d) + } + for _, a := range defects { + for _, c := range defects { + if a.Name != c.Name { + emit(fmt.Sprintf("X_%s__%s", a.Name, c.Name), a, c) + } + } + } + g.Script = b.String() + return g +} + +// refusedRE finds the defect a write-time rule refused: exec enforces some rules even +// under --no-check and rolls the whole script back. +var refusedRE = regexp.MustCompile(Module + `\.S_([A-Z]+)' has validation errors`) + +// Refused returns the defect exec refused to write, or "". +func Refused(execOutput string) string { + if m := refusedRE.FindStringSubmatch(execOutput); m != nil { + return m[1] + } + return "" +} + +// Finding is one error of a checker, on one document. +type Finding struct{ Code, Element string } + +// Findings maps a document name to what a checker reported on it. +type Findings map[string][]Finding + +func (f Findings) codes(doc string) map[string]bool { + out := map[string]bool{} + for _, x := range f[doc] { + out[x.Code] = true + } + return out +} + +// Pair is what a checker reported for a document holding two defects, in one order. +type Pair string + +const ( + Both Pair = "both" + OnlyA Pair = "only first" + OnlyB Pair = "only second" + Neither Pair = "neither" +) + +// Result is the analysis of one checker's findings. +type Result struct { + Defects []string + Signature map[string][]string // defect -> the codes its own document raises + Silent []string // defects whose own document the checker leaves clean + SharedCodes []string // codes two defects raise: they cannot be told apart + Hides map[string][]string // defect -> defects it hides, in both orders + OrderDependent [][2]string + Stages [][]string // defects layered: everything in a stage hides the next ones +} + +// Analyze reads a checker's findings over the generated documents. A defect is visible +// in a document when a code its own single-defect document raises appears there. +func Analyze(defects []Defect, docs map[string][]string, f Findings) Result { + r := Result{Signature: map[string][]string{}, Hides: map[string][]string{}} + owners := map[string][]string{} + for _, d := range defects { + r.Defects = append(r.Defects, d.Name) + sig := sortedCodes(f.codes("S_" + d.Name)) + r.Signature[d.Name] = sig + if len(sig) == 0 { + r.Silent = append(r.Silent, d.Name) + } + for _, c := range sig { + owners[c] = append(owners[c], d.Name) + } + } + for c, o := range owners { + if len(o) > 1 { + r.SharedCodes = append(r.SharedCodes, c) + } + } + sort.Strings(r.SharedCodes) + visible := func(doc, d string) bool { + got := f.codes(doc) + for _, c := range r.Signature[d] { + if got[c] { + return true + } + } + return false + } + outcome := func(a, b string) Pair { + va, vb := visible(fmt.Sprintf("X_%s__%s", a, b), a), visible(fmt.Sprintf("X_%s__%s", a, b), b) + switch { + case va && vb: + return Both + case va: + return OnlyA + case vb: + return OnlyB + } + return Neither + } + hides := map[string]map[string]bool{} + for i, a := range r.Defects { + for _, b := range r.Defects[i+1:] { + if contains(r.Silent, a) || contains(r.Silent, b) { + continue + } + ab, ba := outcome(a, b), outcome(b, a) + // ab is the pair (a first, b second); ba is (b first, a second): swap the + // roles in the second to compare which defect survives. + survivorAB, survivorBA := survivor(ab, a, b), survivor(ba, b, a) + if survivorAB != survivorBA { + r.OrderDependent = append(r.OrderDependent, [2]string{a, b}) + continue + } + if survivorAB == "" || survivorAB == "both" { + continue + } + hider, hidden := survivorAB, a + if hider == a { + hidden = b + } + if hides[hider] == nil { + hides[hider] = map[string]bool{} + } + hides[hider][hidden] = true + } + } + for h, set := range hides { + for x := range set { + r.Hides[h] = append(r.Hides[h], x) + } + sort.Strings(r.Hides[h]) + } + r.Stages = layer(r.Defects, r.Silent, r.Hides) + return r +} + +// survivor names the defect a checker still reports for a pair (first, second), "both" +// when it reports both and "" when neither. +func survivor(p Pair, first, second string) string { + switch p { + case Both: + return "both" + case OnlyA: + return first + case OnlyB: + return second + } + return "" +} + +// layer orders defects into stages: a defect that nobody hides is stage 1, and one +// hidden by a stage n defect is at least stage n+1. Defects the checker leaves clean +// are not placed. +func layer(defects, silent []string, hides map[string][]string) [][]string { + level := map[string]int{} + var depth func(d string, seen map[string]bool) int + hiddenBy := map[string][]string{} + for h, xs := range hides { + for _, x := range xs { + hiddenBy[x] = append(hiddenBy[x], h) + } + } + depth = func(d string, seen map[string]bool) int { + if v, ok := level[d]; ok { + return v + } + if seen[d] { + return 0 + } + seen[d] = true + n := 0 + for _, h := range hiddenBy[d] { + if v := depth(h, seen) + 1; v > n { + n = v + } + } + level[d] = n + return n + } + var stages [][]string + for _, d := range defects { + if contains(silent, d) { + continue + } + n := depth(d, map[string]bool{}) + for len(stages) <= n { + stages = append(stages, nil) + } + stages[n] = append(stages[n], d) + } + return stages +} + +func sortedCodes(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for c := range m { + out = append(out, c) + } + sort.Strings(out) + return out +} + +func contains(xs []string, x string) bool { + for _, y := range xs { + if y == x { + return true + } + } + return false +} diff --git a/scripts/maskmatrix/matrix_test.go b/scripts/maskmatrix/matrix_test.go new file mode 100644 index 0000000000..45d4095d0e --- /dev/null +++ b/scripts/maskmatrix/matrix_test.go @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestGenerate(t *testing.T) { + ds := []Defect{ + {Name: "A", Fragment: "@caption 'T_A' declare $a Integer = $Nope;"}, + {Name: "B", Fragment: "@caption 'T_B' declare $b Integer = 'x';"}, + {Name: "N", NoReturn: true}, + } + g := Generate(ds) + // 3 singles + 3*2 ordered pairs. + if len(g.Docs) != 9 { + t.Errorf("got %d documents, want 9: %v", len(g.Docs), g.Docs) + } + if !strings.HasPrefix(g.Script, "mdl 1;\n") { + t.Errorf("the script must open with the language header (the conformance gate needs it):\n%s", g.Script) + } + if !strings.Contains(g.Script, "Z_baseline") { + t.Error("no clean baseline document") + } + if got := g.Docs["X_B__A"]; !reflect.DeepEqual(got, []string{"B", "A"}) { + t.Errorf("X_B__A holds %v, want [B A] in that order", got) + } + // The pair documents hold both fragments, in the order named. + line := lineOf(g.Script, "Mk.X_B__A") + if i, j := strings.Index(line, "T_B"), strings.Index(line, "T_A"); i < 0 || j < 0 || i > j { + t.Errorf("X_B__A must hold the B fragment before the A fragment: %s", line) + } + // A no-return defect leaves out the final return; the others keep it. + if strings.Contains(lineOf(g.Script, "Mk.S_N "), "return 1;") { + t.Errorf("S_N must have no final return: %s", lineOf(g.Script, "Mk.S_N ")) + } + if !strings.Contains(lineOf(g.Script, "Mk.S_A "), "return 1;") { + t.Errorf("S_A must keep its final return: %s", lineOf(g.Script, "Mk.S_A ")) + } + if strings.Contains(lineOf(g.Script, "Mk.X_A__N"), "return 1;") { + t.Errorf("a pair holding the no-return defect has no final return either: %s", lineOf(g.Script, "Mk.X_A__N")) + } +} + +func lineOf(script, needle string) string { + for _, l := range strings.Split(script, "\n") { + if strings.Contains(l, needle) { + return l + } + } + return "" +} + +func TestRefused(t *testing.T) { + out := "Created microflow: Mk.S_UNDEF\nError: microflow 'Mk.S_FUNC' has validation errors:\n - [MDL044] ...\nNothing was written" + if got := Refused(out); got != "FUNC" { + t.Errorf("Refused = %q, want FUNC", got) + } + if got := Refused("Error: no such project"); got != "" { + t.Errorf("an unrelated failure must not name a defect, got %q", got) + } +} + +func TestParseFindings(t *testing.T) { + raw := []byte(`{"errors":[ + {"code":"CE0109","locations":[{"document-name":"Microflow 'S_UNDEF'","element-name":"Create variable activity 'T_UNDEF'"}]}, + {"code":"CE0149","locations":[{"document-name":"Page 'P_role'","element-name":"Container 'c'"}]}, + {"code":"CE1613","locations":[{"document-name":"Microflow 'M_ref'","element-name":"x"},{"document-name":"Microflow 'M_ref2'","element-name":"y"}]} + ]}`) + f, err := ParseFindings(raw) + if err != nil { + t.Fatal(err) + } + if len(f["S_UNDEF"]) != 1 || f["S_UNDEF"][0].Code != "CE0109" { + t.Errorf("a microflow is keyed by its bare name: %v", f) + } + if len(f["Page 'P_role'"]) != 1 { + t.Errorf("a document that is not a microflow keeps its full name: %v", f) + } + if len(f["M_ref"]) != 1 || len(f["M_ref2"]) != 1 { + t.Errorf("an error with two locations is on both documents: %v", f) + } + if _, err := ParseFindings([]byte("not json")); err == nil { + t.Error("a report that is not JSON must be refused") + } +} + +// staged builds the findings of a checker that reports in stages: a document shows only +// the errors of the earliest stage any of its defects belongs to. +func staged(ds []Defect, docs map[string][]string, stage map[string]int, code map[string]string) Findings { + f := Findings{} + for doc, defects := range docs { + min := 99 + for _, d := range defects { + if stage[d] < min { + min = stage[d] + } + } + for _, d := range defects { + if stage[d] == min { + f[doc] = append(f[doc], Finding{Code: code[d]}) + } + } + } + return f +} + +func defectsNamed(names ...string) []Defect { + var ds []Defect + for _, n := range names { + ds = append(ds, Defect{Name: n}) + } + return ds +} + +func TestAnalyzeFindsTheStages(t *testing.T) { + ds := defectsNamed("DUP", "UNDEF", "TYPE", "LOOP") + g := Generate(ds) + // DUP < UNDEF < {TYPE, LOOP}: the same shape MxBuild showed. + f := staged(ds, g.Docs, + map[string]int{"DUP": 1, "UNDEF": 2, "TYPE": 3, "LOOP": 3}, + map[string]string{"DUP": "CE1", "UNDEF": "CE2", "TYPE": "CE3", "LOOP": "CE4"}) + r := Analyze(ds, g.Docs, f) + want := [][]string{{"DUP"}, {"UNDEF"}, {"TYPE", "LOOP"}} + if !reflect.DeepEqual(r.Stages, want) { + t.Errorf("Stages = %v, want %v", r.Stages, want) + } + if got := r.Hides["DUP"]; !reflect.DeepEqual(got, []string{"LOOP", "TYPE", "UNDEF"}) { + t.Errorf("DUP hides %v", got) + } + if got := r.Hides["UNDEF"]; !reflect.DeepEqual(got, []string{"LOOP", "TYPE"}) { + t.Errorf("UNDEF hides %v", got) + } + if len(r.Hides["TYPE"]) != 0 || len(r.Hides["LOOP"]) != 0 { + t.Errorf("defects reported together hide nothing: %v", r.Hides) + } + if len(r.OrderDependent) != 0 || len(r.Silent) != 0 || len(r.SharedCodes) != 0 { + t.Errorf("unexpected: order %v silent %v shared %v", r.OrderDependent, r.Silent, r.SharedCodes) + } +} + +func TestAnalyzeCheckerThatDoesNotStage(t *testing.T) { + ds := defectsNamed("A", "B", "C") + g := Generate(ds) + f := Findings{} + for doc, defects := range g.Docs { + for _, d := range defects { + f[doc] = append(f[doc], Finding{Code: "CE-" + d}) + } + } + r := Analyze(ds, g.Docs, f) + if len(r.Hides) != 0 || !reflect.DeepEqual(r.Stages, [][]string{{"A", "B", "C"}}) { + t.Errorf("a checker that reports everything has one stage and hides nothing: %v %v", r.Stages, r.Hides) + } +} + +func TestAnalyzeOrderDependence(t *testing.T) { + ds := defectsNamed("A", "B") + g := Generate(ds) + // Whichever defect comes first in a document is the one reported. + f := Findings{ + "S_A": {{Code: "CA"}}, "S_B": {{Code: "CB"}}, + "X_A__B": {{Code: "CA"}}, "X_B__A": {{Code: "CB"}}, + } + r := Analyze(ds, g.Docs, f) + if !reflect.DeepEqual(r.OrderDependent, [][2]string{{"A", "B"}}) { + t.Errorf("OrderDependent = %v, want [[A B]]", r.OrderDependent) + } + if len(r.Hides) != 0 { + t.Errorf("an order-dependent pair is not a 'hides' relation: %v", r.Hides) + } +} + +func TestAnalyzeSilentAndSharedCodes(t *testing.T) { + ds := defectsNamed("A", "B", "C") + g := Generate(ds) + f := Findings{ + "S_A": {{Code: "CX"}}, "S_B": {{Code: "CX"}}, // A and B raise one code + // C's own document is clean: the checker does not see it + } + r := Analyze(ds, g.Docs, f) + if !reflect.DeepEqual(r.Silent, []string{"C"}) { + t.Errorf("Silent = %v, want [C]", r.Silent) + } + if !reflect.DeepEqual(r.SharedCodes, []string{"CX"}) { + t.Errorf("SharedCodes = %v, want [CX]", r.SharedCodes) + } + for _, st := range r.Stages { + if contains(st, "C") { + t.Errorf("a defect the checker leaves clean is not placed in a stage: %v", r.Stages) + } + } +} + +func TestReport(t *testing.T) { + ds := defectsNamed("DUP", "UNDEF") + g := Generate(ds) + f := staged(ds, g.Docs, map[string]int{"DUP": 1, "UNDEF": 2}, map[string]string{"DUP": "CE0111", "UNDEF": "CE0109"}) + var b bytes.Buffer + Report(&b, ds, Analyze(ds, g.Docs, f)) + for _, want := range []string{"DUP CE0111", "DUP hides UNDEF", "no pair differs when swapped", "1 DUP", "2 UNDEF"} { + if !strings.Contains(b.String(), want) { + t.Errorf("report lacks %q:\n%s", want, b.String()) + } + } +} + +func TestCopyProjectLeavesOutTheBinaryAndGit(t *testing.T) { + src, dst := t.TempDir(), filepath.Join(t.TempDir(), "copy") + for path, content := range map[string]string{ + "App.mpr": "mpr", "mprcontents/ab/unit.mxunit": "unit", + "mxcli": "a 110 MB binary", ".git/HEAD": "ref", "modules/mxcli": "a file that merely shares the name", + } { + p := filepath.Join(src, path) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + if err := copyProject(src, dst); err != nil { + t.Fatal(err) + } + for _, kept := range []string{"App.mpr", "mprcontents/ab/unit.mxunit", "modules/mxcli"} { + if _, err := os.Stat(filepath.Join(dst, kept)); err != nil { + t.Errorf("%s was not copied", kept) + } + } + for _, left := range []string{"mxcli", ".git"} { + if _, err := os.Stat(filepath.Join(dst, left)); err == nil { + t.Errorf("%s was copied; only the project root's copy is skipped", left) + } + } +} + +func TestSelectDefectsAndFlags(t *testing.T) { + if _, err := selectDefects("UNDEF,NOPE"); err == nil || !strings.Contains(err.Error(), "NOPE") { + t.Errorf("an unknown defect must be named, got %v", err) + } + if _, err := selectDefects("UNDEF"); err == nil { + t.Error("one defect cannot be compared with anything") + } + if ds, err := selectDefects("TYPE,UNDEF"); err != nil || len(ds) != 2 || ds[0].Name != "TYPE" { + t.Errorf("a subset keeps the order given: %v %v", ds, err) + } + var out, errOut bytes.Buffer + if err := run(nil, &out, &errOut); err == nil || !strings.Contains(err.Error(), "--project") { + t.Errorf("--project and --mx are required, got %v", err) + } +} diff --git a/sdk/javaactions/jssource.go b/sdk/javaactions/jssource.go index 167cfd6aa0..c210112ff9 100644 --- a/sdk/javaactions/jssource.go +++ b/sdk/javaactions/jssource.go @@ -9,64 +9,116 @@ import ( "unicode/utf8" ) -// GenerateJavaScriptSource renders the .js stub Studio Pro writes for a -// JavaScript action: the generated-file banner, an EXTRA CODE section, a JSDoc -// block, and an exported async function wrapping the user code between -// BEGIN/END USER CODE markers. It is pure (depends only on the action's shape), -// so both engines share it for byte-identical output. userCode is placed inside -// the USER CODE markers; when empty a not-implemented stub is emitted. imports -// and extraCode are the sections a regeneration retains from the existing file -// (JavaScriptRetainedSections); both are empty for a new action. +// GenerateJavaScriptSource is JSSource.Generate for an action with no +// documentation and no enumeration values to name. func GenerateJavaScriptSource(actionName string, userCode string, params []*JavaActionParameter, returnType CodeActionReturnType, imports []string, extraCode string) string { + return JSSource{Action: actionName, UserCode: userCode, Params: params, ReturnType: returnType, + Imports: imports, ExtraCode: extraCode}.Generate() +} + +// JSSource is what a JavaScript action's .js file is generated from. +type JSSource struct { + Action string + Documentation string // the action's own, written at the top of the JSDoc + UserCode string + Params []*JavaActionParameter + ReturnType CodeActionReturnType + // Imports and ExtraCode are the sections a regeneration keeps from the + // existing file (JavaScriptRetainedSections); empty for a new action. + Imports []string + ExtraCode string + // EnumValues names an enumeration's values, which the JSDoc spells out as + // a union; nil leaves an enumeration typed as a string. + EnumValues func(qualifiedName string) []string +} + +// importBig is the import mxbuild writes in every JavaScript action file. +const importBig = `import { Big } from "big.js";` + +// Generate renders the .js file exactly as mxbuild writes it — CRLF line +// endings included — so a build leaves the file mxcli wrote alone, and a +// re-run of an unchanged script finds nothing to write (ako/mxcli#2357). The +// shape is measured from mxbuild 11.12.6 (--target=deploy) regenerating +// actions mxcli created (testdata/mxbuild/JS_*_11.12.6.js.golden): the banner +// names the import list among what a regeneration keeps; the Big import is +// always there; the action's documentation opens the JSDoc; an optional +// parameter is `[name]`; Integer, Long and Decimal are Big; an object is +// MxObject and a list MxObject[]; an enumeration is the union of its values. +func (s JSSource) Generate() string { var sb strings.Builder sb.WriteString("// This file was generated by Mendix Studio Pro.\n") sb.WriteString("//\n") sb.WriteString("// WARNING: Only the following code will be retained when actions are regenerated:\n") + sb.WriteString("// - the import list\n") sb.WriteString("// - the code between BEGIN USER CODE and END USER CODE\n") sb.WriteString("// - the code between BEGIN EXTRA CODE and END EXTRA CODE\n") sb.WriteString("// Other code you write will be lost the next time you deploy the project.\n") - sb.WriteString("// Special characters, e.g., é, ö, à, etc. are supported in comments.\n\n") - if len(imports) > 0 { - for _, imp := range imports { - sb.WriteString(imp + "\n") - } - sb.WriteString("\n") + for _, imp := range completeJSImports(s.Imports) { + sb.WriteString(imp + "\n") } + sb.WriteString("\n") sb.WriteString("// BEGIN EXTRA CODE\n") - if strings.TrimSpace(extraCode) != "" { - sb.WriteString(strings.TrimRight(extraCode, "\n") + "\n") + if strings.TrimSpace(s.ExtraCode) != "" { + sb.WriteString(strings.TrimRight(s.ExtraCode, "\n") + "\n") } sb.WriteString("// END EXTRA CODE\n\n") - // JSDoc: one @param per parameter, then the @returns. sb.WriteString("/**\n") - for _, p := range params { - sb.WriteString(fmt.Sprintf(" * @param {%s} %s\n", jsDocType(p.ParameterType), JavaScriptParameterName(p.Name))) + if s.Documentation != "" { + doc := strings.ReplaceAll(strings.ReplaceAll(s.Documentation, "\r\n", "\n"), "\r", "\n") + for _, line := range strings.Split(doc, "\n") { + sb.WriteString(" * " + line + "\n") + } } - sb.WriteString(fmt.Sprintf(" * @returns {%s}\n", jsDocReturnType(returnType))) + for _, p := range s.Params { + name := JavaScriptParameterName(p.Name) + if !p.IsRequired { + name = "[" + name + "]" + } + sb.WriteString(fmt.Sprintf(" * @param {%s} %s\n", s.jsDocType(p.ParameterType), name)) + } + sb.WriteString(fmt.Sprintf(" * @returns {Promise.<%s>}\n", s.jsDocReturn(s.ReturnType))) sb.WriteString(" */\n") - // Exported async function with the action's parameters. - names := make([]string, 0, len(params)) - for _, p := range params { + names := make([]string, 0, len(s.Params)) + for _, p := range s.Params { names = append(names, JavaScriptParameterName(p.Name)) } - sb.WriteString(fmt.Sprintf("export async function %s(%s) {\n", actionName, strings.Join(names, ", "))) + sb.WriteString(fmt.Sprintf("export async function %s(%s) {\n", s.Action, strings.Join(names, ", "))) sb.WriteString("\t// BEGIN USER CODE\n") - if strings.TrimSpace(userCode) == "" { - sb.WriteString("\tthrow new Error(\"JavaScript action '" + actionName + "' was not implemented\");\n") + if strings.TrimSpace(s.UserCode) == "" { + sb.WriteString("\tthrow new Error(\"JavaScript action '" + s.Action + "' was not implemented\");\n") } else { - for _, line := range strings.Split(strings.TrimRight(userCode, "\n"), "\n") { + for _, line := range strings.Split(strings.TrimRight(s.UserCode, "\n"), "\n") { sb.WriteString("\t" + line + "\n") } } sb.WriteString("\t// END USER CODE\n") sb.WriteString("}\n") - return sb.String() + // mxbuild writes CRLF, and rewrites an LF file that is otherwise the same. + out := strings.ReplaceAll(sb.String(), "\r\n", "\n") + return strings.ReplaceAll(out, "\n", "\r\n") +} + +// completeJSImports is the retained import list with the Big import appended +// when it lacks one, deduplicated, in the order found. +func completeJSImports(imports []string) []string { + out := make([]string, 0, len(imports)+1) + seen := map[string]bool{} + for _, imp := range imports { + if imp = strings.TrimSpace(imp); imp != "" && !seen[imp] { + seen[imp] = true + out = append(out, imp) + } + } + if !seen[importBig] { + out = append(out, importBig) + } + return out } // JavaScriptParameterName returns the identifier a parameter has in the .js @@ -94,41 +146,72 @@ func JavaScriptParameterName(name string) string { return string(unicode.ToLower(r)) + name[size:] } -// jsDocType maps a parameter type to a JSDoc type annotation (best-effort). -func jsDocType(t CodeActionParameterType) string { +// jsDocType is a parameter's JSDoc type, as mxbuild writes it. +func (s JSSource) jsDocType(t CodeActionParameterType) string { switch typeNameOf(t) { case "BooleanType": return "boolean" case "IntegerType", "LongType", "DecimalType": - return "number" + return "Big" case "StringType", "StringTemplateParameterType": return "string" case "DateTimeType": return "Date" + case "EnumerationType": + return s.enumUnion(t) + case "EntityType", "FileDocumentType", "TypeParameter": + return "MxObject" + case "ListType": + return "MxObject[]" default: return "*" } } -// jsDocReturnType maps a return type to a JSDoc `Promise.<...>` annotation. -func jsDocReturnType(t CodeActionReturnType) string { +// jsDocReturn is a return type's JSDoc type inside Promise.<…>. +func (s JSSource) jsDocReturn(t CodeActionReturnType) string { if t == nil { - return "Promise." + return "boolean" } switch typeNameOf(t) { case "BooleanType": - return "Promise." + return "boolean" case "IntegerType", "LongType", "DecimalType": - return "Promise." + return "Big" case "StringType": - return "Promise." + return "string" case "DateTimeType": - return "Promise." + return "Date" case "VoidType", "": - return "Promise." + return "void" + case "EnumerationType": + return s.enumUnion(t) + case "ListType": + return "MxObject[]" default: - return "Promise." + return "MxObject" + } +} + +// enumUnion is an enumeration's JSDoc type: undefined or one of its values' +// names — `undefined|"Red"|"Blue"`. Without its values it is a string. +func (s JSSource) enumUnion(t any) string { + qn := "" + if e, ok := t.(*EnumerationType); ok { + qn = e.Enumeration + } + var values []string + if s.EnumValues != nil && qn != "" { + values = s.EnumValues(qn) + } + if len(values) == 0 { + return "string" + } + parts := []string{"undefined"} + for _, v := range values { + parts = append(parts, `"`+v+`"`) } + return strings.Join(parts, "|") } // typeNameOf returns the short Go type name of a code-action type element diff --git a/sdk/javaactions/jssource_golden_test.go b/sdk/javaactions/jssource_golden_test.go index db253ae1d6..19f9d974d8 100644 --- a/sdk/javaactions/jssource_golden_test.go +++ b/sdk/javaactions/jssource_golden_test.go @@ -58,7 +58,7 @@ func TestGenerateJavaScriptSource_SignatureMatchesMxbuild(t *testing.T) { } golden := strings.ReplaceAll(string(b), "\r\n", "\n") imports, extra := JavaScriptRetainedSections(golden) - got := GenerateJavaScriptSource(c.name, "return 1;", c.params, &StringType{}, imports, extra) + got := strings.ReplaceAll(GenerateJavaScriptSource(c.name, "return 1;", c.params, &StringType{}, imports, extra), "\r\n", "\n") wantSig, wantDoc := jsIdentifiers(golden) gotSig, gotDoc := jsIdentifiers(got) diff --git a/sdk/javaactions/jssource_mxbuild_2357_test.go b/sdk/javaactions/jssource_mxbuild_2357_test.go new file mode 100644 index 0000000000..3bfd2da5a7 --- /dev/null +++ b/sdk/javaactions/jssource_mxbuild_2357_test.go @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 + +package javaactions + +import ( + "os" + "path/filepath" + "testing" +) + +// ako/mxcli#2357: a re-run of an unchanged script rewrote every JavaScript +// action's .js file, dropping the action's documentation from the JSDoc. The +// file on disk was mxbuild's, and mxcli's generator wrote another shape. Since +// #2131 changed the user code's indentation, every re-run wrote that other +// shape over mxbuild's. +// +// The goldens *_11.12.6.js.golden are the files mxbuild 11.12.6 +// (--target=deploy) regenerated, byte for byte (CRLF), for actions mxcli +// created. The generator must write the same bytes, so a build leaves the file +// alone and a re-run finds nothing to write. + +func enumValues(qn string) []string { + if qn == "MyFirstModule.Color" { + return []string{"Red", "Blue"} + } + return nil +} + +func requiredParam(name string, t CodeActionParameterType) *JavaActionParameter { + p := param(name, t) + p.IsRequired = true + return p +} + +func TestGenerateJavaScriptSource_MatchesMxbuild_Issue2357(t *testing.T) { + thing := &EntityType{Entity: "MyFirstModule.Thing"} + cases := []struct { + golden string + src JSSource + }{ + {"JS_CopyText_11.12.6.js.golden", JSSource{ + Action: "JS_CopyText", + Documentation: "Copies text to the clipboard (\"Copy PGN\", \"Copy FEN\").\nSecond line of the documentation.", + UserCode: "return true;", + Params: []*JavaActionParameter{param("Text", &StringType{}), requiredParam("Count", &IntegerType{})}, + ReturnType: &BooleanType{}, + }}, + {"JS_Types_11.12.6.js.golden", JSSource{ + Action: "JS_Types", + UserCode: "return null;", + Params: []*JavaActionParameter{ + param("PInt", &IntegerType{}), param("PLong", &LongType{}), param("PDec", &DecimalType{}), + param("PBool", &BooleanType{}), param("PDate", &DateTimeType{}), param("PStr", &StringType{}), + param("PEnum", &EnumerationType{Enumeration: "MyFirstModule.Color"}), + param("PObj", thing), param("PList", &ListType{Entity: "MyFirstModule.Thing"}), + }, + ReturnType: &DecimalType{}, + EnumValues: enumValues, + }}, + {"JS_REnum_11.12.6.js.golden", JSSource{Action: "JS_REnum", UserCode: "return null;", + ReturnType: &EnumerationType{Enumeration: "MyFirstModule.Color"}, EnumValues: enumValues}}, + {"JS_RList_11.12.6.js.golden", JSSource{Action: "JS_RList", UserCode: "return null;", + ReturnType: &ListType{Entity: "MyFirstModule.Thing"}}}, + {"JS_RVoid_11.12.6.js.golden", JSSource{Action: "JS_RVoid", UserCode: "return;", ReturnType: &VoidType{}}}, + } + for _, c := range cases { + t.Run(c.golden, func(t *testing.T) { + want, err := os.ReadFile(filepath.Join("testdata", "mxbuild", c.golden)) + if err != nil { + t.Fatal(err) + } + if got := c.src.Generate(); got != string(want) { + t.Errorf("generated source differs from mxbuild's\n got: %q\nwant: %q", got, want) + } + }) + } +} + +// A regeneration keeps the import list it finds, and adds the Big import only +// when the list lacks it: mxbuild's own file regenerates to itself. +func TestGenerateJavaScriptSource_RegeneratesMxbuildsFile_Issue2357(t *testing.T) { + want, err := os.ReadFile(filepath.Join("testdata", "mxbuild", "JS_CopyText_11.12.6.js.golden")) + if err != nil { + t.Fatal(err) + } + imports, extra := JavaScriptRetainedSections(string(want)) + src := JSSource{ + Action: "JS_CopyText", + Documentation: "Copies text to the clipboard (\"Copy PGN\", \"Copy FEN\").\nSecond line of the documentation.", + UserCode: "return true;", + Params: []*JavaActionParameter{param("Text", &StringType{}), requiredParam("Count", &IntegerType{})}, + ReturnType: &BooleanType{}, + Imports: imports, + ExtraCode: extra, + } + if got := src.Generate(); got != string(want) { + t.Errorf("regenerating mxbuild's file changed it\n got: %q\nwant: %q", got, want) + } +} diff --git a/sdk/javaactions/jssource_test.go b/sdk/javaactions/jssource_test.go index bf6f73bf2a..31dbdb43d6 100644 --- a/sdk/javaactions/jssource_test.go +++ b/sdk/javaactions/jssource_test.go @@ -41,13 +41,14 @@ func TestJavaScriptParameterName_MatchesMxbuild(t *testing.T) { } func TestGenerateJavaScriptSource_ParameterNamesAreLowerFirst(t *testing.T) { - src := GenerateJavaScriptSource("JS_Open", "return Url;", + src := strings.ReplaceAll(GenerateJavaScriptSource("JS_Open", "return Url;", []*JavaActionParameter{param("Url", &StringType{}), param("ReadOnly", &BooleanType{})}, - &StringType{}, nil, "") + &StringType{}, nil, ""), "\r\n", "\n") + // Not marked required, so optional: `[name]`, as mxbuild writes it. for _, want := range []string{ - " * @param {string} url\n", - " * @param {boolean} readOnly\n", + " * @param {string} [url]\n", + " * @param {boolean} [readOnly]\n", "export async function JS_Open(url, readOnly) {\n", } { if !strings.Contains(src, want) { diff --git a/sdk/javaactions/testdata/mxbuild/JS_CopyText_11.12.6.js.golden b/sdk/javaactions/testdata/mxbuild/JS_CopyText_11.12.6.js.golden new file mode 100644 index 0000000000..a59fb19de5 --- /dev/null +++ b/sdk/javaactions/testdata/mxbuild/JS_CopyText_11.12.6.js.golden @@ -0,0 +1,24 @@ +// This file was generated by Mendix Studio Pro. +// +// WARNING: Only the following code will be retained when actions are regenerated: +// - the import list +// - the code between BEGIN USER CODE and END USER CODE +// - the code between BEGIN EXTRA CODE and END EXTRA CODE +// Other code you write will be lost the next time you deploy the project. +import { Big } from "big.js"; + +// BEGIN EXTRA CODE +// END EXTRA CODE + +/** + * Copies text to the clipboard ("Copy PGN", "Copy FEN"). + * Second line of the documentation. + * @param {string} [text] + * @param {Big} count + * @returns {Promise.} + */ +export async function JS_CopyText(text, count) { + // BEGIN USER CODE + return true; + // END USER CODE +} diff --git a/sdk/javaactions/testdata/mxbuild/JS_REnum_11.12.6.js.golden b/sdk/javaactions/testdata/mxbuild/JS_REnum_11.12.6.js.golden new file mode 100644 index 0000000000..8219f39dc2 --- /dev/null +++ b/sdk/javaactions/testdata/mxbuild/JS_REnum_11.12.6.js.golden @@ -0,0 +1,20 @@ +// This file was generated by Mendix Studio Pro. +// +// WARNING: Only the following code will be retained when actions are regenerated: +// - the import list +// - the code between BEGIN USER CODE and END USER CODE +// - the code between BEGIN EXTRA CODE and END EXTRA CODE +// Other code you write will be lost the next time you deploy the project. +import { Big } from "big.js"; + +// BEGIN EXTRA CODE +// END EXTRA CODE + +/** + * @returns {Promise.} + */ +export async function JS_REnum() { + // BEGIN USER CODE + return null; + // END USER CODE +} diff --git a/sdk/javaactions/testdata/mxbuild/JS_RList_11.12.6.js.golden b/sdk/javaactions/testdata/mxbuild/JS_RList_11.12.6.js.golden new file mode 100644 index 0000000000..5dd6551598 --- /dev/null +++ b/sdk/javaactions/testdata/mxbuild/JS_RList_11.12.6.js.golden @@ -0,0 +1,20 @@ +// This file was generated by Mendix Studio Pro. +// +// WARNING: Only the following code will be retained when actions are regenerated: +// - the import list +// - the code between BEGIN USER CODE and END USER CODE +// - the code between BEGIN EXTRA CODE and END EXTRA CODE +// Other code you write will be lost the next time you deploy the project. +import { Big } from "big.js"; + +// BEGIN EXTRA CODE +// END EXTRA CODE + +/** + * @returns {Promise.} + */ +export async function JS_RList() { + // BEGIN USER CODE + return null; + // END USER CODE +} diff --git a/sdk/javaactions/testdata/mxbuild/JS_RVoid_11.12.6.js.golden b/sdk/javaactions/testdata/mxbuild/JS_RVoid_11.12.6.js.golden new file mode 100644 index 0000000000..67f1a3ecf5 --- /dev/null +++ b/sdk/javaactions/testdata/mxbuild/JS_RVoid_11.12.6.js.golden @@ -0,0 +1,20 @@ +// This file was generated by Mendix Studio Pro. +// +// WARNING: Only the following code will be retained when actions are regenerated: +// - the import list +// - the code between BEGIN USER CODE and END USER CODE +// - the code between BEGIN EXTRA CODE and END EXTRA CODE +// Other code you write will be lost the next time you deploy the project. +import { Big } from "big.js"; + +// BEGIN EXTRA CODE +// END EXTRA CODE + +/** + * @returns {Promise.} + */ +export async function JS_RVoid() { + // BEGIN USER CODE + return; + // END USER CODE +} diff --git a/sdk/javaactions/testdata/mxbuild/JS_Types_11.12.6.js.golden b/sdk/javaactions/testdata/mxbuild/JS_Types_11.12.6.js.golden new file mode 100644 index 0000000000..037889ce4d --- /dev/null +++ b/sdk/javaactions/testdata/mxbuild/JS_Types_11.12.6.js.golden @@ -0,0 +1,29 @@ +// This file was generated by Mendix Studio Pro. +// +// WARNING: Only the following code will be retained when actions are regenerated: +// - the import list +// - the code between BEGIN USER CODE and END USER CODE +// - the code between BEGIN EXTRA CODE and END EXTRA CODE +// Other code you write will be lost the next time you deploy the project. +import { Big } from "big.js"; + +// BEGIN EXTRA CODE +// END EXTRA CODE + +/** + * @param {Big} [pInt] + * @param {Big} [pLong] + * @param {Big} [pDec] + * @param {boolean} [pBool] + * @param {Date} [pDate] + * @param {string} [pStr] + * @param {undefined|"Red"|"Blue"} [pEnum] + * @param {MxObject} [pObj] + * @param {MxObject[]} [pList] + * @returns {Promise.} + */ +export async function JS_Types(pInt, pLong, pDec, pBool, pDate, pStr, pEnum, pObj, pList) { + // BEGIN USER CODE + return null; + // END USER CODE +} diff --git a/sql/describe_qualified_2361_test.go b/sql/describe_qualified_2361_test.go new file mode 100644 index 0000000000..fcbd654dbc --- /dev/null +++ b/sql/describe_qualified_2361_test.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sql + +import ( + "context" + "strings" + "testing" +) + +// ako/mxcli#2361: `--tables` prints schema and table apart, so the next step +// is `--describe sales.customers` — which returned zero rows and no error, as +// if the table had no columns. +func TestDescribeTable_SchemaQualified_Issue2361(t *testing.T) { + c := pgConn(t) + if _, err := c.DB.Exec(`create schema if not exists dq; drop table if exists dq.customers; + create table dq.customers (id serial primary key, name varchar(50) not null, email text)`); err != nil { + t.Fatal(err) + } + ctx := context.Background() + + res, err := DescribeTable(ctx, c, "dq.customers") + if err != nil { + t.Fatal(err) + } + if len(res.Rows) != 3 { + t.Errorf("dq.customers: %d columns, want 3", len(res.Rows)) + } + + // CONTROL: the unqualified form, which worked before, still does for a + // name one schema holds. + if _, err := c.DB.Exec(`drop table if exists dq.only_here; create table dq.only_here (a int, b int)`); err != nil { + t.Fatal(err) + } + if res, err := DescribeTable(ctx, c, "only_here"); err != nil || len(res.Rows) != 2 { + t.Errorf("only_here: %v, %+v", err, res) + } + + // A name in two schemas is refused, naming both, rather than listing both + // tables' columns as one. + if _, err := c.DB.Exec(`create schema if not exists dq2; drop table if exists dq2.customers; create table dq2.customers (x int)`); err != nil { + t.Fatal(err) + } + if _, err := DescribeTable(ctx, c, "customers"); err == nil || + !strings.Contains(err.Error(), "dq.customers") || !strings.Contains(err.Error(), "dq2.customers") { + t.Errorf("ambiguous customers: err = %v, want one naming dq.customers and dq2.customers", err) + } + + // A table that does not exist is an error naming it, not an empty list. + _, err = DescribeTable(ctx, c, "dq.nosuch") + if err == nil || !strings.Contains(err.Error(), "dq.nosuch") { + t.Errorf("missing table: err = %v, want one naming dq.nosuch", err) + } +} + +func TestSplitTableName(t *testing.T) { + for in, want := range map[string][2]string{ + "customers": {"", "customers"}, + "sales.customers": {"sales", "customers"}, + `"Sales"."Cust"`: {"Sales", "Cust"}, + "db.sales.customers": {"sales", "customers"}, + } { + s, n := splitTableName(in) + if s != want[0] || n != want[1] { + t.Errorf("splitTableName(%q) = %q, %q; want %q, %q", in, s, n, want[0], want[1]) + } + } +} diff --git a/sql/meta.go b/sql/meta.go index cdca902da2..e2f1e52951 100644 --- a/sql/meta.go +++ b/sql/meta.go @@ -25,13 +25,13 @@ ORDER BY TABLE_SCHEMA, TABLE_NAME`, } // describeQuery maps driver to the query that describes a table's columns. -// The placeholder %s will be replaced with the table name. +// The placeholder %s is replaced with the WHERE filter for the table. var describeQuery = map[DriverName]string{ DriverPostgres: `SELECT column_name, data_type, CASE WHEN is_nullable = 'YES' THEN 'YES' ELSE 'NO' END AS nullable, column_default FROM information_schema.columns -WHERE table_name = '%s' +WHERE %s ORDER BY ordinal_position`, DriverOracle: `SELECT column_name, data_type, data_length, nullable FROM all_tab_columns @@ -41,7 +41,7 @@ ORDER BY column_id`, IS_NULLABLE AS nullable, COLUMN_DEFAULT AS column_default FROM INFORMATION_SCHEMA.COLUMNS -WHERE TABLE_NAME = '%s' +WHERE %s ORDER BY ORDINAL_POSITION`, } @@ -96,32 +96,99 @@ func ShowTables(ctx context.Context, conn *Connection) (*QueryResult, error) { return Execute(ctx, conn, query) } -// DescribeTable returns the column definitions for a table or view. +// DescribeTable returns the column definitions for a table or view, named +// as `table` or `schema.table` — the two columns --tables prints +// (ako/mxcli#2361). A table that is not found is an error, not an empty +// column list. func DescribeTable(ctx context.Context, conn *Connection, table string) (*QueryResult, error) { tmpl, ok := describeQuery[conn.Driver] if !ok { return nil, fmt.Errorf("DESCRIBE TABLE not supported for driver %s", conn.Driver) } - var query string - if conn.Driver == DriverOracle { - // Oracle: split SCHEMA.TABLE into owner + table_name filter - query = fmt.Sprintf(tmpl, oracleTableFilter(table)) - } else { - query = fmt.Sprintf(tmpl, table) + if err := refuseAmbiguousTable(ctx, conn, table); err != nil { + return nil, err } - return Execute(ctx, conn, query) + var filter string + switch conn.Driver { + case DriverOracle: + filter = oracleTableFilter(table) + case DriverSQLServer: + filter = columnsFilter(table, "TABLE_SCHEMA", "TABLE_NAME") + default: + filter = columnsFilter(table, "table_schema", "table_name") + } + res, err := Execute(ctx, conn, fmt.Sprintf(tmpl, filter)) + if err != nil { + return nil, err + } + if len(res.Rows) == 0 { + return nil, fmt.Errorf("table or view %s not found: name it as --tables prints it, table_name or table_schema.table_name", table) + } + return res, nil +} + +// refuseAmbiguousTable refuses an unqualified name that several schemas +// hold: their columns would be listed as one table's. +func refuseAmbiguousTable(ctx context.Context, conn *Connection, table string) error { + schema, name := splitTableName(table) + if schema != "" { + return nil + } + var q string + switch conn.Driver { + case DriverOracle: + q = "SELECT DISTINCT owner FROM all_tab_columns WHERE table_name = UPPER(" + sqlString(name) + ") ORDER BY 1" + case DriverSQLServer: + q = "SELECT DISTINCT TABLE_SCHEMA FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = " + sqlString(name) + " ORDER BY 1" + default: + q = "SELECT DISTINCT table_schema FROM information_schema.columns WHERE table_name = " + sqlString(name) + " ORDER BY 1" + } + res, err := Execute(ctx, conn, q) + if err != nil || len(res.Rows) < 2 { + return nil + } + var names []string + for _, r := range res.Rows { + names = append(names, fmt.Sprintf("%v.%s", r[0], name)) + } + return fmt.Errorf("%s is in more than one schema: name one of %s", name, strings.Join(names, ", ")) +} + +// columnsFilter is the information_schema filter for table, qualified by its +// schema when one is given. +func columnsFilter(table, schemaCol, nameCol string) string { + schema, name := splitTableName(table) + f := fmt.Sprintf("%s = %s", nameCol, sqlString(name)) + if schema != "" { + f = fmt.Sprintf("%s = %s AND %s", schemaCol, sqlString(schema), f) + } + return f } +// splitTableName splits `schema.table` (a leading database part is dropped) +// and removes identifier quotes. +func splitTableName(table string) (schema, name string) { + parts := strings.Split(table, ".") + for i, p := range parts { + parts[i] = strings.Trim(strings.TrimSpace(p), `"[]`+"`") + } + if len(parts) == 1 { + return "", parts[0] + } + return parts[len(parts)-2], parts[len(parts)-1] +} + +func sqlString(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" } + // oracleTableFilter builds a WHERE clause for Oracle's all_tab_columns. // Supports both "TABLE" and "SCHEMA.TABLE" formats. func oracleTableFilter(table string) string { - if i := strings.IndexByte(table, '.'); i >= 0 { - owner := strings.ToUpper(table[:i]) - name := strings.ToUpper(table[i+1:]) - return fmt.Sprintf("owner = '%s' AND table_name = '%s'", owner, name) + schema, name := splitTableName(table) + if schema != "" { + return fmt.Sprintf("owner = %s AND table_name = %s", sqlString(strings.ToUpper(schema)), sqlString(strings.ToUpper(name))) } - return fmt.Sprintf("table_name = UPPER('%s')", table) + return fmt.Sprintf("table_name = UPPER(%s)", sqlString(name)) } // ShowViews returns a list of views for the given connection. diff --git a/sql/query.go b/sql/query.go index 51ee98218e..08f98f0053 100644 --- a/sql/query.go +++ b/sql/query.go @@ -4,6 +4,7 @@ package sql import ( "context" + "database/sql" "fmt" ) @@ -11,6 +12,8 @@ import ( type QueryResult struct { Columns []string Rows [][]any + // RowsAffected is set for a statement run as a write: the rows it changed. + RowsAffected *int64 } // Execute runs a SQL query and returns the result. @@ -20,7 +23,11 @@ func Execute(ctx context.Context, conn *Connection, query string) (*QueryResult, return nil, fmt.Errorf("query failed: %w", err) } defer rows.Close() + return scanRows(rows) +} +// scanRows reads every row of rows. +func scanRows(rows *sql.Rows) (*QueryResult, error) { cols, err := rows.Columns() if err != nil { return nil, fmt.Errorf("failed to get columns: %w", err) diff --git a/sql/readonly.go b/sql/readonly.go new file mode 100644 index 0000000000..c0e3c6cb57 --- /dev/null +++ b/sql/readonly.go @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sql + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "unicode" +) + +// ErrWriteNotAllowed is the refusal of a statement that is not a read when +// writes were not allowed. +var ErrWriteNotAllowed = errors.New("statement is not a read, and writes are not allowed") + +// readKeywords are the leading keywords of statements that read. A statement +// starting with one of them can still write (a data-modifying CTE, EXPLAIN +// ANALYZE of a DELETE), which is why a read also runs in a transaction that +// is read-only and rolled back. +var readKeywords = map[string]bool{ + "select": true, "with": true, "explain": true, "show": true, + "values": true, "table": true, "describe": true, "desc": true, +} + +// ReadStatement reports whether query starts, after comments and opening +// parentheses, with a keyword of a reading statement. +func ReadStatement(query string) bool { + return readKeywords[leadingKeyword(query)] +} + +func leadingKeyword(q string) string { + for { + q = strings.TrimLeftFunc(q, func(r rune) bool { return unicode.IsSpace(r) || r == '(' }) + switch { + case strings.HasPrefix(q, "--"): + if i := strings.IndexByte(q, '\n'); i >= 0 { + q = q[i+1:] + continue + } + return "" + case strings.HasPrefix(q, "/*"): + if i := strings.Index(q, "*/"); i >= 0 { + q = q[i+2:] + continue + } + return "" + } + end := strings.IndexFunc(q, func(r rune) bool { return !unicode.IsLetter(r) && r != '_' }) + if end < 0 { + end = len(q) + } + return strings.ToLower(q[:end]) + } +} + +// Run executes query. Without allowWrite (ako/mxcli#2362) a statement that is +// not a read is refused, and a read runs in a read-only transaction that is +// rolled back, so a write hiding in a read — `with d as (delete …) select …` +// — fails instead of changing data. With allowWrite a non-read statement runs +// as an exec and the result carries the rows it affected. +func Run(ctx context.Context, conn *Connection, query string, allowWrite bool) (*QueryResult, error) { + read := ReadStatement(query) + switch { + case !read && !allowWrite: + kw := leadingKeyword(query) + if kw == "" { + kw = "this statement" + } else { + kw = strings.ToUpper(kw) + } + return nil, fmt.Errorf("%w: %s can change the database, and sql runs read-only unless --write is given", ErrWriteNotAllowed, kw) + case !read: + res, err := conn.DB.ExecContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("statement failed: %w", err) + } + out := &QueryResult{Rows: [][]any{}} + if n, err := res.RowsAffected(); err == nil { + out.RowsAffected = &n + } + return out, nil + case allowWrite: + return Execute(ctx, conn, query) + } + return executeReadOnly(ctx, conn, query) +} + +func executeReadOnly(ctx context.Context, conn *Connection, query string) (*QueryResult, error) { + // PostgreSQL honours a read-only transaction option; the SQL Server driver + // rejects it, and Oracle is told with a statement. The rollback is the + // guard that holds on every driver. + opts := &sql.TxOptions{ReadOnly: conn.Driver == DriverPostgres} + tx, err := conn.DB.BeginTx(ctx, opts) + if err != nil { + return nil, fmt.Errorf("begin read-only transaction: %w", err) + } + defer func() { _ = tx.Rollback() }() + if conn.Driver == DriverOracle { + if _, err := tx.ExecContext(ctx, "SET TRANSACTION READ ONLY"); err != nil { + return nil, fmt.Errorf("set read-only transaction: %w", err) + } + } + rows, err := tx.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("query failed: %w", err) + } + defer rows.Close() + return scanRows(rows) +} diff --git a/sql/readonly_2362_test.go b/sql/readonly_2362_test.go new file mode 100644 index 0000000000..bdcc20c01d --- /dev/null +++ b/sql/readonly_2362_test.go @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 + +package sql + +import ( + "context" + "errors" + "os" + "testing" +) + +// ako/mxcli#2362: `mxcli sql` ran any statement. `delete from sales.orders` +// printed "(0 rows)", which reads as an empty select, and the rows were gone. +// A statement now runs read-only unless writes are allowed. + +func TestReadStatement(t *testing.T) { + for q, want := range map[string]bool{ + "select 1": true, + " SELECT * from t": true, + "-- note\nselect 1": true, + "/* c */ with x as (select 1) select * from x": true, + "(select 1)": true, + "explain select 1": true, + "show search_path": true, + "values (1)": true, + "delete from sales.orders": false, + "update t set a = 1": false, + "insert into t values (1)": false, + "drop table t": false, + "truncate t": false, + "create table t (a int)": false, + "alter table t add b int": false, + "grant select on t to u": false, + "call p()": false, + "exec p": false, + "merge into t using s on (1=1) when matched then delete": false, + "": false, + } { + if got := ReadStatement(q); got != want { + t.Errorf("ReadStatement(%q) = %v, want %v", q, got, want) + } + } +} + +// pgConn connects to the PostgreSQL in MXCLI_TEST_PG_DSN, with sales.orders +// holding rows; skipped where there is none. +func pgConn(t *testing.T) *Connection { + t.Helper() + dsn := os.Getenv("MXCLI_TEST_PG_DSN") + if dsn == "" { + t.Skip("MXCLI_TEST_PG_DSN not set") + } + m := NewManager() + t.Cleanup(m.CloseAll) + if err := m.Connect(DriverPostgres, dsn, "t"); err != nil { + t.Fatal(err) + } + c, _ := m.Get("t") + if _, err := c.DB.Exec(`drop table if exists ro_probe; create table ro_probe(id int); insert into ro_probe values (1),(2)`); err != nil { + t.Fatal(err) + } + return c +} + +func probeCount(t *testing.T, c *Connection) int { + t.Helper() + var n int + if err := c.DB.QueryRow(`select count(*) from ro_probe`).Scan(&n); err != nil { + t.Fatal(err) + } + return n +} + +func TestRun_ReadOnlyRefusesAWrite_Issue2362(t *testing.T) { + c := pgConn(t) + ctx := context.Background() + + _, err := Run(ctx, c, "delete from ro_probe", false) + if !errors.Is(err, ErrWriteNotAllowed) { + t.Fatalf("delete without write: err = %v, want ErrWriteNotAllowed", err) + } + if n := probeCount(t, c); n != 2 { + t.Fatalf("rows after a refused delete = %d, want 2", n) + } + + // A write that reads as a select — a data-modifying CTE — is stopped by the + // read-only transaction, not by the keyword. + _, err = Run(ctx, c, "with d as (delete from ro_probe returning id) select * from d", false) + if err == nil { + t.Errorf("a deleting CTE ran in read-only mode") + } + if n := probeCount(t, c); n != 2 { + t.Fatalf("rows after a read-only CTE delete = %d, want 2", n) + } + + // Reads still work. + res, err := Run(ctx, c, "select id from ro_probe order by id", false) + if err != nil || len(res.Rows) != 2 { + t.Fatalf("select: %v, %+v", err, res) + } +} + +// CONTROL: with writes allowed the delete runs and says how many rows it hit. +func TestRun_WriteAllowedReportsAffectedRows_Issue2362(t *testing.T) { + c := pgConn(t) + res, err := Run(context.Background(), c, "delete from ro_probe where id = 1", true) + if err != nil { + t.Fatal(err) + } + if res.RowsAffected == nil || *res.RowsAffected != 1 { + t.Errorf("RowsAffected = %v, want 1", res.RowsAffected) + } + if n := probeCount(t, c); n != 1 { + t.Errorf("rows after an allowed delete = %d, want 1", n) + } +}