From 29ee41b604ac42d02bc365e8626453fd9eca50c3 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 18:22:05 +0800 Subject: [PATCH 01/11] journeys: the row verb finds Data grid 2.2 rows, whose cells carry no role Cold CMM (9.24.2): Data grid 2.2.x renders rows as div.tr role="row" (display: contents) with div.td cells and no role, and the grid root has no widget-datagrid class. No ROWS selector matched: every row step threw an instrument gap, 110 baseline steps FAILed and journey coverage was INVALID. ROWS gains '.tr[role="row"] > .td'; nthRow takes a row as visible when its first cell is (a display:contents row has no box) and clicks that cell. Controls: journeys-verbs-browser-row-dg22-pass (fixture with the gridcell roles and widget-datagrid class stripped). Old code: the 'dg2.2 cells without a role' case FAILs (and with it the 7 browser-row controls that reject any FAIL line); new code: 7 passed, 0 failed. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/journey-verbs.js | 9 +++++++-- upgrade-toolkit/controls/run-controls.sh | 3 ++- upgrade-toolkit/controls/verbs_browser_control.js | 10 ++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/upgrade-toolkit/bin/journey-verbs.js b/upgrade-toolkit/bin/journey-verbs.js index 1856995..3d8217e 100644 --- a/upgrade-toolkit/bin/journey-verbs.js +++ b/upgrade-toolkit/bin/journey-verbs.js @@ -350,6 +350,8 @@ const ROWS = ['.mx-listview-item', '.mx-templategrid-item', '.widget-gallery-ite '[role="row"]:not(:has([role="columnheader"])) [role="gridcell"]:not(:has(.empty-placeholder))', '[role="row"]:not(:has([role="columnheader"])) [role="cell"]:not(:has(.empty-placeholder))', '.widget-datagrid .td:not(:has(.empty-placeholder))', + // cold CMM (9.24.2): Data grid 2.2.x cells are div.td with no role, and the grid has no widget-datagrid class + '.tr[role="row"] > .td:not(.column-selector):not(:has(.empty-placeholder))', '[role="gridcell"]:not(:has(.empty-placeholder))']; // what counts as a rendered row when no selector found a clickable one (instrument gap vs. empty list) const ANY_ROW = '[role="row"]:not(:has([role="columnheader"])):not(:has(.empty-placeholder)), tbody tr, ' + @@ -430,16 +432,19 @@ async function upload(page, a, name) { // others met an Aborted workflow with the menu hidden. A row that does not exist is a fixture gap, said with the count. const ROW_BOXES = ['.mx-listview-item', '.mx-templategrid-item', '.widget-gallery-item', 'tbody tr', '[role="row"]:not(:has([role="columnheader"])):not(:has(.empty-placeholder))']; +const ROW_CELL = '[role="gridcell"], [role="cell"], .td:not(.column-selector)'; async function nthRow(box, n, ms) { const tries = Math.max(1, Math.ceil((ms || 8000) / 250)); let seen = 0; for (let t = 0; t < tries; t++) { for (const s of ROW_BOXES) { const loc = box.locator(s), vis = []; + // a display: contents row (Data grid 2) has no box of its own: it counts when its first cell is visible for (let i = 0, c = await loc.count().catch(() => 0); i < c && i < 50; i++) - if (await loc.nth(i).isVisible().catch(() => false)) vis.push(loc.nth(i)); + if (await loc.nth(i).isVisible().catch(() => false) || + await loc.nth(i).locator(ROW_CELL).first().isVisible().catch(() => false)) vis.push(loc.nth(i)); if (vis.length > n) { - const cell = vis[n].locator('[role="gridcell"], [role="cell"]').first(); + const cell = vis[n].locator(ROW_CELL).first(); return (await cell.count().catch(() => 0)) ? cell : vis[n]; } seen = Math.max(seen, vis.length); diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 9e633fa..cee2589 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -2581,6 +2581,7 @@ if command -v node > /dev/null; then VB="env NODE_PATH=$VBNP node $here/verbs_browser_control.js" ctl journeys-verbs-browser-row-pass 0 '^ok +dg2 rows \(display: contents\): .*clicked row1' '^ok +dg2 rows load late' '^ok +dg2 cells with role="cell"' '!^FAIL' -- $VB ctl journeys-verbs-browser-row-catch 0 '^ok +dg2 empty placeholder: threw .*the list is empty, a fixture gap' '^ok +rows render, unknown markup: threw .*an instrument gap' '!^FAIL' -- $VB + ctl journeys-verbs-browser-row-dg22-pass 0 '^ok +dg2\.2 cells without a role: returned .*clicked row1' '^ok +dg2\.2 row nth without a role: clicked row2nth' '!^FAIL' -- $VB ctl journeys-verbs-browser-row-nth-pass 0 '^ok +dg2 row nth: row 3 clicked' '^ok +dg2 row nth with dblclick: row 2' '!^FAIL' -- $VB ctl journeys-verbs-browser-row-nth-catch 0 '^ok +dg2 row nth: row 3 of 2 shown: threw: .*row 3 asked for, the list shows 2: .*a fixture gap' '!^FAIL' -- $VB ctl journeys-verbs-browser-row-dblclick-pass 0 '^ok +dg2 double-click trigger: dblclick opens the row: clicked row1' '!^FAIL' -- $VB @@ -2616,7 +2617,7 @@ if command -v node > /dev/null; then # PR-C1 (run 4): a journey that must open an unassigned task takes the list item that states no assignee ctl journeys-verbs-browser-unassigned-pass 0 '^ok +unassigned picks the item showing Assign to me, not the first: picked Appr F42B' '^ok +unassigned picks the item that states no assignee, also without the button' '^ok +unassigned absent takes the first \(the old pick\)' '^ok +unassigned with every item unassigned takes the first' '!^FAIL' -- $VB ctl journeys-verbs-browser-unassigned-catch 0 '^ok +unassigned is not the assigned first item \(catch\): not Appr F42A' '^ok +unassigned with every item assigned keeps the first \(the step guard says it\)' '!^FAIL' -- $VB - else for c in row-pass row-catch tab-pass item-pass item-catch combo-pass combo-catch menu-pass menu-catch fill-pass fill-catch near-pass near-catch ack-pass ack-catch menu-settle-pass menu-settle-catch menu-notice-pass menu-notice-catch tabpane-pass tabpane-catch unassigned-pass unassigned-catch; do skip journeys-verbs-browser-$c "no playwright with a browser"; done; fi + else for c in row-pass row-catch row-dg22-pass tab-pass item-pass item-catch combo-pass combo-catch menu-pass menu-catch fill-pass fill-catch near-pass near-catch ack-pass ack-catch menu-settle-pass menu-settle-catch menu-notice-pass menu-notice-catch tabpane-pass tabpane-catch unassigned-pass unassigned-catch; do skip journeys-verbs-browser-$c "no playwright with a browser"; done; fi else for c in catch pass row-instrument-gap-catch row-hidden-first-pass tab-pass tab-catch date-pass date-catch browser-row-pass browser-row-catch browser-tab-pass browser-item-pass browser-item-catch browser-combo-pass browser-combo-catch browser-menu-pass browser-menu-catch browser-fill-pass browser-fill-catch browser-near-pass browser-near-catch browser-ack-pass browser-ack-catch; do skip journeys-verbs-$c "no node"; done; fi ctl journeys-fill-colon-pass 0 '^"https://idp.example.com:8443/sso\?run=0102"$' -- python3 -c "import sys,json; sys.path.insert(0,'$B'); import run_journeys as r; t=r.run_copy_text(json.dumps({'do':'fill','value':'https://idp.example.com:8443/sso?run=\${RUN_ID}'}), {}, '0102'); print(json.dumps(json.loads(t)['value']))" diff --git a/upgrade-toolkit/controls/verbs_browser_control.js b/upgrade-toolkit/controls/verbs_browser_control.js index 5968ab4..b5f04e2 100644 --- a/upgrade-toolkit/controls/verbs_browser_control.js +++ b/upgrade-toolkit/controls/verbs_browser_control.js @@ -104,6 +104,16 @@ const line = (ok, name, detail) => { if (!ok) fail++; console.log(`${ok ? 'ok ' await rowCase('dg2 rows (display: contents)', grid(rowsHtml('gridcell')), 'row1'); await rowCase('dg2 rows load late (1.5 s)', grid('') + LATE, 'row1'); await rowCase('dg2 cells with role="cell"', grid(rowsHtml('cell')), 'row1'); + // cold CMM (9.24.2): Data grid 2.2.x renders div.td cells with no role and no widget-datagrid class on the grid; 110 + // baseline row clicks threw "an instrument gap" on a healthy Lots page + const DG22 = grid(rowsHtml('gridcell')).replace(/ role="gridcell"/g, '').replace(' widget-datagrid', ''); + await rowCase('dg2.2 cells without a role', DG22, 'row1'); + await page.setContent(DG22.replace("dataset.clicked='row2'", "dataset.clicked='row2nth'")); + try { + await V.row(page, { widget: 'dataGrid21', nth: 1, timeoutMs: 2500 }); + const c = await page.evaluate(() => document.body.dataset.clicked || ''); + line(c === 'row2nth', 'dg2.2 row nth without a role', `clicked ${c || 'nothing'}`); + } catch (e) { line(false, 'dg2.2 row nth without a role', `threw ${e.message.slice(0, 160)}`); } await rowCase('dg2 empty placeholder', grid(EMPTY), /the list is empty, a fixture gap/); await rowCase('rows render, unknown markup', ODD, /row element\(s\) render .* an instrument gap/); // PR-P3: a Data grid 2 with onClickTrigger = double (Purchase Request Product_Overview): the row fires on a double click only From 8b3e17a783d3d5d06564adad76a64ab5efecff46 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 18:22:05 +0800 Subject: [PATCH 02/11] stepupdate: run by hand without a git identity, commit as upgrade-pipeline Cold CMM: repair row 32 says to run stepupdate.sh with a one-line plan. Run by hand under a HOME with no git identity, the DataWidgets step updated, passed idcheck, and then STOPped at "git commit failed" (Author identity unknown). The pipeline exports a fallback identity (git_identity); stepupdate run alone did not. It now sets the same fallback when the app repo has no user.email. Control: stepupdate-no-identity-pass (HOME and XDG_CONFIG_HOME empty, GIT_CONFIG_NOSYSTEM=1, GIT_* unset). Old code: 6 passed, 1 failed (this one); new code: 7 passed, 0 failed. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/stepupdate.sh | 6 ++++++ upgrade-toolkit/controls/run-controls.sh | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/upgrade-toolkit/bin/stepupdate.sh b/upgrade-toolkit/bin/stepupdate.sh index 5a8e9d7..3aa26bb 100755 --- a/upgrade-toolkit/bin/stepupdate.sh +++ b/upgrade-toolkit/bin/stepupdate.sh @@ -154,6 +154,12 @@ fi mpr=$(cd "$app" && ls *.mpr 2>/dev/null) [ "$(echo "$mpr" | grep -c .)" = 1 ] || { echo "stepupdate: $app: expected one .mpr" >&2; exit 2; } git -C "$app" rev-parse --git-dir >/dev/null 2>&1 || { echo "stepupdate: $app is not a git work tree" >&2; exit 2; } +# run by hand (repair row 32) under a HOME with no git identity, every commit failed ("Author identity unknown", cold CMM +# DataWidgets step): the same fallback identity as upgrade-pipeline.sh git_identity +if [ -z "$(git -C "$app" config user.email 2> /dev/null || true)" ]; then + export GIT_AUTHOR_NAME="${GIT_AUTHOR_NAME:-upgrade-pipeline}" GIT_AUTHOR_EMAIL="${GIT_AUTHOR_EMAIL:-upgrade-pipeline@localhost}" + export GIT_COMMITTER_NAME="${GIT_COMMITTER_NAME:-upgrade-pipeline}" GIT_COMMITTER_EMAIL="${GIT_COMMITTER_EMAIL:-upgrade-pipeline@localhost}" +fi if [ -n "$(git -C "$app" status --porcelain --untracked-files=no)" ]; then echo "stepupdate: $app has uncommitted changes; commit or stash first" >&2; exit 2 fi diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index cee2589..9fb712d 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -1597,6 +1597,10 @@ SP="$T/my app" printf '120743 1.6.0 step1-adds\n' > $S/plan-sp.txt ctl stepupdate-space-split-catch 0 'committed' '!No such file' -- bash $B/stepupdate.sh "$SP" $S/plan-sp.txt ctl stepupdate-space-quoted-pass 0 'committed' '!No such file' -- env UPG_UPDATE_CMD="cp $S/{module}/App.mpr '{mpr}'" bash $B/stepupdate.sh "$T/my app2" $S/plan-sp.txt +# cold CMM: run by hand under a HOME with no git identity, the step commit failed ("Author identity unknown", STOP exit 2) +S4=$T/step-noid; mkdir -p $S4/app $S4/home; cp $M/base/App.mpr $S4/app/App.mpr +(cd $S4/app && git init -q && git add App.mpr && git -c user.email=c@c -c user.name=ctl commit -qm base) +ctl stepupdate-no-identity-pass 0 'committed' '!Author identity unknown' '!git commit failed' -- env -u GIT_AUTHOR_NAME -u GIT_AUTHOR_EMAIL -u GIT_COMMITTER_NAME -u GIT_COMMITTER_EMAIL HOME=$S4/home XDG_CONFIG_HOME=$S4/home GIT_CONFIG_NOSYSTEM=1 bash $B/stepupdate.sh $S4/app $S/plan-sp.txt unset UPG_UPDATE_CMD # --- stepupdate: recognised refusals, --no-baseline fallback, resume and keep (E4 13, E4 16, E4 7, E5b 26, E5b 27, E5b 17) - From ad3f45fe141a6ac6debea6123fa456d12916885a Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 18:22:05 +0800 Subject: [PATCH 03/11] smokewalk: a widget's own error box on the page fails the step Cold CMM run 10: after Atlas_Web_Content brought Data grid 2.30 while the DataWidgets 2.x filters stayed, the Lots page showed "The Text filter widget must be placed inside the header of the Data grid 2.0 or Gallery widget" in a red box in place of every filter. landedOk only knew the login form and the "An error occurred" dialog, so the after walk PASSed 23/23. landedOk now also fails on a visible .alert-danger that mentions a widget; validation messages (.mx-validation-message) and app alert text without "widget" still pass. widgetrender.js (journey step "rendered") looks inside one named widget; this is the walk's own check on every page it opens. Control: smokewalk-landedok-widget-error-catch, plus chromium cases in smokewalk_predicates.js. Old code: the widget error box gets null (PASS), exit 1; new code: all landedOk cases ok. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/smokewalk.js | 9 ++++++++- upgrade-toolkit/controls/run-controls.sh | 2 ++ upgrade-toolkit/controls/smokewalk_predicates.js | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/upgrade-toolkit/bin/smokewalk.js b/upgrade-toolkit/bin/smokewalk.js index c5c419b..28b4843 100755 --- a/upgrade-toolkit/bin/smokewalk.js +++ b/upgrade-toolkit/bin/smokewalk.js @@ -3,7 +3,8 @@ // Per role: log in, then for each step go home, click the menu item (or home button), wait until the // expected page is the content form (or a popup opened), and record client errors. // PASS = landed on the expected page, no page error, no console error, no Mendix error dialog, and the document does -// not show the login form or an "An error occurred" dialog (landedOk, T3-6; also checked after the home step). +// not show the login form, an "An error occurred" dialog or a widget's own error box (landedOk, T3-6; also checked +// after the home step). // FAIL = an error, or it landed somewhere else / nothing opened. // INVALID = could not judge (no credentials for the role, login failed, menu link not rendered). // @@ -153,6 +154,12 @@ function landedOk() { const dlg = all('.mx-dialog, .modal-dialog, .mx-dialog-error').filter(shown) .find(e => /An error occurred/i.test(e.textContent || '')); if (dlg) return 'error dialog is showing: ' + String(dlg.textContent).replace(/\s+/g, ' ').trim().slice(0, 160); + // cold CMM: Data grid 2.30 with the DataWidgets 2.x filters rendered "The Text filter widget must be placed inside the + // header of the Data grid 2.0 or Gallery widget" in red boxes in place of every filter, and the step PASSed (23/23). + // A widget's own error box is not an app message; a validation message (.mx-validation-message) is left alone. + const werr = all('.alert-danger').filter(shown) + .find(e => !/(^|\s)mx-validation-message(\s|$)/.test(e.className || '') && /widget/i.test(e.textContent || '')); + if (werr) return 'widget error is showing: ' + String(werr.textContent).replace(/\s+/g, ' ').trim().slice(0, 160); return null; } catch (e) { return 'page could not be inspected: ' + String(e && e.message || e).slice(0, 100); } } diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 9fb712d..c38cd0b 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -1123,6 +1123,8 @@ ctl smokewalk-popup-fixed-catch 0 '^ok +landedWhere fixed popup window' '^ok +la # T3-6: a step is judged by what the browser shows: the login form and the "An error occurred" dialog FAIL, a clean page passes ctl smokewalk-landedok-pass 0 '^ok +landedOk login form page .*FAIL wanted' '^ok +landedOk error dialog .*FAIL wanted' '^ok +landedOk clean page .*PASS wanted' \ '^ok +landedOk hidden login form .*PASS wanted' '^ok +landedOk when inspection throws is not a pass' '!^FAIL' -- node $here/smokewalk_predicates.js +# cold CMM: a page full of widget error boxes ("The Text filter widget must be placed inside the header ...") PASSed 23/23 +ctl smokewalk-landedok-widget-error-catch 0 '^ok +landedOk fake widget error box: got true' '^ok +landedOk fake validation message: got null' '!^FAIL' -- node $here/smokewalk_predicates.js # T6R-9: the compare pools repeated walks (fixtures/smoke/, --rejudge = no browser). A new error the old walks never showed # is a REGRESSION, also when intermittent; the old app's own intermittent error on the new side is FLAKY on both (exit 2) SW="node $B/smokewalk.js"; SF=$FX/smoke diff --git a/upgrade-toolkit/controls/smokewalk_predicates.js b/upgrade-toolkit/controls/smokewalk_predicates.js index 5ac206d..4784e11 100644 --- a/upgrade-toolkit/controls/smokewalk_predicates.js +++ b/upgrade-toolkit/controls/smokewalk_predicates.js @@ -67,6 +67,9 @@ const PAGES = { 'mx error dialog class': ['
An error occurred while ...
', /error dialog is showing/], 'clean page': ['

Home

', null], 'other dialog': ['', null], + 'widget error box (dg2.30 with 2.x filters)': ['
The Text filter widget must be placed inside the header of the Data grid 2.0 or Gallery widget.
', /widget error is showing: The Text filter widget must be placed/], + 'validation message': ['
The widget value is required
', null], + 'app alert text': ['
3 lots expire this week
', null], 'hidden login form': ['

Home

', null], }; function pwLoad() { @@ -99,6 +102,10 @@ function pwLoad() { check('landedOk fake error dialog', /error dialog/.test(landedOk() || ''), true); global.document = fake([hidden('#loginButton'), hidden('input[type=password]'), hidden('.mx-dialog', 'An error occurred'), shown('.mx-dialog', 'Saved')]); check('landedOk fake clean page', landedOk(), null); + global.document = fake([shown('.alert-danger', 'The Date filter widget must be placed inside the header')]); + check('landedOk fake widget error box', /widget error is showing/.test(landedOk() || ''), true); + global.document = fake([Object.assign(shown('.alert-danger', 'widget value required'), { className: 'alert alert-danger mx-validation-message' }), hidden('.alert-danger', 'widget broken')]); + check('landedOk fake validation message', landedOk(), null); global.document = { querySelectorAll() { throw new Error('boom'); } }; check('landedOk when inspection throws is not a pass', /could not be inspected/.test(landedOk() || ''), true); // T8R B2-3: a home step on the plan's login page is SKIPPED as an API-only role, other home pages are walked From da920822c2ea2665b8b834d340e3cbcda7ede717 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 22:06:12 +0800 Subject: [PATCH 04/11] pipeline: export UPG_NAME, so runapp stops the pipeline's own containers Cold WM1 journeys, run next to cold CMM: the pipeline names its containers ${UPG_NAME:-mig}- but never exported UPG_NAME. runapp.py, seeing it unset with another run's cmm-oldjp present, REFUSED to stop mig-oldjp (the FM-9 guard), and newjp then found port 8080 taken ("INFRA port 8080 in use by mig-oldjp"). The pipeline now exports UPG_NAME="${UPG_NAME:-mig}" once, after git_identity; the guard still protects a runapp call made by hand. Control: pipeline-exports-upg-name-pass runs the pipeline's own export line, then runapp stop oldjp with mig-oldjp and cmm-oldjp present. Old code: 5 passed, 1 failed (REFUSED); new code: 6 passed, 0 failed. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 4 ++++ upgrade-toolkit/controls/run-controls.sh | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index b290eb8..67b9554 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -447,6 +447,10 @@ git_identity() { export GIT_COMMITTER_NAME="${GIT_COMMITTER_NAME:-upgrade-pipeline}" GIT_COMMITTER_EMAIL="${GIT_COMMITTER_EMAIL:-upgrade-pipeline@localhost}" } git_identity +# The pipeline names its containers ${UPG_NAME:-mig}- but did not export the name: runapp.py, with UPG_NAME unset, then +# REFUSED to stop mig-oldjp because another run's cmm-oldjp existed (FM-9 guard), and newjp found port 8080 taken (cold WM1 +# journeys, next to cold CMM). Exported, every tool it calls uses the same prefix it does. +export UPG_NAME="${UPG_NAME:-mig}" ensure_gitignore() { local f="$APP/.gitignore" l for l in '.claude/' '.mxcli/' '.stepupdate/' 'deployment/'; do diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index c38cd0b..1c6254d 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -1283,6 +1283,10 @@ ctl runapp-stop-refused-nothing-removed-pass 0 '^0$' -- sh -c "test -f $T/stop.l ctl runapp-stop-with-name-pass 0 'stopping container fm-new' 'stopped fm-new' -- env UPG_NAME=fm "PATH=$T/stopdocker:$PATH" WORKDIR=$T/stopwork STOP_LOG=$T/stop.log STOP_NAMES='mig-new fm-new mig-pg' python3 $B/runapp.py stop new ctl runapp-stop-only-default-pass 0 'stopping container mig-new' -- env -u UPG_NAME "PATH=$T/stopdocker:$PATH" WORKDIR=$T/stopwork STOP_LOG=$T/stop.log STOP_NAMES='mig-new mig-pg' python3 $B/runapp.py stop new ctl runapp-stop-allow-default-pass 0 'stopping container mig-new' -- env -u UPG_NAME UPG_ALLOW_DEFAULT_PREFIX=1 "PATH=$T/stopdocker:$PATH" WORKDIR=$T/stopwork STOP_LOG=$T/stop.log STOP_NAMES='mig-new fm-new' python3 $B/runapp.py stop new +# cold WM1 next to cold CMM: the pipeline used ${UPG_NAME:-mig} without exporting it; runapp stop of mig-oldjp was REFUSED +# (cmm-oldjp exists) and newjp found port 8080 taken. With the pipeline's own UPG_NAME line the stop goes through. +awk '/^export UPG_NAME=/' $B/cloud/upgrade-pipeline.sh > $T/upgname.sh +ctl pipeline-exports-upg-name-pass 0 'stopping container mig-oldjp' '!REFUSED' -- env -u UPG_NAME "PATH=$T/stopdocker:$PATH" WORKDIR=$T/stopwork STOP_LOG=$T/stop.log STOP_NAMES='mig-oldjp cmm-oldjp' bash -c ". $T/upgname.sh; python3 $B/runapp.py stop oldjp" ctl runapp-login-sso-catch 0 '^HARNESS CONDITION: sso: .*replaced by login.old' -- lo sso auto ctl runapp-login-sso-marker-pass 0 'HARNESS CONDITION' 'usernameInput' -- sh -c "cat $LO/sso/harness-login-override.txt $LO/sso/web/login.html; test -f $LO/sso/web/login.html.orig && test -f $LO/sso/web/login.html.gz" ctl runapp-login-sso-nostock-catch 0 'no stock login page' 'INVALID' -- lo sso-nostock auto From 2239f5e4764ee0bea6aca98f6c965b44533340e9 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Wed, 18 Nov 2026 00:49:45 +0800 Subject: [PATCH 05/11] pipeline: every exit harvests the run and writes FEEDBACK-REPORT.md harvest.py (PR #3) only ran when a person remembered steps/report.md 3c, so a run that stopped taught the toolkit nothing (cold WM1). The exit trap now calls bin/feedback_report.py: harvest drafts go to $WORK/learnings/, and FEEDBACK-REPORT.md says where the run stopped, lists the drafts, and explains how to send them back (submit-learnings.sh) or file a toolkit bug (an issue summary of metadata only, checked against the denylist). Rung 2 reports on the whole run folder. The exit code never changes. UPG_HARVEST=0 skips the harvest. Controls: feedback-report-{pass,content-pass,nocode-pass,nocode-says-pass,deny-catch}, pipeline-feedback-on-exit-pass (6 fail on the old code, all pass on the new). Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 16 +- upgrade-toolkit/bin/feedback_report.py | 137 ++++++++++++++++++ upgrade-toolkit/controls/run-controls.sh | 15 ++ upgrade-toolkit/steps/report.md | 4 +- 4 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 upgrade-toolkit/bin/feedback_report.py diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 67b9554..665734c 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -407,12 +407,26 @@ timing_close() { # timing_close EXITCODE { [ -s "$f" ] || printf 'step\tstart_utc\tend_utc\texit\n' > "$f"; printf '%s\t%s\t%s\t%s\n' "$TIMING_STEP" "$TIMING_START" "$end" "$1" >> "$f"; } 2> /dev/null || true TIMING_STEP="" } +# feedback (bin/feedback_report.py): every exit, a stop as much as a PASS, harvests what the run taught into +# $WORK/learnings/ and writes $WORK/FEEDBACK-REPORT.md: where it stopped, the drafts, and how to send them back (PR) or +# file a toolkit bug (issue). Rung 2 reports on the whole run folder. Never changes the exit code. UPG_HARVEST=0 skips the +# harvest; the report still comes. Before this, harvest.py ran only when someone remembered steps/report.md 3c (cold WM1). +feedback_finish() { # feedback_finish EXITCODE STEP + [ "${DRY:-0}" != 1 ] && [ -d "${WORK:-}" ] || return 0 + local fw=$WORK app="${UPG_APP_CODE:-${UPG_APP_NAME:-}}" + [ "$(basename "$WORK")" != rung2 ] || fw=$(dirname "$WORK") + [ -n "$app" ] || app=$(head -n 1 "$fw/state/app-code" 2> /dev/null || true) + python3 -I "$B/feedback_report.py" --work "$fw" --exit "$1" --step "$2" --code "$app" \ + --source "${SOURCE:-}" --target "${TARGET:-}" 2>&1 | sed 's/^/ /' || true +} cleanup() { - local a rc=$? + local a rc=$? st=${TIMING_STEP:-${STEP:-}} + [ "$rc" != 0 ] || st="" timing_close "$rc" || true # the step that is stopping (exit 4, a judgment stop, an error) gets its row with its own exit code # run log (bin/runlog.py): a gate step that stops for judgment still leaves its row; the exit code stays the step's own if [ "${STEP:-}" = gate ] && [ "${DRY:-0}" != 1 ] && [ -z "${RUNLOG_DONE:-}" ]; then record_runlog || true; fi for a in "${APPS_UP[@]:-}"; do [ -z "$a" ] || { runenv; env "${RUNENV[@]}" python3 "$B/runapp.py" stop "$a" > /dev/null 2>&1 || true; }; done + feedback_finish "$rc" "$st" || true } trap cleanup EXIT diff --git a/upgrade-toolkit/bin/feedback_report.py b/upgrade-toolkit/bin/feedback_report.py new file mode 100644 index 0000000..6eba94c --- /dev/null +++ b/upgrade-toolkit/bin/feedback_report.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""feedback_report.py: at the end of every run, harvest what it taught and write FEEDBACK-REPORT.md, no LLM. + +The pipeline calls this from its exit trap, so a run that stops (an environment fault, a judgment stop, a crash) leaves +the same report as one that passes. Before this, bin/harvest.py only ran when a person remembered steps/report.md 3c. + + /learnings/ harvest.py drafts (toolkit; brain drafts in learnings/brain/ when UPG_BRAIN is set) + /FEEDBACK-REPORT.md where the run stopped and why, the drafts, and how to send both back: the learnings + by PR (submit-learnings.sh), a toolkit bug as a GitHub issue with a summary to paste + +The issue summary holds metadata only (versions, step, exit code, the headings of NEEDS-JUDGMENT.md), never log text, +and goes through the same denylist as harvest.py; a hit leaves the summary out and says so. + +usage: feedback_report.py --work DIR --exit N [--step STEP] [--code CODE] [--source V] [--target V] +Never fails the run: exit 0 always, except 2 for a bad call. UPG_HARVEST=0 skips the harvest (the report still comes). +""" +import argparse +import os +import pathlib +import re +import subprocess +import sys + +B = pathlib.Path(__file__).resolve().parent +TK = B.parent + +MEANING = { + 0: "gate PASS, nothing left for a person", + 2: "usage or version-path error", + 3: "missing tool or setup, or an mxcli of another version (an environment fault, not the app)", + 4: "a fixed step failed unexpectedly (likely a toolkit or environment bug: worth an issue)", + 10: "mx check / build / start errors after the upgrade: repair needed", + 11: "journeys INVALID or FAIL on the old or the new app", + 12: "diff or gate items need a comparison by a person", + 13: "a module plan or a module decision is missing", + 14: "the upgrade proposal awaits sign-off", + 15: "the owner paused the run", +} + + +def deny_pattern(): + out = subprocess.run(["bash", "-c", f'. "{B}/private_names.sh" >/dev/null 2>&1; printf %s "${{DENY:-}}"'], + capture_output=True, text=True) + return out.stdout.strip() + + +def toolkit_rev(): + r = subprocess.run(["git", "-C", str(TK), "rev-parse", "--short", "HEAD"], capture_output=True, text=True) + return r.stdout.strip() or "unknown" + + +def issues_url(): + r = subprocess.run(["git", "-C", str(TK), "remote", "get-url", "origin"], capture_output=True, text=True) + m = re.match(r"(?:https://|git@)github\.com[:/]([^/]+/[^/.]+?)(?:\.git)?$", r.stdout.strip()) + return f"https://github.com/{m.group(1)}/issues/new" if m else "the toolkit's GitHub issues page (Issues > New issue)" + + +def main(): + ap = argparse.ArgumentParser(usage=__doc__.split("usage: ")[1].split("\n")[0]) + ap.add_argument("--work", required=True) + ap.add_argument("--exit", type=int, required=True) + ap.add_argument("--step", default="") + ap.add_argument("--code", default="") + ap.add_argument("--source", default="") + ap.add_argument("--target", default="") + a = ap.parse_args() + work = pathlib.Path(a.work) + if not work.is_dir(): + print(f"feedback_report: no work folder {work}", file=sys.stderr) + return 2 + + learn = work / "learnings" + if os.environ.get("UPG_HARVEST", "1") == "0": + harvest = "skipped (UPG_HARVEST=0)" + elif not a.code: + harvest = "skipped: no app code name (set UPG_APP_CODE, or write it to state/app-code, then run bin/harvest.py)" + else: + cmd = [sys.executable, "-I", str(B / "harvest.py"), "--work", str(work), "--code", a.code, "--inbox", str(learn)] + if os.environ.get("UPG_BRAIN"): + cmd += ["--to", "both", "--brain-inbox", str(learn / "brain")] + else: + cmd += ["--to", "toolkit"] + r = subprocess.run(cmd, capture_output=True, text=True) + lines = [l for l in (r.stdout + r.stderr).splitlines() if l.strip()] + harvest = next((l for l in lines if l.startswith("harvest:")), lines[-1] if lines else f"harvest.py exit {r.returncode}") + if r.returncode == 1: + harvest += " (denylist hit: that place was not written)" + drafts = sorted(p.relative_to(work) for p in learn.rglob("*.md")) if learn.is_dir() else [] + + meaning = MEANING.get(a.exit, "unexpected failure (no exit code of the pipeline legend): worth an issue") + nj = work / "NEEDS-JUDGMENT.md" + heads = [l.strip() for l in nj.read_text(errors="replace").splitlines() if l.startswith("## ")][:12] if nj.is_file() else [] + issue = [f"toolkit {toolkit_rev()}", f"upgrade {a.source or '?'} -> {a.target or '?'}", + f"stopped at step: {a.step or 'end of run'}", f"exit {a.exit}: {meaning}"] + if heads: + issue += ["needs judgment:"] + [f" {h[3:]}" for h in heads] + issue_text = "\n".join(issue) + deny = deny_pattern() + if deny and re.search(deny, issue_text, re.I): + issue_text = "(left out: it matches the private denylist; write the issue by hand)" + + rel_learn = "learnings" + out = [ + "# Feedback report", + "", + f"Exit {a.exit} at {a.step or 'the end of the run'}: {meaning}.", + "", + "## What the run taught", + "", + f"{harvest}", + "", + ] + out += [f"- `{d}`" for d in drafts] or ["No drafts."] + out += [ + "", + "## Send it back", + "", + f"1. **Learnings.** Read each draft in `{rel_learn}/`, then copy the toolkit drafts to `contrib/inbox/` of your toolkit", + " checkout and run `bin/submit-learnings.sh toolkit --dry-run`, then without `--dry-run`: one branch, one PR.", + " Brain drafts (`learnings/brain/`) go with `bin/submit-learnings.sh brain`, never to the toolkit.", + f"2. **A toolkit bug** (exit 3 or 4 that is not your machine, a wrong verdict, a crash): open {issues_url()}", + " and paste the summary below. Read it first: no customer, app, host or person names.", + "", + "## Summary for an issue", + "", + "```", + issue_text, + "```", + "", + ] + (work / "FEEDBACK-REPORT.md").write_text("\n".join(out)) + print(f"feedback: {work / 'FEEDBACK-REPORT.md'} ({len(drafts)} draft(s))") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 1c6254d..60c3465 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -7452,6 +7452,21 @@ ctl harvest-denylist-catch 1 'Acmecorp' '!^harvest: wrote' -- sh -c "python3 -I ctl harvest-denylist-wrote-nothing-pass 0 '^none$' -- sh -c "ls $HV/tk-deny/*.md 2>/dev/null || echo none" ctl harvest-bad-code-catch 2 'code' -- python3 -I $B/harvest.py --work $HV/run --code 'a b' --inbox $HV/tk-bad ctl harvest-denied-code-catch 2 'code' -- python3 -I $B/harvest.py --work $HV/run --code Acmecorp --inbox $HV/tk-bad +# feedback_report.py (cold WM1): harvest ran only when a person remembered steps/report.md 3c, so a stopped run taught the +# toolkit nothing. Every exit now harvests into $WORK/learnings and writes FEEDBACK-REPORT.md with an issue summary +# (metadata only, denylisted); the pipeline calls it from its exit trap, so a stop (exit 4 at build) gets one too. +FB=$T/feedback; mkdir -p $FB; cp -R $HV/run $FB/run; cp -R $HV/run $FB/nocode; cp -R $HV/run $FB/deny; cp -R $HV/run $FB/trap +printf '## Acmecorp portal module kept\n' > $FB/deny/NEEDS-JUDGMENT.md +fbr() { env -u UPG_BRAIN -u UPG_HARVEST python3 -I $B/feedback_report.py "$@"; } +ctl feedback-report-pass 0 '^feedback: .*FEEDBACK-REPORT\.md \(1 draft' -- fbr --work $FB/run --exit 4 --step build --code HV1 --source 9.24.0 --target 10.24.0 +ctl feedback-report-content-pass 0 '^Exit 4 at build: a fixed step failed' '^harvest: 3 agent fixes' '^- `learnings/[0-9-]+-HV1-harvest\.md`' 'stopped at step: build' 'upgrade 9\.24\.0 -> 10\.24\.0' 'submit-learnings\.sh toolkit' -- cat $FB/run/FEEDBACK-REPORT.md +ctl feedback-report-nocode-pass 0 '^feedback: .*\(0 draft' -- fbr --work $FB/nocode --exit 13 +ctl feedback-report-nocode-says-pass 0 'skipped: no app code name' '^Exit 13 at the end of the run' -- cat $FB/nocode/FEEDBACK-REPORT.md +fbr --work $FB/deny --exit 13 --code HV1 > /dev/null 2>&1 || true +ctl feedback-report-deny-catch 0 'left out: it matches the private denylist' '!Acmecorp' -- cat $FB/deny/FEEDBACK-REPORT.md +{ awk '/^feedback_finish\(\) \{/,/^}$/' $B/cloud/upgrade-pipeline.sh; awk '/^cleanup\(\) \{/,/^}$/' $B/cloud/upgrade-pipeline.sh + printf '%s\n' 'timing_close() { :; }' "B=$B; WORK=$FB/trap; DRY=0; STEP=build; UPG_APP_CODE=HV1; APPS_UP=()" 'trap cleanup EXIT' 'exit 4'; } > $FB/trap.sh +ctl pipeline-feedback-on-exit-pass 0 '^rc=4$' '^Exit 4 at build' -- sh -c "env -u UPG_BRAIN bash $FB/trap.sh > /dev/null 2>&1; echo rc=\$?; cat $FB/trap/FEEDBACK-REPORT.md" # submit-learnings.sh: a sandbox repo with a local bare remote. The checks refuse a denylisted name, a home folder path, # internal brain text sent to the toolkit, and a brain inbox inside the toolkit repo; a clean draft is pushed as a diff --git a/upgrade-toolkit/steps/report.md b/upgrade-toolkit/steps/report.md index 5d497a8..55339d7 100644 --- a/upgrade-toolkit/steps/report.md +++ b/upgrade-toolkit/steps/report.md @@ -27,7 +27,9 @@ coverage and the human handoff. Keep the machine verdict as evidence; explain th `Run log:` line in `gate-summary.md` and as an information section in `NEEDS-JUDGMENT.md`. The row has two timing columns, "run time (machine)" and "person wait", from `$WORK/state/timing.tsv`; the same numbers are the `Time:` line of `gate-summary.md`. Quote them in the report; `not recorded` means the run has no timing file. -3c. **Harvest.** Before the work folder goes, run `bin/harvest.py --work $WORK --code `: it drafts what the run +3c. **Harvest.** The pipeline does this on every exit (`bin/feedback_report.py`): drafts in `$WORK/learnings/` and + `$WORK/FEEDBACK-REPORT.md` with how to send them back and an issue summary for a toolkit bug. By hand, or with another + code name: `bin/harvest.py --work $WORK --code `: it drafts what the run taught (fixes the agent made by hand, error codes no step names, scripts it wrote, toolkit items of the gate) into `contrib/inbox/` for the toolkit and, for each lesson the brain has no note for, a brain-format draft into `$UPG_BRAIN/inbox/`. No LLM; review the drafts, then `bin/submit-learnings.sh toolkit` and `bin/submit-learnings.sh brain` From 6b791ff6d4e193833279d37747d3fbe3e1871a26 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Wed, 18 Nov 2026 00:49:45 +0800 Subject: [PATCH 06/11] contributing: contributions are open; a bug goes in as an issue from FEEDBACK-REPORT.md The page still said the repo was private and outside contributions were not open, which contradicted the invitation to send PRs. It now invites issues, learnings drafts and PRs under the same bar (a field run or a control), and points bug reports at the issue summary every run now writes. Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc3652a..9f3cfdc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,8 @@ # Contributing -This repo is private. Contributions from outside the team aren't open yet, but the same bar -applies to anyone with access: every change should be backed by a field run or a control, not -a guess. +Contributions are open. Ran the toolkit on your own app and hit a wall, a wrong verdict or a missing step? Send it: +an issue, a learnings draft or a PR are all welcome. One bar for everyone: every change is backed by a field run or a +control, not a guess. ## Three ways in @@ -12,6 +12,9 @@ a guess. made by hand, unknown error codes, scripts and toolkit items into the inbox. Then `upgrade-toolkit/bin/submit-learnings.sh toolkit` runs the leak checks and opens one PR from a `learnings/` branch. 3. **A direct PR** for a finished change, with the checks below. +4. **An issue** for a bug you can't fix yourself. Every run ends with `FEEDBACK-REPORT.md` in its work folder: where it + stopped, the exit code and the drafts it harvested. Paste its summary into a GitHub issue after you have read it for + names, hosts and paths. Toolkit or brain: the toolkit takes generic lessons under code names. Anything internal (customer context, notes from the company brain, `confidentiality: internal`) goes to the brain with `submit-learnings.sh brain`, never here; From 6311d51677b95794040ac4c19076643af29b3bba Mon Sep 17 00:00:00 2001 From: MendixMau Date: Wed, 18 Nov 2026 00:50:54 +0800 Subject: [PATCH 07/11] playbooks: Datagrid 2.30 needs DataWidgets in the same rung; backlog 151-153 from cold CMM widgets.md: Atlas_Web_Content 3.9 swaps in Datagrid 2.30, and DataWidgets 2.x filters then render in red error boxes until DataWidgets is updated in the same rung; plus the Data grid 2.2 DOM shape. BACKLOG: rung 2 plan pre-fill, localedits content id 0 after rung 1, and resumable journeys. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/BACKLOG.md | 3 +++ upgrade-toolkit/playbooks/widgets.md | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/upgrade-toolkit/BACKLOG.md b/upgrade-toolkit/BACKLOG.md index 2529cea..7d34022 100644 --- a/upgrade-toolkit/BACKLOG.md +++ b/upgrade-toolkit/BACKLOG.md @@ -165,6 +165,9 @@ An item is done only when its "done when" check holds on a real app, not when th | 148 | React-client readiness and safety: `dojo_report.py` only runs when a React switch is already decided, so legacy-widget and Dojo-only-package counts surface late instead of in triage for every 10-to-11 or 11-to-11 rung ([dojo-to-react](playbooks/dojo-to-react.md) 1); a non-ES-module pluggable (DropdownContainer 1.2.0) fails the React export but is invisible to `mx check` ([dojo-to-react](playbooks/dojo-to-react.md) 3.1); the React mxbuild bundler can OOM (exit 137) under ~4 GB free container memory with no warning ([dojo-to-react](playbooks/dojo-to-react.md) 3.5) | triage runs `dojo_report.py` on every 10-to-11/11-to-11 rung regardless of client decision; a readiness check flags a non-ES-module pluggable before build; mx10.sh/mx11.sh warns below ~4 GB free before a React build | M | FSM | | 149 | `convert_legacy.py` (dump-mpr-driven Dojo-to-React converter, [dojo-to-react](playbooks/dojo-to-react.md) 2) silently drops `SelectFirst` instead of naming every grid that had it in `CONVERT-REPORT.md`; `run_journeys` reports a client-swap regression without first reconfirming it on a fresh database copy, so Dojo-run state leakage (a destructive journey, an added row) can read as a false regression (FSM: 8 flagged, 0 confirmed after rerun); `runapp` reports a stopped app generically ("app went down") instead of the runtime's own stop reason (FSM: a 3 h trial-license limit, not a crash) | `CONVERT-REPORT.md` lists every `SelectFirst` grid; every REGRESSION gets an automatic fresh-data rerun before being reported; a license/known-limit stop is named, not generic | M | FSM | | 150 | `mxcli widget init` writes `.claude/skills/widgets` into the app folder, which then needs a manual exclude from every app commit | the app's `.gitignore` (or the toolkit's own commit step) excludes it without a manual step | S | FSM | +| 151 | Rung 2 stops at triage (exit 13) for a missing `plan.txt` although rung 1's plan already names most modules: write the rung 2 plan by hand every time (cold CMM run 10) | rung 2 triage pre-fills `plan.txt` from rung 1's plan and the 11.x versions of `playbooks/11-to-11.md`, and stops only for modules it cannot place | M | CMM | +| 152 | `localedits` reports "content id 0" for modules after rung 1 converted the app (cold CMM) | rung 2 localedits resolves the content id from the rung 1 package or says why it cannot | S | CMM | +| 153 | Journeys take 3 h or more on a mid-size app, and a stopped journeys step restarts from the first journey (cold CMM, cold WM1) | journeys resume from the last finished journey; the cockpit shows journeys done/total | M | CMM, WM1 | ## Done (14 Oct, remedy trial round) diff --git a/upgrade-toolkit/playbooks/widgets.md b/upgrade-toolkit/playbooks/widgets.md index d65bd2c..c7f0168 100644 --- a/upgrade-toolkit/playbooks/widgets.md +++ b/upgrade-toolkit/playbooks/widgets.md @@ -157,6 +157,11 @@ resets properties that do not exist in the new version. Seen: version per package (ledger `.stepupdate/widgets.txt`, repeated in the summary). By hand: `python3 $B/widgetdiff.py app/ --rev HEAD~1`. For each replaced package, the pages that use it go on the shotdiff must-open list; a swap the plan did not ask for is restored from the previous commit or recorded as accepted. +- Datagrid swapped, DataWidgets left behind (cold CMM run 10): Atlas_Web_Content 3.9 brought Datagrid 2.30 while the + app kept DataWidgets 2.x. The 2.x filters then render inside a red box ("must be placed inside the header") on every + filtered grid page. Update DataWidgets in the same rung as Atlas_Web_Content (3.11.8 cleared it); smokewalk fails a + page that shows a widget error box. Data grid 2.2 (9.24) rows are `div.tr[role=row]` with `display: contents` and + cells `div.td` with no role and no `widget-datagrid` class on the root; the journeys row verb handles both shapes. ## 4. React-client readiness is not upgrade work, but list it From 9228e32fc07aad9cb3c2287f1ce59ae6f21063e8 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Wed, 18 Nov 2026 01:39:08 +0800 Subject: [PATCH 08/11] pipeline: rung 2 gets the content ids of rung 1's plan for localedits Modules that rung 1 updated carry content id 0 in the MPR, so rung 2's localedits printed UNKNOWN ("give --cid Module=") for each one and the run stopped for a hand-written localedits.args (cold CMM, 7 modules). Before rung 2 starts, rung2_cids adds a --cid=Module=ID line for every module in rung 1's plan.txt that the args file does not already name. Controls: pipeline-rung2-cids-pass, pipeline-rung2-cids-wired-pass (both fail on the old code, pass on the new). Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/BACKLOG.md | 5 ++--- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 13 +++++++++++++ upgrade-toolkit/controls/run-controls.sh | 8 ++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/upgrade-toolkit/BACKLOG.md b/upgrade-toolkit/BACKLOG.md index 7d34022..f2de8ac 100644 --- a/upgrade-toolkit/BACKLOG.md +++ b/upgrade-toolkit/BACKLOG.md @@ -165,9 +165,8 @@ An item is done only when its "done when" check holds on a real app, not when th | 148 | React-client readiness and safety: `dojo_report.py` only runs when a React switch is already decided, so legacy-widget and Dojo-only-package counts surface late instead of in triage for every 10-to-11 or 11-to-11 rung ([dojo-to-react](playbooks/dojo-to-react.md) 1); a non-ES-module pluggable (DropdownContainer 1.2.0) fails the React export but is invisible to `mx check` ([dojo-to-react](playbooks/dojo-to-react.md) 3.1); the React mxbuild bundler can OOM (exit 137) under ~4 GB free container memory with no warning ([dojo-to-react](playbooks/dojo-to-react.md) 3.5) | triage runs `dojo_report.py` on every 10-to-11/11-to-11 rung regardless of client decision; a readiness check flags a non-ES-module pluggable before build; mx10.sh/mx11.sh warns below ~4 GB free before a React build | M | FSM | | 149 | `convert_legacy.py` (dump-mpr-driven Dojo-to-React converter, [dojo-to-react](playbooks/dojo-to-react.md) 2) silently drops `SelectFirst` instead of naming every grid that had it in `CONVERT-REPORT.md`; `run_journeys` reports a client-swap regression without first reconfirming it on a fresh database copy, so Dojo-run state leakage (a destructive journey, an added row) can read as a false regression (FSM: 8 flagged, 0 confirmed after rerun); `runapp` reports a stopped app generically ("app went down") instead of the runtime's own stop reason (FSM: a 3 h trial-license limit, not a crash) | `CONVERT-REPORT.md` lists every `SelectFirst` grid; every REGRESSION gets an automatic fresh-data rerun before being reported; a license/known-limit stop is named, not generic | M | FSM | | 150 | `mxcli widget init` writes `.claude/skills/widgets` into the app folder, which then needs a manual exclude from every app commit | the app's `.gitignore` (or the toolkit's own commit step) excludes it without a manual step | S | FSM | -| 151 | Rung 2 stops at triage (exit 13) for a missing `plan.txt` although rung 1's plan already names most modules: write the rung 2 plan by hand every time (cold CMM run 10) | rung 2 triage pre-fills `plan.txt` from rung 1's plan and the 11.x versions of `playbooks/11-to-11.md`, and stops only for modules it cannot place | M | CMM | -| 152 | `localedits` reports "content id 0" for modules after rung 1 converted the app (cold CMM) | rung 2 localedits resolves the content id from the rung 1 package or says why it cannot | S | CMM | -| 153 | Journeys take 3 h or more on a mid-size app, and a stopped journeys step restarts from the first journey (cold CMM, cold WM1) | journeys resume from the last finished journey; the cockpit shows journeys done/total | M | CMM, WM1 | +| 151 | Rung 2 stops at triage (exit 13) for a missing `plan.txt` although rung 1's plan already names most modules: write the rung 2 plan by hand every time (cold CMM run 10; the content ids now carry over, `rung2_cids`) | rung 2 triage pre-fills `plan.txt` from rung 1's plan and the 11.x versions of `playbooks/11-to-11.md`, and stops only for modules it cannot place | M | CMM | +| 152 | Journeys take 3 h or more on a mid-size app, and a stopped journeys step restarts from the first journey (cold CMM, cold WM1) | journeys resume from the last finished journey; the cockpit shows journeys done/total | M | CMM, WM1 | ## Done (14 Oct, remedy trial round) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 665734c..86baf84 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -411,6 +411,18 @@ timing_close() { # timing_close EXITCODE # $WORK/learnings/ and writes $WORK/FEEDBACK-REPORT.md: where it stopped, the drafts, and how to send them back (PR) or # file a toolkit bug (issue). Rung 2 reports on the whole run folder. Never changes the exit code. UPG_HARVEST=0 skips the # harvest; the report still comes. Before this, harvest.py ran only when someone remembered steps/report.md 3c (cold WM1). +# rung 2 localedits: modules rung 1 updated carry content id 0 in the MPR, so rung 2's localedits printed UNKNOWN ("give +# --cid Module=") for each one and the run stopped for a hand-written localedits.args (cold CMM, 7 modules). The content +# ids are in rung 1's plan.txt: add a --cid=Module=ID line for each plan module the args file does not name yet. +rung2_cids() { # rung2_cids PLAN ARGSFILE + [ -f "$1" ] || return 0 + local cid ver mod + while read -r cid ver mod _; do + case "$cid" in ''|\#*) continue ;; esac + [[ "$cid" =~ ^[0-9]+$ ]] && [ "$cid" != 0 ] && [ -n "$mod" ] || continue + grep -Eq -- "(^|[= ])$mod=" "$2" 2> /dev/null || printf -- '--cid=%s=%s\n' "$mod" "$cid" >> "$2" + done < "$1" +} feedback_finish() { # feedback_finish EXITCODE STEP [ "${DRY:-0}" != 1 ] && [ -d "${WORK:-}" ] || return 0 local fw=$WORK app="${UPG_APP_CODE:-${UPG_APP_NAME:-}}" @@ -1942,6 +1954,7 @@ main() { for s in env-values.json create-spec.json localedits.args interview-mode.txt; do [ ! -f "$WORK/$s" ] || [ -f "$WORK/rung2/$s" ] || cp "$WORK/$s" "$WORK/rung2/$s" done + rung2_cids "$WORK/plan.txt" "$WORK/rung2/localedits.args" fi exec env "RUN=$RUN" "UPG_PHASE=$(echo "$TARGET2" | cut -d. -f1-3)" "UPG_COCKPIT_OUT=$WORK/COCKPIT.html" "${r2[@]}" fi diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 60c3465..eb535ef 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -7467,6 +7467,14 @@ ctl feedback-report-deny-catch 0 'left out: it matches the private denylist' '!A { awk '/^feedback_finish\(\) \{/,/^}$/' $B/cloud/upgrade-pipeline.sh; awk '/^cleanup\(\) \{/,/^}$/' $B/cloud/upgrade-pipeline.sh printf '%s\n' 'timing_close() { :; }' "B=$B; WORK=$FB/trap; DRY=0; STEP=build; UPG_APP_CODE=HV1; APPS_UP=()" 'trap cleanup EXIT' 'exit 4'; } > $FB/trap.sh ctl pipeline-feedback-on-exit-pass 0 '^rc=4$' '^Exit 4 at build' -- sh -c "env -u UPG_BRAIN bash $FB/trap.sh > /dev/null 2>&1; echo rc=\$?; cat $FB/trap/FEEDBACK-REPORT.md" +# rung2_cids (cold CMM): rung 1 updated 7 modules, which then carry content id 0, and rung 2's localedits stopped for a +# hand-written localedits.args. rung 2 now gets --cid lines from rung 1's plan.txt; a module the args already name stays. +R2C=$T/rung2cids; mkdir -p $R2C +printf '%s\n' '# plan' '117187 3.22.0 Atlas_Core' '116540 3.11.8 DataWidgets' '# association targets last' '23513 4.5.0 Administration' > $R2C/plan.txt +printf '%s\n' '--cid=DataWidgets=999' > $R2C/args +awk '/^rung2_cids\(\) \{/,/^}$/' $B/cloud/upgrade-pipeline.sh > $R2C/fn.sh +ctl pipeline-rung2-cids-pass 0 '^--cid=Atlas_Core=117187$' '^--cid=Administration=23513$' '^--cid=DataWidgets=999$' '!DataWidgets=116540' '!plan' -- bash -c ". $R2C/fn.sh; rung2_cids $R2C/plan.txt $R2C/args; cat $R2C/args" +ctl pipeline-rung2-cids-wired-pass 0 'rung2_cids "\$WORK/plan.txt" "\$WORK/rung2/localedits.args"' -- grep -n 'rung2_cids "' $B/cloud/upgrade-pipeline.sh # submit-learnings.sh: a sandbox repo with a local bare remote. The checks refuse a denylisted name, a home folder path, # internal brain text sent to the toolkit, and a brain inbox inside the toolkit repo; a clean draft is pushed as a From ed4ff4dcccca9b29e289a08e5b357f63529799e4 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Wed, 18 Nov 2026 02:07:20 +0800 Subject: [PATCH 09/11] harvest: files the pipeline writes itself are no agent script The first live FEEDBACK-REPORT.md (cold CMM rung 2) listed 37 "scripts the agent wrote", all pipeline output: the patched journey engine copies in /e2e-*/engine (run_journeys.py patches them, so no hash matches a toolkit file), editroute's replay/*.mdl and localedits' saved-edits/. harvest.py now skips those folders. On the same run folder: 37 scripts -> 0. Control: harvest-toolkit-written-pass, and the fixture's harvest-run-pass now carries the three kinds (both fail on the old code, pass on the new). Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/harvest.py | 12 +++++++++++- upgrade-toolkit/controls/run-controls.sh | 4 ++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/upgrade-toolkit/bin/harvest.py b/upgrade-toolkit/bin/harvest.py index e35cb34..b6e0154 100755 --- a/upgrade-toolkit/bin/harvest.py +++ b/upgrade-toolkit/bin/harvest.py @@ -51,6 +51,16 @@ FILE_LIKE = re.compile(r"\.(html|js|java|md|jar|mpk|mpr|css|scss|json|log|txt|py|sh|xml|tsv)$", re.I) SHA = re.compile(r"\s*\(`[0-9a-f]{7,40}`\)") SCRIPT_EXT = {".py", ".sh", ".js", ".mdl"} +# folders the pipeline writes itself, never the agent: the patched journey engine copy (/e2e-*/engine, run_journeys.py), +# editroute's replay MDL and localedits' saved edits. Their files differ from any toolkit file by design (cold CMM: 37 of +# them were listed as scripts the agent wrote). +TOOLKIT_WRITTEN = {"replay", "saved-edits"} + + +def toolkit_written(rel): + parts = Path(rel).parts + return any(p in TOOLKIT_WRITTEN for p in parts[:-1]) or any( + p == "engine" and i > 0 and parts[i - 1].startswith("e2e-") for i, p in enumerate(parts[:-1])) SKIP_DIRS = {".git", "node_modules", "__pycache__", "e2e", "playwright-report", "test-results", "vendor"} TEXT_EXT = {".log", ".md", ".txt", ".out", ".tsv"} MAX_TEXT = 5_000_000 @@ -215,7 +225,7 @@ def harvest(work, brain): if "agent" in r.get("#", ""): found["items"].append({"src": rel, "text": f"agent decision: {r.get('decision', '')}: " f"{r.get('choice', '')} ({r.get('why', '')})"}) - if f.suffix in SCRIPT_EXT: + if f.suffix in SCRIPT_EXT and not toolkit_written(rel): try: h = hashlib.sha256(f.read_bytes()).hexdigest() except OSError: diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index eb535ef..859022e 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -7436,6 +7436,9 @@ hv_changes() { # $1 file, $2 extra why text of the new fix hv_changes $HV/run/CHANGES.md '' printf 'mx check: CE9987 frob widget has no successor\n' > $HV/run/check.log printf '#!/usr/bin/env python3\n# Rewrite the frob widgets on every page of the app\nprint(1)\n' > $HV/run/frob_fix.py +# files the pipeline writes itself are no agent script (cold CMM: 37 listed): a patched engine copy, replay MDL, saved edits +mkdir -p $HV/run/e2e-base/engine $HV/run/replay $HV/run/saved-edits/Mod; printf '// patched engine copy\n' > $HV/run/e2e-base/engine/helpers.js +printf '/* replay of Mod */\n' > $HV/run/replay/Mod.mdl; printf -- '-- CONSTANT Key, as it stands\n' > $HV/run/saved-edits/Mod/constant-Key.mdl printf '| 1 | WARNING: trail: 2 gap(s) in 2 steps |\n' > $HV/run/gate-summary.md printf -- '---\ntitle: mx.logger is gone in 11\nconfidentiality: internal\n---\nmx.logger calls crash on 11, CE9987 too.\n' > $HV/brain/internal/upgrade/mx-logger.md hv_changes $HV/run-deny/CHANGES.md ' for the Acmecorp portal' @@ -7445,6 +7448,7 @@ ctl harvest-run-pass 0 '^harvest: 3 agent fixes \(2 known to the toolkit, 1 new\ ctl harvest-new-fix-catch 0 'decision-frob.*Widget_Fake\.FrobAction\(\)' '!decision-mxlogger.*\[CHANGES' -- sed -n '/^## Fixes the agent made/,/^## Known/p' $HV/tk/2026-01-02-HV1-harvest.md ctl harvest-known-fix-pass 0 'repair-row-13.*\[known: steps/repair\.md row 13\]' 'decision-mxlogger.*\[known: BACKLOG\.md #102' -- cat $HV/tk/2026-01-02-HV1-harvest.md ctl harvest-code-script-item-pass 0 '^- CE9987 ' 'frob_fix\.py \(3 lines\): Rewrite the frob widgets' 'trail: 2 gap' '!CE1613' -- cat $HV/tk/2026-01-02-HV1-harvest.md +ctl harvest-toolkit-written-pass 0 'frob_fix\.py' '!e2e-base/engine' '!replay/Mod' '!saved-edits/' -- cat $HV/tk/2026-01-02-HV1-harvest.md ctl harvest-brain-lookup-pass 0 '^- CE9987 .*brain has a note' 'decision-frob.*brain has no note' -- cat $HV/tk/2026-01-02-HV1-harvest.md ctl harvest-brain-draft-pass 0 'confidentiality: internal' 'status: draft-unreviewed' 'Widget_Fake\.FrobAction\(\)' '!mx\.logger calls crash' '!CE9987' -- cat $HV/bi/2026-01-02-HV1-upgrade.md ctl harvest-scrub-pass 0 '!'"$HV" -- cat $HV/tk/2026-01-02-HV1-harvest.md $HV/bi/2026-01-02-HV1-upgrade.md From 997795221529d4eb5c4e7afbce980aaf6fbb0737 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Wed, 18 Nov 2026 02:13:24 +0800 Subject: [PATCH 10/11] playbooks: 11 removed IContext.setExecutionId; AnyChart maps to Custom chart; backlog 153-154 from cold CMM Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/BACKLOG.md | 2 ++ upgrade-toolkit/playbooks/10-to-11.md | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/upgrade-toolkit/BACKLOG.md b/upgrade-toolkit/BACKLOG.md index f2de8ac..a2c556a 100644 --- a/upgrade-toolkit/BACKLOG.md +++ b/upgrade-toolkit/BACKLOG.md @@ -167,6 +167,8 @@ An item is done only when its "done when" check holds on a real app, not when th | 150 | `mxcli widget init` writes `.claude/skills/widgets` into the app folder, which then needs a manual exclude from every app commit | the app's `.gitignore` (or the toolkit's own commit step) excludes it without a manual step | S | FSM | | 151 | Rung 2 stops at triage (exit 13) for a missing `plan.txt` although rung 1's plan already names most modules: write the rung 2 plan by hand every time (cold CMM run 10; the content ids now carry over, `rung2_cids`) | rung 2 triage pre-fills `plan.txt` from rung 1's plan and the 11.x versions of `playbooks/11-to-11.md`, and stops only for modules it cannot place | M | CMM | | 152 | Journeys take 3 h or more on a mid-size app, and a stopped journeys step restarts from the first journey (cold CMM, cold WM1) | journeys resume from the last finished journey; the cockpit shows journeys done/total | M | CMM, WM1 | +| 153 | No app code name at run start: runlog warns, and harvest and the feedback report skip until a person writes `state/app-code` (cold CMM) | the pipeline derives the code from the app name (or asks once at triage) and writes `state/app-code` for rung 1 and rung 2 | S | CMM | +| 154 | Rung 2 convert stops (exit 13) for a missing `plan-pre.txt` although no update in the plan is pre-10.21 (cold CMM run 12) | triage writes an empty `plan-pre.txt` with a comment when there is nothing to update before the convert | S | CMM | ## Done (14 Oct, remedy trial round) diff --git a/upgrade-toolkit/playbooks/10-to-11.md b/upgrade-toolkit/playbooks/10-to-11.md index b47a4fb..b1036fc 100644 --- a/upgrade-toolkit/playbooks/10-to-11.md +++ b/upgrade-toolkit/playbooks/10-to-11.md @@ -35,8 +35,9 @@ each one stopped the build or the conversion. Check each one by hand on every 10 | 5 | javac: old library copies shadow what CommunityCommons 11.5.1 needs (pdfbox, commons-io) | `jarclash.py`, also inside unused jars | remove the stale jar | | 6 | javac: a STRANDED module whose old library cannot live next to the new one (PDFReader 1.0.0, no newer version) | triage `stranded.md` | port its Java to the new library | | 7 | none visible: vendorlib holds old and new versions side by side until dependencies are resynced (67 to 48 files) | vendorlib after the module updates | resync before the build | -| 8 | runtime only (build, check, data diff clean): legacy Charts widgets `com.mendix.widget.custom.*` (Charts 3.0.1 and older) show "An error occurred in a ... widget", console "Could not find store", from 11.6 (React 19). T7 rung 2 FAIL | pages using `com.mendix.widget.custom.*` ids; a `Charts.mpk` of 3.0.1 or a `ChartsLegacy.mpk` | replace with the Charts 6.x widgets on the 10.24 rung ([widgets](widgets.md) section 1; mxcli can do it, T3) | +| 8 | runtime only (build, check, data diff clean): legacy Charts widgets `com.mendix.widget.custom.*` (Charts 3.0.1 and older) show "An error occurred in a ... widget", console "Could not find store", from 11.6 (React 19). T7 rung 2 FAIL | pages using `com.mendix.widget.custom.*` ids; a `Charts.mpk` of 3.0.1 or a `ChartsLegacy.mpk` | replace with the Charts 6.x widgets on the 10.24 rung ([widgets](widgets.md) section 1; mxcli can do it, T3). An AnyChart maps to `com.mendix.widget.web.customchart.CustomChart` with the same `dataAttribute` and `layoutAttribute`; set no `height` when `heightUnit` is `percentageOfWidth` (cold CMM) | | 9 | `mx convert` to 11 refuses an add-on module built before 10.21: CE1893 "created with 9.24.4, open it in 10.21 to 10.24 first" (T2: 3 vendor add-ons, 3 x CE1893). A vendor `.mxmodule` is a binary and cannot be reopened | `modules/*.mxmodule`: `python3 $B/mxversion.py modules/.mxmodule`, or the "Add-on modules" row of `test_inventory.py`; `repair/missing_deps.py --target 11.x` | the route used in T2: get a NEWER vendor build (built with 10.21 or later), load it on the 10.24 rung (it may need a layout or menu the old app lacks, CE1707: steps/repair.md row 16; a call into the add-on that lost its arguments, CE1571: row 12), run `idcheck.py` old against new (0 RED), then convert. Never convert with the old add-on in place. MC-10, when the vendor's newer build is a bigger version of the module than the app should take: swap ONLY the `modules/*.mxmodule` add-on file for the vendor's build made with 10.24 or later, and keep the module at its installed version, provided the new build's api.json exposes nothing the app does not already call (Asset Console: TeamcenterExtension 1.0.0 kept, add-on 4.2.1 built with 10.24.8) | +| 10 | javac: `cannot find symbol: method setExecutionId(String)` on `IContext` (11.0 removed it, release notes: internal use only) | own and module Java that labels a system context (cold CMM: OpcUaClientMx `SubscriptionManager`) | delete the call and keep the context; the id only labelled it. Note the old argument in a comment for the reviewer | Over-stated by the scanner: the removed Java action base class. The build rewrote 194 of 195 Java action files to the Mendix 11 base class by itself (the one it skipped was dead code). Do not plan hand work for it. From 1e87d94c40f2d8aa7c04e12ce2a4dfca0d68a0f8 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Wed, 18 Nov 2026 02:32:44 +0800 Subject: [PATCH 11/11] widgetdrop: stop before update-widgets drops settings a newer widget brought (repair row 33) Cold CMM: Administration 4.5.0 saved Account_Overview's Roles filter with Data Widgets 3.x in association mode; the kept DataWidgets 2.2.1 made update-widgets rewrite it to the 2.x definition, and the association settings were gone. On 3.11.8 the filter came back in attribute mode: "Unable to get filter store" on 10.24 and 11. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 10 ++ upgrade-toolkit/bin/widgetdrop.py | 130 ++++++++++++++++++ upgrade-toolkit/controls/run-controls.sh | 32 +++++ upgrade-toolkit/playbooks/widgets.md | 5 + upgrade-toolkit/steps/repair.md | 1 + 5 files changed, 178 insertions(+) create mode 100755 upgrade-toolkit/bin/widgetdrop.py diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 86baf84..c92c3ea 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -1189,9 +1189,19 @@ step_update() { nj_add 13 "Decision" "Update the kept module (stepupdate with a one-line plan), or put the old widget back; to accept the skew, create $WORK/widgetskew-ack.txt." nj_park fi + # a module update brought pages saved with a newer widget than the installed package: update-widgets (row 5) would + # drop the settings the old package lacks (cold CMM: Administration 4.5.0's Roles filter lost its association mode) + rc=0 + xo "$WORK/widgetdrop.out" python3 "$B/widgetdrop.py" "$APP" || rc=$? + if [ "$rc" = 1 ] && [ ! -f "$WORK/widgetdrop-ack.txt" ]; then + nj_file 13 "Widget settings that update-widgets would drop: the installed widget package is older than the pages (steps/repair.md row 33)" "$WORK/widgetdrop.out" 10 '^WIDGET-DROP' + nj_add 13 "Decision" "Update the module that ships the widget (stepupdate with a one-line plan) before the build; to accept the loss, create $WORK/widgetdrop-ack.txt." + nj_park + fi else x python3 "$B/repair/jarclash.py" "$APP" --md "$WORK/jarclash.md" x python3 "$B/widgetskew.py" "$APP" --plan "$WORK/plan.txt" + x python3 "$B/widgetdrop.py" "$APP" fi local rc=0 ap=() [ ! -f "$WORK/approved.txt" ] || ap=(--approved "$WORK/approved.txt") diff --git a/upgrade-toolkit/bin/widgetdrop.py b/upgrade-toolkit/bin/widgetdrop.py new file mode 100755 index 0000000..89a1c26 --- /dev/null +++ b/upgrade-toolkit/bin/widgetdrop.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""widgetdrop.py: which widget settings would `mx update-widgets` drop, because the installed widget package lacks them? + + widgetdrop.py [--json out.json] + +Why (cold CMM, 9.24.2 -> 10.24.28): Administration 4.5.0 ships Account_Overview with a Data Widgets 3.x drop-down filter +in association mode (baseType ref, refEntity over System.UserRoles). The plan kept DataWidgets at 2.2.1, whose filter has +no association mode. The build's update-widgets repair (row 5) rewrote the filter to the 2.x definition and the ref +settings were gone; when DataWidgets 3.11.8 came later the filter came back with its default, attribute mode. The page +then showed "Unable to get filter store" in the Roles column on 10.24 and on 11, and update-widgets reported success. + +A pluggable widget in the model carries the definition it was saved with. A property that holds a value other than its +default there, and that the installed widgets/*.mpk no longer declares, is lost by update-widgets: + + WIDGET-DROP Account_Overview drop_downFilter1 (com.mendix.widget.web.datagriddropdownfilter. + DatagridDropdownFilter, installed package lacks it): baseType=ref, refEntity=System.UserRoles + +Fix before update-widgets: update the module that ships the widget (here DataWidgets) in the same rung as the module +whose pages brought the newer definition (steps/repair.md row 33). Excluded documents are skipped. +Exit: 0 = nothing would be dropped, 1 = a drop, 2 = input error. +""" +import argparse, glob, json, os, re, sys, zipfile + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "repair")) +from missing_deps import units # noqa: E402 the .mpr reader (v1 and v2) + + +def installed_keys(proj): + """{widget id: set of property keys} declared by widgets/*.mpk.""" + out = {} + for f in glob.glob(os.path.join(proj, "widgets", "*.mpk")): + try: + with zipfile.ZipFile(f) as z: + for n in z.namelist(): + if not n.endswith(".xml") or n.endswith("package.xml"): + continue + x = z.read(n).decode("utf-8", "replace") + m = re.search(r']*\bid="([^"]+)"', x) + if m: + out[m.group(1)] = set(re.findall(r']*\bkey="([^"]+)"', x)) + except (OSError, zipfile.BadZipFile): + continue + return out + + +def find(d, pred): + if isinstance(d, dict): + if pred(d): + yield d + for v in d.values(): + yield from find(v, pred) + elif isinstance(d, list): + for v in d: + yield from find(v, pred) + + +def value(v): + """The set value of a CustomWidgets$WidgetValue as text, '' when it holds nothing.""" + if not isinstance(v, dict): + return "" + if v.get("PrimitiveValue"): + return v["PrimitiveValue"] + a = v.get("AttributeRef") or {} + if a.get("Attribute"): + return a["Attribute"] + e = v.get("EntityRef") or {} + steps = [s.get("Association") for s in e.get("Steps") or [] if isinstance(s, dict) and s.get("Association")] + if steps or e.get("Entity"): + return "/".join(steps) or e["Entity"] + if v.get("Expression"): + return v["Expression"] + return "" + + +def drops(widget, have): + """[(key, value)] set on this widget instance (its nested objects too) that the installed package does not declare.""" + types = {} + for pt in find(widget.get("Type") or {}, lambda x: x.get("$Type") == "CustomWidgets$WidgetPropertyType"): + vt = pt.get("ValueType") or {} + types[repr(pt.get("$ID"))] = (pt.get("PropertyKey"), str(vt.get("DefaultValue") or "")) + out = [] + for prop in find(widget.get("Object") or {}, lambda x: x.get("$Type") == "CustomWidgets$WidgetProperty"): + key, default = types.get(repr(prop.get("TypePointer")), (None, "")) + val = value(prop.get("Value")) + if key and key not in have and val and val != default: + out.append((key, val)) + return out + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("project"); ap.add_argument("--json") + a = ap.parse_args() + p = a.project + if os.path.isdir(p): + mprs = glob.glob(os.path.join(p, "*.mpr")) + if len(mprs) != 1: + print(f"widgetdrop: {p}: expected one .mpr", file=sys.stderr); return 2 + p = mprs[0] + if not os.path.isfile(p): + print(f"widgetdrop: no project {p}", file=sys.stderr); return 2 + have = installed_keys(os.path.dirname(os.path.abspath(p))) + found, seen = [], 0 + docs = [] + for d in units(p): + if d.get("$Type", "").startswith("Forms$") and d.get("Excluded") is not True: + docs.append(d) + for d in docs: + name = d.get("Name") or "?" + for w in find(d, lambda x: x.get("$Type") == "CustomWidgets$CustomWidget"): + wid = (w.get("Type") or {}).get("WidgetId") + if not wid or wid not in have: # not installed at all: missing_deps.py reports it + continue + seen += 1 + lost = drops(w, have[wid]) + if lost: + found.append({"document": name, "widget": w.get("Name"), "id": wid, "drops": lost}) + lines = [f"# widgetdrop {p}", f"widget instances checked {seen}, installed widget ids {len(have)}"] + for f in found: + lines.append(f"WIDGET-DROP {f['document']} {f['widget']} ({f['id']}, installed package lacks it): " + + ", ".join(f"{k}={v}" for k, v in f["drops"])) + lines.append(f"widgetdrop: {'FAIL' if found else 'PASS'}, {len(found)} of {seen} widget instances would lose settings") + print("\n".join(lines)) + if a.json: + json.dump(found, open(a.json, "w"), indent=1) + return 1 if found else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 859022e..7070458 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -322,6 +322,38 @@ printf '%s\n' 'xo() { o=$1; shift; "$@" > "$o"; }' 'nj_file() { echo "NJ $1 $2"; mkdir -p $WS/w1 $WS/w2; cp $WS/plan.txt $WS/w1/; cp $WS/plan.txt $WS/w2/; : > $WS/w2/widgetskew-ack.txt ctl pipeline-update-widgetskew-catch 0 '^NJ 13 A module update replaced a widget whose styles stay with a kept module \(steps/repair\.md row 32\)$' '^WIDGET-SKEW Datagrid' '^NJ 13 Decision: Update the kept module' '^PARK$' -- bash -c ". $WS/stub.sh; WORK=$WS/w1 APP=$WS/app B=$B; . $WS/nj.sh; echo done" ctl pipeline-update-widgetskew-ack-pass 0 '!^NJ' '!PARK' '^done$' -- bash -c ". $WS/stub.sh; WORK=$WS/w2 APP=$WS/app B=$B; . $WS/nj.sh; echo done" +# widgetdrop.py (cold CMM 10.24.28): Administration 4.5.0's Roles filter was saved with Data Widgets 3.x (association +# mode); update-widgets under the kept DataWidgets 2.2.1 rewrote it to the old definition and the ref settings were lost. +# A set, non-default property the installed package does not declare stops; a default value or a declared key does not +WD=$T/wd; mkdir -p $WD/old/widgets $WD/new/widgets $WD/dflt/widgets +cat > $WD/mk.py <<'EOF' +import bson, sqlite3, sys, uuid, zipfile +d, base, keys = sys.argv[1], sys.argv[2], sys.argv[3].split(",") +wid = "com.mendix.widget.web.datagriddropdownfilter.DatagridDropdownFilter" +def pt(i, k, dv): return {"$ID": i, "$Type": "CustomWidgets$WidgetPropertyType", "PropertyKey": k, "ValueType": {"DefaultValue": dv}} +def pv(i, val): return {"$Type": "CustomWidgets$WidgetProperty", "TypePointer": i, "Value": val} +w = {"$Type": "CustomWidgets$CustomWidget", "Name": "drop_downFilter1", + "Type": {"WidgetId": wid, "ObjectType": {"PropertyTypes": [3, pt("a", "multiSelect", "false"), pt("b", "baseType", "attr"), pt("c", "refEntity", "")]}}, + "Object": {"Properties": [2, pv("a", {"PrimitiveValue": "true"}), pv("b", {"PrimitiveValue": base}), + pv("c", {"EntityRef": {"Steps": [2, {"Association": "System.UserRoles"}]} if base == "ref" else None})]}} +page = {"$Type": "Forms$Page", "Name": "Account_Overview", "Excluded": False, "Content": [w]} +con = sqlite3.connect(f"{d}/app.mpr"); con.execute("create table Unit (UnitID blob, Contents blob)") +con.execute("insert into Unit values (?, ?)", (uuid.uuid4().bytes_le, bson.encode(page))); con.commit() +props = "".join(f'' for k in keys) +with zipfile.ZipFile(f"{d}/widgets/com.mendix.widget.web.DatagridDropdownFilter.mpk", "w") as z: + z.writestr("package.xml", "") + z.writestr("DatagridDropdownFilter.xml", f'{props}') +EOF +python3 $WD/mk.py $WD/old ref multiSelect; python3 $WD/mk.py $WD/new ref multiSelect,baseType,refEntity; python3 $WD/mk.py $WD/dflt attr multiSelect +ctl widgetdrop-catch 1 '^WIDGET-DROP Account_Overview drop_downFilter1 \(com\.mendix\.widget\.web\.datagriddropdownfilter\.DatagridDropdownFilter, installed package lacks it\): baseType=ref, refEntity=System\.UserRoles$' '!multiSelect=' '^widgetdrop: FAIL, 1 of 1 widget' -- python3 $B/widgetdrop.py $WD/old +ctl widgetdrop-declared-pass 0 '!WIDGET-DROP' '^widgetdrop: PASS, 0 of 1 widget' -- python3 $B/widgetdrop.py $WD/new +ctl widgetdrop-default-pass 0 '!WIDGET-DROP' '^widgetdrop: PASS' -- python3 $B/widgetdrop.py $WD/dflt +ctl widgetdrop-noproject-pass 2 'expected one \.mpr' -- python3 $B/widgetdrop.py $WD +# step_update parks on it, unless widgetdrop-ack.txt accepts it +awk '/xo "\$WORK\/widgetdrop.out"/{p=1} p{print} p&&/^ fi$/{exit}' $B/cloud/upgrade-pipeline.sh > $WD/nj.sh +mkdir -p $WD/w1 $WD/w2; : > $WD/w2/widgetdrop-ack.txt +ctl pipeline-update-widgetdrop-catch 0 '^NJ 13 Widget settings that update-widgets would drop: the installed widget package is older than the pages \(steps/repair\.md row 33\)$' '^WIDGET-DROP Account_Overview' '^NJ 13 Decision: Update the module that ships the widget' '^PARK$' -- bash -c ". $WS/stub.sh; WORK=$WD/w1 APP=$WD/old B=$B; . $WD/nj.sh; echo done" +ctl pipeline-update-widgetdrop-ack-pass 0 '!^NJ' '!PARK' '^done$' -- bash -c ". $WS/stub.sh; WORK=$WD/w2 APP=$WD/old B=$B; . $WD/nj.sh; echo done" # the build stop quotes owngone.out when mx check has CE1613 (and only then) mkdir -p $T/og; printf '%s\n' 'crosscalls: own modules 1' 'OWN AuthMod 3.2.0 -> 4.1.0: 1 target(s) own modules use are gone from the 4.1.0 package' ' GONE page AuthMod.Login_Page (used by own Shop.ACT_Open)' > $T/og/owngone.out printf '%s\n' '[error] [CE1613] "The selected page no longer exists." at Microflow ACT_Open' > $T/og/check-1613.log; printf '%s\n' '[error] [CE0117] "x"' > $T/og/check-0117.log diff --git a/upgrade-toolkit/playbooks/widgets.md b/upgrade-toolkit/playbooks/widgets.md index c7f0168..0ecb4fa 100644 --- a/upgrade-toolkit/playbooks/widgets.md +++ b/upgrade-toolkit/playbooks/widgets.md @@ -162,6 +162,11 @@ resets properties that do not exist in the new version. Seen: filtered grid page. Update DataWidgets in the same rung as Atlas_Web_Content (3.11.8 cleared it); smokewalk fails a page that shows a widget error box. Data grid 2.2 (9.24) rows are `div.tr[role=row]` with `display: contents` and cells `div.td` with no role and no `widget-datagrid` class on the root; the journeys row verb handles both shapes. +- Administration 4.x needs DataWidgets 3.x in the same rung (cold CMM): its pages are saved with Data Widgets 3.x + (Account_Overview's Roles filter is a drop-down in association mode). Under a kept DataWidgets 2.x, update-widgets + rewrites the filter to the 2.x definition and drops the association settings; on 3.x later the filter is back in + attribute mode and shows "Unable to get filter store". `bin/widgetdrop.py` names such losses before the build + (steps/repair.md row 33). ## 4. React-client readiness is not upgrade work, but list it diff --git a/upgrade-toolkit/steps/repair.md b/upgrade-toolkit/steps/repair.md index 1afe51d..c5ab0f3 100644 --- a/upgrade-toolkit/steps/repair.md +++ b/upgrade-toolkit/steps/repair.md @@ -43,6 +43,7 @@ keep the fix and continue with the new class. Without `--fixed` a rise is always | 30 | CE2729 "No read access to association 'A_B' in entity 'Other.E' for user role ..." after an update that re-created an association to another module's entity (editroute XASSOC replay) | `work/replay/.mdl` has `member access on ` but no `the other side` part; `mxcli describe entity Other.E` shows the member gone from its grants, the original model has it | The re-created association starts with no member access on BOTH entities. The replay restored only the owner side; since fix 15 it also restores the other side, as only that member merged into each rule (mxcli's grant adds to a role's rule and keeps the rest). Cold CMM run 8: EmployeeManagement.Employee_Account to Administration.Account, ReadOnly for Administrator and User became None, 5 CE2729 on the Employee text boxes. Remedy for a run replayed before the fix (verified, cold CMM run 9): per grant line of the entity in the ORIGINAL model, `grant read (A_B) on entity Other.E to [where ...];` (editroute.member_grant prints these), mxcli exec, commit | | 31 | javac `cannot find symbol` in an updated module's Java (MendixSSO, CommunityCommons) at mxbuild, where the class exists but the member does not | `javacjar.out` (the build stop runs `bin/repair/javacjar.py ` and quotes it): `SYMBOL .`, `HAS `, `HIDDEN BY ` | An older copy of the class sits in another jar, often a fat (shaded, jar-with-dependencies) jar in userlib that bundles its libraries, and javac takes that copy. Cold CMM (10.24.28): MendixSSO 10.6.2 calls `CookieHeaderNames.SameSite` (vendorlib netty-codec-http 4.2.17); userlib/server-examples-0.4.1-SNAPSHOT.jar (OPC UA, 12,588 entries) carries its own older Netty, Guava and BouncyCastle. jardedupe keeps such a jar (only copy of some classes) and jarclash lists it as a note. Remedy (verified, cold CMM run 10: mxbuild passes): `python3 $B/repair/javacjar.py app/ --strip userlib/.jar` (dry run: how many classes a vendorlib jar also carries), then `--apply`, commit `jar:`; the library then runs on the vendorlib versions (newer majors: Netty 4.2, Guava 33): a judgment item, so test what uses the library at the gate. Alternatives: a thin jar plus its separate dependencies, or a newer version of the library | | 32 | After the module updates, data grid (or other widget) pages show their rows without layout: no columns, no header styling; afterwalk passes (the data is on the page), the screens rung says REVIEW | `widgetskew.out` (step_update runs `bin/widgetskew.py --plan plan.txt` and parks): `WIDGET-SKEW -> (by ) ships with `. One module's package replaced a shared widget in widgets/ (`.stepupdate/widgets.txt`), while the module that carries its styles (themesource) was kept at its old version | Update the kept module too: a one-line plan (` `) through `bin/stepupdate.sh` (its local edits are often OS junk, `.DS_Store`); or put the old widget .mpk back from git. Accepting the skew: `widgetskew-ack.txt` in the work dir. Cold CMM: Atlas_Web_Content 3.9.0 brought Datagrid 2.30.6, DataWidgets stayed 3.x (FILES-EDITED on `.DS_Store`); 23 grid pages unstyled | +| 33 | A filter or other pluggable widget that a module update brought loses its settings after the build: the page shows "Unable to get filter store" (or a widget falls back to its default mode) although update-widgets (row 5) succeeded | `widgetdrop.out` (step_update runs `bin/widgetdrop.py ` and parks): `WIDGET-DROP (, installed package lacks it): key=value, ...`. The module's pages were saved with a newer widget than the package in widgets/; update-widgets rewrites them to the old definition and drops every property it lacks | Update the module that ships the widget in the same rung, before the build: a one-line plan through `bin/stepupdate.sh`. Already lost: set the widget again from the module's own package (Studio Pro, or a page export from the cached `-.mpk`); mxcli cannot write a drop-down filter in association mode. Accepting the loss: `widgetdrop-ack.txt` in the work dir. Cold CMM: Administration 4.5.0's Account_Overview Roles filter (baseType ref over System.UserRoles) became an attribute filter under DataWidgets 2.2.1, then broke on 3.11.8 | mxcli v0.28.0 (#1408, #1474), what it changes for the rows above: - `mxcli exec` is all-or-nothing. A script whose statement N fails now changes NOTHING and exits non-zero; before, statements 1 to N-1