From 0478a460c9209a2ead13afbd04cbaa43a1ab6b65 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Mon, 16 Nov 2026 22:19:14 +0800 Subject: [PATCH 01/59] cockpit: a signed plan shows the current wording, and an item the sign-off accepted leaves Blocked on you Cold test WM1: after the toolkit moved to update-instead-of-keep, the signed PROPOSAL.md kept the old "keep" cards (the signed path returned before re-rendering) and seven module cards stayed in Blocked on you while DECISIONS.md held them CONFIRMED with the sign-off. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cockpitpage.py | 4 ++++ upgrade-toolkit/bin/proposal.py | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/upgrade-toolkit/bin/cockpitpage.py b/upgrade-toolkit/bin/cockpitpage.py index 536c20c..44a33f4 100644 --- a/upgrade-toolkit/bin/cockpitpage.py +++ b/upgrade-toolkit/bin/cockpitpage.py @@ -567,6 +567,10 @@ def add(it): for o in opts: o["writes"] = [{"decisions_md": {"id": pid, "name": it["name"], "kind": it["kind"], "option": o["label"]}}] is_assumed = bool(row and "ASSUMED" in (row[5] if len(row) > 5 else "")) or not row + # an item the sign-off accepted at its recommendation is answered (cold test WM1: seven module cards stayed in + # Blocked on you while DECISIONS.md held them CONFIRMED with the sign-off) + if not last.get(pid) and signed and row and len(row) > 5 and row[5] in ("CONFIRMED", "DECIDED"): + last[pid] = {"item": pid, "choice": row[2], "by": row[4], "via": "sign-off"} if is_assumed and not last.get(pid): assumed.append(pid) items.append({"id": pid, "kind": "plan-choice", "hidden": True, "dot": "warn", "title": f"{it['name']} ({it['kind']})", "step": "propose", diff --git a/upgrade-toolkit/bin/proposal.py b/upgrade-toolkit/bin/proposal.py index becc91f..195f302 100644 --- a/upgrade-toolkit/bin/proposal.py +++ b/upgrade-toolkit/bin/proposal.py @@ -777,6 +777,14 @@ def accept_answers(signer): print(f"proposal: PAUSED by the owner: {so[1]}. Resume: remove the Status line and sign, then rerun the pipeline.") return 15 if so and so[0] == "signed" and prev_hash == hashval: + # same plan, newer wording (cold test WM1: the signed cards kept saying "keep old version" after the toolkit + # recommended the update): re-render the items, the person's sign-off section is copied over as written + i = old.find("\n## 6. Sign-off\n") + cur = render(F, hashval, None, old[i + len("\n## 6. Sign-off\n"):].strip("\n")) if i >= 0 else old + if cur != old and parse_signoff(cur) == so: + with open(out_path + ".tmp", "w", encoding="utf-8") as f: + f.write(cur) + os.replace(out_path + ".tmp", out_path) n = accept_answers(so[1]) approve_steps(a.work, sids, "approved", so[1], "signoff", decisions(a.work)) print(f"proposal: signed by {so[1]} on {so[2]}" + (f"; {n} unanswered section-3 item(s) took their recommendation (DECISIONS.md)" if n else "")) From bfc9e761d0265f14fcdacad6f5132fe7c47c0f82 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Mon, 16 Nov 2026 22:43:04 +0800 Subject: [PATCH 02/59] editroute: replay the change, not the element; route again after a routed update REPLAY builds each element by a three-way merge of mxcli describe (installed package, original app, updated app), sets the app's property values on the new version when the vendor rewrote the lines around them, and holds any result mxcli check refuses. A module the routed update keeps again is routed again (routes add up); the same reason twice holds. Cold test WM1 Atlas_Core: 7 of 7 elements merge and pass check (was: replay refused, exit 13). Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 11 +- upgrade-toolkit/bin/editroute.py | 173 +++++++++++++++--- upgrade-toolkit/controls/editroute_control.sh | 40 +++- upgrade-toolkit/controls/run-controls.sh | 10 +- 4 files changed, 198 insertions(+), 36 deletions(-) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index cfa10af..8d44f91 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -988,8 +988,17 @@ route_kept() { xo "$WORK/editroute.out" python3 "$B/editroute.py" plan --work "$WORK" --app "$APP" --original "$WORK/original.mpr" --plan "$WORK/plan.txt" "${le[@]}" || rc=$? cat "$WORK/editroute.out" >> "$LOG" [ "$rc" = 0 ] || { note "editroute: no kept module could be routed, they stay kept (see $WORK/editroute.out)"; return 0; } - run_stepupdate "$WORK/route-plan.txt" routed + local pass + for pass in 1 2 3; do # a routed update can keep a module for a second reason (WM1 Email_Connector: diagram-only, then 3 associations) + run_stepupdate "$WORK/route-plan.txt" routed + [ -s "$APP/.stepupdate/kept.txt" ] && [ "$pass" -lt 3 ] || break + rc=0; xo "$WORK/editroute.out" python3 "$B/editroute.py" plan --work "$WORK" --app "$APP" --original "$WORK/original.mpr" --plan "$WORK/plan.txt" "${le[@]}" || rc=$? + cat "$WORK/editroute.out" >> "$LOG" + [ "$rc" = 0 ] || break + done rc=0; xo "$WORK/editroute-collect.out" python3 "$B/editroute.py" collect --work "$WORK" || rc=$? + cat "$WORK/editroute-collect.out" >> "$LOG" + ! grep -q '^HELD ' "$WORK/editroute-collect.out" || note "editroute: $(grep -c '^HELD ' "$WORK/editroute-collect.out") replayed element(s) held for a person: $(grep '^HELD ' "$WORK/editroute-collect.out" | head -3 | tr '\n' ';')" note "editroute: $(grep -c '^ROUTE' "$WORK/editroute.out") kept module(s) routed, $(grep -c ' HOLD:' "$WORK/editroute.out") held for a person (edit-routes.json)" } diff --git a/upgrade-toolkit/bin/editroute.py b/upgrade-toolkit/bin/editroute.py index 837f328..7928bc0 100755 --- a/upgrade-toolkit/bin/editroute.py +++ b/upgrade-toolkit/bin/editroute.py @@ -10,8 +10,12 @@ Routes, per refused module (the module is updated in all of them except HOLD): COSMETIC every refused microflow/nanoflow/rule equals the installed package once diagram geometry (@position, @anchor, @curve, @size) is ignored: forced, nothing to replay - REPLAY pages, layouts, snippets, images and constants: saved as MDL by the forced update (--save-edits) and replayed onto the - new version (replay/.mdl); mixed with COSMETIC logic, only the UI elements are replayed + REPLAY pages, layouts, snippets, images and constants: the CHANGE is replayed onto the new version, not the old element + (replay/.mdl): a three-way merge per element of `mxcli describe` (base = installed package, ours = the + original app, theirs = the updated app), each result passed through `mxcli check` first; mixed with COSMETIC + logic, only the UI elements are replayed. Why (cold test WM1, Atlas_Core): the saved whole element lost the + layout's master-layout link (empty Atlas_Default, every page would lose the company design) and carried the old + Sprintr widget's properties onto the new widget; replaying it was refused by mxcli (MDL081, MDL-WIDGET01) UNREAD mxcli could not read some elements and localedits shows no domain or logic edit: forced, whatever it could save replayed FILES theme/JavaScript files could not be compared (installed package unreadable): the saved copies under .stepupdate/saved// stay, the update may commit (accept-unsaved.txt) @@ -22,12 +26,16 @@ HOLD a real logic edit, Java user code, a GUID change, a removal something still uses, a tool or network fault: it stays a card for a person, with the evidence. The recommendation there is still to update (move the edit to _Ext). + A module the routed update kept again is routed again (the pipeline runs plan up to three times): routes add up per module + (Email_Connector, WM1: COSMETIC, then XASSOC for the 3 associations the app's Email module owns into it). The same reason + twice is a HOLD. + editroute.py plan --work W --app A --original W/original.mpr --plan W/plan.txt writes the route files, prints ROUTE lines editroute.py collect --work W after the routed update: builds replay/ Files: W/edit-routes.json (one row per module), W/route-plan.txt (the plan lines of the routed modules, plan order), and appends to W/force-ui.txt, W/accept-unsaved.txt, W/approved.txt, W/approved-step.txt. Exit 0 = something routed, 1 = nothing. """ -import argparse, glob, json, os, re, subprocess, sys +import argparse, difflib, glob, json, os, re, subprocess, sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import localedits # noqa: E402 (MXCLI, CACHE, fetch_pkg) @@ -150,18 +158,38 @@ def cmd_plan(a): led = {} cache = os.environ.get("UPG_CACHE") or localedits.CACHE mpr = os.path.join(app, next((f for f in os.listdir(app) if f.endswith(".mpr")), "")) - rows = [] + try: # an earlier pass of this rung: its rows stay, a module kept again adds its new route + prev = {x["module"]: x for x in json.load(open(os.path.join(W, "edit-routes.json")))} + except (OSError, ValueError): + prev = {} + rows, now = dict(prev), [] for mod, why in kept: r, reason, extra = route_module(mod, why, S, led.get(mod), a.original, cache, mpr) - rows.append({"module": mod, "route": r, "reason": reason, "kept_because": why, **extra}) + p = prev.get(mod) + if p and p.get("kept_because") == why and r != "HOLD": + r, reason, extra = "HOLD", f"the routed update kept it again for the same reason: {why}", {} + e = led.get(mod) or {} + if r in ("REPLAY", "UNREAD") and e.get("cid") and e.get("version"): + extra["pkg"] = localedits.fetch_pkg(e["cid"], e["version"], cache)[0] + row = {"module": mod, "route": r, "reason": reason, "kept_because": why, "original": a.original, "app_mpr": mpr, **extra} + if p and p["route"] != "HOLD" and r != "HOLD": + for k in ("ui", "cosmetic", "assocs", "vendor"): + row[k] = sorted(set(p.get(k, [])) | set(row.get(k, []))) + row["pkg"] = row.get("pkg") or p.get("pkg") + row["route"] = "+".join(dict.fromkeys(p["route"].split("+") + [r])) + row["reason"] = p["reason"] + "; then " + reason + rows[mod] = row + now.append(row) print(f"ROUTE {mod} {r}: {reason}") - json.dump(rows, open(os.path.join(W, "edit-routes.json"), "w"), indent=1) + json.dump(list(rows.values()), open(os.path.join(W, "edit-routes.json"), "w"), indent=1) + rows = now go = {x["module"] for x in rows if x["route"] != "HOLD"} plan = [ln for ln in rd(a.plan).splitlines() if len(ln.split()) >= 3 and not ln.startswith("#") and ln.split()[2] in go] with open(os.path.join(W, "route-plan.txt"), "w") as f: f.write("# modules the update kept, routed by editroute.py (edit-routes.json)\n" + "".join(ln + "\n" for ln in plan)) - append(os.path.join(W, "force-ui.txt"), [f"{x['module']} # editroute {x['route']}" for x in rows if x["route"] in ("COSMETIC", "REPLAY", "UNREAD", "XASSOC")]) - append(os.path.join(W, "accept-unsaved.txt"), [f"{x['module']} # editroute FILES: saved copies under .stepupdate/saved/{x['module']}" for x in rows if x["route"] == "FILES"]) + append(os.path.join(W, "force-ui.txt"), [f"{x['module']} # editroute {x['route']}" for x in rows + if {"COSMETIC", "REPLAY", "UNREAD", "XASSOC"} & set(x["route"].split("+"))]) + append(os.path.join(W, "accept-unsaved.txt"), [f"{x['module']} # editroute FILES: saved copies under .stepupdate/saved/{x['module']}" for x in rows if "FILES" in x["route"].split("+")]) append(os.path.join(W, "approved.txt"), [f"{qn} # editroute VENDOR: the new {x['module']} drops it, nothing outside the module refers to it" for x in rows for qn in x.get("vendor", [])]) append(os.path.join(W, "approved-step.txt"), [f"{qn} # editroute XASSOC: removed by the {x['module']} update, replayed with its original id" @@ -169,36 +197,133 @@ def cmd_plan(a): return 0 if go else 1 +def norm(t): + return [ln.rstrip() for ln in (t or "").splitlines() if ln.strip() and not ln.startswith("mdl ")] + + +def merge_element(kind, qn, base_mpr, orig, app_mpr, tmp): + """-> (mdl or None, how): the app's change carried onto the updated element, per the REPLAY rules in the docstring.""" + ours = describe(orig, kind, qn) + if not ours: + return None, "unreadable in the original model" + base = describe(base_mpr, kind, qn) if base_mpr else None + if base is not None and norm(ours) == norm(base): + return "", "no edit (equal to the installed package)" + theirs = describe(app_mpr, kind, qn) + if base is None: + return ours, "added by the app: its version" + if theirs is None or norm(theirs) == norm(base): + return ours, "the vendor did not change it: the app's version" + paths = [] + for n, t in (("ours", ours), ("base", base), ("theirs", theirs)): + paths.append(os.path.join(tmp, f"{qn}.{n}")) + with open(paths[-1], "w") as f: + f.write("\n".join(norm(t)) + "\n") + r = subprocess.run(["git", "merge-file", "-p", *paths], capture_output=True, text=True) + if r.returncode == 0: + return "mdl 1;\n" + r.stdout, "merged: the app's change on the new version" + props = prop_patch(norm(base), norm(ours), norm(theirs)) + if props is not None: + return "mdl 1;\n" + "\n".join(props) + "\n", "the app's property values set on the new version" + b, o = set(norm(base)), norm(ours) + if sum(1 for ln in o if ln in b) < 0.3 * max(len(b), 1): + return ours, "the app rewrote the whole element: its version kept, the vendor's change to it dropped" + return None, "the app's change and the vendor's change overlap (merge conflict)" + + +PROP = re.compile(r"^(\s*)([\w.]+):\s") + + +def prop_patch(base, ours, theirs): + """theirs with the app's changed property values, when every app change is a `key: value` line whose key also stands once + in theirs (WM1 FeedbackWidget: only `sprintrapp` differed; the vendor rewrote the lines around it, so a line merge collides).""" + sm = difflib.SequenceMatcher(None, base, ours, autojunk=False) + sets = {} + for op, i1, i2, j1, j2 in sm.get_opcodes(): + if op == "equal": + continue + if op != "replace" or i2 - i1 != j2 - j1: + return None + for bl, ol in zip(base[i1:i2], ours[j1:j2]): + mb, mo = PROP.match(bl), PROP.match(ol) + if not (mb and mo and mb.group(2) == mo.group(2)): + return None + sets[mo.group(2)] = ol + out = list(theirs) + for key, line in sets.items(): + hits = [i for i, ln in enumerate(out) if (m := PROP.match(ln)) and m.group(2) == key] + if len(hits) != 1: + return None + comma = out[hits[0]].rstrip().endswith(",") + out[hits[0]] = line.rstrip().rstrip(",") + ("," if comma else "") + return out if sets else None + + +def checked(mdl, app_mpr, tmp, qn): + p = os.path.join(tmp, f"{qn}.check.mdl") + with open(p, "w") as f: + f.write(mdl) + rc, out = mx(["check", p, "-p", app_mpr]) + return rc == 0, " ".join(ln.strip() for ln in out.splitlines() if "✗" in ln or "error" in ln.lower())[:300] + + def cmd_collect(a): W = os.path.abspath(a.work) try: - rows = json.load(open(os.path.join(W, "edit-routes.json"))) + allrows = json.load(open(os.path.join(W, "edit-routes.json"))) except (OSError, ValueError): return 1 os.makedirs(os.path.join(W, "replay"), exist_ok=True) + tmp = os.path.join(W, "replay", ".merge") + os.makedirs(tmp, exist_ok=True) n = 0 - for x in rows: - mod, r = x["module"], x["route"] + for x in allrows: + mod, routes = x["module"], set(x["route"].split("+")) d = os.path.join(W, "saved-edits", mod) - if r == "REPLAY" or r == "UNREAD": + parts, held = [], [] + if routes & {"REPLAY", "UNREAD"}: keep = {re.sub(r"\W", "", e.split(" ", 1)[1]).lower() for e in x.get("ui", [])} - files = [f for f in sorted(glob.glob(os.path.join(d, "*.mdl"))) if not os.path.basename(f).startswith("association-") - and (r == "UNREAD" or re.sub(r"\W", "", os.path.basename(f)[:-4].split("-", 1)[-1]).lower() in keep)] - elif r == "XASSOC": - files = sorted(glob.glob(os.path.join(d, "association-*.mdl"))) + for f in sorted(glob.glob(os.path.join(d, "*.mdl"))): + b = os.path.basename(f)[:-4] + if b.startswith("association-") or "-" not in b: + continue + kind, name = b.split("-", 1) + if "UNREAD" not in routes and re.sub(r"\W", "", name).lower() not in keep: + continue + qn = f"{mod}.{name}" + if kind in ("layout", "snippet", "page", "constant") and x.get("original") and x.get("app_mpr"): + mdl, how = merge_element(kind, qn, x.get("pkg"), x["original"], x["app_mpr"], tmp) + else: + mdl, how = rd(f), "saved element (no describe for this kind)" + if mdl is None: + held.append(f"{kind} {name}: {how}") + continue + if not mdl.strip(): + continue + ok, why = checked(mdl, x["app_mpr"], tmp, qn) if x.get("app_mpr") else (True, "") + if not ok: + held.append(f"{kind} {name}: mxcli check refuses the result: {why}") + continue + parts.append((f"{kind} {name}: {how}", mdl)) + if "XASSOC" in routes: + for f in sorted(glob.glob(os.path.join(d, "association-*.mdl"))): + parts.append((os.path.basename(f), rd(f))) with open(os.path.join(W, "replay", f"{mod}.guids"), "w") as g: g.write("".join(q + "\n" for q in x.get("assocs", []))) - else: - continue - if not files: - print(f"COLLECT {mod}: nothing saved under {d}") + for h in held: + print(f"HELD {mod} {h}") + x["held"] = held + if not parts: + if routes & {"REPLAY", "UNREAD", "XASSOC"}: + print(f"COLLECT {mod}: nothing to replay") continue with open(os.path.join(W, "replay", f"{mod}.mdl"), "w") as out: - out.write(f"mdl 1;\n/* replay of {mod} (editroute {r}), from {d} */\n") - for f in files: - out.write(f"\n/* {os.path.basename(f)} */\n" + re.sub(r"^mdl 1;\s*", "", rd(f).strip(), flags=re.M) + "\n") - print(f"COLLECT {mod}: {len(files)} saved element(s) -> replay/{mod}.mdl") + out.write(f"mdl 1;\n/* replay of {mod} (editroute {x['route']}) */\n") + for label, mdl in parts: + out.write(f"\n/* {label} */\n" + re.sub(r"^mdl 1;\s*", "", mdl.strip(), flags=re.M) + "\n") + print(f"COLLECT {mod}: {len(parts)} element(s) -> replay/{mod}.mdl" + (f", {len(held)} held for a person" if held else "")) n += 1 + json.dump(allrows, open(os.path.join(W, "edit-routes.json"), "w"), indent=1) return 0 if n else 1 diff --git a/upgrade-toolkit/controls/editroute_control.sh b/upgrade-toolkit/controls/editroute_control.sh index 63fea43..16fec54 100644 --- a/upgrade-toolkit/controls/editroute_control.sh +++ b/upgrade-toolkit/controls/editroute_control.sh @@ -2,23 +2,37 @@ # editroute_control.sh CASE: controls of bin/editroute.py (kept modules routed to an update) and the cockpitcheck keep-recommended # rule. A stub mxcli answers describe and refs, so no project or network is needed. # plan one kept module per route; prints the ROUTE lines and the files editroute writes -# collect the replay files built from saved edits after the routed update +# collect the replay files after the routed update: the app's change merged onto the new element, never the saved whole element +# reroute a routed update keeps a module for a second reason: routed again, routes add up; the same reason twice is a HOLD # gate cockpitcheck on a module card that recommends the old version, with and without an end date set -u B=$(cd "$(dirname "$0")/../bin" && pwd) T=$(mktemp -d); trap 'rm -rf "$T"' EXIT W=$T/work; A=$T/app; S=$A/.stepupdate; C=$T/cache -mkdir -p "$W" "$S" "$C/7-1.0.0" -touch "$A/App.mpr" "$C/7-1.0.0.mpk" "$C/7-1.0.0/project.mpr" +mkdir -p "$W" "$S" "$C/7-1.0.0" "$T/orig" +touch "$A/App.mpr" "$T/orig/App.mpr" "$C/7-1.0.0.mpk" "$C/7-1.0.0/project.mpr" cat > "$T/mxcli" <<'PY' #!/usr/bin/env python3 import sys a = sys.argv[1:] if a[0] == "describe": - mpr, qn = a[2], a[4] + mpr, kind, qn = a[2], a[3], a[4] pkg = mpr.endswith("project.mpr") + side = "base" if pkg else "ours" if "/orig/" in mpr else "theirs" + if kind in ("layout", "snippet", "constant"): # base = installed package, ours = original app, theirs = updated app + el = {"Uimod.L1": {"base": ["class: 'a',", "title: 'x'"], "ours": ["class: 'mine',", "title: 'x'"], "theirs": ["class: 'a',", "title: 'y'"]}, + "Uimod.K": {"base": ["value: '1'"], "ours": ["value: '2'"], "theirs": ["value: '1'"]}, + "Uimod.Added": {"ours": ["class: 'own'"]}, + "Uimod.Bad": {"base": ["class: 'a'"], "ours": ["class: 'BROKEN'"], "theirs": ["class: 'a'"]}}.get(qn, {}).get(side) + if el: + print(f"mdl 1;\ncreate or modify {kind} {qn} (\n" + "\n".join(" " + x for x in el) + "\n) {\n};") + sys.exit(0) body = "logic A" if pkg or "Cosmetic" in qn else "logic B" print(("@position(1, 1)\n" if pkg else "@position(9, 9)\n@anchor(from: right, to: left)\n") + body) +elif a[0] == "check": + bad = "BROKEN" in open(a[1]).read() + print(" ✗ class BROKEN is not a class [MDL000]" if bad else "ok") + sys.exit(1 if bad else 0) elif a[0] == "refs": print(f"References to {a[3]}\n| Caller |" if "Used" in a[3] else f"(no references found to attribute {a[3]})") PY @@ -28,7 +42,7 @@ refuse() { printf 'refusing to update: %s element(s) have been changed locally a local m=$2; shift 2; for e in "$@"; do printf ' %s\n' "$e" >> "$S/step$n-$m.log"; done; printf '\n - Save them first: --save-edits \n' >> "$S/step$n-$m.log"; } n=1; refuse 1 Cosmo "MICROFLOW CosmeticFlow" n=2; refuse 2 Realmod "MICROFLOW RealFlow" -n=3; refuse 3 Uimod "LAYOUT L1" "CONSTANT K" +n=3; refuse 3 Uimod "LAYOUT L1" "CONSTANT K" "LAYOUT Added" "LAYOUT Bad" printf '5 element(s) could not be read, so it cannot be shown that the module is unedited.\n' > "$S/step4-Unread.log" touch "$S/step5-Filesmod.log" "$S/step6-Vendmod.log" "$S/step7-Usedmod.log" "$S/step8-Target.log" "$S/step9-Javamod.log" printf 'RED REMOVED attribute Vendmod.E.Gone (column/table dropped; intended?)\n' > "$S/step6-Vendmod.idcheck" @@ -52,16 +66,22 @@ json.dump([{"name": m, "cid": 7, "version": "1.0.0", "domain": [], "ui": [], "lo PY for m in Cosmo Realmod Uimod Unread Filesmod Vendmod Usedmod Target Javamod; do echo "7 2.0.0 $m"; done > "$W/plan.txt" case "${1:-}" in - plan) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$A/App.mpr" --plan "$W/plan.txt"; echo "rc=$?" + plan) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt"; echo "rc=$?" for f in route-plan.txt force-ui.txt accept-unsaved.txt approved.txt approved-step.txt; do sed "s/^/$f: /" "$W/$f"; done ;; - collect) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$A/App.mpr" --plan "$W/plan.txt" > /dev/null + collect) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null mkdir -p "$W/saved-edits/Uimod" "$W/saved-edits/Target" - printf 'mdl 1;\ncreate or modify layout Uimod.L1;\n' > "$W/saved-edits/Uimod/layout-L1.mdl" - printf 'mdl 1;\ncreate or modify constant Uimod.K;\n' > "$W/saved-edits/Uimod/constant-K.mdl" - printf 'mdl 1;\ncreate or modify page Uimod.NotRefused;\n' > "$W/saved-edits/Uimod/page-NotRefused.mdl" + for e in layout-L1 constant-K layout-Added layout-Bad page-NotRefused; do printf 'mdl 1;\n-- saved whole element %s\n' "$e" > "$W/saved-edits/Uimod/$e.mdl"; done printf 'mdl 1;\ncreate or modify association Other.A_B from Other.X to Target.Y;\n' > "$W/saved-edits/Target/association-Other_A_B.mdl" python3 "$B/editroute.py" collect --work "$W"; echo "rc=$?" sed 's/^/Uimod.mdl: /' "$W/replay/Uimod.mdl"; sed 's/^/Target.mdl: /' "$W/replay/Target.mdl"; sed 's/^/Target.guids: /' "$W/replay/Target.guids" ;; + reroute) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null + # the routed update keeps Cosmo again, now for associations another module owns into it + printf 'Cosmo idcheck REVIEW (persistence loss or removal: accept it in approved.txt with a reason, or keep), see x\n' > "$S/kept.txt" + printf 'RED REMOVED association Other.C_D (column/table dropped; intended?)\n' > "$S/step10-Cosmo.idcheck"; touch "$S/step10-Cosmo.log" + python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt"; echo "rc=$?" + python3 -c "import json,sys; [print('routes:', x['module'], x['route']) for x in json.load(open(sys.argv[1])) if x['module'] in ('Cosmo','Uimod')]" "$W/edit-routes.json" + sed 's/^/route-plan.txt: /' "$W/route-plan.txt" + python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt"; echo "rc=$?" ;; gate) python3 - "$B" <<'PY' import sys sys.path.insert(0, sys.argv[1]) diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 56573be..385320b 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6625,7 +6625,7 @@ ctl cockpit-check-running-no-stopbar-pass 0 'stopbar: 0' '^0$' -- $CK check-runn ER="bash $here/editroute_control.sh" ctl editroute-cosmetic-pass 0 '^ROUTE Cosmo COSMETIC: only diagram layout differs' '^force-ui.txt: Cosmo ' -- $ER plan ctl editroute-real-logic-held-catch 0 '^ROUTE Realmod HOLD: real logic edit: microflow RealFlow \(move it to Realmod_Ext' '!route-plan.txt: .* Realmod$' -- $ER plan -ctl editroute-ui-constant-replay-pass 0 '^ROUTE Uimod REPLAY: 2 UI' '^force-ui.txt: Uimod ' -- $ER plan +ctl editroute-ui-constant-replay-pass 0 '^ROUTE Uimod REPLAY: 4 UI' '^force-ui.txt: Uimod ' -- $ER plan ctl editroute-unread-unedited-pass 0 '^ROUTE Unread UNREAD' -- $ER plan ctl editroute-files-accept-pass 0 '^ROUTE Filesmod FILES' '^accept-unsaved.txt: Filesmod ' -- $ER plan ctl editroute-vendor-drop-unused-pass 0 '^ROUTE Vendmod VENDOR' '^approved.txt: Vendmod.E.Gone # editroute VENDOR' -- $ER plan @@ -6633,6 +6633,14 @@ ctl editroute-vendor-drop-used-catch 0 '^ROUTE Usedmod HOLD' '!approved.txt: Use ctl editroute-xassoc-pass 0 '^ROUTE Target XASSOC' '^approved-step.txt: Other.A_B ' -- $ER plan ctl editroute-java-held-catch 0 '^ROUTE Javamod HOLD' '!route-plan.txt: .* Javamod$' -- $ER plan ctl editroute-collect-pass 0 'Uimod.mdl: create or modify layout Uimod.L1' 'Uimod.mdl: create or modify constant Uimod.K' '!NotRefused' 'Target.mdl: create or modify association Other.A_B' '^Target.guids: Other.A_B$' -- $ER collect +# cold test WM1 Atlas_Core: a saved whole element lost the master-layout link and carried the old widget's properties; the change +# is merged onto the new element instead, and a result mxcli check refuses is held, never executed +ctl editroute-collect-prop-merge-pass 0 "^Uimod.mdl: class: 'mine',$" "^Uimod.mdl: title: 'y'$" '!saved whole element' -- $ER collect +ctl editroute-collect-app-added-pass 0 "layout Added: added by the app" "^Uimod.mdl: class: 'own'$" "constant K: the vendor did not change it" -- $ER collect +ctl editroute-collect-check-held-catch 0 '^HELD Uimod layout Bad: mxcli check refuses' '!BROKEN.*Uimod.mdl|Uimod.mdl:.*BROKEN' -- $ER collect +# cold test WM1 Email_Connector: diagram-only, then the routed update kept it for 3 associations; routed again, routes add up +ctl editroute-reroute-adds-route-pass 0 '^ROUTE Cosmo XASSOC' '^routes: Cosmo COSMETIC\+XASSOC$' '^route-plan.txt: 7 2.0.0 Cosmo$' -- $ER reroute +ctl editroute-reroute-same-reason-held-catch 0 '^ROUTE Cosmo HOLD: the routed update kept it again for the same reason' '^rc=1$' -- $ER reroute ctl cockpit-check-keep-recommended-catch 0 "'keep old version \(kept list\); ': CAUGHT" "'keep old version until 2027-03': ok" "'update to the latest version a': ok" -- $ER gate ctl cockpit-db-pattern-pass 0 'has claude\.use\("db"\)' 'has claude\.use\("user"\)' 'decisions"\)\.doc\(it\.id\)\.set' 'signoffs"\)\.doc\(await user\.id\(\)\)\.set' 'decisions"\)\.onSnapshot' 'never reads window\.claude\.db' -- $CK pattern ctl cockpit-live-script-only-with-flag-pass 0 'live: polls stamp 1' 'standalone without --live: polls stamp 0' 'content page: polls stamp 0' -- $CK live-script From de12f74c67cb618f4f5cca5b6d0888e07093970d Mon Sep 17 00:00:00 2001 From: MendixMau Date: Mon, 16 Nov 2026 22:49:44 +0800 Subject: [PATCH 03/59] editroute XASSOC: associations from the original model with their member access; never replay a whole entity Data test 2026-11-16 (WM1, real Postgres): save-edits wrote no file for the 4 associations the app's Email module owns into Email_Connector; a re-created association lost its member access; a saved whole entity put 5 widened vendor attributes back to String(200). Without the original GUID, AppNote 0/3 and links 0/3 after a green deploy; with it 3/3. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/editroute.py | 27 ++++++++++++++++--- upgrade-toolkit/controls/editroute_control.sh | 9 ++++++- upgrade-toolkit/controls/run-controls.sh | 4 +++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/upgrade-toolkit/bin/editroute.py b/upgrade-toolkit/bin/editroute.py index 7928bc0..aabe9d7 100755 --- a/upgrade-toolkit/bin/editroute.py +++ b/upgrade-toolkit/bin/editroute.py @@ -285,7 +285,9 @@ def cmd_collect(a): keep = {re.sub(r"\W", "", e.split(" ", 1)[1]).lower() for e in x.get("ui", [])} for f in sorted(glob.glob(os.path.join(d, "*.mdl"))): b = os.path.basename(f)[:-4] - if b.startswith("association-") or "-" not in b: + if b.startswith(("association-", "entity-", "microflow-", "nanoflow-")) or "-" not in b: + # a saved whole entity puts the vendor's widened attributes back (EmailTemplate: 5 of them String(2000) + # to String(200), data test 2026-11-16); logic goes to _Ext, never back into the vendor module continue kind, name = b.split("-", 1) if "UNREAD" not in routes and re.sub(r"\W", "", name).lower() not in keep: @@ -306,8 +308,27 @@ def cmd_collect(a): continue parts.append((f"{kind} {name}: {how}", mdl)) if "XASSOC" in routes: - for f in sorted(glob.glob(os.path.join(d, "association-*.mdl"))): - parts.append((os.path.basename(f), rd(f))) + # from the ORIGINAL model, not save-edits: on WM1 the forced update wrote no association-*.mdl for the 4 + # associations the app's Email module owns into Email_Connector (data test 2026-11-16), so nothing was replayed + owners = {} + for q in x.get("assocs", []): + f = os.path.join(d, f"association-{q.replace('.', '_')}.mdl") + mdl = describe(x["original"], "association", q) if x.get("original") else None + mdl = mdl or (rd(f) if os.path.exists(f) else None) + if not mdl: + held.append(f"association {q}: not in the original model or the saved edits") + continue + parts.append((f"association {q}: from the original model, original id restored after", mdl)) + m = re.search(r"\bfrom\s+([\w.]+)\s+to\b", mdl) + if m: + owners.setdefault(m.group(1), []).append(q.split(".", 1)[1]) + # a re-created association has no member access (ReadWrite became None; a build error on a non-persistent + # owner, data test 2026-11-16): the owner's grant lines that name it go in again + for ent, names in owners.items(): + g = [ln.strip() for ln in (describe(x["original"], "entity", ent) or "").splitlines() + if ln.strip().startswith("grant ") and any(re.search(rf"\b{re.escape(n)}\b", ln) for n in names)] + if g: + parts.append((f"member access on {ent} for {len(names)} association(s)", "\n".join(g))) with open(os.path.join(W, "replay", f"{mod}.guids"), "w") as g: g.write("".join(q + "\n" for q in x.get("assocs", []))) for h in held: diff --git a/upgrade-toolkit/controls/editroute_control.sh b/upgrade-toolkit/controls/editroute_control.sh index 16fec54..9f4be20 100644 --- a/upgrade-toolkit/controls/editroute_control.sh +++ b/upgrade-toolkit/controls/editroute_control.sh @@ -27,6 +27,13 @@ if a[0] == "describe": if el: print(f"mdl 1;\ncreate or modify {kind} {qn} (\n" + "\n".join(" " + x for x in el) + "\n) {\n};") sys.exit(0) + if kind == "association": # only the original model still has it + if "/orig/" in mpr and qn == "Other.A_B": + print(f"mdl 1;\ncreate or modify association {qn}\nfrom Other.X to Target.Y;") + sys.exit(0) + if kind == "entity" and qn == "Other.X": + print("create or modify entity Other.X (\n);\ngrant read (A_B, Name), write (A_B) on entity Other.X to Other.User;\ngrant read (Name) on entity Other.X to Other.Guest;") + sys.exit(0) body = "logic A" if pkg or "Cosmetic" in qn else "logic B" print(("@position(1, 1)\n" if pkg else "@position(9, 9)\n@anchor(from: right, to: left)\n") + body) elif a[0] == "check": @@ -70,7 +77,7 @@ case "${1:-}" in for f in route-plan.txt force-ui.txt accept-unsaved.txt approved.txt approved-step.txt; do sed "s/^/$f: /" "$W/$f"; done ;; collect) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null mkdir -p "$W/saved-edits/Uimod" "$W/saved-edits/Target" - for e in layout-L1 constant-K layout-Added layout-Bad page-NotRefused; do printf 'mdl 1;\n-- saved whole element %s\n' "$e" > "$W/saved-edits/Uimod/$e.mdl"; done + for e in layout-L1 constant-K layout-Added layout-Bad page-NotRefused entity-Whole; do printf 'mdl 1;\n-- saved whole element %s\n' "$e" > "$W/saved-edits/Uimod/$e.mdl"; done printf 'mdl 1;\ncreate or modify association Other.A_B from Other.X to Target.Y;\n' > "$W/saved-edits/Target/association-Other_A_B.mdl" python3 "$B/editroute.py" collect --work "$W"; echo "rc=$?" sed 's/^/Uimod.mdl: /' "$W/replay/Uimod.mdl"; sed 's/^/Target.mdl: /' "$W/replay/Target.mdl"; sed 's/^/Target.guids: /' "$W/replay/Target.guids" ;; diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 385320b..4819158 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6637,6 +6637,10 @@ ctl editroute-collect-pass 0 'Uimod.mdl: create or modify layout Uimod.L1' 'Uimo # is merged onto the new element instead, and a result mxcli check refuses is held, never executed ctl editroute-collect-prop-merge-pass 0 "^Uimod.mdl: class: 'mine',$" "^Uimod.mdl: title: 'y'$" '!saved whole element' -- $ER collect ctl editroute-collect-app-added-pass 0 "layout Added: added by the app" "^Uimod.mdl: class: 'own'$" "constant K: the vendor did not change it" -- $ER collect +# data test 2026-11-16: save-edits wrote no association file; a re-created association lost its member access; a saved +# whole entity put widened vendor attributes back +ctl editroute-collect-assoc-from-original-pass 0 'association Other.A_B: from the original model' '^Target.mdl: grant read \(A_B, Name\), write \(A_B\) on entity Other.X to Other.User;$' '!Other.Guest' -- $ER collect +ctl editroute-collect-no-whole-entity-catch 0 '!entity-Whole' '!saved whole element entity' -- $ER collect ctl editroute-collect-check-held-catch 0 '^HELD Uimod layout Bad: mxcli check refuses' '!BROKEN.*Uimod.mdl|Uimod.mdl:.*BROKEN' -- $ER collect # cold test WM1 Email_Connector: diagram-only, then the routed update kept it for 3 associations; routed again, routes add up ctl editroute-reroute-adds-route-pass 0 '^ROUTE Cosmo XASSOC' '^routes: Cosmo COSMETIC\+XASSOC$' '^route-plan.txt: 7 2.0.0 Cosmo$' -- $ER reroute From 590007812f945af8525a51fd9586517a65885da8 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Mon, 16 Nov 2026 22:50:22 +0800 Subject: [PATCH 04/59] pipeline: a rerun of the update step starts with fresh route files (earlier rows moved aside) Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 8d44f91..eb81b14 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -983,6 +983,9 @@ assoc_unreplayed() { route_kept() { if [ "$DRY" = 1 ]; then x python3 "$B/editroute.py" plan --work "$WORK" --app "$APP" --original "$WORK/original.mpr" --plan "$WORK/plan.txt"; return 0; fi [ -s "$APP/.stepupdate/kept.txt" ] || return 0 + # a rerun of the step starts fresh: the earlier run's rows would read as "kept again for the same reason" and hold all + local old="$WORK/.editroute-prev-$(date +%Y%m%d%H%M%S)" f + for f in edit-routes.json replay; do [ ! -e "$WORK/$f" ] || { mkdir -p "$old"; mv "$WORK/$f" "$old/"; }; done local rc=0 le=() [ ! -s "$WORK/localedits.json" ] || le=(--localedits "$WORK/localedits.json") xo "$WORK/editroute.out" python3 "$B/editroute.py" plan --work "$WORK" --app "$APP" --original "$WORK/original.mpr" --plan "$WORK/plan.txt" "${le[@]}" || rc=$? From 6d73740d7a969cd2d1051b10996ed7c7de54de41 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Mon, 16 Nov 2026 23:00:11 +0800 Subject: [PATCH 05/59] editroute score: did each route work, per run and in a ledger across runs Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 5 ++ upgrade-toolkit/bin/editroute.py | 57 ++++++++++++++++++- upgrade-toolkit/controls/editroute_control.sh | 8 +++ upgrade-toolkit/controls/run-controls.sh | 2 + 4 files changed, 70 insertions(+), 2 deletions(-) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index eb81b14..ce3af2b 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -1011,6 +1011,11 @@ step_update() { run_stepupdate "$WORK/plan.txt" post-convert route_kept replay_edits + if [ "$DRY" != 1 ] && [ -s "$WORK/edit-routes.json" ]; then # did each route work, here and across runs (route ledger) + local sl="${UPG_ROUTE_LEDGER:-${UPG_CACHE:-$HOME/.cache/upgrade-toolkit}/route-ledger.tsv}"; mkdir -p "$(dirname "$sl")" + python3 "$B/editroute.py" score --work "$WORK" --app "$APP" --plan "$WORK/plan.txt" --ledger "$sl" > "$WORK/editroute-score.out" 2>&1 || true + note "$(tail -1 "$WORK/editroute-score.out") (editroute-score.tsv; all runs: $sl)" + fi assoc_unreplayed if [ "$DRY" != 1 ]; then [ ! -s "$APP/.stepupdate/kept.txt" ] || cp "$APP/.stepupdate/kept.txt" "$WORK/kept.txt" diff --git a/upgrade-toolkit/bin/editroute.py b/upgrade-toolkit/bin/editroute.py index aabe9d7..9892952 100755 --- a/upgrade-toolkit/bin/editroute.py +++ b/upgrade-toolkit/bin/editroute.py @@ -30,12 +30,14 @@ (Email_Connector, WM1: COSMETIC, then XASSOC for the 3 associations the app's Email module owns into it). The same reason twice is a HOLD. + editroute.py score --work W --app A --plan W/plan.txt [--ledger F] did each route work: editroute-score.tsv, exit 1 when a + kept module did not end on the plan's version (held, replay failed, or kept again) editroute.py plan --work W --app A --original W/original.mpr --plan W/plan.txt writes the route files, prints ROUTE lines editroute.py collect --work W after the routed update: builds replay/ Files: W/edit-routes.json (one row per module), W/route-plan.txt (the plan lines of the routed modules, plan order), and appends to W/force-ui.txt, W/accept-unsaved.txt, W/approved.txt, W/approved-step.txt. Exit 0 = something routed, 1 = nothing. """ -import argparse, difflib, glob, json, os, re, subprocess, sys +import argparse, difflib, glob, json, os, re, subprocess, sys, time sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import localedits # noqa: E402 (MXCLI, CACHE, fetch_pkg) @@ -348,6 +350,54 @@ def cmd_collect(a): return 0 if n else 1 +def cmd_score(a): + """Did each route work? One row per module the update kept, judged on the app's own git log and the run's files.""" + W, app = os.path.abspath(a.work), os.path.abspath(a.app) + try: + rows = json.load(open(os.path.join(W, "edit-routes.json"))) + except (OSError, ValueError): + print("editroute-score: no edit-routes.json, nothing was routed") + return 0 + r = subprocess.run(["git", "-C", app, "log", "--format=%s"], capture_output=True, text=True) + log = r.stdout.splitlines() if r.returncode == 0 else [] + want = {} + for ln in rd(a.plan).splitlines() if a.plan else []: + f = ln.split() + if len(f) >= 3 and f[0].isdigit(): + want[f[2]] = f[1] + kept = {ln.split(" ", 1)[0] for ln in rd(os.path.join(app, ".stepupdate", "kept.txt")).splitlines() if ln.strip()} + out, tot = [], {"modules": 0, "routed": 0, "on_target": 0, "held_modules": 0, "held_elements": 0, "replay_failed": 0} + for x in rows: + mod = x["module"] + got = next((m.group(1) for ln in log if (m := re.match(rf"step: {re.escape(mod)} (\S+) \(stepupdate, idcheck PASS", ln))), "") + replayed = any(ln.startswith(f"replay: {mod}") for ln in log) + rp = os.path.join(W, "replay", f"{mod}.mdl") + failed = os.path.exists(rp) and not replayed and os.path.exists(os.path.join(W, f"replay-{mod}.out")) + on = bool(got) and mod not in kept and (not want.get(mod) or got == want[mod]) + held = len(x.get("held") or []) + verdict = ("HOLD" if x["route"] == "HOLD" else "REPLAY FAILED" if failed else "ON TARGET" if on + else "KEPT" if mod in kept else "OFF TARGET") + tot["modules"] += 1; tot["routed"] += x["route"] != "HOLD"; tot["on_target"] += on + tot["held_modules"] += x["route"] == "HOLD"; tot["held_elements"] += held; tot["replay_failed"] += failed + out.append([mod, x["route"], got or "-", want.get(mod, "-"), "yes" if replayed else "no" if os.path.exists(rp) else "-", + str(held), verdict, x.get("reason", "")[:160]]) + head = ["module", "route", "installed", "plan_target", "replayed", "held_elements", "verdict", "reason"] + with open(os.path.join(W, "editroute-score.tsv"), "w") as f: + f.write("\t".join(head) + "\n" + "".join("\t".join(o) + "\n" for o in out)) + if a.ledger: # one line per module per run, across apps: the route that was chosen and whether it worked + new = not os.path.exists(a.ledger) + with open(a.ledger, "a") as f: + if new: + f.write("\t".join(["date", "app", "work"] + head) + "\n") + for o in out: + f.write("\t".join([time.strftime("%Y-%m-%d"), a.label or os.path.basename(app), W] + o) + "\n") + for o in out: + print(f"SCORE {o[0]} {o[1]} -> {o[6]} (installed {o[2]}, plan {o[3]}, held {o[5]})") + print(f"editroute-score: {tot['on_target']}/{tot['modules']} kept module(s) ended on the plan's version, " + f"{tot['held_modules']} held, {tot['held_elements']} element(s) held, {tot['replay_failed']} replay(s) failed") + return 0 if tot["on_target"] == tot["modules"] else 1 + + def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) sp = ap.add_subparsers(dest="cmd", required=True) @@ -356,8 +406,11 @@ def main(): p.add_argument("--original", required=True); p.add_argument("--plan", required=True); p.add_argument("--localedits") c = sp.add_parser("collect") c.add_argument("--work", required=True) + s = sp.add_parser("score") + s.add_argument("--work", required=True); s.add_argument("--app", required=True); s.add_argument("--plan") + s.add_argument("--ledger", help="append the rows here too (cross-run: UPG_ROUTE_LEDGER)"); s.add_argument("--label") a = ap.parse_args() - return cmd_plan(a) if a.cmd == "plan" else cmd_collect(a) + return {"plan": cmd_plan, "collect": cmd_collect, "score": cmd_score}[a.cmd](a) if __name__ == "__main__": diff --git a/upgrade-toolkit/controls/editroute_control.sh b/upgrade-toolkit/controls/editroute_control.sh index 9f4be20..ec43d12 100644 --- a/upgrade-toolkit/controls/editroute_control.sh +++ b/upgrade-toolkit/controls/editroute_control.sh @@ -3,6 +3,7 @@ # rule. A stub mxcli answers describe and refs, so no project or network is needed. # plan one kept module per route; prints the ROUTE lines and the files editroute writes # collect the replay files after the routed update: the app's change merged onto the new element, never the saved whole element +# score did each route work: on the plan's version, held, or replay failed; one ledger line per module across runs # reroute a routed update keeps a module for a second reason: routed again, routes add up; the same reason twice is a HOLD # gate cockpitcheck on a module card that recommends the old version, with and without an end date set -u @@ -81,6 +82,13 @@ case "${1:-}" in printf 'mdl 1;\ncreate or modify association Other.A_B from Other.X to Target.Y;\n' > "$W/saved-edits/Target/association-Other_A_B.mdl" python3 "$B/editroute.py" collect --work "$W"; echo "rc=$?" sed 's/^/Uimod.mdl: /' "$W/replay/Uimod.mdl"; sed 's/^/Target.mdl: /' "$W/replay/Target.mdl"; sed 's/^/Target.guids: /' "$W/replay/Target.guids" ;; + score) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null + git -C "$A" init -q; git -C "$A" commit -q --allow-empty -m "step: Cosmo 2.0.0 (stepupdate, idcheck PASS, forced)" + sed -i.bak '/^Cosmo /d' "$S/kept.txt" # stepupdate takes a module it committed out of kept.txt (E6-12) + printf '7 2.0.0 Cosmo\n7 2.0.0 Uimod\n' >> "$W/plan.txt" + mkdir -p "$W/replay"; echo x > "$W/replay/Uimod.mdl"; echo "MDL081" > "$W/replay-Uimod.out" + python3 "$B/editroute.py" score --work "$W" --app "$A" --plan "$W/plan.txt" --ledger "$T/ledger.tsv" --label app1; echo "rc=$?" + sed 's/^/ledger: /' "$T/ledger.tsv" | cut -f1-3,5,6,11 | head -3 ;; reroute) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null # the routed update keeps Cosmo again, now for associations another module owns into it printf 'Cosmo idcheck REVIEW (persistence loss or removal: accept it in approved.txt with a reason, or keep), see x\n' > "$S/kept.txt" diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 4819158..1e377ef 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6643,6 +6643,8 @@ ctl editroute-collect-assoc-from-original-pass 0 'association Other.A_B: from th ctl editroute-collect-no-whole-entity-catch 0 '!entity-Whole' '!saved whole element entity' -- $ER collect ctl editroute-collect-check-held-catch 0 '^HELD Uimod layout Bad: mxcli check refuses' '!BROKEN.*Uimod.mdl|Uimod.mdl:.*BROKEN' -- $ER collect # cold test WM1 Email_Connector: diagram-only, then the routed update kept it for 3 associations; routed again, routes add up +# does routing work across real runs: a route that ends off the plan's version is counted, never hidden +ctl editroute-score-pass 0 '^SCORE Cosmo COSMETIC -> ON TARGET' '^SCORE Uimod REPLAY -> REPLAY FAILED' '^SCORE Realmod HOLD -> HOLD' 'ended on the plan.s version' '^rc=1$' '^ledger: .*app1' -- $ER score ctl editroute-reroute-adds-route-pass 0 '^ROUTE Cosmo XASSOC' '^routes: Cosmo COSMETIC\+XASSOC$' '^route-plan.txt: 7 2.0.0 Cosmo$' -- $ER reroute ctl editroute-reroute-same-reason-held-catch 0 '^ROUTE Cosmo HOLD: the routed update kept it again for the same reason' '^rc=1$' -- $ER reroute ctl cockpit-check-keep-recommended-catch 0 "'keep old version \(kept list\); ': CAUGHT" "'keep old version until 2027-03': ok" "'update to the latest version a': ok" -- $ER gate From 081799b1a2779676806130ea0ee5e3d9a4cb1ef0 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Mon, 16 Nov 2026 23:12:18 +0800 Subject: [PATCH 06/59] editroute: brain notes on every route (brainnotes.py, hit@k measured), reset on a rerun, score - brainnotes.py: the company brain's practice notes for a run event, no model; --eval measures hit@k on a labelled set. Each route row carries its notes, the cockpit card shows them. No brain: nothing, exit 0. - editroute reset, first thing in the update step: an earlier routing's lines leave the decision files. WM1 rerun: 'Atlas_Core # editroute REPLAY' forced Atlas_Core before routing and its edits were never merged. - grep -c with no match no longer fails the update step. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/brainnotes.py | 122 ++++++++++++++++++ upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 9 +- upgrade-toolkit/bin/editroute.py | 55 +++++++- upgrade-toolkit/bin/evidence.py | 1 + upgrade-toolkit/controls/editroute_control.sh | 19 +++ upgrade-toolkit/controls/run-controls.sh | 4 + 6 files changed, 204 insertions(+), 6 deletions(-) create mode 100644 upgrade-toolkit/bin/brainnotes.py diff --git a/upgrade-toolkit/bin/brainnotes.py b/upgrade-toolkit/bin/brainnotes.py new file mode 100644 index 0000000..39b13c6 --- /dev/null +++ b/upgrade-toolkit/bin/brainnotes.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""brainnotes.py: the company brain's notes that belong next to a run event (a held module, a route, a failed check). + +Why (cold test WM1, 2026-11-16): the toolkit read only the brain's version tables. The brain also holds the practice and +pitfall notes for exactly the events a run stops on (never edit a Marketplace module, hide Atlas layouts in the company +UI module, PDFBox 2 jars next to PDFBox 3 after a Community Commons update), and no card showed them. The data test hit +the PDFBox case by hand while the brain already had the note. + +Search, no model: every note's title, frontmatter keywords and body are scored against the query words (idf weighted; +title x3, keywords x2, body x1), plain-word stems ("layouts" = "layout"). The brain is found by brainpath.py and only read. +No brain on this machine: prints nothing, exit 0 (the toolkit runs without it). + + brainnotes.py "query text" [-k 3] [--brain DIR] [--json] top notes: score, path under the brain, title + brainnotes.py --eval gold.tsv [-k 3] [--brain DIR] hit@k against a labelled set (id, slugs, why; slugs + comma-separated paths without .md, or NONE); the queries + come from a second file column 'text' or --queries FILE +""" +import argparse, json, math, os, re, sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +STOP = set("a an the and or of to in on for with is are be by it its at as from this that what which how should we our " + "not no do does after before into out than then when would must can there their them one two new old".split()) + + +def stem(w): + for suf in ("ies", "es", "s"): + if len(w) > 4 and w.endswith(suf): + return w[: -len(suf)] + ("y" if suf == "ies" else "") + return w + + +def words(t): + return [stem(w) for w in re.findall(r"[a-z0-9_]+", (t or "").lower()) if w not in STOP and len(w) > 1] + + +def load(brain): + notes = [] + for root, _, files in os.walk(os.path.join(brain, "internal")): + for f in files: + if not f.endswith(".md") or f in ("INDEX.md", "INTERNAL-ONLY.md"): + continue + p = os.path.join(root, f) + t = open(p, encoding="utf-8", errors="replace").read() + fm, body = "", t + if t.startswith("---"): + j = t.find("\n---", 3) + fm, body = t[3:j], t[j + 4:] + title = (re.search(r"^title:\s*(.+)$", fm, re.M) or [None, ""])[1].strip("'\" ") + kw = (re.search(r"^keywords:\s*(.+)$", fm, re.M) or [None, ""])[1] + notes.append({"path": os.path.relpath(p, os.path.join(brain, "internal"))[:-3], "title": title, + "f": (set(words(title)), set(words(kw)), words(body))}) + return notes + + +def search(notes, q, k=3): + qs = set(words(q)) + n = len(notes) or 1 + df = {w: sum(1 for x in notes if w in x["f"][0] or w in x["f"][1] or w in x["f"][2]) for w in qs} + out = [] + for x in notes: + t, kw, body = x["f"] + bset = set(body) + s = sum(math.log(1 + n / (1 + df[w])) * (3 * (w in t) + 2 * (w in kw) + (w in bset)) for w in qs) + if s > 0: + out.append((round(s, 1), x["path"], x["title"])) + return sorted(out, key=lambda r: -r[0])[:k] + + +def brain_dir(arg): + if arg: + return arg + try: + import brainpath + return brainpath.find()[0] + except Exception: + return None + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("query", nargs="?") + ap.add_argument("-k", type=int, default=3) + ap.add_argument("--brain"); ap.add_argument("--json", action="store_true") + ap.add_argument("--eval"); ap.add_argument("--queries") + a = ap.parse_args() + b = brain_dir(a.brain) + if not b or not os.path.isdir(os.path.join(b, "internal")): + return 0 + notes = load(b) + if not a.eval: + r = search(notes, a.query or "", a.k) + print(json.dumps([{"score": s, "path": p, "title": t} for s, p, t in r]) if a.json + else "\n".join(f"{s}\t{p}\t{t}" for s, p, t in r)) + return 0 + q = {} + if a.queries: + for ln in open(a.queries): + if "|" in ln: + i, t = ln.split("|", 1) + q[i.strip()] = t.strip() + hit = tot = none_ok = nones = 0 + for ln in open(a.eval): + f = ln.rstrip("\n").split("\t") + if len(f) < 2 or f[0] in ("id", "") or f[0] not in q: + continue + gold = {g.strip() for g in f[1].split(",") if g.strip() and g.strip() != "NONE"} + got = search(notes, q[f[0]], a.k) + paths = [p for _, p, _ in got] + if not gold: + nones += 1 + print(f"{f[0]}\tNONE\t(top: {paths[:1]})") + continue + tot += 1 + ok = bool(gold & set(paths)) + hit += ok + print(f"{f[0]}\t{'HIT ' if ok else 'MISS'}\tgold {sorted(gold)}\tgot {paths}") + print(f"brainnotes hit@{a.k}: {hit}/{tot} situations with a gold note ({nones} with none in the brain)") + return 0 if tot and hit == tot else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index ce3af2b..2e741e2 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -983,9 +983,6 @@ assoc_unreplayed() { route_kept() { if [ "$DRY" = 1 ]; then x python3 "$B/editroute.py" plan --work "$WORK" --app "$APP" --original "$WORK/original.mpr" --plan "$WORK/plan.txt"; return 0; fi [ -s "$APP/.stepupdate/kept.txt" ] || return 0 - # a rerun of the step starts fresh: the earlier run's rows would read as "kept again for the same reason" and hold all - local old="$WORK/.editroute-prev-$(date +%Y%m%d%H%M%S)" f - for f in edit-routes.json replay; do [ ! -e "$WORK/$f" ] || { mkdir -p "$old"; mv "$WORK/$f" "$old/"; }; done local rc=0 le=() [ ! -s "$WORK/localedits.json" ] || le=(--localedits "$WORK/localedits.json") xo "$WORK/editroute.out" python3 "$B/editroute.py" plan --work "$WORK" --app "$APP" --original "$WORK/original.mpr" --plan "$WORK/plan.txt" "${le[@]}" || rc=$? @@ -1001,13 +998,15 @@ route_kept() { done rc=0; xo "$WORK/editroute-collect.out" python3 "$B/editroute.py" collect --work "$WORK" || rc=$? cat "$WORK/editroute-collect.out" >> "$LOG" - ! grep -q '^HELD ' "$WORK/editroute-collect.out" || note "editroute: $(grep -c '^HELD ' "$WORK/editroute-collect.out") replayed element(s) held for a person: $(grep '^HELD ' "$WORK/editroute-collect.out" | head -3 | tr '\n' ';')" - note "editroute: $(grep -c '^ROUTE' "$WORK/editroute.out") kept module(s) routed, $(grep -c ' HOLD:' "$WORK/editroute.out") held for a person (edit-routes.json)" + ! grep -q '^HELD ' "$WORK/editroute-collect.out" || note "editroute: $(grep -c '^HELD ' "$WORK/editroute-collect.out" || true) replayed element(s) held for a person: $(grep '^HELD ' "$WORK/editroute-collect.out" | head -3 | tr '\n' ';')" + note "editroute: $(grep -c '^ROUTE' "$WORK/editroute.out" || true) kept module(s) routed, $(grep -c ' HOLD:' "$WORK/editroute.out" || true) held for a person (edit-routes.json)" } # update: module updates with the post-convert plan; kept modules are routed (route_kept), the rest decided at the gate. step_update() { [ -f "$WORK/plan.txt" ] || [ "$DRY" = 1 ] || { nj_add 13 "plan.txt missing" "$WORK/plan.txt does not exist"; nj_flush; } + # a rerun starts as the first run did: an earlier routing's force-ui lines would force modules before they are routed + [ "$DRY" = 1 ] || { python3 "$B/editroute.py" reset --work "$WORK" | tee -a "$LOG" > /dev/null || true; } run_stepupdate "$WORK/plan.txt" post-convert route_kept replay_edits diff --git a/upgrade-toolkit/bin/editroute.py b/upgrade-toolkit/bin/editroute.py index 9892952..7c5a88d 100755 --- a/upgrade-toolkit/bin/editroute.py +++ b/upgrade-toolkit/bin/editroute.py @@ -30,6 +30,7 @@ (Email_Connector, WM1: COSMETIC, then XASSOC for the 3 associations the app's Email module owns into it). The same reason twice is a HOLD. + editroute.py reset --work W first thing in the update step: an earlier routing's lines and files out of the way editroute.py score --work W --app A --plan W/plan.txt [--ledger F] did each route work: editroute-score.tsv, exit 1 when a kept module did not end on the plan's version (held, replay failed, or kept again) editroute.py plan --work W --app A --original W/original.mpr --plan W/plan.txt writes the route files, prints ROUTE lines @@ -174,6 +175,7 @@ def cmd_plan(a): if r in ("REPLAY", "UNREAD") and e.get("cid") and e.get("version"): extra["pkg"] = localedits.fetch_pkg(e["cid"], e["version"], cache)[0] row = {"module": mod, "route": r, "reason": reason, "kept_because": why, "original": a.original, "app_mpr": mpr, **extra} + row["brain"] = brain_for(mod, r, reason) if p and p["route"] != "HOLD" and r != "HOLD": for k in ("ui", "cosmetic", "assocs", "vendor"): row[k] = sorted(set(p.get(k, [])) | set(row.get(k, []))) @@ -199,6 +201,28 @@ def cmd_plan(a): return 0 if go else 1 +HINT = {"HOLD": "edited Marketplace module extension module local edits", "REPLAY": "layout page snippet edited in Marketplace module", + "XASSOC": "module update drops associations another module owns data", "VENDOR": "module update removes attribute data loss", + "FILES": "theme styling files edited in module overwritten by update", "UNREAD": "Marketplace module update local edits", + "COSMETIC": "Marketplace module update local edits"} +_NOTES = None + + +def brain_for(mod, route, reason, k=2): + """[{path, title}] of the company brain's notes for this route (brainnotes.py); [] without a brain on this machine.""" + global _NOTES + try: + import brainnotes + if _NOTES is None: + b = brainnotes.brain_dir(None) + _NOTES = brainnotes.load(b) if b and os.path.isdir(os.path.join(b, "internal")) else [] + q = f"{mod.replace('_', ' ')} {HINT.get(route.split('+')[0], '')} {reason}" + r = brainnotes.search(_NOTES, q, k) if _NOTES else [] + return [{"path": p, "title": t} for sc, p, t in r if sc >= 0.5 * r[0][0]] # a far weaker second match is noise + except Exception: + return [] + + def norm(t): return [ln.rstrip() for ln in (t or "").splitlines() if ln.strip() and not ln.startswith("mdl ")] @@ -350,6 +374,33 @@ def cmd_collect(a): return 0 if n else 1 +def cmd_reset(a): + """A rerun of the update step starts as the first run did: editroute's own lines leave the decision files (a person's + lines stay), the earlier routes and replays move aside. Why (WM1 rerun, 2026-11-16): the earlier run's 'Atlas_Core # + editroute REPLAY' in force-ui.txt made the first update pass force Atlas_Core before any routing, so its edits were saved + and never merged back: every page on the vendor layouts again.""" + W = os.path.abspath(a.work) + n = 0 + for f in ("force-ui.txt", "accept-unsaved.txt", "approved.txt", "approved-step.txt"): + p = os.path.join(W, f) + lines = rd(p).splitlines(True) + keep = [ln for ln in lines if "# editroute " not in ln] + if len(keep) != len(lines): + n += len(lines) - len(keep) + with open(p, "w") as fh: + fh.writelines(keep) + old = os.path.join(W, ".editroute-prev-" + time.strftime("%Y%m%d%H%M%S")) + moved = [] + for f in ("edit-routes.json", "replay", "editroute-score.tsv"): + if os.path.exists(os.path.join(W, f)): + os.makedirs(old, exist_ok=True) + os.replace(os.path.join(W, f), os.path.join(old, f)) + moved.append(f) + print(f"editroute-reset: {n} line(s) of an earlier routing removed from the decision files" + + (f", {', '.join(moved)} moved to {os.path.basename(old)}" if moved else "")) + return 0 + + def cmd_score(a): """Did each route work? One row per module the update kept, judged on the app's own git log and the run's files.""" W, app = os.path.abspath(a.work), os.path.abspath(a.app) @@ -409,8 +460,10 @@ def main(): s = sp.add_parser("score") s.add_argument("--work", required=True); s.add_argument("--app", required=True); s.add_argument("--plan") s.add_argument("--ledger", help="append the rows here too (cross-run: UPG_ROUTE_LEDGER)"); s.add_argument("--label") + r = sp.add_parser("reset") + r.add_argument("--work", required=True) a = ap.parse_args() - return {"plan": cmd_plan, "collect": cmd_collect, "score": cmd_score}[a.cmd](a) + return {"plan": cmd_plan, "collect": cmd_collect, "score": cmd_score, "reset": cmd_reset}[a.cmd](a) if __name__ == "__main__": diff --git a/upgrade-toolkit/bin/evidence.py b/upgrade-toolkit/bin/evidence.py index f1f83c2..d8a5476 100644 --- a/upgrade-toolkit/bin/evidence.py +++ b/upgrade-toolkit/bin/evidence.py @@ -107,6 +107,7 @@ def for_item(work, name): for x in json.loads(_rd(os.path.join(work, "edit-routes.json")) or "[]"): if x.get("module") == name: what.insert(0, ("held for a person: " if x["route"] == "HOLD" else f"the pipeline updates it anyway ({x['route']}): ") + x["reason"]) + what[1:1] = [f"company brain (internal): {b['title']} ({b['path']})" for b in x.get("brain") or []] files.insert(0, _rel(work, os.path.join(work, "edit-routes.json"))) return {"what": what, "files": files} diff --git a/upgrade-toolkit/controls/editroute_control.sh b/upgrade-toolkit/controls/editroute_control.sh index ec43d12..89ac2e4 100644 --- a/upgrade-toolkit/controls/editroute_control.sh +++ b/upgrade-toolkit/controls/editroute_control.sh @@ -4,6 +4,8 @@ # plan one kept module per route; prints the ROUTE lines and the files editroute writes # collect the replay files after the routed update: the app's change merged onto the new element, never the saved whole element # score did each route work: on the plan's version, held, or replay failed; one ledger line per module across runs +# brain each route carries the company brain's notes for it, shown on the card; hit@k measured on a labelled set +# reset a rerun of the update step drops an earlier routing's own lines (a person's stay) and moves its files aside # reroute a routed update keeps a module for a second reason: routed again, routes add up; the same reason twice is a HOLD # gate cockpitcheck on a module card that recommends the old version, with and without an end date set -u @@ -89,6 +91,23 @@ case "${1:-}" in mkdir -p "$W/replay"; echo x > "$W/replay/Uimod.mdl"; echo "MDL081" > "$W/replay-Uimod.out" python3 "$B/editroute.py" score --work "$W" --app "$A" --plan "$W/plan.txt" --ledger "$T/ledger.tsv" --label app1; echo "rc=$?" sed 's/^/ledger: /' "$T/ledger.tsv" | cut -f1-3,5,6,11 | head -3 ;; + brain) Bd="$T/fake-brain"; mkdir -p "$Bd/baseline/versions" "$Bd/internal/architecture" "$Bd/internal/ux-ui" "$Bd/internal/upgrade" + touch "$Bd/baseline/versions/studio-pro-upgrade-10.md" + printf -- '---\ntitle: Never edit Marketplace modules, use an extension module\nkeywords: [extension module, local edits]\n---\nPut changes in a separate module.\n' > "$Bd/internal/architecture/ext.md" + printf -- '---\ntitle: Hide Atlas layouts in the company UI module\nkeywords: [Atlas_Core layouts, design system]\n---\nsettings.json excludes.\n' > "$Bd/internal/ux-ui/atlas.md" + printf -- '---\ntitle: Java dependency sync behind a proxy\nkeywords: [gradle, proxy]\n---\nSet the proxy.\n' > "$Bd/internal/upgrade/proxy.md" + printf 'id\tslugs\twhy\nS1\tarchitecture/ext\tx\nS2\tux-ui/atlas\tx\nS3\tNONE\tx\n' > "$T/gold.tsv" + printf 'S1 | edited Marketplace module, local edits refuse the update\nS2 | Atlas_Core layouts point at the company design system\nS3 | nothing like this\n' > "$T/q.txt" + UPG_BRAIN="$Bd" python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null + python3 -c "import json,sys; [print('brain:', x['module'], [b['path'] for b in x['brain']]) for x in json.load(open(sys.argv[1])) if x['module'] in ('Realmod','Uimod')]" "$W/edit-routes.json" + UPG_BRAIN="$Bd" python3 "$B/evidence.py" --work "$W" Realmod | head -3 + UPG_BRAIN="$Bd" python3 "$B/brainnotes.py" --eval "$T/gold.tsv" --queries "$T/q.txt"; echo "rc=$?" + UPG_BRAIN="$T/none" python3 "$B/brainnotes.py" "anything"; echo "no brain rc=$?" ;; + reset) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null + echo "Handmod # a person forced this" >> "$W/force-ui.txt"; mkdir -p "$W/replay" + python3 "$B/editroute.py" reset --work "$W"; echo "rc=$?" + sed 's/^/force-ui.txt: /' "$W/force-ui.txt"; grep -c 'editroute' "$W/approved-step.txt" | sed 's/^/approved-step editroute lines: /' + ls "$W" | sed 's/^/work: /' ;; reroute) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null # the routed update keeps Cosmo again, now for associations another module owns into it printf 'Cosmo idcheck REVIEW (persistence loss or removal: accept it in approved.txt with a reason, or keep), see x\n' > "$S/kept.txt" diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 1e377ef..13a69fb 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6645,6 +6645,10 @@ ctl editroute-collect-check-held-catch 0 '^HELD Uimod layout Bad: mxcli check re # cold test WM1 Email_Connector: diagram-only, then the routed update kept it for 3 associations; routed again, routes add up # does routing work across real runs: a route that ends off the plan's version is counted, never hidden ctl editroute-score-pass 0 '^SCORE Cosmo COSMETIC -> ON TARGET' '^SCORE Uimod REPLAY -> REPLAY FAILED' '^SCORE Realmod HOLD -> HOLD' 'ended on the plan.s version' '^rc=1$' '^ledger: .*app1' -- $ER score +# the brain's practice notes reach the card that needs them (WM1: the PDFBox note existed, nobody saw it); measured, not assumed +ctl editroute-brain-notes-pass 0 "^brain: Realmod \['architecture/ext'" "^brain: Uimod \['ux-ui/atlas'" '^- company brain \(internal\): Never edit Marketplace modules' 'brainnotes hit@3: 2/2' '^rc=0$' '^no brain rc=0$' -- $ER brain +# WM1 rerun: the earlier run's 'Atlas_Core # editroute REPLAY' forced Atlas_Core before routing; its edits were never merged back +ctl editroute-reset-pass 0 '^editroute-reset: [1-9][0-9]* line' '^force-ui.txt: Handmod # a person forced this$' '!^force-ui.txt: .*# editroute' '^approved-step editroute lines: 0$' '!^work: edit-routes.json$' '^rc=0$' -- $ER reset ctl editroute-reroute-adds-route-pass 0 '^ROUTE Cosmo XASSOC' '^routes: Cosmo COSMETIC\+XASSOC$' '^route-plan.txt: 7 2.0.0 Cosmo$' -- $ER reroute ctl editroute-reroute-same-reason-held-catch 0 '^ROUTE Cosmo HOLD: the routed update kept it again for the same reason' '^rc=1$' -- $ER reroute ctl cockpit-check-keep-recommended-catch 0 "'keep old version \(kept list\); ': CAUGHT" "'keep old version until 2027-03': ok" "'update to the latest version a': ok" -- $ER gate From a901480bb83c3051278ba19c72f910bd8d4e915a Mon Sep 17 00:00:00 2001 From: MendixMau Date: Mon, 16 Nov 2026 23:13:16 +0800 Subject: [PATCH 07/59] pipeline: a decision file with only comments no longer fails the update step; brainnotes splits Atlas_Core and keeps version digits After editroute reset emptied force-ui.txt, grep -v matched nothing and pipefail stopped the WM1 rerun at the force-ui read. brainnotes hit@3 on the 19-situation labelled set: 15/19 -> 17/19. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/brainnotes.py | 3 ++- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/upgrade-toolkit/bin/brainnotes.py b/upgrade-toolkit/bin/brainnotes.py index 39b13c6..546af48 100644 --- a/upgrade-toolkit/bin/brainnotes.py +++ b/upgrade-toolkit/bin/brainnotes.py @@ -30,7 +30,8 @@ def stem(w): def words(t): - return [stem(w) for w in re.findall(r"[a-z0-9_]+", (t or "").lower()) if w not in STOP and len(w) > 1] + # Atlas_Core is "Atlas Core" in a title; a version digit ("Atlas 4") is a word of its own + return [stem(w) for w in re.findall(r"[a-z0-9]+", (t or "").lower()) if w not in STOP and (len(w) > 1 or w.isdigit())] def load(brain): diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 2e741e2..4bb6d14 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -885,14 +885,14 @@ run_stepupdate() { # run_stepupdate PLAN label # always KEPT, since --unattended keeps every refused module. $WORK/force-ui.txt names them, one per line. local fu=() if [ -f "$WORK/force-ui.txt" ]; then - local mods; mods=$(grep -vE '^[[:space:]]*(#|$)' "$WORK/force-ui.txt" | awk '{print $1}' | paste -sd, -) + local mods; mods=$( { grep -vE '^[[:space:]]*(#|$)' "$WORK/force-ui.txt" || true; } | awk '{print $1}' | paste -sd, -) [ -z "$mods" ] || fu=(--force-ui "$mods" --saved-edits "$WORK/saved-edits") fi # R-hook (pr-rerun): $WORK/accept-unsaved.txt names modules whose update may commit although the installed package cannot # prove their theme/JavaScript files unedited (stepupdate --accept-unsaved; NanoflowCommons 6.0.0 is not in the catalog) local au=() if [ -f "$WORK/accept-unsaved.txt" ]; then - local amods; amods=$(grep -vE '^[[:space:]]*(#|$)' "$WORK/accept-unsaved.txt" | awk '{print $1}' | paste -sd, -) + local amods; amods=$( { grep -vE '^[[:space:]]*(#|$)' "$WORK/accept-unsaved.txt" || true; } | awk '{print $1}' | paste -sd, -) [ -z "$amods" ] || au=(--accept-unsaved "$amods") fi # F14: approved-step.txt holds step-only acceptances (a removal the replay restores): the step idcheck reads them, From facba738d06a6be61dc95f6fdba90f0a8e464fab Mon Sep 17 00:00:00 2001 From: MendixMau Date: Mon, 16 Nov 2026 23:43:54 +0800 Subject: [PATCH 08/59] editroute: an edited layout in a Marketplace module repoints the app's pages; check with the mxcli exec runs mxcli never writes a layout into a Marketplace module (any build). WM1 rerun: Atlas_Default and Atlas_TopBar were rewritten to sit on the app's design-module layouts; the merged replay stopped the run. Now: a layout redirected to another module's layout becomes ALTER PAGES SET LAYOUT (pages in Marketplace modules keep the vendor layout), a layout the app added and nothing uses is not carried, any other edited layout is held for a person. Collect checks with the mxcli exec runs: the container's (2026-10-13) refused what the local one passed. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 4 +- upgrade-toolkit/bin/editroute.py | 40 ++++++++++++++++++- upgrade-toolkit/controls/editroute_control.sh | 10 +++-- upgrade-toolkit/controls/run-controls.sh | 10 +++-- 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 4bb6d14..1cf6290 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -996,7 +996,9 @@ route_kept() { cat "$WORK/editroute.out" >> "$LOG" [ "$rc" = 0 ] || break done - rc=0; xo "$WORK/editroute-collect.out" python3 "$B/editroute.py" collect --work "$WORK" || rc=$? + # check with the mxcli that exec runs (the container's may be older than the local one) + local chk="$B/mxcli-container.sh"; [ -z "${MXCLI:-}" ] || [ ! -x "$MXCLI" ] || chk=$MXCLI + rc=0; xo "$WORK/editroute-collect.out" env UPG_MXCLI_CHECK="$chk" python3 "$B/editroute.py" collect --work "$WORK" || rc=$? cat "$WORK/editroute-collect.out" >> "$LOG" ! grep -q '^HELD ' "$WORK/editroute-collect.out" || note "editroute: $(grep -c '^HELD ' "$WORK/editroute-collect.out" || true) replayed element(s) held for a person: $(grep '^HELD ' "$WORK/editroute-collect.out" | head -3 | tr '\n' ';')" note "editroute: $(grep -c '^ROUTE' "$WORK/editroute.out" || true) kept module(s) routed, $(grep -c ' HOLD:' "$WORK/editroute.out" || true) held for a person (edit-routes.json)" diff --git a/upgrade-toolkit/bin/editroute.py b/upgrade-toolkit/bin/editroute.py index 7c5a88d..eefc87c 100755 --- a/upgrade-toolkit/bin/editroute.py +++ b/upgrade-toolkit/bin/editroute.py @@ -285,11 +285,38 @@ def prop_patch(base, ours, theirs): return out if sets else None +LAYOUT_OF = re.compile(r"^\s*Layout:\s*([\w]+\.[\w]+)", re.M | re.I) + + +def layout_route(qn, ours, orig): + """-> (mdl or None, how) for an edited layout inside a Marketplace module. mxcli never writes a layout there (any build: + the update would overwrite it), so the merged layout cannot go back in. WM1 (rerun 2026-11-16): Atlas_Default and + Atlas_TopBar were rewritten to sit on DISW_DesignSystem layouts; the replay stopped the run. What the app meant is + "pages use the company layout": repoint the pages to it. mxcli skips and names pages inside Marketplace modules; + those show the vendor layout after the update.""" + m = LAYOUT_OF.search(ours or "") + if m and m.group(1).split(".")[0] != qn.split(".")[0]: + return (f"mdl 1;\nALTER PAGES SET LAYOUT = {m.group(1)} WHERE LAYOUT = {qn};\n", + f"the app redirected it to {m.group(1)}: the app's pages repointed there (pages in Marketplace modules " + "keep the vendor layout)") + rc, out = mx(["refs", "-p", orig, qn]) + if rc == 0 and "no references found" in out: + return "", "used by nothing: not carried" + return None, "mxcli never writes a layout into a Marketplace module: copy it to the app's own module, repoint its pages" + + def checked(mdl, app_mpr, tmp, qn): p = os.path.join(tmp, f"{qn}.check.mdl") with open(p, "w") as f: f.write(mdl) - rc, out = mx(["check", p, "-p", app_mpr]) + # the mxcli that exec runs, when the pipeline names it: WM1's container mxcli (2026-10-13) refused a layout the local + # one (with mendixlabs/mxcli#1411) passed, so check said yes and the replay stopped the run + pre = os.environ.get("UPG_MXCLI_CHECK", "").split() + if pre: + r = subprocess.run(pre + ["check", p, "-p", app_mpr], capture_output=True, text=True) + rc, out = r.returncode, (r.stdout or "") + (r.stderr or "") + else: + rc, out = mx(["check", p, "-p", app_mpr]) return rc == 0, " ".join(ln.strip() for ln in out.splitlines() if "✗" in ln or "error" in ln.lower())[:300] @@ -319,7 +346,16 @@ def cmd_collect(a): if "UNREAD" not in routes and re.sub(r"\W", "", name).lower() not in keep: continue qn = f"{mod}.{name}" - if kind in ("layout", "snippet", "page", "constant") and x.get("original") and x.get("app_mpr"): + if kind == "layout" and x.get("original"): + ours = describe(x["original"], kind, qn) + base = describe(x["pkg"], kind, qn) if x.get("pkg") else None + if ours and base and norm(ours) == norm(base): + continue + mdl, how = layout_route(qn, ours, x["original"]) if ours else (None, "unreadable in the original model") + if mdl is not None and not mdl.strip(): + print(f"SKIP {mod} {kind} {name}: {how}") + continue + elif kind in ("layout", "snippet", "page", "constant") and x.get("original") and x.get("app_mpr"): mdl, how = merge_element(kind, qn, x.get("pkg"), x["original"], x["app_mpr"], tmp) else: mdl, how = rd(f), "saved element (no describe for this kind)" diff --git a/upgrade-toolkit/controls/editroute_control.sh b/upgrade-toolkit/controls/editroute_control.sh index 89ac2e4..b9513fc 100644 --- a/upgrade-toolkit/controls/editroute_control.sh +++ b/upgrade-toolkit/controls/editroute_control.sh @@ -26,7 +26,11 @@ if a[0] == "describe": el = {"Uimod.L1": {"base": ["class: 'a',", "title: 'x'"], "ours": ["class: 'mine',", "title: 'x'"], "theirs": ["class: 'a',", "title: 'y'"]}, "Uimod.K": {"base": ["value: '1'"], "ours": ["value: '2'"], "theirs": ["value: '1'"]}, "Uimod.Added": {"ours": ["class: 'own'"]}, - "Uimod.Bad": {"base": ["class: 'a'"], "ours": ["class: 'BROKEN'"], "theirs": ["class: 'a'"]}}.get(qn, {}).get(side) + "Uimod.Bad": {"base": ["class: 'a'"], "ours": ["class: 'BROKEN'"], "theirs": ["class: 'a'"]}, + # layouts: one redirected to the app's design module, one the app added and nothing uses, one edited and used + "Uimod.Redir": {"base": ["class: 'a',", "title: 'x'"], "ours": ["Layout: Brand.Main"], "theirs": ["class: 'a',", "title: 'y'"]}, + "Uimod.Spare": {"ours": ["class: 'copy'"]}, + "Uimod.UsedOwn": {"base": ["class: 'a'"], "ours": ["class: 'own'"], "theirs": ["class: 'a'"]}}.get(qn, {}).get(side) if el: print(f"mdl 1;\ncreate or modify {kind} {qn} (\n" + "\n".join(" " + x for x in el) + "\n) {\n};") sys.exit(0) @@ -52,7 +56,7 @@ refuse() { printf 'refusing to update: %s element(s) have been changed locally a local m=$2; shift 2; for e in "$@"; do printf ' %s\n' "$e" >> "$S/step$n-$m.log"; done; printf '\n - Save them first: --save-edits \n' >> "$S/step$n-$m.log"; } n=1; refuse 1 Cosmo "MICROFLOW CosmeticFlow" n=2; refuse 2 Realmod "MICROFLOW RealFlow" -n=3; refuse 3 Uimod "LAYOUT L1" "CONSTANT K" "LAYOUT Added" "LAYOUT Bad" +n=3; refuse 3 Uimod "SNIPPET L1" "CONSTANT K" "SNIPPET Added" "SNIPPET Bad" "LAYOUT Redir" "LAYOUT Spare" "LAYOUT UsedOwn" printf '5 element(s) could not be read, so it cannot be shown that the module is unedited.\n' > "$S/step4-Unread.log" touch "$S/step5-Filesmod.log" "$S/step6-Vendmod.log" "$S/step7-Usedmod.log" "$S/step8-Target.log" "$S/step9-Javamod.log" printf 'RED REMOVED attribute Vendmod.E.Gone (column/table dropped; intended?)\n' > "$S/step6-Vendmod.idcheck" @@ -80,7 +84,7 @@ case "${1:-}" in for f in route-plan.txt force-ui.txt accept-unsaved.txt approved.txt approved-step.txt; do sed "s/^/$f: /" "$W/$f"; done ;; collect) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null mkdir -p "$W/saved-edits/Uimod" "$W/saved-edits/Target" - for e in layout-L1 constant-K layout-Added layout-Bad page-NotRefused entity-Whole; do printf 'mdl 1;\n-- saved whole element %s\n' "$e" > "$W/saved-edits/Uimod/$e.mdl"; done + for e in snippet-L1 constant-K snippet-Added snippet-Bad layout-Redir layout-Spare layout-UsedOwn page-NotRefused entity-Whole; do printf 'mdl 1;\n-- saved whole element %s\n' "$e" > "$W/saved-edits/Uimod/$e.mdl"; done printf 'mdl 1;\ncreate or modify association Other.A_B from Other.X to Target.Y;\n' > "$W/saved-edits/Target/association-Other_A_B.mdl" python3 "$B/editroute.py" collect --work "$W"; echo "rc=$?" sed 's/^/Uimod.mdl: /' "$W/replay/Uimod.mdl"; sed 's/^/Target.mdl: /' "$W/replay/Target.mdl"; sed 's/^/Target.guids: /' "$W/replay/Target.guids" ;; diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 13a69fb..3b101c0 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6625,23 +6625,25 @@ ctl cockpit-check-running-no-stopbar-pass 0 'stopbar: 0' '^0$' -- $CK check-runn ER="bash $here/editroute_control.sh" ctl editroute-cosmetic-pass 0 '^ROUTE Cosmo COSMETIC: only diagram layout differs' '^force-ui.txt: Cosmo ' -- $ER plan ctl editroute-real-logic-held-catch 0 '^ROUTE Realmod HOLD: real logic edit: microflow RealFlow \(move it to Realmod_Ext' '!route-plan.txt: .* Realmod$' -- $ER plan -ctl editroute-ui-constant-replay-pass 0 '^ROUTE Uimod REPLAY: 4 UI' '^force-ui.txt: Uimod ' -- $ER plan +ctl editroute-ui-constant-replay-pass 0 '^ROUTE Uimod REPLAY: 7 UI' '^force-ui.txt: Uimod ' -- $ER plan ctl editroute-unread-unedited-pass 0 '^ROUTE Unread UNREAD' -- $ER plan ctl editroute-files-accept-pass 0 '^ROUTE Filesmod FILES' '^accept-unsaved.txt: Filesmod ' -- $ER plan ctl editroute-vendor-drop-unused-pass 0 '^ROUTE Vendmod VENDOR' '^approved.txt: Vendmod.E.Gone # editroute VENDOR' -- $ER plan ctl editroute-vendor-drop-used-catch 0 '^ROUTE Usedmod HOLD' '!approved.txt: Usedmod' -- $ER plan ctl editroute-xassoc-pass 0 '^ROUTE Target XASSOC' '^approved-step.txt: Other.A_B ' -- $ER plan ctl editroute-java-held-catch 0 '^ROUTE Javamod HOLD' '!route-plan.txt: .* Javamod$' -- $ER plan -ctl editroute-collect-pass 0 'Uimod.mdl: create or modify layout Uimod.L1' 'Uimod.mdl: create or modify constant Uimod.K' '!NotRefused' 'Target.mdl: create or modify association Other.A_B' '^Target.guids: Other.A_B$' -- $ER collect +ctl editroute-collect-pass 0 'Uimod.mdl: create or modify snippet Uimod.L1' 'Uimod.mdl: create or modify constant Uimod.K' '!NotRefused' 'Target.mdl: create or modify association Other.A_B' '^Target.guids: Other.A_B$' -- $ER collect # cold test WM1 Atlas_Core: a saved whole element lost the master-layout link and carried the old widget's properties; the change # is merged onto the new element instead, and a result mxcli check refuses is held, never executed ctl editroute-collect-prop-merge-pass 0 "^Uimod.mdl: class: 'mine',$" "^Uimod.mdl: title: 'y'$" '!saved whole element' -- $ER collect -ctl editroute-collect-app-added-pass 0 "layout Added: added by the app" "^Uimod.mdl: class: 'own'$" "constant K: the vendor did not change it" -- $ER collect +ctl editroute-collect-app-added-pass 0 "snippet Added: added by the app" "^Uimod.mdl: class: 'own'$" "constant K: the vendor did not change it" -- $ER collect # data test 2026-11-16: save-edits wrote no association file; a re-created association lost its member access; a saved # whole entity put widened vendor attributes back ctl editroute-collect-assoc-from-original-pass 0 'association Other.A_B: from the original model' '^Target.mdl: grant read \(A_B, Name\), write \(A_B\) on entity Other.X to Other.User;$' '!Other.Guest' -- $ER collect ctl editroute-collect-no-whole-entity-catch 0 '!entity-Whole' '!saved whole element entity' -- $ER collect -ctl editroute-collect-check-held-catch 0 '^HELD Uimod layout Bad: mxcli check refuses' '!BROKEN.*Uimod.mdl|Uimod.mdl:.*BROKEN' -- $ER collect +# WM1 rerun: mxcli never writes a layout into a Marketplace module; the app's redirect to its design module becomes a page repoint +ctl editroute-collect-layout-repoint-pass 0 '^Uimod.mdl: ALTER PAGES SET LAYOUT = Brand.Main WHERE LAYOUT = Uimod.Redir;$' '^SKIP Uimod layout Spare: used by nothing' '^HELD Uimod layout UsedOwn: mxcli never writes a layout' '!create or modify layout' -- $ER collect +ctl editroute-collect-check-held-catch 0 '^HELD Uimod snippet Bad: mxcli check refuses' '!BROKEN.*Uimod.mdl|Uimod.mdl:.*BROKEN' -- $ER collect # cold test WM1 Email_Connector: diagram-only, then the routed update kept it for 3 associations; routed again, routes add up # does routing work across real runs: a route that ends off the plan's version is counted, never hidden ctl editroute-score-pass 0 '^SCORE Cosmo COSMETIC -> ON TARGET' '^SCORE Uimod REPLAY -> REPLAY FAILED' '^SCORE Realmod HOLD -> HOLD' 'ended on the plan.s version' '^rc=1$' '^ledger: .*app1' -- $ER score From d4b02a6c56ec8942c13cddc36401ac9fb7eb2a41 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 00:15:17 +0800 Subject: [PATCH 09/59] editroute: read the idcheck file the kept reason names, not the newest step's WM1 rerun 4: Email_Connector's kept reason named step6's idcheck (3 RED association removals, an XASSOC case); editroute read step10, a refusal with no idcheck, said 'no RED rows read' and held it. The old Email_Connector snippet then opened the new MxModelReflection select page (a Data Grid 2) from a reference selector: CE0620. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/editroute.py | 6 +++++- upgrade-toolkit/controls/editroute_control.sh | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/upgrade-toolkit/bin/editroute.py b/upgrade-toolkit/bin/editroute.py index eefc87c..9668a71 100755 --- a/upgrade-toolkit/bin/editroute.py +++ b/upgrade-toolkit/bin/editroute.py @@ -97,7 +97,11 @@ def route_module(mod, why, S, ledit, original, cache, mpr): if re.search(r"NETWORK|update failed|Java user code|javasource", why): return "HOLD", why, {} if "idcheck REVIEW" in why: - idc = log[:-4] + ".idcheck" if log else "" + # the file the kept reason names: a later step of the same module (WM1: step10, a refusal) has no idcheck, and + # reading its log gave "no RED rows read", so Email_Connector was held and its old snippet broke the build (CE0620) + m = re.search(r"see \S*?(step\d+-[\w.]+\.idcheck)", why) + idc = os.path.join(S, m.group(1)) if m and os.path.exists(os.path.join(S, m.group(1))) else \ + next((c for c in (p[:-4] + ".idcheck" for p in reversed(logs)) if os.path.exists(c)), "") rows, own, xassoc, bad = red_rows(idc), [], [], [] for finding, kind, qn in rows: if finding == "REMOVED" and kind in ("attribute", "entity") and qn.split(".")[0] == mod: diff --git a/upgrade-toolkit/controls/editroute_control.sh b/upgrade-toolkit/controls/editroute_control.sh index b9513fc..9bf6797 100644 --- a/upgrade-toolkit/controls/editroute_control.sh +++ b/upgrade-toolkit/controls/editroute_control.sh @@ -62,6 +62,7 @@ touch "$S/step5-Filesmod.log" "$S/step6-Vendmod.log" "$S/step7-Usedmod.log" "$S/ printf 'RED REMOVED attribute Vendmod.E.Gone (column/table dropped; intended?)\n' > "$S/step6-Vendmod.idcheck" printf 'RED REMOVED attribute Usedmod.E.Used (column/table dropped; intended?)\n' > "$S/step7-Usedmod.idcheck" printf 'RED REMOVED association Other.A_B (column/table dropped; intended?)\n' > "$S/step8-Target.idcheck" +printf 'refusing to update: 1 element(s) have been changed locally\n' > "$S/step12-Target.log" # a later step, no idcheck (WM1) cat > "$S/kept.txt" <<'K' Cosmo local edits refuse the update Realmod local edits refuse the update @@ -70,7 +71,7 @@ Unread elements could not be read, module not shown unedited Filesmod FILES-UNKNOWN: update replaced 1 theme/javascript/java file(s), unproven unedited Vendmod idcheck REVIEW (persistence loss or removal: accept it in approved.txt with a reason, or keep), see x Usedmod idcheck REVIEW (persistence loss or removal: accept it in approved.txt with a reason, or keep), see x -Target idcheck REVIEW (persistence loss or removal: accept it in approved.txt with a reason, or keep), see x +Target idcheck REVIEW (persistence loss or removal: accept it in approved.txt with a reason, or keep), see .stepupdate/step8-Target.idcheck Javamod local Java user code edits (FILES-EDITED javasource) K python3 - "$W/localedits.json" <<'PY' From b63a1912c45f6d378c6e1b00502901cc0d374bb5 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 00:39:52 +0800 Subject: [PATCH 10/59] editroute: XASSOC carries dropped associations without data too WM1 rerun 5: Email.EmailNP_EmailTemplate (non-persistent owner) is an info row in idcheck; only RED rows were replayed, so the app's data view and create activity lost it (CE1613). Score went 6/7 -> 7/7 with the previous fix. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/editroute.py | 8 ++++++-- upgrade-toolkit/controls/editroute_control.sh | 2 +- upgrade-toolkit/controls/run-controls.sh | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/upgrade-toolkit/bin/editroute.py b/upgrade-toolkit/bin/editroute.py index 9668a71..2676e6f 100755 --- a/upgrade-toolkit/bin/editroute.py +++ b/upgrade-toolkit/bin/editroute.py @@ -81,8 +81,8 @@ def refusal(log): return [(k, n.strip()) for k, n in rows], int(u[-1]) if u else 0 -def red_rows(idc): - return [m.groups() for m in re.finditer(r"^RED\s+(\w+)\s+(\w+)\s+([\w.]+)", rd(idc), re.M)] +def red_rows(idc, sev="RED"): + return [m.groups() for m in re.finditer(rf"^(?:{sev})\s+(\w+)\s+(\w+)\s+([\w.]+)", rd(idc), re.M)] def unused(original, kind, qn): @@ -112,6 +112,10 @@ def route_module(mod, why, S, ledit, original, cache, mpr): bad.append(f"{finding} {kind} {qn}") if bad or not rows: return "HOLD", "idcheck finding still used or not a removal: " + ", ".join(bad or ["no RED rows read"]), {} + # an association with no data (a non-persistent owner) is an info row, yet the app still uses it: WM1's + # Email.EmailNP_EmailTemplate, dropped silently, broke a data view and a create activity (CE1613) + xassoc += [qn for f, k, qn in red_rows(idc, "info") if f == "REMOVED" and k == "association" + and qn.split(".")[0] != mod and qn not in xassoc] if xassoc: return "XASSOC", f"the update drops {len(xassoc)} association(s) other modules own into {mod}; replayed with their original ids", \ {"assocs": xassoc, "vendor": own} diff --git a/upgrade-toolkit/controls/editroute_control.sh b/upgrade-toolkit/controls/editroute_control.sh index 9bf6797..3504c4a 100644 --- a/upgrade-toolkit/controls/editroute_control.sh +++ b/upgrade-toolkit/controls/editroute_control.sh @@ -61,7 +61,7 @@ printf '5 element(s) could not be read, so it cannot be shown that the module is touch "$S/step5-Filesmod.log" "$S/step6-Vendmod.log" "$S/step7-Usedmod.log" "$S/step8-Target.log" "$S/step9-Javamod.log" printf 'RED REMOVED attribute Vendmod.E.Gone (column/table dropped; intended?)\n' > "$S/step6-Vendmod.idcheck" printf 'RED REMOVED attribute Usedmod.E.Used (column/table dropped; intended?)\n' > "$S/step7-Usedmod.idcheck" -printf 'RED REMOVED association Other.A_B (column/table dropped; intended?)\n' > "$S/step8-Target.idcheck" +printf 'RED REMOVED association Other.A_B (column/table dropped; intended?)\ninfo REMOVED association Other.Np_B (no data)\n' > "$S/step8-Target.idcheck" printf 'refusing to update: 1 element(s) have been changed locally\n' > "$S/step12-Target.log" # a later step, no idcheck (WM1) cat > "$S/kept.txt" <<'K' Cosmo local edits refuse the update diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 3b101c0..fa17a50 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6643,6 +6643,8 @@ ctl editroute-collect-assoc-from-original-pass 0 'association Other.A_B: from th ctl editroute-collect-no-whole-entity-catch 0 '!entity-Whole' '!saved whole element entity' -- $ER collect # WM1 rerun: mxcli never writes a layout into a Marketplace module; the app's redirect to its design module becomes a page repoint ctl editroute-collect-layout-repoint-pass 0 '^Uimod.mdl: ALTER PAGES SET LAYOUT = Brand.Main WHERE LAYOUT = Uimod.Redir;$' '^SKIP Uimod layout Spare: used by nothing' '^HELD Uimod layout UsedOwn: mxcli never writes a layout' '!create or modify layout' -- $ER collect +# WM1 rerun 5: a dropped association with no data (non-persistent owner) is an info row, and the app still used it (CE1613) +ctl editroute-xassoc-no-data-assoc-pass 0 '^ROUTE Target XASSOC: the update drops 2 association' '^approved-step.txt: Other.Np_B ' -- $ER plan ctl editroute-collect-check-held-catch 0 '^HELD Uimod snippet Bad: mxcli check refuses' '!BROKEN.*Uimod.mdl|Uimod.mdl:.*BROKEN' -- $ER collect # cold test WM1 Email_Connector: diagram-only, then the routed update kept it for 3 associations; routed again, routes add up # does routing work across real runs: a route that ends off the plan's version is counted, never hidden From 293ef08a16a3989db5a97c700dd56a486cb61531 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 01:10:05 +0800 Subject: [PATCH 11/59] pipeline: build repair runs jardedupe with --updated-from-git WM1 rerun 6: stepupdate's jar cleanup was refused (stale vendorlib) and the build repair ran without the flag, so the old userlib pdfbox/fontbox 2.0.24 stayed next to vendorlib 3.0.7 and javac failed (Misc.java:643). With the flag the 7 SHADOWED jars resolve as leftovers of the updated modules. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 6 ++++-- upgrade-toolkit/controls/run-controls.sh | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 1cf6290..3329a6a 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -1077,8 +1077,10 @@ step_build() { if [ "$vrc" = 0 ]; then note "mx sync-java-dependencies failed (exit $rc); vendorlib filled by vendorlib_mvn.sh (host Maven)" else note "mx sync-java-dependencies failed (exit $rc), vendorlib_mvn.sh exit $vrc (3 = version range to pin), recorded NOT RUN; the build decides"; fi fi - rc=0; x python3 "$B/repair/jardedupe.py" "$APP" || rc=$? - x python3 "$B/repair/jardedupe.py" "$APP" --apply || true + # --updated-from-git: the 'step:' commits prove which modules moved; without it the old userlib copies they left behind + # stay (WM1: pdfbox 2.0.24 next to vendorlib 3.0.7, javac failed in CommunityCommons Misc.java:643) + rc=0; x python3 "$B/repair/jardedupe.py" "$APP" --updated-from-git || rc=$? + x python3 "$B/repair/jardedupe.py" "$APP" --updated-from-git --apply || true commit_app "repair: sync java dependencies, jardedupe" repair repair-row-2-3 # row 4: missing or broken packages are noted (mx check reports what matters); DEPRECATED-11 builds clean but fails at # runtime on 11 (T7-29), so it stops here diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index fa17a50..72a051c 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -481,6 +481,8 @@ for f in bcprov-jdk18on-1.85.jar commons-io-2.21.0.jar xercesImpl-2.12.2.jar; do ctl jardedupe-catch 1 'bcprov-jdk18on-1\.78' 'commons-io-2\.11\.0' 'KEEP.*xercesImpl' -- python3 $B/repair/jardedupe.py $J ctl jardedupe-apply-stale-refused 2 'REFUSED: vendorlib is stale' -- python3 $B/repair/jardedupe.py $J --apply ctl jardedupe-apply 0 'vendorlib current: stated with --vendorlib-synced' -- python3 $B/repair/jardedupe.py $J --apply --vendorlib-synced +# WM1 rerun 6: the build repair ran jardedupe without --updated-from-git; pdfbox 2 stayed next to pdfbox 3, javac failed +ctl pipeline-jardedupe-updated-from-git-pass 0 '^2$' -- bash -c "grep -c 'jardedupe.py\" \"\\\$APP\" --updated-from-git' $B/cloud/upgrade-pipeline.sh" ctl jardedupe-pass 0 '!bcprov-jdk18on-1\.78' -- python3 $B/repair/jardedupe.py $J ctl jardedupe-keeps-own-and-newer 0 -- bash -c "[ -f $J/userlib/own-helper-1.0.jar ] && [ -f $J/userlib/xercesImpl-2.12.2-sp1.jar ] && [ ! -f $J/userlib/bcprov-jdk18on-1.78.jar.SAML20.RequiredLib ]" # U4: -sources and -javadoc jars are their own artifact: never compared with the class jar, never kept over it From 9040a80a3706a0b88f4bc1077852174569d0f123 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 01:49:13 +0800 Subject: [PATCH 12/59] build: map module roles the updates added; after walk follows the baseline's microflow steps WM1 rerun 7: Atlas_Core 4 brings Atlas_Core.Administrator/User; its layout's language selector calls DS_Account_CurrentUser, allowed for those only. Nothing mapped them, mx check stayed clean, and the smoke walk failed Administration.Account_Overview 'for security reasons'. modroles.py maps roles NEW in this run, of an updated module, by name to the user roles; the guest role is never granted (GUEST line), no name match is UNMATCHED. The after walk skipped microflow steps the baseline ran: 4 false REGRESSIONs. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 16 +++- upgrade-toolkit/bin/repair/modroles.py | 90 +++++++++++++++++++ upgrade-toolkit/controls/modroles_control.sh | 29 ++++++ upgrade-toolkit/controls/run-controls.sh | 10 +++ 4 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 upgrade-toolkit/bin/repair/modroles.py create mode 100644 upgrade-toolkit/controls/modroles_control.sh diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 3329a6a..23df2de 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -452,7 +452,9 @@ import json, sys roles, out, pw = sys.argv[1], sys.argv[2], sys.argv[3] json.dump({r: {"user": "smoke_" + r.lower(), "pass": pw} for r in roles.split(",") if r}, open(out, "w"), indent=1)' "$1" "$2" "$SMOKE_PW" } -smoke_mf() { [ "${UPG_SMOKE_MICROFLOWS:-0}" != 1 ] || echo --microflows; } # UPG_SMOKE_MICROFLOWS (header) +# UPG_SMOKE_MICROFLOWS (header); the after walk also follows the baseline: a baseline that ran microflow steps compared +# with a walk that skipped them gave 4 false REGRESSIONs on WM1 (rerun 7, env not set on the rerun) +smoke_mf() { if [ "${UPG_SMOKE_MICROFLOWS:-0}" = 1 ] || grep -q '"landed": *"microflow ran"' "$WORK/smoke-before.json" 2> /dev/null; then echo --microflows; fi; } user_args() { # user_args ROLES -> sets UARGS for smokewalk.js UARGS=(); local r IFS=',' read -ra _rl <<< "$1" @@ -1082,6 +1084,18 @@ step_build() { rc=0; x python3 "$B/repair/jardedupe.py" "$APP" --updated-from-git || rc=$? x python3 "$B/repair/jardedupe.py" "$APP" --updated-from-git --apply || true commit_app "repair: sync java dependencies, jardedupe" repair repair-row-2-3 + # module roles an update ADDED, mapped by name (WM1: Atlas_Core 4's language selector needs Atlas_Core.User; unmapped, + # every page on an Atlas layout failed "for security reasons" while mx check stayed clean). Guest role: never granted. + if [ -f "$WORK/original.mpr" ]; then + rm -f "$WORK/modroles.mdl" + xo "$WORK/modroles.txt" python3 "$B/repair/modroles.py" "$MPR" --original "$WORK/original.mpr" --mdl "$WORK/modroles.mdl" || true + if [ "$DRY" != 1 ] && [ -s "$WORK/modroles.mdl" ]; then + rc=0; xo "$WORK/modroles-exec.out" mxcli_run exec "$WORK/modroles.mdl" -p "$MPR" || rc=$? + [ "$rc" = 0 ] || note "modroles: exec failed (exit $rc), see $WORK/modroles-exec.out" + commit_app "repair: map module roles the updates added" repair repair-modroles + fi + [ "$DRY" = 1 ] || note "$(tail -1 "$WORK/modroles.txt") (see $WORK/modroles.txt)" + fi # row 4: missing or broken packages are noted (mx check reports what matters); DEPRECATED-11 builds clean but fails at # runtime on 11 (T7-29), so it stops here rc=0; xo "$WORK/missing_deps.txt" python3 "$B/repair/missing_deps.py" "$APP" --target "$TARGET" || rc=$? diff --git a/upgrade-toolkit/bin/repair/modroles.py b/upgrade-toolkit/bin/repair/modroles.py new file mode 100644 index 0000000..c228890 --- /dev/null +++ b/upgrade-toolkit/bin/repair/modroles.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""modroles.py: module roles a Marketplace update ADDED, mapped to the app's user roles by name. + +Why (cold test WM1, 2026-11-17): Atlas_Core 3 had no module roles; Atlas_Core 4 brings Atlas_Core.Administrator and +Atlas_Core.User, and its layout's language selector calls Atlas_Core.DS_Account_CurrentUser, allowed for those two only. +mxcli's update restored "0 role grant(s)" (there were none), so every page on an Atlas layout failed for every user +role: "Executing action failed for security reasons". The build was clean; only the smoke walk saw it. Company brain: +UI modules' user/administrator module roles must be mapped to the app's user roles. + +Rule: only roles that are NEW in this run (absent from the original model) of a module with a 'step: ' +commit, and held by no user role. Such a role Mod.R goes to every user role that already holds a module role named R in +another module (Administrator holds Administration.Administrator, so it gets Atlas_Core.Administrator). No name match: +UNMATCHED, nothing granted, a person decides. The guest (anonymous) user role is never granted: GUEST, a person decides. A role the app left unmapped on purpose is never new, so never touched. + + modroles.py APP.mpr --original ORIGINAL.mpr [--mdl OUT.mdl] [--mxcli CMD] + prints GRANT / GUEST / UNMATCHED lines and a summary; --mdl writes the ALTER USER ROLE statements (no grants: no file) +""" +import argparse, os, re, shlex, subprocess, sys + + +def mx(cmd, mpr, q): + r = subprocess.run(shlex.split(cmd) + ["-p", mpr, "-c", q], capture_output=True, text=True) + return r.stdout + + +def table_col(out, col=0): + rows = [] + for ln in out.splitlines(): + if not ln.startswith("|") or set(ln) <= set("|-"): + continue + rows.append([c.strip() for c in ln.strip("|").split("|")][col]) + return rows[1:] # drop the header + + +def module_roles(cmd, mpr): + return {r for r in table_col(mx(cmd, mpr, "show module roles")) if "." in r} + + +def user_roles(cmd, mpr): + out = {} + for u in table_col(mx(cmd, mpr, "show user roles")): + d = mx(cmd, mpr, f"describe user role {u}") # "describe userrole X" is a parse error in -c + m = re.search(r"ModuleRoles:\s*\(([^)]*)\)", d) + out[u] = [x.strip() for x in m.group(1).split(",") if x.strip()] if m else [] + return out + + +def updated_modules(app_dir): + r = subprocess.run(["git", "-C", app_dir, "log", "--format=%s"], capture_output=True, text=True) + return {m.group(1) for m in re.finditer(r"^step: (\w+) \S+", r.stdout, re.M)} + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("mpr"); ap.add_argument("--original", required=True) + ap.add_argument("--mdl"); ap.add_argument("--mxcli", default=os.environ.get("UPG_MXCLI_READ", "mxcli")) + a = ap.parse_args() + now, orig = module_roles(a.mxcli, a.mpr), module_roles(a.mxcli, a.original) + users = user_roles(a.mxcli, a.mpr) + held = {r for rs in users.values() for r in rs} + moved = updated_modules(os.path.dirname(os.path.abspath(a.mpr))) + m = re.search(r"Guest User Role:\s*(\w+)", mx(a.mxcli, a.mpr, "describe app security")) + guest = m.group(1) if m else None + grants, unmatched = [], [] + for r in sorted(now - orig - held): + mod, name = r.split(".", 1) + if mod not in moved: + continue + to = [u for u, rs in sorted(users.items()) if any(x.split(".", 1)[1].lower() == name.lower() + and x.split(".", 1)[0] != mod for x in rs)] + gu = guest in to + if gu: # what anonymous visitors may run is a person's call + to.remove(guest) + print(f"GUEST {r}: the guest role {guest} holds a '{name}' role elsewhere; not granted, decide by hand") + if to: + for u in to: + grants.append((u, r)) + print(f"GRANT {r} -> user role {u} (new in the {mod} update; {u} holds a '{name}' role in another module)") + else: + unmatched.append(r) + gu or print(f"UNMATCHED {r}: new in the {mod} update, no user role holds a '{name}' role elsewhere; map it by hand") + if grants and a.mdl: + with open(a.mdl, "w") as f: + f.write("mdl 1;\n" + "".join(f"ALTER USER ROLE {u} ADD MODULE ROLES ({r});\n" for u, r in grants)) + print(f"modroles: {len(grants)} grant(s), {len(unmatched)} unmatched new module role(s)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/controls/modroles_control.sh b/upgrade-toolkit/controls/modroles_control.sh new file mode 100644 index 0000000..3e9df8e --- /dev/null +++ b/upgrade-toolkit/controls/modroles_control.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# modroles_control.sh: bin/repair/modroles.py on a fake model (WM1 2026-11-17: Atlas_Core 4's new module roles unmapped, +# every page on an Atlas layout failed "for security reasons"). Prints the script's output; run-controls.sh greps it. +set -euo pipefail +B=$(cd "$(dirname "$0")/../bin" && pwd) +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT +mkdir -p "$T/app"; : > "$T/app/App.mpr"; : > "$T/original.mpr" +git -C "$T/app" init -q; git -C "$T/app" -c user.name=c -c user.email=c@c commit -q --allow-empty -m "step: Atlas_Core 4.1.0 (stepupdate)" +git -C "$T/app" -c user.name=c -c user.email=c@c commit -q --allow-empty -m "step: Widgets 2.0.0 (stepupdate)" +cat > "$T/mxcli" <<'EOF' +#!/usr/bin/env bash +# fake: -p MPR -c QUERY +mpr=$2 q=$4 +roles() { printf '| Qualified Name | Module | Role |\n|---|---|---|\n'; for r in "$@"; do printf '| %s | x | x |\n' "$r"; done; } +case "$q" in + "show module roles") + if [[ $mpr == *original.mpr ]]; then roles Administration.Administrator Administration.User Legacy.Viewer + else roles Administration.Administrator Administration.User Legacy.Viewer Atlas_Core.Administrator Atlas_Core.User \ + Atlas_Core.Designer Widgets.Reader Other.User; fi ;; + "show user roles") printf '| Name | Module Roles |\n|---|---|\n| Administrator | 2 |\n| User | 1 |\n| Anonymous | 1 |\n' ;; + "describe user role Administrator") echo " ModuleRoles: (Administration.Administrator, Administration.User)," ;; + "describe user role User") echo " ModuleRoles: (Administration.User)," ;; + "describe user role Anonymous") echo " ModuleRoles: (Administration.User)," ;; + "describe app security") echo "Guest User Role: Anonymous" ;; +esac +EOF +chmod +x "$T/mxcli" +python3 "$B/repair/modroles.py" "$T/app/App.mpr" --original "$T/original.mpr" --mdl "$T/out.mdl" --mxcli "$T/mxcli" +[ ! -f "$T/out.mdl" ] || sed 's/^/mdl: /' "$T/out.mdl" diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 72a051c..5a8f6cf 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -481,6 +481,16 @@ for f in bcprov-jdk18on-1.85.jar commons-io-2.21.0.jar xercesImpl-2.12.2.jar; do ctl jardedupe-catch 1 'bcprov-jdk18on-1\.78' 'commons-io-2\.11\.0' 'KEEP.*xercesImpl' -- python3 $B/repair/jardedupe.py $J ctl jardedupe-apply-stale-refused 2 'REFUSED: vendorlib is stale' -- python3 $B/repair/jardedupe.py $J --apply ctl jardedupe-apply 0 'vendorlib current: stated with --vendorlib-synced' -- python3 $B/repair/jardedupe.py $J --apply --vendorlib-synced +# WM1 rerun 7: baseline ran microflow steps, the after walk skipped them: 4 false REGRESSIONs +ctl pipeline-smoke-mf-follows-baseline-pass 0 '^--microflows$' -- bash -c "WORK=\$(mktemp -d); echo '{\"landed\": \"microflow ran\"}' > \$WORK/smoke-before.json; eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf" +ctl pipeline-smoke-mf-off-without-baseline-pass 0 '!microflows' -- bash -c "WORK=\$(mktemp -d); eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf; echo done" +# WM1 2026-11-17: Atlas_Core 4's new module roles unmapped; every Atlas page failed for security reasons (smoke walk only) +MR="bash $B/../controls/modroles_control.sh" +ctl modroles-grant-by-name-pass 0 '^mdl: ALTER USER ROLE Administrator ADD MODULE ROLES \(Atlas_Core.Administrator\);' '^mdl: ALTER USER ROLE User ADD MODULE ROLES \(Atlas_Core.User\);' '^modroles: 3 grant' -- $MR +ctl modroles-guest-held-catch 0 '^GUEST Atlas_Core.User' '!ALTER USER ROLE Anonymous' -- $MR +ctl modroles-unmatched-catch 0 '^UNMATCHED Atlas_Core.Designer' '^UNMATCHED Widgets.Reader' -- $MR +ctl modroles-keeps-old-and-unmoved-pass 0 '!Legacy.Viewer' '!Other.User' -- $MR +ctl pipeline-runs-modroles-pass 0 '^1$' -- bash -c "grep -c 'repair/modroles.py\" \"\\\$MPR\" --original' $B/cloud/upgrade-pipeline.sh" # WM1 rerun 6: the build repair ran jardedupe without --updated-from-git; pdfbox 2 stayed next to pdfbox 3, javac failed ctl pipeline-jardedupe-updated-from-git-pass 0 '^2$' -- bash -c "grep -c 'jardedupe.py\" \"\\\$APP\" --updated-from-git' $B/cloud/upgrade-pipeline.sh" ctl jardedupe-pass 0 '!bcprov-jdk18on-1\.78' -- python3 $B/repair/jardedupe.py $J From e4aa1c16839528a38cbb423b7439cd79b297e4ea Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 04:02:00 +0800 Subject: [PATCH 13/59] diff: approve System-module platform drops by a baseline count of 0 WM1 9.24 -> 10.24 gate: 4 open RED idcheck rows, all System (DataBrokerEntityMetadata, WorkflowActivity. IsMigrationActivity): Mendix 10 drops them. FSM approved the same rows by hand after counting 0 rows. platform_drops.py counts rows / non-null values on the baseline snapshot and approves only on 0, with the count as the reason; any other count or a failed query stays open (KEEP). App and Marketplace modules are never touched. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 7 ++ upgrade-toolkit/bin/platform_drops.py | 64 +++++++++++++++++++ .../controls/platform_drops_control.sh | 28 ++++++++ upgrade-toolkit/controls/run-controls.sh | 6 ++ 4 files changed, 105 insertions(+) create mode 100644 upgrade-toolkit/bin/platform_drops.py create mode 100644 upgrade-toolkit/controls/platform_drops_control.sh diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 23df2de..186b60f 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -1535,6 +1535,13 @@ step_diff() { [ ! -f "$WORK/allow-effective.txt" ] || allowf=allow-effective.txt local notesf=gate-notes.txt; [ ! -f "$WORK/gate-notes-effective.txt" ] || notesf=gate-notes-effective.txt # the person's notes plus the heal's behaviour-change lines [ ! -f "$WORK/log-allow-effective.txt" ] || logallowf=log-allow-effective.txt + # platform drops (System module) with no data in the baseline are approved by count, not by a person (WM1, FSM: 9->10 + # drops System.DataBrokerEntityMetadata; 0 rows); any other count stays open for a person + local bdb; bdb=$(awk '$2 == "snapshot" && $3 == "before" { d = $1 } END { print d }' "$WORK/db-ledger.txt" 2> /dev/null) + if [ -n "$bdb" ] && [ -f "$(mda_old)" ] && [ -f "$(mda_new)" ]; then + xo "$WORK/platform-drops.out" python3 "$B/platform_drops.py" "$(mda_old)" "$(mda_new)" --approved "$WORK/approved.txt" --psql "$(psql_str "$bdb")" || true + [ "$DRY" = 1 ] || note "$(tail -1 "$WORK/platform-drops.out") (see $WORK/platform-drops.out)" + fi for f in APPROVED=approved.txt CONST_OVERRIDES=env-values.json CONST_ALLOW=const-allow.txt ALLOW=$allowf LOG_ALLOW=$logallowf SMOKE_SHOT_ACCEPT=shot-accept.txt GATE_NOTES=$notesf HEAL_JSON=heal.json; do k=${f%%=*}; [ ! -f "$WORK/${f#*=}" ] || kv+=("$k=$WORK/${f#*=}") done diff --git a/upgrade-toolkit/bin/platform_drops.py b/upgrade-toolkit/bin/platform_drops.py new file mode 100644 index 0000000..7d55f5a --- /dev/null +++ b/upgrade-toolkit/bin/platform_drops.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""platform_drops.py: a System-module element the platform removes, approved only when the baseline holds no data in it. + +Why (cold test WM1, 2026-11-17): the 9.24 -> 10.24 gate stopped on 4 RED idcheck rows, all in the System module +(System.DataBrokerEntityMetadata and its attributes, System.WorkflowActivity.IsMigrationActivity). Mendix 10 drops them; +the app cannot keep them. FSM approved the same rows by hand after counting 0 rows, and the 9-to-10 playbook says the +same. The count is the proof, so it is made here, on the baseline snapshot, and written as the reason. + +Rule: only RED REMOVED rows of the System module. Entity: approved when its table has 0 rows. Attribute: approved when +its column holds 0 non-null values (or its table is gone and had 0 rows). Any other count, or a query that fails: +not approved, printed as KEEP with the count, a person decides. App and Marketplace modules are never touched here. + + platform_drops.py OLD.mda NEW.mda --approved approved.txt --psql "docker exec -i PG psql -U mendix -d DB" [--dry-run] + appends one line per approval: ' # platform drop (platform_drops.py): in ' +""" +import argparse, os, re, shlex, subprocess, sys + +B = os.path.dirname(os.path.abspath(__file__)) +ROW = re.compile(r"^RED\s+REMOVED\s+(entity|attribute)\s+(System\.[\w.]+)", re.M) +TAG = "# platform drop (platform_drops.py)" + + +def count(psql, sql): + r = subprocess.run(shlex.split(psql) + ["-tAc", sql], capture_output=True, text=True) + out = r.stdout.strip() + return int(out) if r.returncode == 0 and out.isdigit() else None + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("old"); ap.add_argument("new") + ap.add_argument("--approved", required=True); ap.add_argument("--psql", required=True) + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--idcheck-out", help="read the idcheck rows from this file instead of running idcheck.py") + a = ap.parse_args() + rows = open(a.idcheck_out).read() if a.idcheck_out else subprocess.run( + [sys.executable, os.path.join(B, "idcheck.py"), a.old, a.new], capture_output=True, text=True).stdout + have = open(a.approved).read() if os.path.exists(a.approved) else "" + db = a.psql.split()[-1] + new, kept = [], 0 + for kind, qn in ROW.findall(rows): + if re.search(rf"^{re.escape(qn)}\s", have, re.M): + continue + parts = qn.split(".") + table = f'"system${parts[1].lower()}"' + if kind == "entity": + n, what = count(a.psql, f"select count(*) from {table}"), "rows" + else: + n, what = count(a.psql, f'select count(*) from {table} where "{parts[2].lower()}" is not null'), "non-null values" + if n == 0: + new.append(f"{qn} {TAG}: Mendix removes it; 0 {what} in {db}") + print(f"APPROVE {qn}: 0 {what} in {db}") + else: + kept += 1 + print(f"KEEP {qn}: {'count failed' if n is None else f'{n} {what}'} in {db}; a person decides") + if new and not a.dry_run: + with open(a.approved, "a") as f: + f.write(("" if not have or have.endswith("\n") else "\n") + "\n".join(new) + "\n") + print(f"platform_drops: {len(new)} approved by count, {kept} kept for a person") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/controls/platform_drops_control.sh b/upgrade-toolkit/controls/platform_drops_control.sh new file mode 100644 index 0000000..fb719ec --- /dev/null +++ b/upgrade-toolkit/controls/platform_drops_control.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# platform_drops_control.sh: bin/platform_drops.py with fake idcheck rows and a fake psql (WM1 2026-11-17: System-module +# drops of Mendix 10 approved only on a count of 0). Prints the script's output and the approved file. +set -euo pipefail +B=$(cd "$(dirname "$0")/../bin" && pwd) +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT +cat > "$T/rows" <<'R' +RED REMOVED entity System.Gone (column/table dropped; intended?) +RED REMOVED attribute System.Kept.HasData (column/table dropped; intended?) +RED REMOVED attribute System.Broken.Col (column/table dropped; intended?) +RED REMOVED attribute Shop.Order.Note (column/table dropped; intended?) +RED REMOVED entity System.Already (column/table dropped; intended?) +info REMOVED attribute System.Info.Col (no data) +R +printf 'System.Already # a person: fine\n' > "$T/approved.txt" +cat > "$T/psql" <<'P' +#!/usr/bin/env bash +q=${*: -1} +case "$q" in + *'"system$gone"'*) echo 0 ;; + *'"system$kept"'*) echo 3 ;; + *'"system$broken"'*) echo 'ERROR: relation does not exist' >&2; exit 1 ;; + *) echo 0 ;; +esac +P +chmod +x "$T/psql" +python3 "$B/platform_drops.py" old new --idcheck-out "$T/rows" --approved "$T/approved.txt" --psql "$T/psql -d basedb" +sed 's/^/approved: /' "$T/approved.txt" diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 5a8f6cf..ba1b5b7 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -481,6 +481,12 @@ for f in bcprov-jdk18on-1.85.jar commons-io-2.21.0.jar xercesImpl-2.12.2.jar; do ctl jardedupe-catch 1 'bcprov-jdk18on-1\.78' 'commons-io-2\.11\.0' 'KEEP.*xercesImpl' -- python3 $B/repair/jardedupe.py $J ctl jardedupe-apply-stale-refused 2 'REFUSED: vendorlib is stale' -- python3 $B/repair/jardedupe.py $J --apply ctl jardedupe-apply 0 'vendorlib current: stated with --vendorlib-synced' -- python3 $B/repair/jardedupe.py $J --apply --vendorlib-synced +# WM1 2026-11-17: the 9->10 gate stopped on System-module drops; approved only when the baseline count is 0 +PD="bash $B/../controls/platform_drops_control.sh" +ctl platform-drops-zero-approved-pass 0 '^approved: System.Gone # platform drop \(platform_drops.py\): Mendix removes it; 0 rows in basedb$' '^platform_drops: 1 approved by count, 2 kept' -- $PD +ctl platform-drops-data-kept-catch 0 '^KEEP System.Kept.HasData: 3 non-null values' '!approved: System.Kept' -- $PD +ctl platform-drops-query-fail-kept-catch 0 '^KEEP System.Broken.Col: count failed' '!approved: System.Broken' -- $PD +ctl platform-drops-app-and-info-untouched-pass 0 '!Shop.Order' '!System.Info' '!APPROVE System.Already' -- $PD # WM1 rerun 7: baseline ran microflow steps, the after walk skipped them: 4 false REGRESSIONs ctl pipeline-smoke-mf-follows-baseline-pass 0 '^--microflows$' -- bash -c "WORK=\$(mktemp -d); echo '{\"landed\": \"microflow ran\"}' > \$WORK/smoke-before.json; eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf" ctl pipeline-smoke-mf-off-without-baseline-pass 0 '!microflows' -- bash -c "WORK=\$(mktemp -d); eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf; echo done" From 4fba119741637991eb92fc8c50e433fde725cdc7 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 04:11:46 +0800 Subject: [PATCH 14/59] heal: a NEW_ENDPOINT row is a new surface, not lost data WM1: counts_of('status 404 -> 200') read the HTTP codes as row counts 404 -> 200, so three new /rest-doc/ API docs of updated modules were classed real-regression with 'rows or columns of the old data are missing'. They stay open (likely-real: a person accepts a new surface or switches it off) with a reason that says what they are. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/heal.py | 3 +++ upgrade-toolkit/controls/lib/mkheal.py | 1 + upgrade-toolkit/controls/run-controls.sh | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/upgrade-toolkit/bin/heal.py b/upgrade-toolkit/bin/heal.py index af19240..e960ea5 100644 --- a/upgrade-toolkit/bin/heal.py +++ b/upgrade-toolkit/bin/heal.py @@ -168,6 +168,9 @@ def data_rows(self): def data_row(self, r): c, det = r["check"], r["detail"] id_ = "data:" + c + if r.get("verdict") == "NEW_ENDPOINT": # WM1: "status 404 -> 200" read as counts 404 -> 200, called lost data + return self.row(id_, c, "likely-real", f"{det}: a new surface the old app did not have (often an updated module's " + "API doc); nothing was lost, but a person accepts it in writing or switches it off") n = counts_of(det) if "gone" in det or "lost" in det or "dropped" in det or (n and n[1] < n[0]): return self.row(id_, c, "real-regression", f"{det}: rows or columns of the old data are missing after the upgrade") diff --git a/upgrade-toolkit/controls/lib/mkheal.py b/upgrade-toolkit/controls/lib/mkheal.py index 780a3fb..fda23de 100644 --- a/upgrade-toolkit/controls/lib/mkheal.py +++ b/upgrade-toolkit/controls/lib/mkheal.py @@ -19,6 +19,7 @@ def put(rel, text): "- REVIEW log.start.NEW_WARNING: WARNING Configuration: Using derived value for SessionKeepAliveUpdatesInterval (SessionTimeout 30000)\n" "- REVIEW log.start.NEW_ERROR: ERROR Shop failed to start the queue\n" "- REVIEW log.start.NEW_WARNING: WARNING Something else changed\n" + "- NEW_ENDPOINT http./rest-doc/shop/openapi.json: status 404 -> 200: endpoint did not exist before\n" "- PASS Shop.Other: same\n") put("twin-old.md", "# twin\n\n## All rows\n- FAIL Shop.Item.changedDate: 3 rows differ\n") J = { diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index ba1b5b7..38a0e5d 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6451,10 +6451,12 @@ hcls() { python3 -I -c " import json,sys for r in json.load(open('$HL/h1/heal.json'))['rows']: if r['id']==sys.argv[1]: print(r['id'], '=>', r['class'], '/', r['heal_kind'], '/', r['evidence'])" "$1"; } -ctl heal-classes-line-pass 0 'classes: startup-noise 1, no-test-data 1, no-click-path 1, outside-call 1, test-setup 4, real-regression 3, likely-real 2' 'CHANGED 6' -- cat $HL/plan1.out +ctl heal-classes-line-pass 0 'classes: startup-noise 1, no-test-data 1, no-click-path 1, outside-call 1, test-setup 4, real-regression 3, likely-real 3' 'CHANGED 6' -- cat $HL/plan1.out ctl heal-startup-noise-evidence-catch 0 'data:Shop\.Item\.changedDate => startup-noise / allow / start noise: .* also differs between two starts of the SAME old app' -- hcls data:Shop.Item.changedDate ctl heal-allow-line-reason-pass 0 '^Shop\.Item\.changedDate 3 rows differ # start noise: ' '!Shop\.Customer' '!Shop\.Order' -- cat $HL/h1/heal-allow.txt ctl heal-no-twin-no-noise-catch 0 'data:Shop\.Customer\.name => likely-real' '!startup-noise' -- hcls data:Shop.Customer.name +# WM1 2026-11-17: 'status 404 -> 200' was read as counts 404 -> 200 and called lost data (real-regression) +ctl heal-new-endpoint-not-lost-data-catch 0 'data:http\./rest-doc/shop/openapi\.json => likely-real / None / .*a new surface the old app did not have' '!real-regression' '!missing after the upgrade' -- hcls data:http./rest-doc/shop/openapi.json ctl heal-lost-rows-real-regression-catch 0 'data:Shop\.Order\.total => real-regression / None' 'missing after the upgrade' -- hcls data:Shop.Order.total ctl heal-harness-log-accepted-catch 0 'test-setup / logallow / known harness line: .*SessionTimeout 30000' -- hcls 'log:log.start.NEW_WARNING:WARNING_Configuration_Using_derived_value_for_Se' ctl heal-log-allow-line-pass 0 '^WARNING Configuration: Using derived value for SessionKeepAliveUpdatesInterval\* # test harness: ' '!Something else' -- cat $HL/h1/heal-log-allow.txt From 8af37a07902e57926b8a5e7c5bb4f672c2b10103 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 04:48:56 +0800 Subject: [PATCH 15/59] pipeline: platform drops step survives a work folder without db-ledger.txt awk on a missing ledger exits 2 and set -e ended step_diff (7 dry-run controls: pipeline-diff-*, pipeline-ladder-*, pipeline-dry-diff-gate-pass). A run without a baseline snapshot would have died the same way. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 186b60f..dbc979f 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -1537,7 +1537,7 @@ step_diff() { [ ! -f "$WORK/log-allow-effective.txt" ] || logallowf=log-allow-effective.txt # platform drops (System module) with no data in the baseline are approved by count, not by a person (WM1, FSM: 9->10 # drops System.DataBrokerEntityMetadata; 0 rows); any other count stays open for a person - local bdb; bdb=$(awk '$2 == "snapshot" && $3 == "before" { d = $1 } END { print d }' "$WORK/db-ledger.txt" 2> /dev/null) + local bdb; bdb=$(awk '$2 == "snapshot" && $3 == "before" { d = $1 } END { print d }' "$WORK/db-ledger.txt" 2> /dev/null || true) # no ledger: awk exit 2 must not end the step under set -e if [ -n "$bdb" ] && [ -f "$(mda_old)" ] && [ -f "$(mda_new)" ]; then xo "$WORK/platform-drops.out" python3 "$B/platform_drops.py" "$(mda_old)" "$(mda_new)" --approved "$WORK/approved.txt" --psql "$(psql_str "$bdb")" || true [ "$DRY" = 1 ] || note "$(tail -1 "$WORK/platform-drops.out") (see $WORK/platform-drops.out)" From aaf319f455ac02682671f2013cf7d3b7001c2b5f Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 05:04:21 +0800 Subject: [PATCH 16/59] controls: drop PATH entries with a space before any control runs Stubs are prepended with an unquoted env PATH=$X:$PATH; VS Code's 'Application Support/.../copilot-chat' entry split it and 11 controls failed exit 127 on a healthy toolkit. Full suite: 56 -> 35 failed. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/controls/run-controls.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 38a0e5d..ec8bc1c 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -109,6 +109,9 @@ export UPG_LOCK=$T/test.lock UPG_LOCK_POLL=1 # controls never wait on the real # container exports JAVA_TOOL_OPTIONS: the container-wrapper controls then saw the run's own docker arguments. Controls set # their own environment; these never come from the caller. unset UPG_EXTRA UPG_ENV_FILE UPG_CA UPG_DRYRUN UPG_NAME +# Controls prepend stubs with an unquoted `env PATH=$X:$PATH`; a PATH entry with a space (VS Code's "Application Support/ +# .../copilot-chat/debugCommand") split it, and 11 controls failed exit 127 on a healthy toolkit (2026-11-17) +PATH=$(printf '%s' "$PATH" | tr ':' '\n' | grep -v ' ' | paste -sd: -); export PATH export UPG_COCKPIT_OPEN=0 # a control run never opens browser tabs (the live-cockpit controls turn it on with a stub opener) npass=0; nfail=0; nskip=0; failed=""; skipped="" From 53eeff3c3cf385844a7bb5bc54f2eb8186204517 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 05:12:42 +0800 Subject: [PATCH 17/59] controls: run the same on macOS as on Linux - one spelling of the temp folder (no '/T//', /private/var resolved): gate-config-relative-*, mxrun-* - TZ=UTC: fixtures are stamped in UTC (journeys-archive-keep on a +08 machine) - no bytecode under controls/ (controls-fixtures-not-ignored), BSD wc padding trimmed - the timing fake clock answers date +%s without using a tick (b4ddb02 added the cockpit's running marker): timing-*, runlog-timing-columns, pipeline-autorunlog-ok-row - GNU-only sed -i replaced by the existing sedi helper: runlog-backfill-*, release-impact-orphan-* Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/controls/run-controls.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index ec8bc1c..60c3206 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -101,7 +101,11 @@ fast=0; only="" while [ $# -gt 0 ]; do case "$1" in --fast) fast=1; shift;; --only) only=$2; shift 2;; *) echo "usage: run-controls.sh [--fast] [--only REGEX]"; exit 2;; esac done -T=$(mktemp -d "${TMPDIR:-/tmp}/upg-controls.XXXXXX") +# One spelling of the temp folder: TMPDIR ends in '/' on macOS ('/T//' in paths) and /var is /private/var; tools that +# resolve their paths then never matched the expected ones (gate-config-relative-*, mxrun-*). Fixtures are stamped in UTC: +# on a +08 machine the archive and timing controls read local time as UTC. Controls never write bytecode under controls/. +T=$(cd "$(mktemp -d "${TMPDIR:-/tmp}"/upg-controls.XXXXXX)" && pwd -P) +export TZ=UTC PYTHONDONTWRITEBYTECODE=1 trap 'rm -rf "$T"' EXIT mkdir -p "$T/out" "$T/m" "$T/field" export UPG_LOCK=$T/test.lock UPG_LOCK_POLL=1 # controls never wait on the real heavy lock (except the start control) @@ -735,7 +739,7 @@ ctl scsscase-input-catch 2 -- python3 $B/repair/scsscase.py $T/empty-nothing # B4T1-3, B4T2-2, B4T4-1, B4T5-2, B4T7-2, B4T8-1: a fixture that .gitignore swallows (*.log) is never committed, so its # controls FAIL in every fresh clone. No file under controls/ may be ignored (needs git; SKIP outside a work tree). if git -C "$here" rev-parse --git-dir >/dev/null 2>&1; then - IGN='n=$(git -C "$1" ls-files -o -i --exclude-standard . | tee /dev/stderr | wc -l); echo "ignored under controls: $n"' + IGN='n=$(git -C "$1" ls-files -o -i --exclude-standard . | tee /dev/stderr | wc -l | tr -d " "); echo "ignored under controls: $n"' ctl controls-fixtures-not-ignored-pass 0 '^ignored under controls: 0$' -- bash -c "$IGN" _ "$here" # catch: a file git still ignores (.DS_Store) placed under fixtures is found (the probe is removed again) mkdir -p "$here/fixtures/zz-ignore-probe" && : > "$here/fixtures/zz-ignore-probe/.DS_Store" @@ -1799,9 +1803,9 @@ hgapp() { # hgapp NAME: App.mpr 4.0.0 + files, then the install commit (4.1.0, hgit $HG/$1 add -A .; hgit $HG/$1 commit -qm "step: Administration 4.1.0 (stepupdate, content 23513)" } hgapp hclean; echo notes > $HG/hclean/README.md; hgit $HG/hclean add README.md; hgit $HG/hclean commit -qm "docs" -sed -i 's/\r$//' $HG/hclean/themesource/administration/web/_variables.scss; hgit $HG/hclean commit -qam "line ends only" +sedi 's/\r$//' $HG/hclean/themesource/administration/web/_variables.scss; hgit $HG/hclean commit -qam "line ends only" ctl themefiles-history-clean-pass 0 '^FILES-UNEDITED-BY-HISTORY Administration since [0-9a-f]{7} [0-9-]+ "step: Administration 4\.1\.0 \(stepupdate' '[0-9]+ files, provenance: toolkit step' '!FILES-EDITED' '!FILES-UNKNOWN' -- python3 $B/themefiles.py history $HG/hclean Administration --mpr App.mpr -hgapp hedit; sed -i 's/f26522/00ff00/; s/^\$brand-primary: *[^;]*/$brand-primary: #00ff00/' $HG/hedit/themesource/administration/web/_variables.scss; echo '$brand-extra: #123;' >> $HG/hedit/themesource/administration/web/_variables.scss; hgit $HG/hedit commit -qam "brand colour" +hgapp hedit; sedi 's/f26522/00ff00/; s/^\$brand-primary: *[^;]*/$brand-primary: #00ff00/' $HG/hedit/themesource/administration/web/_variables.scss; echo '$brand-extra: #123;' >> $HG/hedit/themesource/administration/web/_variables.scss; hgit $HG/hedit commit -qam "brand colour" ctl themefiles-history-edited-catch 1 '^FILES-EDITED themesource/administration/web/_variables\.scss \(by history, since [0-9a-f]{7}\)' '!FILES-UNEDITED' -- python3 $B/themefiles.py history $HG/hedit Administration --mpr App.mpr --json $HG/hedit.json ctl themefiles-history-edited-vars-pass 0 'brand-extra: #123;' -- cat $HG/hedit.json git clone -q --depth 1 file://$HG/hclean $HG/hshallow @@ -3196,7 +3200,7 @@ ctl release-impact-auto-one-mpr-pass 0 'facts: 1[0-9] counted from .*facts-clean ctl release-impact-auto-one-mpr-flags-pass 0 'facts: 1[0-9] counted from' '!auto cannot use: runtime_os' '!auto cannot use: db_vendor' 'dotnet-ide-host' 'other-db-version: row is about postgresql 13' -- sh -c "$RIN --auto --from-mpr $M/facts-clean/App.mpr --db-vendor postgresql --db-version 16 --runtime-os linux --out $RI/one2.md; cat $RI/one2.md" ctl release-impact-one-mpr-no-auto-catch 1 'go together' -- $RIN --from-mpr $M/facts-clean/App.mpr --out $RI/one3.md # T2F-4: an entry whose row the brain reworded is listed, kept across reruns, and returns with its row -cp -r $RN $RI/brain-n19b; sed -i 's/Intel-based Macs are no longer supported/Intel Macs are dropped/' $RI/brain-n19b/baseline/versions/studio-pro-upgrade-11.md +cp -r $RN $RI/brain-n19b; sedi 's/Intel-based Macs are no longer supported/Intel Macs are dropped/' $RI/brain-n19b/baseline/versions/studio-pro-upgrade-11.md [ -f $RI/n19.md ] && sed -E 's/^(\| 4 \|.*\|) +\|$/\1 handled: evidence\/mac.md (arm64 checked) |/' $RI/n19.md > $RI/n19h.md ctl release-impact-orphan-catch 0 'orphan entries: 1 handled entries matched no row' '^## Entries that matched no row \(1\)' '\| o1 \| Studio Pro \| 11\.6\.8 \| Studio Pro on macOS now runs natively on Apple Silicon \(arm64\)\. Intel-based Macs are no longer supported\. \| handled: evidence/mac\.md' -- sh -c "python3 $B/release_impact.py --brain $RI/brain-n19b --from 11.6.3 --to 11.12.3 --auto --facts $RF/facts-n19.json --out $RI/n19h.md; cat $RI/n19h.md" ctl release-impact-orphan-check-pass 1 '^auto: 10, human: 0, open: 2$' '!OPEN .*o1' -- python3 $B/release_impact.py --check $RI/n19h.md @@ -5882,7 +5886,7 @@ ctl runlog-code-name-accepted-pass 0 'rc=0' '!customer name' 'rows=1' \ -- sh -c "python3 -I $B/runlog.py --work $RL/w1 --log $RL/log6.md --app CODE-A --type 'template app' 2>&1 >/dev/null; echo rc=\$?; echo rows=\$(grep -c '^| 2' $RL/log6.md)" # backfill: a root with two run folders (one copy of the fixture, one more); dry run prints and writes nothing mkdir -p $RL/root/a/work $RL/root/b/work; cp -R $RLF/. $RL/root/a/work/; cp -R $RLF/. $RL/root/b/work/ -sed -i 's/ 20261013041248\./ 20261013999999./' $RL/root/b/work/gate-summary.md +sedi 's/ 20261013041248\./ 20261013999999./' $RL/root/b/work/gate-summary.md ctl runlog-backfill-dry-run-catch 0 '^ADD +a/work' '^ADD +b/work' '2 run folders under .*: 2 to add' 'Dry run: nothing written' 'log=absent' \ -- sh -c "python3 -I $B/runlog.py --backfill $RL/root --log $RL/log7.md --app CODE-A; [ -e $RL/log7.md ] && echo log=present || echo log=absent" ctl runlog-backfill-write-pass 0 '^ADDED +a/work' '2 added' 'rows=2' \ @@ -5902,7 +5906,7 @@ mkdir -p $RL/xroot/x1 $RL/xroot/x2; cp -R $RLF/. $RL/xroot/x1/; printf '# journa ctl runlog-explain-skipped-reason-catch 0 '^x1: WOULD-ADD' '^x2: SKIPPED no gate-summary\.md and no SCORECARD verdict' 'log=absent' \ -- sh -c "python3 -I $B/runlog.py --explain $RL/xroot --log $RL/log12.md --app CODE-A; [ -e $RL/log12.md ] && echo log=present || echo log=absent" mkdir -p $RL/sroot/a $RL/sroot/b; cp -R $RLF/. $RL/sroot/a/; cp -R $RLF/. $RL/sroot/b/ -sed -i 's/ 20261013041248\./ 20261015999999./; s/2026-10-13T/2026-10-15T/' $RL/sroot/b/gate-summary.md +sedi 's/ 20261013041248\./ 20261015999999./; s/2026-10-13T/2026-10-15T/' $RL/sroot/b/gate-summary.md ctl runlog-summary-latest-per-app-idempotent-pass 0 '^- \*\*CODE-A\*\* \(template app\): latest run 2026-10-15; highest version reached 11\.12\.5; .*; 2 runs$' 'same=yes' 'apps=1' \ -- sh -c "L=$RL/log13.md; python3 -I $B/runlog.py --backfill $RL/sroot --log \$L --app CODE-A --type 'template app' --write >/dev/null; python3 -I $B/runlog.py --summary --log \$L >/dev/null; a=\$(md5sum < \$L); python3 -I $B/runlog.py --summary --log \$L >/dev/null; [ \"\$a\" = \"\$(md5sum < \$L)\" ] && echo same=yes; grep '^- \*\*' \$L; echo apps=\$(grep -c '^- \*\*' \$L)" # the pipeline hook: record_runlog and cleanup, extracted and run under the pipeline's own set -Eeu / ERR trap @@ -5932,7 +5936,8 @@ set -Eeu trap 'exit 4' ERR WORK=$1; TIMES=$2; shift 2 . "$FN" -date() { local n; n=$(cat "$WORK/.tick" 2> /dev/null || echo 0); n=$((n + 1)); echo "$n" > "$WORK/.tick"; sed -n "${n}p" "$TIMES"; } +# the cockpit's running marker asks `date +%s` (b4ddb02); it must not use up a tick of the listed clock +date() { [ "${1:-}" != +%s ] || { echo 0; return; }; local n; n=$(cat "$WORK/.tick" 2> /dev/null || echo 0); n=$((n + 1)); echo "$n" > "$WORK/.tick"; sed -n "${n}p" "$TIMES"; } mkdir -p "$WORK/state" "$WORK/logs" ONLY=""; FROM=""; DRY=${DRY:-0}; NJ_TMP="$WORK/state/nj.tmp"; NJ_CODE=0; RUN=$WORK; SOURCE=10.24.0.1; TARGET=11.12.5; ARGS_ORIG=(); SELF=pipeline; STEP=""; LOG=$WORK/logs/x.log; APPS_UP=() mxcli_pin() { :; }; proposal_recheck() { :; }; cockpit_regen() { :; } From e13f33fdfc8ad706ecd2d2ac746e5cb84aa99fc6 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 05:27:17 +0800 Subject: [PATCH 18/59] cockpitmatrix: no personal name in a comment (customer-names-tracked-toolkit-pass) Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cockpitmatrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upgrade-toolkit/bin/cockpitmatrix.py b/upgrade-toolkit/bin/cockpitmatrix.py index 9b08864..f7ca568 100644 --- a/upgrade-toolkit/bin/cockpitmatrix.py +++ b/upgrade-toolkit/bin/cockpitmatrix.py @@ -2061,7 +2061,7 @@ def render(model, matrix): '
\n

Since you last looked

\n' f'
    {h_feed(matrix)}
\n
\n \n' f' \n \n\n{qdlgs}\n' '\n' From fe66a7b4f944aed088a309c14f89d1db227c0efa Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 05:29:26 +0800 Subject: [PATCH 19/59] controls: bt7 runs against a stand-in app server (AD-1 app_alive made every role INVALID on --port 1) Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/controls/run-controls.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 60c3206..252c5c8 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6297,7 +6297,13 @@ else: json.dump({"results": [{"rung": "ui", "name": "login", "verdict": v, "detail": d}], "walks": []}, open("artifacts/journey-findings.json", "w")) EOF2 chmod +x $BT7/bin/node -bt7() { local w=$1 refusals=$2; shift 2; rm -f $BT7/count.*; env PATH=$BT7/bin:$PATH BT7_COUNT=$BT7/count BT7_REFUSALS=$refusals python3 $B/run_journeys.py $BT7/gen --label before --port 1 --admin-port 2 --admin-pass x --users $BT7/users.json --psql "$SHOP" --work $BT7/$w "$@"; } +# AD-1 (app_alive): run_journeys.py first asks whether the app answers at all; on --port 1 nothing did, so every role was +# INVALID "app down" before the stub runner was reached. A plain HTTP server stands in for the app. +bt7() { local w=$1 refusals=$2 port srv rc=0; shift 2; rm -f $BT7/count.* + port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])') + python3 -m http.server "$port" --bind 127.0.0.1 --directory $BT7 > /dev/null 2>&1 & srv=$! + for _ in 1 2 3 4 5 6 7 8 9 10; do curl -s -o /dev/null "http://127.0.0.1:$port/" && break; sleep 0.3; done + env PATH=$BT7/bin:$PATH BT7_COUNT=$BT7/count BT7_REFUSALS=$refusals python3 $B/run_journeys.py $BT7/gen --label before --port "$port" --admin-port 2 --admin-pass x --users $BT7/users.json --psql "$SHOP" --work $BT7/$w "$@" || rc=$?; kill $srv 2> /dev/null; wait $srv 2> /dev/null; return $rc; } bt7sum() { python3 -c "import json; r=json.load(open('$BT7/$1/before/summary.json'))['roles']; print({k: (v['login'], v.get('session_retries', 0)) for k, v in sorted(r.items())})"; } ctl bt7-retry-after-refusal-catch 0 'session cap: RoleB refused by the runtime.s session limit; waiting 1s for sessions to expire, retry 1 of 2' -- bash -c "$(declare -f bt7); BT7=$BT7 B=$B SHOP=\"$SHOP\"; bt7 w1 1 --session-wait 1 >/dev/null 2>$BT7/w1.err; cat $BT7/w1.err | grep 'session cap'" ctl bt7-retry-recovers-login-catch 0 "^{'RoleA': \('PASS', 0\), 'RoleB': \('PASS', 1\)}\$" -- bt7sum w1 From c1698e77b896cf9199994ec9d24ad98a9d2ff1d7 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 05:38:34 +0800 Subject: [PATCH 20/59] controls: last stale fixtures (macOS and gate drift) - toolcheck-host-bsd-sed-catch: the nogsed PATH was made but never used; a host with Homebrew gsed never warned - usercode-no-package-catch: /bin/false does not exist on macOS - jarid-apply-old-sbom-refused: BSD touch has no -d DATE; touch -t, and a python utime for '+1 hour' - remedy-apply-release-todo-pass: the release rung's number moved when the gate gained a rung Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/controls/run-controls.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 252c5c8..53faeea 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -601,9 +601,9 @@ cp -r $T/jarid $T/jarid-b; cp -r $T/jarid $T/jarid-c; cp -r $T/jarid $T/jarid-d ctl jarid-apply-stale-refused 2 'REFUSED: vendorlib is stale' -- python3 $B/repair/jardedupe.py $T/jarid-b --apply ctl jarid-apply-stale-removes-nothing 0 -- bash -c "cd $T/jarid-b/userlib; [ -f org.apache.commons.io-2.3.0.jar ] && [ -f commons-io-2.11.0.jar.CommunityCommons.RequiredLib ] && [ -f org.apache.commons.lang3.jar ]" # an SBOM OLDER than the last module change is no proof (the sync ran for the previous module set) -echo '{}' > $T/jarid-c/vendorlib-sbom.json; touch -d '2001-01-01' $T/jarid-c/vendorlib-sbom.json +echo '{}' > $T/jarid-c/vendorlib-sbom.json; touch -t 200101010000 $T/jarid-c/vendorlib-sbom.json ctl jarid-apply-old-sbom-refused 2 'vendorlib-sbom\.json is older than the last module change' -- python3 $B/repair/jardedupe.py $T/jarid-c --apply -echo '{}' > $T/jarid-d/vendorlib-sbom.json; touch -d '+1 hour' $T/jarid-d/vendorlib-sbom.json +echo '{}' > $T/jarid-d/vendorlib-sbom.json; python3 -c "import os, sys, time; t = time.time() + 3600; os.utime(sys.argv[1], (t, t))" $T/jarid-d/vendorlib-sbom.json ctl jarid-apply-current-sbom-pass 0 'vendorlib current: vendorlib-sbom\.json is newer than the last module change' 'removed 4 jars and 3 markers' -- python3 $B/repair/jardedupe.py $T/jarid-d --apply ctl jarid-apply-result 0 -- bash -c "cd $T/jarid-d/userlib; [ ! -f org.apache.commons.io-2.3.0.jar ] && [ ! -f commons-io-2.11.0.jar ] && [ ! -f org.apache.commons.lang3.jar ] && [ ! -f org.apache.commons.io-2.3.0.jar.TcConnector.RequiredLib ] && [ -f commons-codec-1.15.jar ] && [ -f own-helper-1.0.jar ] && [ ! -f commons-codec.commons-codec-1.15.jar ]" ctl jarid-clean-after-apply 0 '!REMOVE' -- python3 $B/repair/jardedupe.py $T/jarid-d @@ -1944,7 +1944,7 @@ ctl stepupdate-java-action-identical-usercode-forced-pass 0 'IDENTICAL: user cod ctl stepupdate-java-action-commit-pass 0 'step: JavaAct 2.0 \(stepupdate, idcheck PASS, convert-made Java action metadata forced, user code identical\)' -- git -C $S/ja log -1 --format=%s javaapp jc '\t\tLog.info("local tweak"); return Login.start(this.ctx);' ctl stepupdate-java-action-changed-usercode-keep-catch 1 'DIFFERENT: Java code of JavaAct 1\.0' 'changed user code: javasource/javaact/actions/StartLogin\.java' 'local LOGIC edits \(JAVA_ACTION' 'KEPT: JavaAct' '!committed [0-9a-f]{7}' '0 committed, 1 kept' -- env UPG_CACHE=$UC/cache bash $B/stepupdate.sh $S/jc $S/jc.plan --localedits $UC/le.json --unattended -ctl usercode-no-package-catch 2 'CANNOT COMPARE: installed package JavaAct 1\.1 not cached' -- env UPG_CACHE=$UC/cache UPG_MXCLI=/bin/false python3 $B/usercode.py $S/ja JavaAct --cid 999 --version 1.1 +ctl usercode-no-package-catch 2 'CANNOT COMPARE: installed package JavaAct 1\.1 not cached' -- env UPG_CACHE=$UC/cache UPG_MXCLI=$(command -v false) python3 $B/usercode.py $S/ja JavaAct --cid 999 --version 1.1 if [ -d "$LAB/f1/runs/e4/work" ] && [ -d "$LAB/e5/run2/work" ]; then # field: replay the real refusal output (E4 ExcelImporter local edits, E5b NanoflowCommons pre-9.18 baseline) newapp fe4 '1 1.0 ExcelImporter\n' @@ -2592,7 +2592,8 @@ for d in $(echo "$PATH" | tr ':' ' '); do for f in "$d"/*; do n=${f##*/}; case $ ctl toolcheck-host-no-timeout-catch 0 'WARNING, no GNU timeout or gtimeout' 'host OK' -- env PATH=$NT UPG_PYTHON=$T/py-ok zsh $B/toolcheck.sh --host ctl toolcheck-host-timeout-pass 0 'host OK' '!no GNU timeout' -- env UPG_PYTHON=$T/py-ok zsh $B/toolcheck.sh --host NS=$T/nogsed; mkdir -p $NS $T/bsdsed; printf '#!/bin/sh\nexit 1\n' > $T/bsdsed/sed; chmod +x $T/bsdsed/sed -ctl toolcheck-host-bsd-sed-catch 0 'WARNING, sed is not GNU sed' -- env PATH=$T/bsdsed:$NT UPG_PYTHON=$T/py-ok zsh $B/toolcheck.sh --host +for f in "$NT"/*; do [ "${f##*/}" = gsed ] || ln -s "$(readlink "$f")" "$NS/${f##*/}" 2>/dev/null; done # a host with Homebrew gsed never warned +ctl toolcheck-host-bsd-sed-catch 0 'WARNING, sed is not GNU sed' -- env PATH=$T/bsdsed:$NS UPG_PYTHON=$T/py-ok zsh $B/toolcheck.sh --host # B4T2-1, B4N1-3: setup-container.sh pre-places the ANTLR jar (Maven Central 429 -> mirror; a proxy page is no jar), never # takes a credential on a command line, and runs toolcheck AFTER the pip/apt step: REFUSED or no rsync = exit 1 (it ran # before the pip step with `|| true`: "REFUSED" for modules installed a step later, exit 0 with modules or rsync missing) @@ -5780,7 +5781,7 @@ ctl remedy-apply-fail-branch-kept-pass 0 '^main$' 'remedy-fx-card-a' 'model v1' ctl remedy-apply-tried-again-catch 2 'already tried: fx-card A FAIL' -- rapply fx-card A ctl remedy-apply-check-fail-catch 1 'mx check rc 0, error lines 1: FAIL' 'fx-card B3 FAIL .*remedy-check-fx-card-b3\.log' -- rapply fx-card B3 'CHECK_CMD=echo "[error] CE0001 broken"' ctl remedy-apply-release-pause-catch 5 'rung release INVALID' 'PAUSED for a person \(release\): .*remedy-todo-fx-card-b2\.md' '!recorded in' -- rapply GATE_STUB_RC_release_impact=1 fx-card B2 -ctl remedy-apply-release-todo-pass 0 '^## Release rows' '\| 10 \| release \| INVALID \|' 'finish fx-card B2 PASS' -- cat $AW/remedy-todo-fx-card-b2.md +ctl remedy-apply-release-todo-pass 0 '^## Release rows' '\| [0-9]+ \| release \| INVALID \|' 'finish fx-card B2 PASS' -- cat $AW/remedy-todo-fx-card-b2.md ctl remedy-apply-finish-fail-catch 1 'fx-card B2 FAIL .* finished by a person: rows stay open' 'app back on main' -- rapply --finish fx-card B2 FAIL --note 'rows stay open' ctl remedy-apply-eye-pause-catch 5 'rung idcheck PASS' 'PAUSED for a person \(eye\)' -- rapply fx-card B ctl remedy-apply-eye-todo-pass 0 '^## By eye' 'eye ' -- cat $AW/remedy-todo-fx-card-b.md From b56c7a178c26f6adb74eda82765f84cdd4de68cc Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 06:33:49 +0800 Subject: [PATCH 21/59] fetch: read the zip listing once (unzip | grep -q under pipefail lost to SIGPIPE) grep -q quits at the first match; unzip then died of SIGPIPE, pipefail made the check false, and deployment/ (the build output, 858 MB on DOCGEN) was unpacked after all, with no note. bt9-fetch-* failed 1 run in 3; after: 8 of 8. The note control now shows the fetch log when it fails. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 9 ++++++--- upgrade-toolkit/controls/run-controls.sh | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index dbc979f..d0a07c2 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -503,10 +503,13 @@ step_fetch() { must mkdir -p "$APP" # BT-9: a package may ship the build output deployment/ next to the .mpr (DOCGEN: 858 MB). mxbuild writes it again and # ensure_gitignore ignores it, so it is not unpacked; the zip itself is untouched. Only the folder beside the .mpr. - local mprpath dpre="" - mprpath=$(unzip -Z1 "$APP_ZIP" 2> /dev/null | grep -E '\.mpr$' | awk -F/ '{print NF, $0}' | sort -n | head -n 1 | cut -d' ' -f2- || true) + local mprpath dpre="" listing + # the listing is read once: `unzip -Z1 | grep -q` under pipefail failed whenever grep quit first (SIGPIPE), and then + # deployment/ was unpacked after all (bt9-fetch-* controls, 1 run in 3) + listing=$(unzip -Z1 "$APP_ZIP" 2> /dev/null || true) + mprpath=$(printf '%s\n' "$listing" | grep -E '\.mpr$' | awk -F/ '{print NF, $0}' | sort -n | head -n 1 | cut -d' ' -f2- || true) case "$mprpath" in */*) dpre="${mprpath%/*}/" ;; esac - if [ "$DRY" != 1 ] && unzip -Z1 "$APP_ZIP" 2> /dev/null | grep -q "^${dpre}deployment/."; then + if [ "$DRY" != 1 ] && grep -q "^${dpre}deployment/." <<< "$listing"; then must unzip -q -o "$APP_ZIP" -d "$APP" -x "${dpre}deployment/*" note "fetch: the package ships a deployment/ folder (build output); not unpacked, mxbuild writes it again (BT-9)" else diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 53faeea..dce5aef 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6192,8 +6192,8 @@ echo big > $BT9Z/src/shop/deployment/model/model.mdp; echo j > $BT9Z/src/shop/ja (cd $BT9Z/src && zip -qr ../shop.mpk shop) mkdir -p $BT9Z/run1 $BT9Z/run2; cp -r $BTR/tools $BT9Z/run1/tools; cp -r $BTR/tools $BT9Z/run2/tools bt9fetch() { env PATH=$BTR/bin:$PATH MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-zip $BT9Z/shop.mpk --source 10.24.26.1 --target 11.15.0.1 --work $1/work --only fetch; } -ctl bt9-fetch-skips-deployment-catch 2 '^theme/a.css$' '^javasource/deployment/Keep.java$' '!deployment/model' -- sh -c "$(declare -f bt9fetch); BTR=$BTR B=$B BT9Z=$BT9Z; bt9fetch $BT9Z/run1 >/dev/null 2>&1; cd $BT9Z/run1/app/shop && find . -type f ! -path './.git/*' | sed 's#^\./##' | sort; cd $BT9Z/run1/app/shop; bash -c 'exit 2'" -ctl bt9-fetch-note-pass 0 'fetch: the package ships a deployment/ folder' -- cat $BT9Z/run1/work/state/notes.txt +ctl bt9-fetch-skips-deployment-catch 2 '^theme/a.css$' '^javasource/deployment/Keep.java$' '!deployment/model' -- sh -c "$(declare -f bt9fetch); BTR=$BTR B=$B BT9Z=$BT9Z; bt9fetch $BT9Z/run1 > $BT9Z/run1.log 2>&1; cd $BT9Z/run1/app/shop && find . -type f ! -path './.git/*' | sed 's#^\./##' | sort; cd $BT9Z/run1/app/shop; bash -c 'exit 2'" +ctl bt9-fetch-note-pass 0 'fetch: the package ships a deployment/ folder' -- sh -c "cat $BT9Z/run1/work/state/notes.txt || tail -25 $BT9Z/run1.log" # BT-11: a module in both plan-pre.txt and plan.txt: the post-convert line starts from the pre-convert target. Golden: the real # DOCGEN plan-pre.txt and plan.txt (Atlas_Core 3.21.1 then 4.5.1, DataWidgets once); installed versions from its PROPOSAL.md. From b3eac0c6d0f077f6957047a5b12e2a46b9053b1f Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 06:45:53 +0800 Subject: [PATCH 22/59] pipeline: two more grep -q pipes under pipefail - replay skip: git log | grep -qxF could lose to SIGPIPE on a long log, and a module already replayed was replayed again - journeys stop: grep | grep -vq under '!' could read a SIGPIPE as 'only SCREEN REVIEW lines' and stop 12 (a comparison job) for a run that had FAIL lines; it is a count now Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index d0a07c2..950e40d 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -931,7 +931,8 @@ replay_edits() { # replay_edits [pre]: pre = only modules the post-convert plan mod=$(basename "$f"); mod=${mod%.*} if [ "${1:-}" = pre ] && grep -qE "^[0-9]+ [^ ]+ $mod\$" "$WORK/plan.txt" 2> /dev/null; then continue; fi # one replay per module per rung: the 10.24 rung's replay must not skip the 11 rung's (editroute writes a new replay/ each rung) - if [ "$DRY" != 1 ] && git_app log --format=%s 2> /dev/null | grep -qxF "replay: $mod${UPG_PHASE:+ ($UPG_PHASE)}"; then note "replay $mod skipped: replayed already"; continue; fi + # no -q: grep must read the whole log, or git dies of SIGPIPE and pipefail makes the check false + if [ "$DRY" != 1 ] && git_app log --format=%s 2> /dev/null | grep -xF "replay: $mod${UPG_PHASE:+ ($UPG_PHASE)}" > /dev/null; then note "replay $mod skipped: replayed already"; continue; fi if [ "$DRY" != 1 ] && grep -q "^$mod " "$APP/.stepupdate/kept.txt" 2> /dev/null; then note "replay $mod skipped: the module was kept, its edits are in place"; continue; fi rc=0 case "$f" in @@ -1288,7 +1289,9 @@ step_journeys() { local code=11 # only SCREEN REVIEW lines and no FAIL/REGRESSION/INVALID journey: a comparison job (12), else a journey fix (11) if [ -f "$WORK/journeys-after.out" ] && grep -q 'SCREEN REVIEW' "$WORK/journeys-after.out" && - ! grep -E 'REGRESSION|NOT IN BASELINE|INVALID|\bFAIL\b' "$WORK/journeys-after.out" | grep -vq 'SCREEN REVIEW'; then code=12; fi + [ "$(grep -E 'REGRESSION|NOT IN BASELINE|INVALID|\bFAIL\b' "$WORK/journeys-after.out" | grep -vc 'SCREEN REVIEW' || true)" = 0 ]; then code=12; fi + # a count, not `| grep -vq`: under pipefail an early-quitting grep -q made the first grep die of SIGPIPE, and `!` turned + # that into "only SCREEN REVIEW lines" (stop 12) for a run that had FAIL lines nj_file "$code" "Journeys on the NEW app (exit $rc): REGRESSION/FAIL is a gate FAIL, INVALID blocks DONE, SCREEN REVIEW needs both images opened and a reason in shot-accept.txt" "$WORK/journeys-after.out" 60 'REGRESSION|NOT IN BASELINE|INVALID|FAIL|SCREEN|todo|skipped|journeys:' nj_add "$code" "Evidence" "summary: $WORK/e2e-after/after/summary.json ; plan: $WORK/journeys/plan.json ; screens: $WORK/e2e-after" nj_flush From bf9fe758b1f5a5a0e7b6738c38021ed5cb2d716d Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 07:07:47 +0800 Subject: [PATCH 23/59] platform_drops: re-count editroute VENDOR approvals on the baseline editroute approves a vendor drop when nothing outside the module refers to it. No references is not no data: WM1's MendixSSO.AuthRequest.Nonce held 5 values, the ID check let it through, only the data rung saw the loss. Each such approval is now counted: 0 stays, data or a failed count comments the line out (HELD) and the gate stops for a person. WM1 baseline dry run: 'HELD MendixSSO.AuthRequest.Nonce: 5 non-null values in upg_run_base'. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/platform_drops.py | 42 +++++++++++++++---- .../controls/platform_drops_control.sh | 11 +++-- upgrade-toolkit/controls/run-controls.sh | 4 ++ 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/upgrade-toolkit/bin/platform_drops.py b/upgrade-toolkit/bin/platform_drops.py index 7d55f5a..2b64384 100644 --- a/upgrade-toolkit/bin/platform_drops.py +++ b/upgrade-toolkit/bin/platform_drops.py @@ -8,7 +8,12 @@ Rule: only RED REMOVED rows of the System module. Entity: approved when its table has 0 rows. Attribute: approved when its column holds 0 non-null values (or its table is gone and had 0 rows). Any other count, or a query that fails: -not approved, printed as KEEP with the count, a person decides. App and Marketplace modules are never touched here. +not approved, printed as KEEP with the count, a person decides. App and Marketplace modules are never approved here. + +Vendor drops (WM1 rerun 9): editroute.py approves a Marketplace attribute or entity the new version drops when nothing +outside the module refers to it ('# editroute VENDOR'). "No references" is not "no data": MendixSSO.AuthRequest.Nonce held +5 values, the ID check let it through, and only the data rung saw the loss. So each such approval is counted here too. +0 = it stays. Data, or a count that fails = the line is commented out (HELD), and the gate stops on it for a person. platform_drops.py OLD.mda NEW.mda --approved approved.txt --psql "docker exec -i PG psql -U mendix -d DB" [--dry-run] appends one line per approval: ' # platform drop (platform_drops.py): in ' @@ -18,6 +23,7 @@ B = os.path.dirname(os.path.abspath(__file__)) ROW = re.compile(r"^RED\s+REMOVED\s+(entity|attribute)\s+(System\.[\w.]+)", re.M) TAG = "# platform drop (platform_drops.py)" +VENDOR = re.compile(r"^([A-Za-z]\w*\.\w+(?:\.\w+)?)\s+# editroute VENDOR\b.*$", re.M) def count(psql, sql): @@ -26,6 +32,15 @@ def count(psql, sql): return int(out) if r.returncode == 0 and out.isdigit() else None +def sql_for(qn): + """(sql, what) counting the data in a removed entity (rows) or attribute (non-null values).""" + parts = qn.split(".") + table = f'"{parts[0].lower()}${parts[1].lower()}"' + if len(parts) == 2: + return f"select count(*) from {table}", "rows" + return f'select count(*) from {table} where "{parts[2].lower()}" is not null', "non-null values" + + def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("old"); ap.add_argument("new") @@ -41,22 +56,33 @@ def main(): for kind, qn in ROW.findall(rows): if re.search(rf"^{re.escape(qn)}\s", have, re.M): continue - parts = qn.split(".") - table = f'"system${parts[1].lower()}"' - if kind == "entity": - n, what = count(a.psql, f"select count(*) from {table}"), "rows" - else: - n, what = count(a.psql, f'select count(*) from {table} where "{parts[2].lower()}" is not null'), "non-null values" + sql, what = sql_for(qn) + n = count(a.psql, sql) if n == 0: new.append(f"{qn} {TAG}: Mendix removes it; 0 {what} in {db}") print(f"APPROVE {qn}: 0 {what} in {db}") else: kept += 1 print(f"KEEP {qn}: {'count failed' if n is None else f'{n} {what}'} in {db}; a person decides") + held = [] + for m in VENDOR.finditer(have): + qn = m.group(1) + sql, what = sql_for(qn) + n = count(a.psql, sql) + if n == 0: + print(f"VENDOR-OK {qn}: 0 {what} in {db}") + continue + got = "count failed" if n is None else f"{n} {what}" + held.append((m.group(0), f"# HELD by platform_drops.py: {got} in {db}; a person decides (was: {m.group(0)})")) + print(f"HELD {qn}: {got} in {db}; the vendor drop loses it, a person decides") + if not a.dry_run and held: + for line, repl in held: + have = have.replace(line, repl, 1) + open(a.approved, "w").write(have) if new and not a.dry_run: with open(a.approved, "a") as f: f.write(("" if not have or have.endswith("\n") else "\n") + "\n".join(new) + "\n") - print(f"platform_drops: {len(new)} approved by count, {kept} kept for a person") + print(f"platform_drops: {len(new)} approved by count, {kept} kept for a person, {len(held)} vendor drop(s) held for data") return 0 diff --git a/upgrade-toolkit/controls/platform_drops_control.sh b/upgrade-toolkit/controls/platform_drops_control.sh index fb719ec..fd7dd5c 100644 --- a/upgrade-toolkit/controls/platform_drops_control.sh +++ b/upgrade-toolkit/controls/platform_drops_control.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # platform_drops_control.sh: bin/platform_drops.py with fake idcheck rows and a fake psql (WM1 2026-11-17: System-module -# drops of Mendix 10 approved only on a count of 0). Prints the script's output and the approved file. +# drops of Mendix 10 approved only on a count of 0; +# rerun 9: an editroute VENDOR approval re-counted, MendixSSO.AuthRequest.Nonce held 5 values). Prints the script's output and the approved file. set -euo pipefail B=$(cd "$(dirname "$0")/../bin" && pwd) T=$(mktemp -d); trap 'rm -rf "$T"' EXIT @@ -12,14 +13,18 @@ RED REMOVED attribute Shop.Order.Note (column/table dropped; intended?) RED REMOVED entity System.Already (column/table dropped; intended?) info REMOVED attribute System.Info.Col (no data) R -printf 'System.Already # a person: fine\n' > "$T/approved.txt" +printf '%s\n' 'System.Already # a person: fine' \ + 'MendixSSO.AuthRequest.Nonce # editroute VENDOR: the new MendixSSO drops it, nothing outside the module refers to it' \ + 'Vend.Old # editroute VENDOR: the new Vend drops it, nothing outside the module refers to it' \ + 'Vend.Odd.Col # editroute VENDOR: the new Vend drops it, nothing outside the module refers to it' > "$T/approved.txt" cat > "$T/psql" <<'P' #!/usr/bin/env bash q=${*: -1} case "$q" in *'"system$gone"'*) echo 0 ;; *'"system$kept"'*) echo 3 ;; - *'"system$broken"'*) echo 'ERROR: relation does not exist' >&2; exit 1 ;; + *'"system$broken"'*|*'"vend$odd"'*) echo 'ERROR: relation does not exist' >&2; exit 1 ;; + *'"mendixsso$authrequest" where "nonce"'*) echo 5 ;; *) echo 0 ;; esac P diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index dce5aef..b6293a9 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -494,6 +494,10 @@ ctl platform-drops-zero-approved-pass 0 '^approved: System.Gone # platform drop ctl platform-drops-data-kept-catch 0 '^KEEP System.Kept.HasData: 3 non-null values' '!approved: System.Kept' -- $PD ctl platform-drops-query-fail-kept-catch 0 '^KEEP System.Broken.Col: count failed' '!approved: System.Broken' -- $PD ctl platform-drops-app-and-info-untouched-pass 0 '!Shop.Order' '!System.Info' '!APPROVE System.Already' -- $PD +# rerun 9: editroute approved MendixSSO.AuthRequest.Nonce as a VENDOR drop ("nothing refers to it"); it held 5 values +ctl platform-drops-vendor-data-held-catch 0 '^HELD MendixSSO.AuthRequest.Nonce: 5 non-null values' '^approved: # HELD by platform_drops.py: 5 non-null values' '!^approved: MendixSSO' -- $PD +ctl platform-drops-vendor-empty-stays-pass 0 '^VENDOR-OK Vend.Old: 0 rows' '^approved: Vend.Old # editroute VENDOR' -- $PD +ctl platform-drops-vendor-count-fail-held-catch 0 '^HELD Vend.Odd.Col: count failed' '!^approved: Vend.Odd' '2 vendor drop\(s\) held for data$' -- $PD # WM1 rerun 7: baseline ran microflow steps, the after walk skipped them: 4 false REGRESSIONs ctl pipeline-smoke-mf-follows-baseline-pass 0 '^--microflows$' -- bash -c "WORK=\$(mktemp -d); echo '{\"landed\": \"microflow ran\"}' > \$WORK/smoke-before.json; eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf" ctl pipeline-smoke-mf-off-without-baseline-pass 0 '!microflows' -- bash -c "WORK=\$(mktemp -d); eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf; echo done" From c8270f59f1c436604badc9694df366c448df8bc8 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 07:30:20 +0800 Subject: [PATCH 24/59] replay: a failed replay is held, not a stop The replay's own changes are undone (the tree was clean before it; untracked files it made are removed, paths relative to the app), the module goes to replay-held.txt, and the build, journeys and gate still run and give evidence. The gate stops 13 'upgrade complete except held replays' until a person fixes the replay or acknowledges it in replay-held-ack.txt. A dirty tree before the replay keeps the old immediate stop: nothing can be undone safely. 5 controls (replay_hold_control.sh): 4 catch the old pipeline. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 29 ++++++++++-- .../controls/replay_hold_control.sh | 44 +++++++++++++++++++ upgrade-toolkit/controls/run-controls.sh | 7 +++ 3 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 upgrade-toolkit/controls/replay_hold_control.sh diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 950e40d..6a02462 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -30,7 +30,7 @@ # 10 mx check / mxbuild / quick start errors after convert+update: repair needed (steps/repair.md) # 11 journeys INVALID or FAIL on the old app or the new one: fix the journey or write the todo (steps/journeys.md) # 12 diff / gate REVIEW, FAIL or INVALID items need a comparison by a person (steps/gate.md). A FAIL is never a pass. -# 13 module plan missing, or a module kept / a data-bearing finding needs a decision (steps/upgrade.md) +# 13 module plan missing, or a module kept / a replay held / a data-bearing finding needs a decision (steps/upgrade.md) # 14 the upgrade proposal awaits sign-off ($WORK/PROPOSAL.md section 6, steps/propose.md); also when a plan input changed # after the sign-off (the sign-off is cleared). Nothing in the app model has changed yet. # 15 the owner paused the run on purpose (`Status: PAUSED ` in PROPOSAL.md); NEEDS-JUDGMENT.md has the resume command @@ -919,7 +919,8 @@ run_stepupdate() { # run_stepupdate PLAN label # "create or modify entity" drops every attribute it does not list) # $WORK/replay/.patch file edits (theme SCSS, JavaScript), a diff against the OLD package with paths relative to # the app folder, applied with git apply onto the NEW files: copying the old files back would -# undo the update. A patch that does not apply is a judgment stop, never a partial apply. +# undo the update. A patch that does not apply is never applied in part: like a failed .mdl it is +# HELD (replay-held.txt, the gate stops 13), its own changes undone. # A module that was kept (not updated) is not replayed: its edits are still there. The total idcheck that follows compares # the result with the original, so a replay that misses a model edit stays visible. replay_edits() { # replay_edits [pre]: pre = only modules the post-convert plan does not touch (their final version came @@ -934,7 +935,8 @@ replay_edits() { # replay_edits [pre]: pre = only modules the post-convert plan # no -q: grep must read the whole log, or git dies of SIGPIPE and pipefail makes the check false if [ "$DRY" != 1 ] && git_app log --format=%s 2> /dev/null | grep -xF "replay: $mod${UPG_PHASE:+ ($UPG_PHASE)}" > /dev/null; then note "replay $mod skipped: replayed already"; continue; fi if [ "$DRY" != 1 ] && grep -q "^$mod " "$APP/.stepupdate/kept.txt" 2> /dev/null; then note "replay $mod skipped: the module was kept, its edits are in place"; continue; fi - rc=0 + rc=0 clean=1 + [ "$DRY" = 1 ] || [ -z "$(git_app status --porcelain 2> /dev/null)" ] || clean=0 case "$f" in *.mdl) xo "$WORK/replay-$mod.out" mxcli_run exec "$f" -p "$MPR" || rc=$? # F14 (Purchase Request 10.24, real run): MendixSSO 10.6.2 dropped the six associations PurchaseApprovals @@ -960,7 +962,19 @@ replay_edits() { # replay_edits [pre]: pre = only modules the post-convert plan note "replay $mod: context changed, applied without context (each removed line unique in its file)" else rc=1; fi ;; esac - [ "$rc" = 0 ] || { nj_file 13 "replay of $mod failed (exit $rc): $f" "$WORK/replay-$mod$([ "${f##*.}" = patch ] && echo -patch).out" 30; nj_flush; } + local rlog="$WORK/replay-$mod$([ "${f##*.}" = patch ] && echo -patch).out" + if [ "$rc" != 0 ] && [ "$clean" = 0 ]; then + nj_file 13 "replay of $mod failed (exit $rc) on a tree that was not clean before it, so it cannot be undone: $f" "$rlog" 30; nj_flush + elif [ "$rc" != 0 ]; then + # a failed replay is held, not a stop (WM1): its own changes are undone (the tree was clean before it), the build, + # journeys and gate still run and give evidence, and the gate stops 13 on replay-held.txt until a person decides + git_app checkout -q -- . 2> /dev/null || true + git_app ls-files -z --others --exclude-standard 2> /dev/null | (cd "$APP" && xargs -0 rm -f --) # paths are relative to the app + printf '%s exit %s: %s (log: %s)\n' "$mod" "$rc" "$f" "$rlog" >> "$WORK/replay-held.txt" + note "replay $mod HELD (exit $rc): its changes undone, the run goes on; the gate stops on $WORK/replay-held.txt" + continue + fi + [ ! -f "$WORK/replay-held.txt" ] || { grep -v "^$mod exit " "$WORK/replay-held.txt" > "$WORK/replay-held.txt.n" || true; mv "$WORK/replay-held.txt.n" "$WORK/replay-held.txt"; } commit_app "replay: $mod${UPG_PHASE:+ ($UPG_PHASE)}" replay "replay-$mod" note "replayed the local edits of $mod from $f" done @@ -1610,6 +1624,12 @@ step_gate() { nj_add 13 "Decision" "Update them by hand (steps/upgrade.md, playbooks/edited-modules.md), or acknowledge the list: copy $WORK/kept.txt to $WORK/kept-ack.txt (then the verdict says 'complete except kept modules')." nj_flush fi + # held replays: the module is updated but its local edits are not back (replay_edits) + if [ "$DRY" != 1 ] && [ -s "$WORK/replay-held.txt" ] && [ ! -f "$WORK/replay-held-ack.txt" ]; then + nj_file 13 "upgrade complete except held replays (the local edits of these modules are NOT in the app)" "$WORK/replay-held.txt" 40 + nj_add 13 "Decision" "Fix the replay (its log is named per line) and rerun --from upgrade, or acknowledge the loss in writing: copy $WORK/replay-held.txt to $WORK/replay-held-ack.txt with a reason per line." + nj_flush + fi write_summary record_runlog || true } @@ -1621,6 +1641,7 @@ write_summary() { printf 'No judgment stop remained. Evidence (all under %s):\n' "$WORK" printf -- '- build: check-*.log, build-%s.log, quick start log logs/runtime-new.log\n- gate: gate-summary.md (one row per rung)\n- identity: idcheck.md, idcheck-total.md\n- constants: constants.md\n- data: diff.md (before.json, after.json)\n- start log: logdiff.md\n- smoke walk: smoke-after.json, shotdiff-smoke.md\n- journeys: e2e-after/, journeys-after.out\n- release impact: release-impact.md (check clean)\n\n' "$TARGET" if [ -s "$WORK/kept.txt" ]; then printf 'upgrade complete except kept modules (acknowledged): see kept.txt\n\n'; fi + if [ -s "$WORK/replay-held.txt" ]; then printf 'local edits NOT replayed (acknowledged): see replay-held-ack.txt\n\n'; fi case "$(python3 "$B/proposal.py" --status --work "$WORK" 2> /dev/null)" in signed|none) ;; *) printf 'proposal not signed: %s/PROPOSAL.md has no Signed-off line (unattended run); the verdict is not an owner-approved upgrade.\n\n' "$WORK" ;; diff --git a/upgrade-toolkit/controls/replay_hold_control.sh b/upgrade-toolkit/controls/replay_hold_control.sh new file mode 100644 index 0000000..9ab0d52 --- /dev/null +++ b/upgrade-toolkit/controls/replay_hold_control.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# replay_hold_control.sh CASE: replay_edits and step_gate from the pipeline on a real git app with a fake mxcli. +# A failed replay is held (its changes undone, the run goes on) and the gate stops 13 on replay-held.txt. +# fail the replay writes half the model and a new unit, then fails +# dirty the same, on a tree that was not clean before the replay (cannot be undone: the old stop) +# ok the replay succeeds after an earlier run held it (the held line goes) +# gate step_gate with a held replay: stop 13 +# gate-ack the same with replay-held-ack.txt: the gate goes through +set -euo pipefail +P=$(cd "$(dirname "$0")/../bin/cloud" && pwd)/upgrade-pipeline.sh +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT +APP=$T/app WORK=$T/work DRY=0 MPR=$T/app/App.mpr B=$T/b +mkdir -p "$APP" "$WORK/replay" "$WORK/state" "$B" +echo orig > "$MPR" +git -C "$APP" init -q; git -C "$APP" add -A; git -C "$APP" -c user.name=c -c user.email=c@c commit -qm original +echo 'create page Mod.P;' > "$WORK/replay/Mod.mdl" +git_app() { git -C "$APP" -c user.name=c -c user.email=c@c "$@"; } +xo() { local o=$1; shift; "$@" > "$o" 2>&1; } +note() { echo "note: $*"; } +Q="" +nj_file() { echo "STOP $1: $2"; Q=$1; } +nj_add() { echo "STOP $1: $2"; Q=$1; } +nj_flush() { [ -z "$Q" ] || { echo "EXIT $Q"; exit "$Q"; }; } +commit_app() { git_app add -A; git_app commit -qm "$1"; echo "commit: $1"; } +write_summary() { :; } +record_runlog() { :; } +case "$1" in + fail|dirty) mxcli_run() { echo half > "$MPR"; echo unit > "$APP/new.unit"; echo "MDL081 refused" >&2; return 3; } ;; + *) mxcli_run() { echo replayed > "$MPR"; } ;; +esac +eval "$(sed -n '/^replay_edits() {/,/^}/p' "$P")" +eval "$(sed -n '/^step_gate() {/,/^}/p' "$P")" +case "$1" in + dirty) echo stray > "$APP/stray.txt"; replay_edits ;; + ok) echo "Mod exit 3: old (log: x)" > "$WORK/replay-held.txt"; echo "Other exit 1: y (log: z)" >> "$WORK/replay-held.txt"; replay_edits ;; + gate|gate-ack) echo 'Mod exit 3: r (log: l)' > "$WORK/replay-held.txt"; : > "$WORK/release-impact.md"; printf '#!/bin/sh\n' > "$B/release_impact.py" + [ "$1" = gate ] || echo 'Mod exit 3: accepted, the page edit is redone by hand' > "$WORK/replay-held-ack.txt" + step_gate; echo "gate went through"; exit 0 ;; + *) replay_edits ;; +esac +echo "mpr: $(cat "$MPR")" +echo "untracked: $(git_app ls-files --others --exclude-standard | tr '\n' ' ')" +[ ! -f "$WORK/replay-held.txt" ] || sed 's/^/held: /' "$WORK/replay-held.txt" +echo "commits: $(git_app log --format=%s | tr '\n' '|')" diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index b6293a9..b903799 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -498,6 +498,13 @@ ctl platform-drops-app-and-info-untouched-pass 0 '!Shop.Order' '!System.Info' '! ctl platform-drops-vendor-data-held-catch 0 '^HELD MendixSSO.AuthRequest.Nonce: 5 non-null values' '^approved: # HELD by platform_drops.py: 5 non-null values' '!^approved: MendixSSO' -- $PD ctl platform-drops-vendor-empty-stays-pass 0 '^VENDOR-OK Vend.Old: 0 rows' '^approved: Vend.Old # editroute VENDOR' -- $PD ctl platform-drops-vendor-count-fail-held-catch 0 '^HELD Vend.Odd.Col: count failed' '!^approved: Vend.Odd' '2 vendor drop\(s\) held for data$' -- $PD +# a failed replay is held: its changes undone, the run goes on, the gate stops 13 until a person fixes or acknowledges it +RH="bash $B/../controls/replay_hold_control.sh" +ctl replay-hold-undone-catch 0 '^note: replay Mod HELD \(exit 3\)' '^mpr: orig$' '^untracked: $' '^held: Mod exit 3: ' '^commits: original\|$' '!^STOP' -- $RH fail +ctl replay-hold-dirty-tree-stops-catch 13 '^STOP 13: replay of Mod failed \(exit 3\) on a tree that was not clean' '!HELD' -- $RH dirty +ctl replay-hold-cleared-on-success-pass 0 '^commit: replay: Mod$' '^held: Other exit 1' '!^held: Mod' '^mpr: replayed$' -- $RH ok +ctl replay-hold-gate-stops-catch 13 '^STOP 13: upgrade complete except held replays' '!gate went through' -- $RH gate +ctl replay-hold-gate-ack-pass 0 '^gate went through$' '!^STOP' -- $RH gate-ack # WM1 rerun 7: baseline ran microflow steps, the after walk skipped them: 4 false REGRESSIONs ctl pipeline-smoke-mf-follows-baseline-pass 0 '^--microflows$' -- bash -c "WORK=\$(mktemp -d); echo '{\"landed\": \"microflow ran\"}' > \$WORK/smoke-before.json; eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf" ctl pipeline-smoke-mf-off-without-baseline-pass 0 '!microflows' -- bash -c "WORK=\$(mktemp -d); eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf; echo done" From bb5e4dd1634abf1d5ba726895e5a15744cadfd8e Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 07:54:10 +0800 Subject: [PATCH 25/59] idcheck: MODULE_RECREATED is info when every element identity is kept mxcli's update gives a module's documents new unit ids but keeps the identities of its entities, attributes and associations, which the tables hang on. WM1: 28 of 28 MODULE_RECREATED lines had no GUID change, yet each was a warn saying the module was deleted and imported. Now: info with the count kept; warn (NS-7) when an identity changed or the module has none to compare. Fixture reid_module.py --guids makes the real delete and re-import. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/evidence.py | 2 +- upgrade-toolkit/bin/idcheck.py | 18 ++++++++++-- upgrade-toolkit/bin/stepupdate.sh | 2 +- upgrade-toolkit/controls/lib/reid_module.py | 31 +++++++++++++++++++-- upgrade-toolkit/controls/run-controls.sh | 8 ++++-- upgrade-toolkit/steps/upgrade.md | 2 ++ 6 files changed, 54 insertions(+), 9 deletions(-) diff --git a/upgrade-toolkit/bin/evidence.py b/upgrade-toolkit/bin/evidence.py index d8a5476..0bf7aea 100644 --- a/upgrade-toolkit/bin/evidence.py +++ b/upgrade-toolkit/bin/evidence.py @@ -70,7 +70,7 @@ def for_item(work, name): r = re.match(r"RED\s+(\w+)\s+(\w+)\s+([\w.]+)", ln) if r: rows.append(f"{r.group(2)} {r.group(3)}: {r.group(1).lower()} by the new version, its data is lost") - elif ln.startswith("warn MODULE_RECREATED"): + elif ln.startswith("warn MODULE_RECREATED"): # the info form (element identities kept) is no data row rows.append("the new version replaces the module as a whole (all ids new), pages that use it are rewritten too") if rows: what += rows diff --git a/upgrade-toolkit/bin/idcheck.py b/upgrade-toolkit/bin/idcheck.py index c054b0c..9c08136 100755 --- a/upgrade-toolkit/bin/idcheck.py +++ b/upgrade-toolkit/bin/idcheck.py @@ -36,6 +36,8 @@ warn MODULE_RECREATED (MPR on both sides) a module with 3 or more units shares no unit id with its old self: it was deleted and re-imported, not updated (NS-7: DataWidgets 2.27.6 in notification-service cc318c0b, every unit new and 12 own pages rewritten, reverted the next day). A warning: the exit code does not change. + info instead when every element identity of the module is kept (WM1: mxcli's update copies documents + with new unit ids, the data is safe); warn when one changed or the module has none to compare. RED = the element holds data: persistent entity, stored attribute of a persistent entity, association whose both ends are persistent. """ @@ -373,9 +375,19 @@ def main(): if rec: lines.append("") for m, no, nn in rec: - lines.append(f"warn MODULE_RECREATED {m}: all {no} unit ids new ({nn} units now): the module was deleted and imported " - f"again, not updated in place; documents that use it (pages, snippets) were rewritten too: review them, " - f"or redo the update with Studio Pro's module update (NS-7)") + both = [(k, q) for k in ("entity", "attribute", "association") for q in old[k] if old[k][q]["module"] == m and q in new[k]] + kept = sum(old[k][q]["guid"] == new[k][q]["guid"] for k, q in both) + if both and kept == len(both): + # WM1 (mxcli marketplace update): every module "recreated", yet every element identity kept; the tables hang on + # those identities, so the data is safe. Only the documents have new unit ids. + lines.append(f"info MODULE_RECREATED {m}: all {no} unit ids new ({nn} units now), but all {kept} element identities " + f"kept (entities, attributes, associations): an in-place copy update, the data is safe; the app's own " + f"pages that use {m} may still be rewritten: diff them (git show --stat)") + else: + lines.append(f"warn MODULE_RECREATED {m}: all {no} unit ids new ({nn} units now)" + + (f", {kept} of {len(both)} element identities kept" if both else ", no element identities to compare") + + ": the module was deleted and imported again, not updated in place; documents that use it (pages, " + "snippets) were rewritten too: review them, or redo the update with Studio Pro's module update (NS-7)") verdict = "FAIL" if any(x["finding"] == "GUID_CHANGED" for x in open_red) else ("REVIEW" if open_red else "PASS") lines += ["", f"verdict: {verdict} (FAIL = open data-bearing GUID change; REVIEW = open data-bearing removals or" " persistence loss only: confirm each is intended, then add it to the approved list / allow-list)"] diff --git a/upgrade-toolkit/bin/stepupdate.sh b/upgrade-toolkit/bin/stepupdate.sh index 2e7fd7c..a2e4971 100755 --- a/upgrade-toolkit/bin/stepupdate.sh +++ b/upgrade-toolkit/bin/stepupdate.sh @@ -517,7 +517,7 @@ PY echo "STOP: idcheck input error after $mod"; exit 2 fi # NS-7: idcheck warns when the update re-created a module (every unit id new) instead of updating it in place - grep -E '^warn ' "$app/.stepupdate/step$n-$mod.idcheck" | sed 's/^/ /' + grep -E '^(warn|info MODULE_RECREATED) ' "$app/.stepupdate/step$n-$mod.idcheck" | sed 's/^/ /' # NS-6: a module import also replaces shared widgets/*.mpk (notification-service cdd62cc5: the Email Connector import # swapped Datagrid 2.28.3 -> 2.30.6 and 8 more; reverted on day 111). Name every widget the step changed. wout=$(python3 "$bin/widgetdiff.py" "$app" 2>/dev/null) diff --git a/upgrade-toolkit/controls/lib/reid_module.py b/upgrade-toolkit/controls/lib/reid_module.py index f40fc0d..429fee7 100644 --- a/upgrade-toolkit/controls/lib/reid_module.py +++ b/upgrade-toolkit/controls/lib/reid_module.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 """reid_module.py: give every unit of one module a new UnitID, as a delete plus re-import of the module does (NS-7). - reid_module.py (in place; MPR v1, run it on a copy) + reid_module.py [--guids] (in place; MPR v1, run it on a copy) + +--guids also gives every element of the module's domain model (entity, attribute, association) a new GUID: a real +delete and re-import, where the data goes too. Without it only the unit ids change, as mxcli's update does (WM1). notification-service cc318c0b: the DataWidgets 2.27.6 import re-created the module, so its ModuleImpl, folder, enumeration, security, domain model, settings and JavaScript action units all got new ids. The new ids here are @@ -13,6 +16,23 @@ import bson path, mod = sys.argv[1], sys.argv[2] +guids = "--guids" in sys.argv[3:] + + +def regen(x): + """New GUIDs (uuid5 of the old, same BSON type) everywhere under x; the number replaced.""" + n = 0 + if isinstance(x, dict): + for k, v in x.items(): + if k == "GUID" and isinstance(v, (bytes, bytearray)): + b = uuid.uuid5(uuid.NAMESPACE_OID, "reguid:" + bytes(v).hex()).bytes_le + x[k] = type(v)(b, v.subtype) if hasattr(v, "subtype") else b + n += 1 + else: + n += regen(v) + elif isinstance(x, list): + n = sum(regen(i) for i in x) + return n con = sqlite3.connect(path) rows = [(bytes(u), bytes(c) if c else None, b) for u, c, b in con.execute("select UnitID, ContainerID, Contents from Unit")] parent = {u: c for u, c, _ in rows} @@ -27,9 +47,16 @@ inside.add(u); break seen.add(x); x = parent.get(x) new = {u: uuid.uuid5(uuid.NAMESPACE_OID, "reid:" + u.hex()).bytes_le for u in inside} +ng = 0 +if guids: + for u, _, b in rows: + if u in inside and b and bson.decode(b).get("$Type") == "DomainModels$DomainModel": + d = bson.decode(b) + ng += regen(d) + con.execute("update Unit set Contents = ? where UnitID = ?", (bson.encode(d), u)) for u, c, _ in rows: nu, nc = new.get(u, u), new.get(c, c) if c else c if (nu, nc) != (u, c): con.execute("update Unit set UnitID = ?, ContainerID = ? where UnitID = ?", (nu, nc, u)) con.commit() -print(f"reid_module: {mod}: {len(inside)} units got new ids") +print(f"reid_module: {mod}: {len(inside)} units got new ids" + (f", {ng} element GUIDs new" if guids else "")) diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index b903799..7d19f8a 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -1465,11 +1465,15 @@ ctl stepupdate-widgets-unchanged-pass 0 'committed' '!WIDGETS' -- bash $B/stepup ctl widgetdiff-bad-rev-catch 2 'not a commit' -- python3 $B/widgetdiff.py $S/wg --rev no-such-rev # NS-7: an update that deletes and re-imports a module (cc318c0b, DataWidgets 2.27.6): every unit id of it is new mkdir -p $S/recreate; cp $M/base/App.mpr $S/recreate/App.mpr; python3 $here/lib/reid_module.py $S/recreate/App.mpr Administration > /dev/null -ctl idcheck-module-recreated-catch 0 '^warn MODULE_RECREATED Administration: all 5 unit ids new \(5 units now\)' '^verdict: PASS' -- python3 $B/idcheck.py $M/base/App.mpr $S/recreate/App.mpr +# WM1: mxcli's update gives new unit ids but keeps every element identity: info, the data is safe +ctl idcheck-module-recreated-ids-kept-info-pass 0 '^info MODULE_RECREATED Administration: all 5 unit ids new \(5 units now\), but all 7 element identities kept' '!^warn MODULE_RECREATED' '^verdict: PASS' -- python3 $B/idcheck.py $M/base/App.mpr $S/recreate/App.mpr +# a real delete and re-import: the element GUIDs are new too: warn, and the GUID changes are RED +mkdir -p $S/recreate-g; cp $M/base/App.mpr $S/recreate-g/App.mpr; python3 $here/lib/reid_module.py $S/recreate-g/App.mpr Administration --guids > /dev/null +ctl idcheck-module-recreated-catch 1 '^warn MODULE_RECREATED Administration: all 5 unit ids new \(5 units now\), 0 of 7 element identities kept' '!^info MODULE_RECREATED' 'GUID_CHANGED' -- python3 $B/idcheck.py $M/base/App.mpr $S/recreate-g/App.mpr ctl idcheck-module-updated-pass 0 '!MODULE_RECREATED' -- python3 $B/idcheck.py $M/base/App.mpr $M/step1-adds/App.mpr ctl idcheck-module-recreated-json-pass 0 '"module": "Administration"' -- sh -c "python3 $B/idcheck.py $M/base/App.mpr $S/recreate/App.mpr --json $T/recreate.json > /dev/null && cat $T/recreate.json" newapp rc '1 1.0 Recreate\n' -ctl stepupdate-module-recreated-catch 0 '^ +warn MODULE_RECREATED Administration' 'committed' -- bash $B/stepupdate.sh $S/rc $S/rc.plan +ctl stepupdate-module-recreated-catch 0 '^ +info MODULE_RECREATED Administration: .*element identities kept' 'committed' -- bash $B/stepupdate.sh $S/rc $S/rc.plan # NS-10/NS-13: notification-service 3fbf1566 is the 11.12.4 convert (model units, Badge.mpk and 22 more without their # appstore.id entry, widgets-appstore-metadata.json, regenerated JSDoc) squashed together with toolkit files for k in cn cnok cnw; do rm -rf $S/$k; mkdir -p $S/$k/mprcontents/0a $S/$k/javascriptsource/nanoflowcommons/actions diff --git a/upgrade-toolkit/steps/upgrade.md b/upgrade-toolkit/steps/upgrade.md index 38ef1ed..47bcd6f 100644 --- a/upgrade-toolkit/steps/upgrade.md +++ b/upgrade-toolkit/steps/upgrade.md @@ -16,6 +16,8 @@ Goal: reach the target version with every step proven not to change a runtime id re-imported instead of updated in place; NS-7, notification-service cc318c0b: DataWidgets 2.27.6 rewrote 12 own pages and was reverted the next day). After a MODULE_RECREATED line, diff the own pages the step touched (`git show --stat HEAD`) before going on, or revert the step and update the module with Studio Pro. + `info MODULE_RECREATED` is the same with every element identity kept (WM1: mxcli's update copies the documents + with new unit ids; 28 of 28 such lines had no GUID change): the data is safe, only the page diff remains. - approved.txt: one fnmatch pattern per line on the element or "FINDING kind name", then `# reason`. Only for findings accepted in writing. - Local edits refusal ("Save them first: --save-edits"). stepupdate names the edited element kinds. UI only From 7180efa14531b9f0708f0d38a9119f259073852c Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 08:12:33 +0800 Subject: [PATCH 26/59] datamove: count gate for a run-once data move (scenario D) datamove.py check: per 'Src.Entity.Attr -> Dst.Entity.Attr' line in datamove.txt, the non-null values in the target after must be at least those in the source on the baseline. MOVED / LOST / UNCOUNTED; exit 1 unless all MOVED. Real DBs (read only): drt_base 2 nonces, drt_c (vendor dropped the column) count fails -> UNCOUNTED, refused. Not wired into the pipeline yet: where the move runs (a deploy before the update, or SQL) is still open. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/datamove.py | 53 ++++++++++++++++++++ upgrade-toolkit/controls/datamove_control.sh | 24 +++++++++ upgrade-toolkit/controls/run-controls.sh | 6 +++ 3 files changed, 83 insertions(+) create mode 100644 upgrade-toolkit/bin/datamove.py create mode 100644 upgrade-toolkit/controls/datamove_control.sh diff --git a/upgrade-toolkit/bin/datamove.py b/upgrade-toolkit/bin/datamove.py new file mode 100644 index 0000000..cd5dbb8 --- /dev/null +++ b/upgrade-toolkit/bin/datamove.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""datamove.py: prove a run-once data move (scenario D) moved every value: count in the target = count in the baseline. + +Why (data test 2026-11-16, scenario D; WM1 MendixSSO.AuthRequest.Nonce): data the app keeps in a vendor module must move +to _Ext before the update drops the column. However the move runs (RUN-ONCE-DESIGN.md: a deploy before the +update, or SQL), the gate needs the same proof: every non-null value in the source on the baseline is in the target after. + + datamove.py check --plan datamove.txt --before-psql "docker exec -i PG psql -U mendix -d BASE" --after-psql "... -d AFTER" + +datamove.txt: one move per line, '#' comments: Src.Entity.Attr -> Dst.Entity.Attr [# reason] +Per line: MOVED (target >= source), LOST (target < source), or UNCOUNTED (a count failed). Exit 0 when every line is +MOVED, 1 on LOST or UNCOUNTED, 2 on a malformed plan. +""" +import argparse, os, re, sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from platform_drops import count, sql_for + +LINE = re.compile(r"^\s*([A-Za-z]\w*\.\w+\.\w+)\s*->\s*([A-Za-z]\w*\.\w+\.\w+)\s*(?:#.*)?$") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + sub = ap.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("check") + c.add_argument("--plan", required=True); c.add_argument("--before-psql", required=True); c.add_argument("--after-psql", required=True) + a = ap.parse_args() + moves = [] + for n, ln in enumerate(open(a.plan), 1): + if not ln.strip() or ln.lstrip().startswith("#"): + continue + m = LINE.match(ln) + if not m: + print(f"datamove: {a.plan}:{n}: not 'Src.Entity.Attr -> Dst.Entity.Attr': {ln.strip()}", file=sys.stderr) + return 2 + moves.append(m.groups()) + bad = 0 + for src, dst in moves: + nb, na = count(a.before_psql, sql_for(src)[0]), count(a.after_psql, sql_for(dst)[0]) + if nb is None or na is None: + verdict, bad = "UNCOUNTED", bad + 1 + elif na < nb: + verdict, bad = "LOST", bad + 1 + else: + verdict = "MOVED" + show = lambda x: "count failed" if x is None else str(x) + print(f"{verdict} {src} -> {dst}: {show(nb)} before, {show(na)} after") + print(f"datamove: {len(moves) - bad} of {len(moves)} moves proven") + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/controls/datamove_control.sh b/upgrade-toolkit/controls/datamove_control.sh new file mode 100644 index 0000000..46c64d9 --- /dev/null +++ b/upgrade-toolkit/controls/datamove_control.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# datamove_control.sh CASE: bin/datamove.py check with a fake psql (base and after databases told apart by -d). +# moved every value in the target lost the target has fewer +# fail a count fails bad a malformed plan line +set -euo pipefail +B=$(cd "$(dirname "$0")/../bin" && pwd) +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT +cat > "$T/psql" <<'P' +#!/usr/bin/env bash +db=$2 q=${*: -1} +case "$db:$q" in + base:*'"mendixsso$authrequest" where "nonce"'*) echo 5 ;; + after:*'"mendixsso_ext$authrequestext" where "nonce"'*) echo "${AFTER_N:-5}" ;; + *) echo 'ERROR: relation does not exist' >&2; exit 1 ;; +esac +P +chmod +x "$T/psql" +case "$1" in + bad) echo 'MendixSSO.AuthRequest.Nonce to MendixSSO_Ext.AuthRequestExt.Nonce' > "$T/plan" ;; + fail) echo 'MendixSSO.AuthRequest.Nonce -> Other.Gone.Col' > "$T/plan" ;; + *) printf '# the SSO nonce moves to the extension module\nMendixSSO.AuthRequest.Nonce -> MendixSSO_Ext.AuthRequestExt.Nonce # scenario D\n' > "$T/plan" ;; +esac +[ "$1" != lost ] || export AFTER_N=3 +python3 "$B/datamove.py" check --plan "$T/plan" --before-psql "$T/psql -d base" --after-psql "$T/psql -d after" diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 7d19f8a..8190509 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -505,6 +505,12 @@ ctl replay-hold-dirty-tree-stops-catch 13 '^STOP 13: replay of Mod failed \(exit ctl replay-hold-cleared-on-success-pass 0 '^commit: replay: Mod$' '^held: Other exit 1' '!^held: Mod' '^mpr: replayed$' -- $RH ok ctl replay-hold-gate-stops-catch 13 '^STOP 13: upgrade complete except held replays' '!gate went through' -- $RH gate ctl replay-hold-gate-ack-pass 0 '^gate went through$' '!^STOP' -- $RH gate-ack +# scenario D: a run-once data move is proven by count (target after >= source on the baseline) +DM="bash $B/../controls/datamove_control.sh" +ctl datamove-moved-pass 0 '^MOVED MendixSSO.AuthRequest.Nonce -> MendixSSO_Ext.AuthRequestExt.Nonce: 5 before, 5 after$' '^datamove: 1 of 1 moves proven$' -- $DM moved +ctl datamove-lost-catch 1 '^LOST .*: 5 before, 3 after$' -- $DM lost +ctl datamove-count-fail-catch 1 '^UNCOUNTED .*count failed after$' -- $DM fail +ctl datamove-bad-plan-catch 2 "not 'Src.Entity.Attr -> Dst.Entity.Attr'" -- $DM bad # WM1 rerun 7: baseline ran microflow steps, the after walk skipped them: 4 false REGRESSIONs ctl pipeline-smoke-mf-follows-baseline-pass 0 '^--microflows$' -- bash -c "WORK=\$(mktemp -d); echo '{\"landed\": \"microflow ran\"}' > \$WORK/smoke-before.json; eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf" ctl pipeline-smoke-mf-off-without-baseline-pass 0 '!microflows' -- bash -c "WORK=\$(mktemp -d); eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf; echo done" From 132323a668c7f2b82dd31988e7e1db182ba37b71 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 09:49:29 +0800 Subject: [PATCH 27/59] editroute: copy Marketplace pages the menu opens onto the company layout ALTER PAGES SET LAYOUT skips pages inside Marketplace modules, so they keep the vendor layout. WM1 gate 2026-11-17: the Accounts menu opened Administration.Account_Overview on plain Atlas 4 (sidebar and header gone, the mx logo over the title; screens REVIEW). nav_copies: per Marketplace page on the redirected layout that a navigation profile opens, copy it into the module where the app already keeps copies of that vendor module's pages (same title; WM1: DISW_DesignSystem_Extention.MyAccount_Edit), on the company layout, grants mapped by role name, the menu pointed at the copy. No such module: HELD for a person. Tried on a copy of WM1's 10.24 app: check passed, exec created the page and the grant, the navigation differs from before only in the Accounts line. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/editroute.py | 76 +++++++++++++++++++ upgrade-toolkit/controls/editroute_control.sh | 22 ++++++ upgrade-toolkit/controls/run-controls.sh | 4 + 3 files changed, 102 insertions(+) diff --git a/upgrade-toolkit/bin/editroute.py b/upgrade-toolkit/bin/editroute.py index 2676e6f..a044065 100755 --- a/upgrade-toolkit/bin/editroute.py +++ b/upgrade-toolkit/bin/editroute.py @@ -313,6 +313,74 @@ def layout_route(qn, ours, orig): return None, "mxcli never writes a layout into a Marketplace module: copy it to the app's own module, repoint its pages" +def table(out): + """mxcli's | a | b | tables -> list of dicts (header row first).""" + rows = [[c.strip() for c in ln.strip().strip("|").split("|")] for ln in out.splitlines() + if ln.lstrip().startswith("|") and not re.match(r"^\s*\|[-| ]+\|\s*$", ln)] + return [dict(zip(rows[0], r)) for r in rows[1:]] if rows else [] + + +def nav_copies(qn, company, orig, app_mpr): + """-> [(label, mdl)] for the Marketplace pages on the redirected layout qn that the navigation opens. ALTER PAGES skips + them, so they keep the vendor layout: WM1's Accounts menu opened Administration.Account_Overview on plain Atlas 4, + sidebar and header gone, the mx logo over the title (screens REVIEW, 2026-11-17). Copy each into the module where + the app already keeps its copies of that vendor module's pages (a page with the same title: WM1's + DISW_DesignSystem_Extention.MyAccount_Edit), on the company layout, grants mapped by role name, the menu pointed at + the copy. No such module: not copied, named in the label for a person.""" + rc, out = mx(["-p", orig, "-c", "SHOW MODULES"]) + mods = table(out) if rc == 0 else [] + market = {m["Module"] for m in mods if m.get("Source", "").startswith("Marketplace")} + rc, out = mx(["refs", "-p", orig, qn]) + pages = [r["SourceName"] for r in table(out) if r.get("SourceType") == "PAGE" and r.get("RefKind") == "layout" + and r["SourceName"].split(".")[0] in market] if rc == 0 else [] + if not pages: + return [] + rc, out = mx(["-p", orig, "-c", "SHOW PAGES"]) + allp = table(out) if rc == 0 else [] + res = [] + for page in pages: + rc, out = mx(["refs", "-p", orig, page]) + navs = sorted({r["SourceName"].split(".", 1)[1] for r in table(out) if r.get("SourceType") == "NAVIGATION"}) + if not navs: + continue + vmod, name = page.split(".", 1) + titles = {p["Title"] for p in allp if p["Module"] == vmod and p.get("Title")} + homes = [p for p in allp if p["Module"] not in market | {vmod, "System"} and p.get("Title") in titles] + if not homes: + res.append((f"page {page}: opened from the navigation, keeps the vendor layout; no app module holds copies of " + f"{vmod} pages, copy it by hand", "")) + continue + tmod, folder = homes[0]["Module"], homes[0]["Folder"] + src = describe(app_mpr, "page", page) or describe(orig, "page", page) + if not src: + continue + copy = f"{tmod}.{name}" + rc, out = mx(["-p", app_mpr, "-c", f"SHOW MODULE ROLES IN {tmod}"]) + troles = {r["Role"] for r in table(out)} if rc == 0 else set() + body, lost = [], [] + for ln in src.splitlines(): + g = re.match(rf"^grant view on page {re.escape(page)} to (.+);$", ln.strip()) + if g: + keep = [r.strip() for r in g.group(1).split(",") if r.strip().split(".")[-1] in troles] + lost += [r.strip() for r in g.group(1).split(",") if r.strip().split(".")[-1] not in troles] + if keep: + body.append(f"grant view on page {copy} to " + ", ".join(f"{tmod}.{r.split('.')[-1]}" for r in keep) + ";") + continue + ln = re.sub(rf"\bpage {re.escape(page)}\b", f"page {copy}", ln) + ln = re.sub(r"folder '[^']*'", f"folder '{folder}'", ln, count=1) if ln.startswith("create ") else ln + body.append(re.sub(rf"^(\s*Layout:\s*){re.escape(qn)}\b", rf"\g<1>{company}", ln)) + parts = ["\n".join(body)] + for nav in navs: + n = describe(app_mpr, "navigation", nav) or "" + n2 = re.sub(rf"\bshow page {re.escape(page)}\b", f"show page {copy}", n) + if n2 != n: + parts.append(n2) + res.append((f"page {page}: opened from the navigation, copied to {copy} on {company}, menu pointed at the copy" + + (f"; no {tmod} role for {', '.join(lost)}: not granted" if lost else ""), + "mdl 1;\n" + "\n".join(re.sub(r"^mdl 1;\s*", "", p.strip(), flags=re.M) for p in parts) + "\n")) + return res + + def checked(mdl, app_mpr, tmp, qn): p = os.path.join(tmp, f"{qn}.check.mdl") with open(p, "w") as f: @@ -363,6 +431,14 @@ def cmd_collect(a): if mdl is not None and not mdl.strip(): print(f"SKIP {mod} {kind} {name}: {how}") continue + m = LAYOUT_OF.search(ours or "") if mdl else None + for label, cmdl in nav_copies(qn, m.group(1), x["original"], x.get("app_mpr") or x["original"]) if m else []: + if not cmdl: + held.append(label) + elif not (ok_why := checked(cmdl, x["app_mpr"], tmp, label.split()[1]) if x.get("app_mpr") else (True, ""))[0]: + held.append(f"{label}: mxcli check refuses the result: {ok_why[1]}") + else: + parts.append((label, cmdl)) elif kind in ("layout", "snippet", "page", "constant") and x.get("original") and x.get("app_mpr"): mdl, how = merge_element(kind, qn, x.get("pkg"), x["original"], x["app_mpr"], tmp) else: diff --git a/upgrade-toolkit/controls/editroute_control.sh b/upgrade-toolkit/controls/editroute_control.sh index 3504c4a..3f163c8 100644 --- a/upgrade-toolkit/controls/editroute_control.sh +++ b/upgrade-toolkit/controls/editroute_control.sh @@ -17,7 +17,29 @@ touch "$A/App.mpr" "$T/orig/App.mpr" "$C/7-1.0.0.mpk" "$C/7-1.0.0/project.mpr" cat > "$T/mxcli" <<'PY' #!/usr/bin/env python3 import sys +import os a = sys.argv[1:] +nav = os.environ.get("NAVCOPY") # navcopy cases: a Marketplace page on the redirected layout, opened from the menu +if nav and a[0] == "-p": + q = a[3] + if q == "SHOW MODULES": + print("| Module | Source |\n|---|---|\n| Uimod | Marketplace v1.0.0 |\n| Brand | |") + elif q == "SHOW PAGES": + print("| Qualified Name | Module | Name | Folder | Title |\n|---|---|---|---|---|\n| Uimod.Acct | Uimod | Acct | Admin | Accounts |\n" + "| Uimod.Edit | Uimod | Edit | Admin | Edit Account |\n" + f"| Brand.MyEdit | Brand | MyEdit | Pages | {'Edit Account' if nav == 'home' else 'Something else'} |") + elif q == "SHOW MODULE ROLES IN Brand": + print("| Qualified Name | Module | Role |\n|---|---|---|\n| Brand.Administrator | Brand | Administrator |") + sys.exit(0) +if nav and a[0] == "refs" and a[3] in ("Uimod.Redir", "Uimod.Acct"): + print("| SourceType | SourceName | RefKind |\n|---|---|---|\n" + ("| PAGE | Uimod.Acct | layout |" if a[3] == "Uimod.Redir" + else "| NAVIGATION | Navigation.Responsive | menu_item |")) + sys.exit(0) +if nav and a[0] == "describe" and a[3] in ("page", "navigation"): + print("mdl 1;\ncreate or modify page Uimod.Acct folder 'Admin' (\n Title: 'Accounts',\n Layout: Uimod.Redir\n) {\n};\n" + "grant view on page Uimod.Acct to Uimod.Administrator, Uimod.Auditor;" if a[3] == "page" else + "mdl 1;\ncreate or modify navigation Responsive\n{\n menu item 'Accounts' ( OnClick: show page Uimod.Acct with (DisabledDuringExecution: false) )\n}") + sys.exit(0) if a[0] == "describe": mpr, kind, qn = a[2], a[3], a[4] pkg = mpr.endswith("project.mpr") diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 8190509..f29225b 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6699,6 +6699,10 @@ ctl editroute-collect-assoc-from-original-pass 0 'association Other.A_B: from th ctl editroute-collect-no-whole-entity-catch 0 '!entity-Whole' '!saved whole element entity' -- $ER collect # WM1 rerun: mxcli never writes a layout into a Marketplace module; the app's redirect to its design module becomes a page repoint ctl editroute-collect-layout-repoint-pass 0 '^Uimod.mdl: ALTER PAGES SET LAYOUT = Brand.Main WHERE LAYOUT = Uimod.Redir;$' '^SKIP Uimod layout Spare: used by nothing' '^HELD Uimod layout UsedOwn: mxcli never writes a layout' '!create or modify layout' -- $ER collect +# WM1 gate 2026-11-17: ALTER PAGES skips Marketplace pages, so the Accounts menu opened Administration.Account_Overview on plain +# Atlas 4 (screens REVIEW); the page is copied to the app's module for copies, on the company layout, the menu pointed at it +ctl editroute-collect-nav-page-copied-pass 0 "^Uimod.mdl: create or modify page Brand.Acct folder 'Pages' \\($" '^Uimod.mdl: Layout: Brand.Main$' '^Uimod.mdl: grant view on page Brand.Acct to Brand.Administrator;$' '^Uimod.mdl: menu item .Accounts. \( OnClick: show page Brand.Acct ' 'no Brand role for Uimod.Auditor: not granted' -- env NAVCOPY=home $ER collect +ctl editroute-collect-nav-page-no-home-held-catch 0 '^HELD Uimod page Uimod.Acct: opened from the navigation, keeps the vendor layout; no app module holds copies' '!Brand.Acct' -- env NAVCOPY=nohome $ER collect # WM1 rerun 5: a dropped association with no data (non-persistent owner) is an info row, and the app still used it (CE1613) ctl editroute-xassoc-no-data-assoc-pass 0 '^ROUTE Target XASSOC: the update drops 2 association' '^approved-step.txt: Other.Np_B ' -- $ER plan ctl editroute-collect-check-held-catch 0 '^HELD Uimod snippet Bad: mxcli check refuses' '!BROKEN.*Uimod.mdl|Uimod.mdl:.*BROKEN' -- $ER collect From acdabb21abd8e49f8b0be391de16072d2ce06d5b Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 10:25:21 +0800 Subject: [PATCH 28/59] pipeline: Interview mode auto parks judgment stops and asks them in one grouped round WM1 cold test 2026-11-17: 16 judgment stops, one question each, most answered with the toolkit's recommendation. Interview mode auto (UPG_INTERVIEW_MODE=auto or interview-mode.txt) takes the recommendations at propose like unattended, and parks the stops nothing later depends on (journeys, idcheck, smoke walk, diff, gate: nj_park) in parked/.md; the run goes on. At the end bin/parked.py writes one NEEDS-JUDGMENT.md, grouped (modules and data, journeys, screens and gate) plus the ASSUMED proposal lines, and the run exits with the first parked code. Build or start failures, a missing plan or a replay that cannot be undone still stop at once. A step that runs again parks afresh; a run with a parked item never writes the PASS summary. Rung 2 inherits the mode and lists rung 1's parked items too. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 64 +++++++++++---- upgrade-toolkit/bin/parked.py | 77 +++++++++++++++++++ .../controls/replay_hold_control.sh | 2 + upgrade-toolkit/controls/run-controls.sh | 11 +++ upgrade-toolkit/steps/propose.md | 6 ++ 5 files changed, 145 insertions(+), 15 deletions(-) create mode 100644 upgrade-toolkit/bin/parked.py diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 6a02462..4f03d6d 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -36,6 +36,10 @@ # 15 the owner paused the run on purpose (`Status: PAUSED ` in PROPOSAL.md); NEEDS-JUDGMENT.md has the resume command # Interview mode unattended (env UPG_INTERVIEW_MODE=unattended, or a line `Interview mode: unattended` in $WORK/interview-mode.txt): # the propose step signs nothing, records ASSUMED (unattended) in DECISIONS.md, goes on, and every report says 'proposal not signed'. +# Interview mode auto (UPG_INTERVIEW_MODE=auto, or `Interview mode: auto`): the same at propose, and a judgment stop that nothing +# after it depends on (journeys, idcheck, smoke walk, diff, gate: nj_park) is parked in $WORK/parked/.md and the run goes on; +# at the end NEEDS-JUDGMENT.md lists every parked item grouped by kind (bin/parked.py) and the run exits with the first code. +# A build or start failure, a missing plan or a replay that cannot be undone still stops at once (nj_flush). # Decisions (bin/decisions.py): at the start of the run and of every step the pipeline applies $WORK/decisions.json (the page buttons # and the chat share it); after every step, and when a step stops, it regenerates $WORK/COCKPIT.html (bin/cockpitpage.py). # Hidden options for rung 2 (set by this script): --db NAME --old-mda PATH --seed-from DB @@ -241,6 +245,31 @@ nj_flush() { # write NEEDS-JUDGMENT.md and exit with the first recorded code say "STOP $STEP: needs judgment (exit $NJ_CODE), see $WORK/NEEDS-JUDGMENT.md" exit "$NJ_CODE" } +# WM1 cold test 2026-11-17: 16 stops, one question each, most answered with the recommendation. Auto mode parks the stops +# nothing later depends on and asks them all at the end (park_finish). +nj_park() { # like nj_flush, but in Interview mode auto the items go to $WORK/parked/$STEP.md and the step goes on + [ "$NJ_CODE" != 0 ] || return 0 + if [ "$DRY" = 1 ] || ! interview_auto; then nj_flush; return 0; fi + mkdir -p "$WORK/parked" + { printf '\n' "$NJ_CODE" "$STEP"; cat "$NJ_TMP"; } >> "$WORK/parked/$STEP.md" + say "PARKED $STEP: exit $NJ_CODE parked (Interview mode: auto), the run goes on; $WORK/parked/$STEP.md" + : > "$NJ_TMP"; NJ_CODE=0 +} +parked_any() { [ -n "$(ls "$WORK/parked" 2> /dev/null)" ] || { [ "$(basename "$WORK")" = rung2 ] && [ -n "$(ls "$WORK/../parked" 2> /dev/null)" ]; }; } +park_finish() { # end of the run: every parked item in one NEEDS-JUDGMENT.md, exit with the first parked code + [ "$DRY" != 1 ] && parked_any || return 0 + local also=() out first code step + [ "$(basename "$WORK")" != rung2 ] || [ ! -d "$WORK/../parked" ] || also=(--also "$(cd "$WORK/.." && pwd)/parked") + out=$(python3 "$B/parked.py" report --work "$WORK" --steps "${STEPS[*]}" ${also[@]+"${also[@]}"}) || return 0 + first=$(printf '%s\n' "$out" | tail -n 1); code=$(echo "$first" | cut -d' ' -f2); step=$(echo "$first" | cut -d' ' -f3) + STEP=$step + { printf '%s\n' "$out" | sed '$d' + printf '\nRerun once, after the answers, from the first parked step (%s; a rung-1 step: the rung-1 command with --from %s):\n\n %s\n' "$step" "$step" "$(rerun_cmd)" + } > "$WORK/NEEDS-JUDGMENT.md" + cockpit_regen + say "STOP auto: $(grep -c '^` and the nj_add text after it. Output: the +grouped markdown on stdout; the last line `first: N S DIR` names the first parked block in step order (rung 1 first): the +exit code of the run and the step to rerun from. Exit 1 when nothing is parked (no output). +""" +import argparse, os, re, sys + +OPEN = re.compile(r"^$") +KINDS = [("13", "Modules and data"), ("11", "Journeys"), ("12", "Screens, comparisons and the gate")] + + +def blocks(d, steps): + out = [] + for s in steps: + p = os.path.join(d, s + ".md") + if not os.path.isfile(p): + continue + cur = None + for ln in open(p, encoding="utf-8", errors="replace").read().splitlines(): + m = OPEN.match(ln) + if m: + cur = {"code": m.group(1), "step": m.group(2), "dir": d, "text": []} + out.append(cur) + elif cur is not None: + cur["text"].append(ln) + return out + + +def assumed(work): + p = os.path.join(work, "DECISIONS.md") + return [ln for ln in open(p, encoding="utf-8", errors="replace").read().splitlines() if "ASSUMED" in ln] if os.path.isfile(p) else [] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + sub = ap.add_subparsers(dest="cmd", required=True) + r = sub.add_parser("report") + r.add_argument("--work", required=True); r.add_argument("--steps", required=True); r.add_argument("--also") + a = ap.parse_args() + steps = a.steps.split() + rung = lambda d: "rung 1" if a.also and d == a.also else "this rung" if a.also else "" + bl = (blocks(a.also, steps) if a.also else []) + blocks(os.path.join(a.work, "parked"), steps) + if not bl: + return 1 + print(f"# NEEDS JUDGMENT: {len(bl)} parked item(s) (Interview mode: auto), answer them in one round\n") + print("The run went on past each of these; nothing below is passed. Decide each in writing (the allow/approved/ack files") + print("its text names, or the cockpit page), then rerun once from the step named in the last line.\n") + for code, title in KINDS + [(c, f"Other (exit {c})") for c in sorted({b["code"] for b in bl} - {k for k, _ in KINDS})]: + group = [b for b in bl if b["code"] == code] + if not group: + continue + print(f"# {title}: {len(group)}\n") + for b in group: + where = " ".join(x for x in (rung(b["dir"]), "step", b["step"]) if x) + print(f"") + print("\n".join(b["text"]).strip("\n").replace("\n## ", "\n### ").replace("## ", "### ", 1) + "\n") + asm = assumed(a.work) + (assumed(os.path.dirname(a.also)) if a.also else []) + if asm: + print(f"# Proposal: {len(asm)} recommendation(s) taken without a sign-off (ASSUMED), confirm them\n") + print("\n".join(f"- {x}" for x in asm) + "\n") + f = bl[0] + print(f"first: {f['code']} {f['step']} {f['dir']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/controls/replay_hold_control.sh b/upgrade-toolkit/controls/replay_hold_control.sh index 9ab0d52..e4c12c9 100644 --- a/upgrade-toolkit/controls/replay_hold_control.sh +++ b/upgrade-toolkit/controls/replay_hold_control.sh @@ -21,6 +21,8 @@ Q="" nj_file() { echo "STOP $1: $2"; Q=$1; } nj_add() { echo "STOP $1: $2"; Q=$1; } nj_flush() { [ -z "$Q" ] || { echo "EXIT $Q"; exit "$Q"; }; } +nj_park() { nj_flush; } # Interview mode is not auto here: a parkable stop stops +parked_any() { false; } commit_app() { git_app add -A; git_app commit -qm "$1"; echo "commit: $1"; } write_summary() { :; } record_runlog() { :; } diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index f29225b..ab01223 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -1624,6 +1624,17 @@ mkdir -p $NJ/w13; printf '1 1.0 Administration\n' > $NJ/w13/plan.txt ctl pipeline-no-plan-pre-catch 0 'needs judgment \(exit 13\)' 'plan-pre.txt missing on a 10.x to 11 run' '!mx convert exit' -- bash -c "MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-dir $NJ/app --source 10.24.26.1 --target 11.15.0.1 --work $NJ/w13 --from convert --only convert; cat $NJ/w13/NEEDS-JUDGMENT.md" mkdir -p $NJ/w13b; printf '1 1.0 Administration\n' > $NJ/w13b/plan.txt; printf '# nothing before the convert\n' > $NJ/w13b/plan-pre.txt ctl pipeline-plan-pre-stated-pass 0 '!plan-pre.txt missing' 'convert' -- bash -c "MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-dir $NJ/app --source 10.24.26.1 --target 11.15.0.1 --work $NJ/w13b --from convert --only convert; cat $NJ/w13b/NEEDS-JUDGMENT.md 2>/dev/null; true" +# WM1 cold test 2026-11-17: 16 judgment stops, one question each. Interview mode auto parks a stop nothing later depends on +# (here the gate's release-impact and kept-modules stops), goes on, and asks every parked item in one grouped list at the end +for d in ga gm; do mkdir -p $NJ/$d; printf 'Admin 1.0\n' > $NJ/$d/kept.txt; printf '# release impact\n' > $NJ/$d/release-impact.md; done +printf 'Interview mode: auto\n' > $NJ/ga/interview-mode.txt +ctl pipeline-auto-parks-all-pass 12 '^PARKED gate: exit 12 parked' '^PARKED gate: exit 13 parked' '2 parked item\(s\) to answer in one round \(exit 12\)' '^# Modules and data: 1$' '^# Screens, comparisons and the gate: 1$' '--from gate$' '!PASS gate' -- bash -c "MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-dir $NJ/app --source 10.24.26.1 --target 11.15.0.1 --work $NJ/ga --only gate; rc=\$?; cat $NJ/ga/NEEDS-JUDGMENT.md; exit \$rc" +ctl pipeline-auto-off-stops-first-catch 12 'needs judgment \(exit 12\)' '!PARKED' '!kept modules' -- bash -c "MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-dir $NJ/app --source 10.24.26.1 --target 11.15.0.1 --work $NJ/gm --only gate; rc=\$?; cat $NJ/gm/NEEDS-JUDGMENT.md; exit \$rc" +# a stop that later steps depend on is never parked: the plan-pre stop of F13 still ends the run in auto mode +mkdir -p $NJ/w13a; printf '1 1.0 Administration\n' > $NJ/w13a/plan.txt +ctl pipeline-auto-hard-stop-still-stops-catch 0 'needs judgment \(exit 13\)' '!PARKED' '!mx convert exit' -- bash -c "UPG_INTERVIEW_MODE=auto MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-dir $NJ/app --source 10.24.26.1 --target 11.15.0.1 --work $NJ/w13a --from convert --only convert; true" +# a step that runs again parks afresh: the gate rerun after the acks leaves nothing parked and reaches the PASS summary +ctl pipeline-auto-rerun-clears-pass 0 'PASS gate' '!PARKED' '!parked item' -- bash -c "cp $NJ/ga/kept.txt $NJ/ga/kept-ack.txt; printf '## Studio Pro\n\n| 1 | 11.15 | behaviour | x | not relevant: control fixture |\n' > $NJ/ga/release-impact.md; MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-dir $NJ/app --source 10.24.26.1 --target 11.15.0.1 --work $NJ/ga --only gate" newapp o11b '1 1.0 Old11Base\n' ctl stepupdate-pre1021-baseline-pass 0 'installed Old11Base cannot serve as the local-edit baseline \(range from 10.21.0 to 10.24.99\)' 'committed' '!SKIP' -- bash $B/stepupdate.sh $S/o11b $S/o11b.plan --unattended newapp o11 '1 1.0 Old11Pkg\n' diff --git a/upgrade-toolkit/steps/propose.md b/upgrade-toolkit/steps/propose.md index 75b1774..681194f 100644 --- a/upgrade-toolkit/steps/propose.md +++ b/upgrade-toolkit/steps/propose.md @@ -31,6 +31,12 @@ Signing an item nobody answered accepts its recommendation (recorded as a P-n li records `ASSUMED (unattended)` P-lines, goes on, and the gate summary and SUMMARY.md say "proposal not signed". A person who later signs and answers the P-n lines closes them; the ASSUMED lines block the owner sign-off until then. +`Interview mode: auto` does the same at propose and also parks the later judgment stops that nothing after them depends on +(journeys, idcheck, smoke walk, diff, gate) in `$W/parked/.md`; the run goes on. At the end NEEDS-JUDGMENT.md lists every +parked item and every ASSUMED line, grouped (modules and data, journeys, screens and gate), for one round of answers and one +rerun from the first parked step (bin/parked.py). A build or start failure, a missing plan or a replay that cannot be undone +still stops at once. A run with a parked item never ends in PASS. + Section-3 items that fit a pattern in playbooks/rearchitecture.md carry its options, effort and recommendation and cite `[RA-n, status]` (bin/proposal.py); the answer updates that entry (steps/decide.md). ## Step records From 1879f45a3144633f20325c9dbc1cb7d29d8f6991 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 10:41:14 +0800 Subject: [PATCH 29/59] editroute: the menu copy leaves out APP TITLE and APP ICON WM1 rerun 2026-11-17: the Account_Overview copy was HELD because the container mxcli (78bdfa5) refuses the APP TITLE/APP ICON clauses the local mxcli's describe of the navigation prints. An omitted clause keeps the stored value, so the copy drops them; checked with the container mxcli against the WM1 model: Check passed. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/editroute.py | 4 +++- upgrade-toolkit/controls/editroute_control.sh | 2 +- upgrade-toolkit/controls/run-controls.sh | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/upgrade-toolkit/bin/editroute.py b/upgrade-toolkit/bin/editroute.py index a044065..247f5fe 100755 --- a/upgrade-toolkit/bin/editroute.py +++ b/upgrade-toolkit/bin/editroute.py @@ -373,8 +373,10 @@ def nav_copies(qn, company, orig, app_mpr): for nav in navs: n = describe(app_mpr, "navigation", nav) or "" n2 = re.sub(rf"\bshow page {re.escape(page)}\b", f"show page {copy}", n) + # WM1 2026-11-17: the container mxcli (78bdfa5) refuses the APP TITLE/APP ICON clauses the local describe prints; + # an omitted clause leaves the stored value alone (ako/mxcli#1756), so the copy drops them if n2 != n: - parts.append(n2) + parts.append(re.sub(r"(?im)^\s*app (title|icon)\b.*\n", "", n2)) res.append((f"page {page}: opened from the navigation, copied to {copy} on {company}, menu pointed at the copy" + (f"; no {tmod} role for {', '.join(lost)}: not granted" if lost else ""), "mdl 1;\n" + "\n".join(re.sub(r"^mdl 1;\s*", "", p.strip(), flags=re.M) for p in parts) + "\n")) diff --git a/upgrade-toolkit/controls/editroute_control.sh b/upgrade-toolkit/controls/editroute_control.sh index 3f163c8..89733db 100644 --- a/upgrade-toolkit/controls/editroute_control.sh +++ b/upgrade-toolkit/controls/editroute_control.sh @@ -38,7 +38,7 @@ if nav and a[0] == "refs" and a[3] in ("Uimod.Redir", "Uimod.Acct"): if nav and a[0] == "describe" and a[3] in ("page", "navigation"): print("mdl 1;\ncreate or modify page Uimod.Acct folder 'Admin' (\n Title: 'Accounts',\n Layout: Uimod.Redir\n) {\n};\n" "grant view on page Uimod.Acct to Uimod.Administrator, Uimod.Auditor;" if a[3] == "page" else - "mdl 1;\ncreate or modify navigation Responsive\n{\n menu item 'Accounts' ( OnClick: show page Uimod.Acct with (DisabledDuringExecution: false) )\n}") + "mdl 1;\ncreate or modify navigation Responsive\n app title 'Uimod App'\n app icon Brand.Icons.Logo\n{\n menu item 'Accounts' ( OnClick: show page Uimod.Acct with (DisabledDuringExecution: false) )\n}") sys.exit(0) if a[0] == "describe": mpr, kind, qn = a[2], a[3], a[4] diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index ab01223..9679f95 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6712,7 +6712,7 @@ ctl editroute-collect-no-whole-entity-catch 0 '!entity-Whole' '!saved whole elem ctl editroute-collect-layout-repoint-pass 0 '^Uimod.mdl: ALTER PAGES SET LAYOUT = Brand.Main WHERE LAYOUT = Uimod.Redir;$' '^SKIP Uimod layout Spare: used by nothing' '^HELD Uimod layout UsedOwn: mxcli never writes a layout' '!create or modify layout' -- $ER collect # WM1 gate 2026-11-17: ALTER PAGES skips Marketplace pages, so the Accounts menu opened Administration.Account_Overview on plain # Atlas 4 (screens REVIEW); the page is copied to the app's module for copies, on the company layout, the menu pointed at it -ctl editroute-collect-nav-page-copied-pass 0 "^Uimod.mdl: create or modify page Brand.Acct folder 'Pages' \\($" '^Uimod.mdl: Layout: Brand.Main$' '^Uimod.mdl: grant view on page Brand.Acct to Brand.Administrator;$' '^Uimod.mdl: menu item .Accounts. \( OnClick: show page Brand.Acct ' 'no Brand role for Uimod.Auditor: not granted' -- env NAVCOPY=home $ER collect +ctl editroute-collect-nav-page-copied-pass 0 "^Uimod.mdl: create or modify page Brand.Acct folder 'Pages' \\($" '^Uimod.mdl: Layout: Brand.Main$' '^Uimod.mdl: grant view on page Brand.Acct to Brand.Administrator;$' '^Uimod.mdl: menu item .Accounts. \( OnClick: show page Brand.Acct ' 'no Brand role for Uimod.Auditor: not granted' '!^Uimod.mdl: app (title|icon)' -- env NAVCOPY=home $ER collect ctl editroute-collect-nav-page-no-home-held-catch 0 '^HELD Uimod page Uimod.Acct: opened from the navigation, keeps the vendor layout; no app module holds copies' '!Brand.Acct' -- env NAVCOPY=nohome $ER collect # WM1 rerun 5: a dropped association with no data (non-persistent owner) is an info row, and the app still used it (CE1613) ctl editroute-xassoc-no-data-assoc-pass 0 '^ROUTE Target XASSOC: the update drops 2 association' '^approved-step.txt: Other.Np_B ' -- $ER plan From e1b677ff31c232ec974cbdf0eb326f261e38631a Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 11:14:01 +0800 Subject: [PATCH 30/59] playbook 9-to-10: no mxcli oql on the 9.24 runtime, the read-only SQL stand-in and its proof Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/playbooks/9-to-10.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/upgrade-toolkit/playbooks/9-to-10.md b/upgrade-toolkit/playbooks/9-to-10.md index 0cb6609..0c0c547 100644 --- a/upgrade-toolkit/playbooks/9-to-10.md +++ b/upgrade-toolkit/playbooks/9-to-10.md @@ -19,6 +19,19 @@ Route: 9.x must be on 9.24 first, then convert to 10.24, then update modules ONC SKILL.md, NR1-4), with only convert/step/build/jar commits since the "original" commit, is CONVERT-MADE and forced (T1-15, T3-10, T4-3, T8-12). Commit the app's first commit as "original" and the convert as "convert ...". +- No data reads on the OLD app through `mxcli oql` (WM1 CLI test, 2026-11-17; mendixlabs/mxcli#1879): the 9.24 runtime + has no `preview_execute_oql` admin action, so every call answers `{"result":-5,"message":"Action not found."}` and + mxcli's hint about the live-preview dev flags cannot help. Seeds, row counts, outcome checks and fixture-manifest.sh + on the old app are an instrument FAULT, so the run has no "before" data. One call tells: post `preview_execute_oql` + with `SELECT COUNT(*) AS n FROM System.Session` to the admin port; 10.24 answers `{"n":"5"}`-style data. + Way round, until mxcli has one: a stand-in for mxcli (`MXCLI=`) that passes every other command to the real + mxcli and answers `oql` with a mapped SQL query, run read-only (`begin transaction read only; select + coalesce(json_agg(t),'[]'::json) from () t; rollback;`) on the database behind that admin port, printing + mxcli's JSON with every value as a string. Tables are `"module$entity"`, columns lower case, an association is its + own table `"module$assoc"` with `"module$parentid"`/`"module$childid"` (check with `\d`, never guess). A query with + no mapping exits 1, so its check reads INVALID, never PASS; count forms (`SELECT COUNT(*) AS n FROM M.E`) can be + derived. Prove the stand-in first: every mapped query through real mxcli and through the stand-in on the 10.24 + side, after the journeys changed data, row for row equal (WM1: 10/10). Reports name the verdicts that came from SQL. - Coverage gap of localedits on 9.24 (E5b-7): pages and layouts are compared only when both models can be read by the same mxcli; a 9.24 app against a 10.x package often cannot. Run localedits again right after the convert, on 10.24, before the post-convert updates: that run sees the page, layout and logic layers. From 183d0447717aad43829c729ab1814611dc3c95ab Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 11:28:24 +0800 Subject: [PATCH 31/59] editroute: no menu copy of a page that holds a widget mxcli cannot write WM1 rerun 2026-11-17: the Account_Overview copy got the company layout and the menu, but its classic data grid (Forms$DataGrid, NOT re-executable in describe) was dropped: an empty page where the user list was. mxcli cannot set a page's layout in place either (unsupported page-level property: Layout). Such a page is HELD with the widget named and the by-hand route (Studio Pro copy onto the company layout, menu pointed at it). Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/editroute.py | 8 ++++++++ upgrade-toolkit/controls/editroute_control.sh | 5 +++-- upgrade-toolkit/controls/run-controls.sh | 2 ++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/upgrade-toolkit/bin/editroute.py b/upgrade-toolkit/bin/editroute.py index 247f5fe..d2bac92 100755 --- a/upgrade-toolkit/bin/editroute.py +++ b/upgrade-toolkit/bin/editroute.py @@ -354,6 +354,14 @@ def nav_copies(qn, company, orig, app_mpr): src = describe(app_mpr, "page", page) or describe(orig, "page", page) if not src: continue + # WM1 2026-11-17: Account_Overview holds a classic data grid mxcli cannot write; describe marks it NOT re-executable + # and the copy came out without its user list (menu back, page empty). A copy that would drop a widget is no copy + if "NOT re-executable" in src: + w = re.search(r"--\s*(\S+ \(\w+\))\s*-- NOT re-executable", src) + res.append((f"page {page}: opened from the navigation, keeps the vendor layout; a copy would drop " + f"{w.group(1) if w else 'a widget'} (mxcli cannot write it): copy it by hand in Studio Pro onto " + f"{company} and point the menu at the copy", "")) + continue copy = f"{tmod}.{name}" rc, out = mx(["-p", app_mpr, "-c", f"SHOW MODULE ROLES IN {tmod}"]) troles = {r["Role"] for r in table(out)} if rc == 0 else set() diff --git a/upgrade-toolkit/controls/editroute_control.sh b/upgrade-toolkit/controls/editroute_control.sh index 89733db..527fc02 100644 --- a/upgrade-toolkit/controls/editroute_control.sh +++ b/upgrade-toolkit/controls/editroute_control.sh @@ -27,7 +27,7 @@ if nav and a[0] == "-p": elif q == "SHOW PAGES": print("| Qualified Name | Module | Name | Folder | Title |\n|---|---|---|---|---|\n| Uimod.Acct | Uimod | Acct | Admin | Accounts |\n" "| Uimod.Edit | Uimod | Edit | Admin | Edit Account |\n" - f"| Brand.MyEdit | Brand | MyEdit | Pages | {'Edit Account' if nav == 'home' else 'Something else'} |") + f"| Brand.MyEdit | Brand | MyEdit | Pages | {'Something else' if nav == 'nohome' else 'Edit Account'} |") elif q == "SHOW MODULE ROLES IN Brand": print("| Qualified Name | Module | Role |\n|---|---|---|\n| Brand.Administrator | Brand | Administrator |") sys.exit(0) @@ -36,7 +36,8 @@ if nav and a[0] == "refs" and a[3] in ("Uimod.Redir", "Uimod.Acct"): else "| NAVIGATION | Navigation.Responsive | menu_item |")) sys.exit(0) if nav and a[0] == "describe" and a[3] in ("page", "navigation"): - print("mdl 1;\ncreate or modify page Uimod.Acct folder 'Admin' (\n Title: 'Accounts',\n Layout: Uimod.Redir\n) {\n};\n" + grid = " -- Forms$DataGrid (grid12) -- NOT re-executable: mxcli cannot author this widget\n" if nav == "grid" else "" + print("mdl 1;\ncreate or modify page Uimod.Acct folder 'Admin' (\n Title: 'Accounts',\n Layout: Uimod.Redir\n) {\n" + grid + "};\n" "grant view on page Uimod.Acct to Uimod.Administrator, Uimod.Auditor;" if a[3] == "page" else "mdl 1;\ncreate or modify navigation Responsive\n app title 'Uimod App'\n app icon Brand.Icons.Logo\n{\n menu item 'Accounts' ( OnClick: show page Uimod.Acct with (DisabledDuringExecution: false) )\n}") sys.exit(0) diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 9679f95..b0dd9a1 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6714,6 +6714,8 @@ ctl editroute-collect-layout-repoint-pass 0 '^Uimod.mdl: ALTER PAGES SET LAYOUT # Atlas 4 (screens REVIEW); the page is copied to the app's module for copies, on the company layout, the menu pointed at it ctl editroute-collect-nav-page-copied-pass 0 "^Uimod.mdl: create or modify page Brand.Acct folder 'Pages' \\($" '^Uimod.mdl: Layout: Brand.Main$' '^Uimod.mdl: grant view on page Brand.Acct to Brand.Administrator;$' '^Uimod.mdl: menu item .Accounts. \( OnClick: show page Brand.Acct ' 'no Brand role for Uimod.Auditor: not granted' '!^Uimod.mdl: app (title|icon)' -- env NAVCOPY=home $ER collect ctl editroute-collect-nav-page-no-home-held-catch 0 '^HELD Uimod page Uimod.Acct: opened from the navigation, keeps the vendor layout; no app module holds copies' '!Brand.Acct' -- env NAVCOPY=nohome $ER collect +# WM1 2026-11-17: the copy of a page with a widget mxcli cannot write came out without it (Account_Overview, no user list) +ctl editroute-collect-nav-page-unwritable-widget-held-catch 0 '^HELD Uimod page Uimod.Acct: opened from the navigation, keeps the vendor layout; a copy would drop Forms\$DataGrid \(grid12\)' '!create or modify page Brand.Acct' '!show page Brand.Acct' -- env NAVCOPY=grid $ER collect # WM1 rerun 5: a dropped association with no data (non-persistent owner) is an info row, and the app still used it (CE1613) ctl editroute-xassoc-no-data-assoc-pass 0 '^ROUTE Target XASSOC: the update drops 2 association' '^approved-step.txt: Other.Np_B ' -- $ER plan ctl editroute-collect-check-held-catch 0 '^HELD Uimod snippet Bad: mxcli check refuses' '!BROKEN.*Uimod.mdl|Uimod.mdl:.*BROKEN' -- $ER collect From 186c0f1af05ed82321f90173e27e097984a77f2e Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 11:35:03 +0800 Subject: [PATCH 32/59] playbook 9-to-10: data reads on 9.24 go through the toolkit's own Postgres route, not a hand stand-in The pipeline already reads data through UPG_PSQL (pgdata.js + pgmap.py), which works on the 9.24 database as on 10.24 (WM1 rerun: 97 entities, 111 associations mapped). The hand OQL-to-SQL stand-in from the CLI test covers only the queries typed into it; outside the pipeline use the same route. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/playbooks/9-to-10.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/upgrade-toolkit/playbooks/9-to-10.md b/upgrade-toolkit/playbooks/9-to-10.md index 0c0c547..e8f83aa 100644 --- a/upgrade-toolkit/playbooks/9-to-10.md +++ b/upgrade-toolkit/playbooks/9-to-10.md @@ -24,14 +24,12 @@ Route: 9.x must be on 9.24 first, then convert to 10.24, then update modules ONC mxcli's hint about the live-preview dev flags cannot help. Seeds, row counts, outcome checks and fixture-manifest.sh on the old app are an instrument FAULT, so the run has no "before" data. One call tells: post `preview_execute_oql` with `SELECT COUNT(*) AS n FROM System.Session` to the admin port; 10.24 answers `{"n":"5"}`-style data. - Way round, until mxcli has one: a stand-in for mxcli (`MXCLI=`) that passes every other command to the real - mxcli and answers `oql` with a mapped SQL query, run read-only (`begin transaction read only; select - coalesce(json_agg(t),'[]'::json) from () t; rollback;`) on the database behind that admin port, printing - mxcli's JSON with every value as a string. Tables are `"module$entity"`, columns lower case, an association is its - own table `"module$assoc"` with `"module$parentid"`/`"module$childid"` (check with `\d`, never guess). A query with - no mapping exits 1, so its check reads INVALID, never PASS; count forms (`SELECT COUNT(*) AS n FROM M.E`) can be - derived. Prove the stand-in first: every mapped query through real mxcli and through the stand-in on the 10.24 - side, after the journeys changed data, row for row equal (WM1: 10/10). Reports name the verdicts that came from SQL. + The pipeline needs nothing for this: run_journeys.py sets `UPG_PSQL` and the engine asks the app's Postgres + (bin/pgdata.js + bin/pgmap.py, names read from the runtime's `mendixsystem$` tables, the same on 9.24; WM1 rerun: + 97 entities, 111 associations on the 9.24 database). Driving an e2e engine by hand outside the pipeline, give it + the same route (`UPG_PSQL`, or `pgmap.py query --psql CMD ""`); never hand-map OQL to SQL. pgmap refuses + what it cannot translate (exit 1: the check is INVALID, never PASS). A hand mapping was proved once (WM1 CLI test: + 10/10 queries equal to real mxcli on 10.24), but it covers only the queries someone typed in. - Coverage gap of localedits on 9.24 (E5b-7): pages and layouts are compared only when both models can be read by the same mxcli; a 9.24 app against a 10.x package often cannot. Run localedits again right after the convert, on 10.24, before the post-convert updates: that run sees the page, layout and logic layers. From 44d9111a41a2cd779f27f59a58b9de2a1e68d127 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 11:55:36 +0800 Subject: [PATCH 33/59] data compare: a vendor module's own change and the toolkit's admin login are noted, not FAIL WM1 rerun 2026-11-17: the gate card said Data compare: Failed on 4 rows, none a loss of the app's data: MendixSSO 4.2 -> 10.6 dropped AuthRequest.Nonce, MxModelReflection 7 -> 9 rewrote its Module rows, and System.User.changedDate/Password changed on MxAdmin alone (runapp.py create_admin_user at every start). - diff.py --module-jumps: a column gone or values changed in an attribute of a module the run updated is PASS* with the jump named; rows lost stay FAIL in any module; twin-old evidence still wins over this reason - snapshot.py records System.User figures without the local admin row (UPG_ADMIN_USER, default MxAdmin); diff.py notes a change on that row alone as the toolkit's own login - gate.sh and heal pass module-jumps.txt when the run has one On the WM1 databases (fresh snapshot of a copy of the frozen db vs after_r5, --db-only): FAIL 7 -> 0, PASS 355. Controls: 5 new (3 fail on the old diff.py, 2 guards hold on both), diff group 31/31. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 33 +++++++++--- upgrade-toolkit/bin/diff.py | 22 +++++++- upgrade-toolkit/bin/gate.sh | 2 +- upgrade-toolkit/bin/snapshot.py | 13 ++++- upgrade-toolkit/controls/run-controls.sh | 52 +++++++++++++++++++ 5 files changed, 113 insertions(+), 9 deletions(-) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 4f03d6d..d4763e4 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -492,11 +492,29 @@ user_args() { # user_args ROLES -> sets UARGS for smokewalk.js mda_old() { if [ -n "$OLD_MDA" ]; then echo "$OLD_MDA"; else echo "$WORK/old-$SOURCE.mda"; fi; } mda_new() { echo "$WORK/new-$TARGET.mda"; } -# DB names: working, _frozen never started. +# DB names: working, _frozen never started. Cold CMM (2026-11-17): every run folder named "run" (the README's own +# example) got upg_run_*, and a second app's baseline reused the first app's frozen database without a word. The default +# name carries a hash of the run folder and is kept in $WORK/state/db-prefix; a run whose db-ledger.txt predates that keeps +# its old name. The frozen database carries its run folder as a comment (frozen_owner), checked before every reuse. +RUNP=$(cd "$RUN" 2> /dev/null && pwd -P || echo "$RUN") if [ -n "$DB_ARG" ]; then X=$DB_ARG -else X="upg_$(basename "$RUN" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9_\n' '_')"; fi +elif [ -s "$WORK/state/db-prefix" ]; then X=$(head -n1 "$WORK/state/db-prefix") +else + X="upg_$(basename "$RUN" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9_\n' '_' | cut -c1-24)" + [ -f "$WORK/db-ledger.txt" ] || X="${X}_$(printf '%s' "$RUNP" | cksum | awk '{printf "%06x", $1 % 16777216}')" +fi X_FROZEN="${X}_frozen" [[ "$X" =~ ^[a-z0-9_]+$ ]] || usage_die "database name '$X' must be lower case letters, digits, underscore (use --db)" +[ "$DRY" = 1 ] || [ -s "$WORK/state/db-prefix" ] || { mkdir -p "$WORK/state" && echo "$X" > "$WORK/state/db-prefix"; } +frozen_stamp() { must docker exec "$PGC" psql -U mendix -qc "COMMENT ON DATABASE $X_FROZEN IS 'upgrade-toolkit run $RUNP'"; } +frozen_owner() { # refuse a frozen database that another run made: its comment names another run folder, or it has none and + # this run's ledger never used a copy of it (a run from before the stamp adopts its own one here) + local own + own=$(docker exec "$PGC" psql -U mendix -tAc "SELECT coalesce(shobj_description(oid, 'pg_database'), '') FROM pg_database WHERE datname='$X_FROZEN'" 2> /dev/null || true) + [ "$own" != "upgrade-toolkit run $RUNP" ] || return 0 + if [ -z "$own" ] && grep -q "^${X}_" "$WORK/db-ledger.txt" 2> /dev/null; then frozen_stamp; return 0; fi + die4 "frozen database $X_FROZEN exists but this run did not make it (${own:-no owner stamp}; this run: $RUNP). Another app's data would be the baseline: give this run its own name with --db upg_, or drop that database if it is yours." +} preflight() { local t @@ -620,8 +638,9 @@ Decide, in writing: (a) wrap it in a host app (an app with a home page, a theme [ "$rc" != 1 ] || note "RENAMED modules in $WORK/renamed.md: not version updates, a decision item in the proposal" rc=0; xo "$WORK/xmod.txt" python3 "$B/idcheck.py" "$APP" --xmod || rc=$? # scanner (steps/triage.md item 6): optional, lives in the company brain checkout - # the optional scanner: $EA, $UPG_SCANNER, or UPG_SCANNER= in the private ~/.config/upgrade-toolkit/local.env - local EA="${EA:-${UPG_SCANNER:-$(sed -n 's/^UPG_SCANNER=//p' "$HOME/.config/upgrade-toolkit/local.env" 2>/dev/null)}}" + # the optional scanner: $EA, $UPG_SCANNER, or UPG_SCANNER= in the private ~/.config/upgrade-toolkit/local.env; `|| true`: without + # that file sed exits 2 and the ERR trap printed "unexpected failure at line 624" in every triage (cold WM1 #12, cold CMM) + local EA="${EA:-${UPG_SCANNER:-$(sed -n 's/^UPG_SCANNER=//p' "$HOME/.config/upgrade-toolkit/local.env" 2>/dev/null || true)}}" if [ -f "$EA/scripts/scan.py" ]; then rc=0; (cd "$APP" && x python3 "$EA/scripts/scan.py" --project . --target "$TARGET" --from-version auto --data "$EA/data" --out "$WORK/findings.json") || rc=$? if [ "$rc" = 0 ]; then @@ -675,6 +694,7 @@ step_baseline() { commit_app "build: regenerated stubs" config build-stubs fi if db_exists "$X_FROZEN"; then + frozen_owner note "reusing existing frozen database $X_FROZEN" else if [ -n "$SEED_FROM" ]; then copy_db "$SEED_FROM" "$X"; fi @@ -685,7 +705,7 @@ step_baseline() { app_up old "$old" "$SOURCE" "$X" || die4 "old app did not start after the seed (log: $LOG)" app_down old fi - copy_db "$X" "$X_FROZEN" + copy_db "$X" "$X_FROZEN"; frozen_stamp fi # endpoint probe on its own copy (steps/triage.md item 8): drop listed paths that answer 404 on the old app if [ -s "$WORK/endpoints.txt" ] && [ ! -f "$WORK/endpoints.probed" ]; then @@ -1485,7 +1505,8 @@ step_heal() { must python3 "$B/heal.py" grids --model "$WORK/original.mpr" --out "$WORK/heal-grids.json" [ ! -f "$WORK/allow.txt" ] || al=(--allow "$WORK/allow.txt") [ ! -f "$WORK/log-allow.txt" ] || la=(--log-allow "$WORK/log-allow.txt") - xo "$WORK/heal-pre-diff.out" python3 "$B/diff.py" "$WORK/before.json" "$WORK/after.json" --md "$WORK/heal-pre-diff.md" ${al[@]+"${al[@]}"} ${la[@]+"${la[@]}"} || true + local jm=(); [ -f "$WORK/module-jumps.txt" ] && jm=(--module-jumps "$WORK/module-jumps.txt") + xo "$WORK/heal-pre-diff.out" python3 "$B/diff.py" "$WORK/before.json" "$WORK/after.json" --md "$WORK/heal-pre-diff.md" ${al[@]+"${al[@]}"} ${la[@]+"${la[@]}"} ${jm[@]+"${jm[@]}"} || true heal_twin for rnd in 1 2; do xo "$WORK/heal-plan-$rnd.out" python3 "$B/heal.py" plan --work "$WORK" --round "$rnd" || die4 "heal.py plan failed (log: $WORK/heal-plan-$rnd.out)" diff --git a/upgrade-toolkit/bin/diff.py b/upgrade-toolkit/bin/diff.py index 5148e26..aab721f 100755 --- a/upgrade-toolkit/bin/diff.py +++ b/upgrade-toolkit/bin/diff.py @@ -90,8 +90,16 @@ def main(): help="compare the db section only (the mutant control: a db-only snapshot of a copy; E6-11, E7-18)") ap.add_argument("--emit-allow", dest="emit_allow", help="the restart control (PV-7): write one allow.txt line per FAIL or REVIEW row of this diff, as start noise") + ap.add_argument("--module-jumps", dest="module_jumps", + help="module-jumps.txt (\"Mod a -> b\" per updated Marketplace module): a column gone or values changed in " + "such a module is the vendor's update, PASS* with the jump named, not FAIL") a = ap.parse_args() B, A = json.load(open(a.before)), json.load(open(a.after)) + jumps = {} + if a.module_jumps and os.path.isfile(a.module_jumps): + for l in open(a.module_jumps): + m, _, v = l.strip().partition(" ") + if m and v: jumps[m] = v.strip() allow = [] if a.allow: for l in open(a.allow): @@ -102,10 +110,16 @@ def main(): rows = [] def add(check, verdict, detail): + # WM1 2026-11-17: MendixSSO 4.2 -> 10.6 dropped AuthRequest.Nonce and MxModelReflection 7 -> 9 rewrote its own rows; + # both read FAIL on the gate card though nothing of the app's data was lost. Only a vendor module's own attribute, + # and only when the run updated that module; rows lost stay FAIL whatever the module if verdict in ("FAIL", "REVIEW", "INVALID", "NEW_ENDPOINT"): for cpat, dpat, why in allow: if fnmatch.fnmatch(check, cpat) and (not dpat or fnmatch.fnmatch(detail, dpat)): verdict, detail = "PASS*", f"{detail} (accepted: {why})"; break + mod = check.split(".")[2] if check.startswith("db.attr.") else "" + if verdict == "FAIL" and mod in jumps and detail.startswith(("column gone", "values changed")): + verdict, detail = "PASS*", f"{detail} (noted: the {mod} update {jumps[mod]} changed its own attribute, not the app's data)" rows.append((verdict, check, detail)) for side, s in (("before", B), ("after", A)): @@ -153,7 +167,13 @@ def add(check, verdict, detail): if rb["nonnull"] == 0: continue if ra is None: add(cid, "FAIL" if rb["nonnull"] else "INFO", f"column gone ({rb['nonnull']} non-null values before)") elif ra["nonnull"] < rb["nonnull"]: add(cid, "FAIL", f"non-null {rb['nonnull']} -> {ra['nonnull']}" + (" (named users)" if anon and at in anb["attrs"] else hint if at.startswith("System.User.") else "")) - elif ra["md5"] != rb["md5"]: add(cid, "FAIL", f"values changed (non-null {rb['nonnull']} -> {ra['nonnull']})") + elif ra["md5"] != rb["md5"]: + nb_, na_ = (anb.get("noadmin") or {}).get(at), (ana.get("noadmin") or {}).get(at) + if nb_ and na_ and nb_ == na_: # snapshot.py: the same column without the local admin's row + add(cid, "PASS*", f"values changed (non-null {rb['nonnull']} -> {ra['nonnull']}) on the local admin row alone " + "(noted: runapp.py sets that user's password at every start, the toolkit's own login)") + else: + add(cid, "FAIL", f"values changed (non-null {rb['nonnull']} -> {ra['nonnull']})") else: add(cid, "PASS", f"{rb['nonnull']} values identical") for at in sorted(set(db_a["attributes"]) - set(db_b["attributes"])): add(f"db.attr.{at}", "INFO", f"new column, {db_a['attributes'][at]['nonnull']} non-null") diff --git a/upgrade-toolkit/bin/gate.sh b/upgrade-toolkit/bin/gate.sh index 2e79f52..6075619 100755 --- a/upgrade-toolkit/bin/gate.sh +++ b/upgrade-toolkit/bin/gate.sh @@ -293,7 +293,7 @@ cmd_rung() { build) CMD=(bash -c 'grep -q "BUILD SUCCEEDED" "$1" || exit 1; n=$(grep -c "^ERROR at" "$1"); [ "$n" = 0 ] || { echo "$n ERROR at lines"; exit 1; }' _ "$V_BUILD_LOG");; idcheck) CMD=("$PY" "$B/idcheck.py" "$V_OLD_MDA" "$V_NEW_MDA"); opt --approved "$V_APPROVED"; CMD+=(--md "$G/idcheck.md");; const) CMD=("$PY" "$B/constcheck.py" "$V_OLD_MDA" "$V_NEW_MDA"); opt --overrides "$V_CONST_OVERRIDES"; opt --allow "$V_CONST_ALLOW"; CMD+=(--md "$G/constants.md");; - data) CMD=("$PY" "$B/diff.py" "$V_BEFORE_JSON" "$V_AFTER_JSON"); opt --allow "$V_ALLOW"; opt --log-allow "$V_LOG_ALLOW"; CMD+=(--md "$G/diff.md");; + data) CMD=("$PY" "$B/diff.py" "$V_BEFORE_JSON" "$V_AFTER_JSON"); opt --allow "$V_ALLOW"; opt --log-allow "$V_LOG_ALLOW"; [ -f "$V_WORK/module-jumps.txt" ] && CMD+=(--module-jumps "$V_WORK/module-jumps.txt"); CMD+=(--md "$G/diff.md");; rest) CMD=("$PY" "$B/restcontract.py" "$V_REST_BEFORE" "$V_REST_AFTER");; logdiff) CMD=("$PY" "$B/logdiff.py" "$V_OLD_LOG" "$V_NEW_LOG"); opt --allow "$V_LOG_ALLOW"; CMD+=(--md "$G/logdiff.md");; smoke) diff --git a/upgrade-toolkit/bin/snapshot.py b/upgrade-toolkit/bin/snapshot.py index 6522cc2..6ae4b4e 100755 --- a/upgrade-toolkit/bin/snapshot.py +++ b/upgrade-toolkit/bin/snapshot.py @@ -94,6 +94,9 @@ def db_snapshot(psql): return out +ADMIN = re.sub(r"[^\w.@-]", "", os.environ.get("UPG_ADMIN_USER", "MxAdmin")) + + def anonymous_snapshot(psql, ents, attrs, cols): """{"users": n anonymous rows, "total": n, "attrs": {"System.User.X": {...}}, "assoc": {name: {...}}} for the rows of System.User that are not anonymous; {} when System.User or IsAnonymous cannot be found.""" @@ -107,12 +110,20 @@ def anonymous_snapshot(psql, ents, attrs, cols): named = f'coalesce("{acol}", false) = false' n_anon = int(q(psql, f'select count(*) from "{ut}" where not ({named})')[0][0]) n_total = int(q(psql, f'select count(*) from "{ut}"')[0][0]) - res = {"users": n_anon, "total": n_total, "attrs": {}, "assoc": {}} + res = {"users": n_anon, "total": n_total, "attrs": {}, "assoc": {}, "noadmin": {}} + # WM1 2026-11-17: runapp.py sets the local admin's password at every start (create_admin_user), which changes that + # one row's Password and changedDate; the same figures without that row let diff.py tell it from a real change + ncol = next((c for an, c in attrs.get(uid, []) if an == "Name"), None) + ncol = ncol if ncol in cols[ut] else None for an, c in attrs.get(uid, []): if c in cols[ut]: n, h = q(psql, f'select count("{c}"), md5(coalesce(string_agg(coalesce("{c}"::text,\'~\'), \'|\' order by id),\'\')) ' f'from "{ut}" where {named}')[0] res["attrs"][f"System.User.{an}"] = {"nonnull": int(n), "md5": h} + if ncol: + n, h = q(psql, f'select count("{c}"), md5(coalesce(string_agg(coalesce("{c}"::text,\'~\'), \'|\' order by id),\'\')) ' + f'from "{ut}" where {named} and coalesce("{ncol}", \'\') <> \'{ADMIN}\'')[0] + res["noadmin"][f"System.User.{an}"] = {"nonnull": int(n), "md5": h} for name, table, pcol, ccol, peid in q(psql, 'select association_name, table_name, parent_column_name, child_column_name, parent_entity_id ' 'from "mendixsystem$association"'): if peid != uid or table not in cols or ccol not in cols[table]: diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index b0dd9a1..383403a 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -320,6 +320,29 @@ EOF ctl diff-anonymous-users-grouped-pass 0 'PASS\* \| db\.known\.anonymous_users \| 18 anonymous System\.User rows gone \(18 -> 0\)' '!FAIL \| db\.(attr|assoc|entity)\.System\.User' 'FAIL 0 \| REVIEW 0' -- python3 $B/diff.py $T/anon-ok-b.json $T/anon-ok-a.json --allow $FX/allow/narrow.txt ctl diff-anonymous-named-lost-catch 1 'FAIL \| db\.attr\.System\.User\.Name \| non-null 3 -> 2 \(named users\)' 'FAIL \| db\.entity\.System\.User \| named user rows 3 -> 2' -- python3 $B/diff.py $T/anon-named-lost-b.json $T/anon-named-lost-a.json --allow $FX/allow/narrow.txt ctl diff-anonymous-old-snapshot-catch 1 'FAIL \| db\.entity\.System\.User \| rows 21 -> 3 \(if these are Anonymous_\* users' '!known\.anonymous' -- python3 $B/diff.py $T/anon-ok-old-b.json $T/anon-ok-old-a.json --allow $FX/allow/narrow.txt +# WM1 2026-11-17: the gate card said "Data compare: Failed" on 4 rows that were no loss: a vendor module's own attribute +# changed by its update (MendixSSO, MxModelReflection) and the local admin's row (runapp.py sets its password at start). +python3 - "$T" <<'EOF' +import json, sys +t = sys.argv[1] +for nm in ("jump", "jump-rows", "admin", "admin-other"): + b = json.load(open(f"{t}/sb.json")); a = json.load(open(f"{t}/sa.json")) + for d, h in ((b, "v1"), (a, "v2")): + d["db"]["entities"]["Vend.Thing"] = 5 if d is b or nm != "jump-rows" else 3 + d["db"]["attributes"]["Vend.Thing.Field"] = {"nonnull": 5, "md5": h if nm.startswith("jump") else "same"} + d["db"]["entities"]["System.User"] = 3 + d["db"]["attributes"]["System.User.changedDate"] = {"nonnull": 3, "md5": "c" + h if nm.startswith("admin") else "same"} + rest = "same" if nm == "admin" or d is b else "other" + d["db"]["anonymous"] = {"users": 0, "total": 3, "attrs": {"System.User.changedDate": d["db"]["attributes"]["System.User.changedDate"]}, + "assoc": {}, "noadmin": {"System.User.changedDate": {"nonnull": 2, "md5": rest}}} + json.dump(b, open(f"{t}/{nm}-b.json", "w")); json.dump(a, open(f"{t}/{nm}-a.json", "w")) +open(f"{t}/jumps.txt", "w").write("Vend 1.0.0 -> 2.0.0\n") +EOF +ctl diff-module-jump-attr-noted-pass 0 'PASS\* \| db\.attr\.Vend\.Thing\.Field \| values changed .*the Vend update 1\.0\.0 -> 2\.0\.0' 'FAIL 0 \| REVIEW 0' -- python3 $B/diff.py $T/jump-b.json $T/jump-a.json --allow $FX/allow/narrow.txt --module-jumps $T/jumps.txt +ctl diff-module-jump-attr-unlisted-catch 1 'FAIL \| db\.attr\.Vend\.Thing\.Field \| values changed' -- python3 $B/diff.py $T/jump-b.json $T/jump-a.json --allow $FX/allow/narrow.txt +ctl diff-module-jump-rows-lost-catch 1 'FAIL \| db\.entity\.Vend\.Thing \| rows 5 -> 3' -- python3 $B/diff.py $T/jump-rows-b.json $T/jump-rows-a.json --allow $FX/allow/narrow.txt --module-jumps $T/jumps.txt +ctl diff-admin-row-noted-pass 0 'PASS\* \| db\.attr\.System\.User\.changedDate \| .*local admin row alone' 'FAIL 0 \| REVIEW 0' -- python3 $B/diff.py $T/admin-b.json $T/admin-a.json --allow $FX/allow/narrow.txt +ctl diff-admin-other-user-catch 1 'FAIL \| db\.attr\.System\.User\.changedDate \| values changed' '!local admin' -- python3 $B/diff.py $T/admin-other-b.json $T/admin-other-a.json --allow $FX/allow/narrow.txt ctl diff-owner-catch 1 'FAIL \| db\.identity\.assoc:System\.owner' -- python3 $B/diff.py $T/sb.json $T/s-owner-lost.json --allow $FX/allow/narrow.txt ctl diff-owner-pass 0 '!System\.owner' 'FAIL 0 \| REVIEW 0' -- python3 $B/diff.py $T/sb.json $T/s-owner-order.json --allow $FX/allow/narrow.txt ctl diff-new-endpoint-catch 2 'NEW_ENDPOINT \| http\./rest-doc/notify/openapi\.json' -- python3 $B/diff.py $T/sb.json $T/sa.json --allow $FX/allow/email-only.txt @@ -514,6 +537,10 @@ ctl datamove-bad-plan-catch 2 "not 'Src.Entity.Attr -> Dst.Entity.Attr'" -- $DM # WM1 rerun 7: baseline ran microflow steps, the after walk skipped them: 4 false REGRESSIONs ctl pipeline-smoke-mf-follows-baseline-pass 0 '^--microflows$' -- bash -c "WORK=\$(mktemp -d); echo '{\"landed\": \"microflow ran\"}' > \$WORK/smoke-before.json; eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf" ctl pipeline-smoke-mf-off-without-baseline-pass 0 '!microflows' -- bash -c "WORK=\$(mktemp -d); eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf; echo done" +# cold WM1 #12 and cold CMM: without ~/.config/upgrade-toolkit/local.env the scanner lookup's sed failed inside $(...) and the ERR +# trap printed "unexpected failure (exit 1) at line 624 in step 'triage'" on every machine but the author's; the run went on +ctl pipeline-scanner-lookup-no-local-env-pass 0 '^EA=\[\]$' '!unexpected failure' -- bash -c "set -Eeuo pipefail; trap 'echo unexpected failure \$? >&2; exit 4' ERR; HOME=\$(mktemp -d); unset EA UPG_SCANNER; eval \"f() { \$(grep 'upgrade-toolkit/local.env\" 2>' \${PIPE:-$B/cloud/upgrade-pipeline.sh} | grep -v '^ *#'); echo \\\"EA=[\\\$EA]\\\"; }\"; f" +ctl pipeline-scanner-lookup-local-env-pass 0 '^EA=\[/opt/scan\]$' -- bash -c "set -Eeuo pipefail; trap 'echo unexpected failure \$? >&2; exit 4' ERR; HOME=\$(mktemp -d); mkdir -p \$HOME/.config/upgrade-toolkit; echo UPG_SCANNER=/opt/scan > \$HOME/.config/upgrade-toolkit/local.env; unset EA UPG_SCANNER; eval \"f() { \$(grep 'upgrade-toolkit/local.env\" 2>' \${PIPE:-$B/cloud/upgrade-pipeline.sh} | grep -v '^ *#'); echo \\\"EA=[\\\$EA]\\\"; }\"; f" # WM1 2026-11-17: Atlas_Core 4's new module roles unmapped; every Atlas page failed for security reasons (smoke walk only) MR="bash $B/../controls/modroles_control.sh" ctl modroles-grant-by-name-pass 0 '^mdl: ALTER USER ROLE Administrator ADD MODULE ROLES \(Atlas_Core.Administrator\);' '^mdl: ALTER USER ROLE User ADD MODULE ROLES \(Atlas_Core.User\);' '^modroles: 3 grant' -- $MR @@ -1577,6 +1604,31 @@ mkdir -p $V/papp; touch $V/papp/App.mpr # F2: an --app-dir outside the run folder is refused up front (exit 3), not left for mxbuild to fail on at baseline mkdir -p $V/outside/oapp; touch $V/outside/oapp/App.mpr ctl pipeline-appdir-outside-catch 3 'is outside the run folder' '!step ' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/outside/oapp --source 10.24.26.1 --target 11.15.0.1 --work $V/pw2/work +# cold CMM 2026-11-17: two run folders both named "run" shared the database names upg_run_*; the second app's baseline said +# "reusing existing frozen database upg_run_frozen" and started on the first app's data. Names now differ per run folder, +# and a frozen database whose owner stamp names another run (or that has no stamp and no ledger line here) is refused +mkdir -p $V/dbn1/run/app $V/dbn2/run/app; touch $V/dbn1/run/app/App.mpr $V/dbn2/run/app/App.mpr +ctl pipeline-db-name-per-run-catch 0 '^2$' -- bash -c "for d in dbn1 dbn2; do bash \${PIPE:-$B/cloud/upgrade-pipeline.sh} --app-dir $V/\$d/run/app --source 10.24.26.1 --target 11.15.0.1 --work $V/\$d/run/work --only baseline --dry-run 2>&1 | sed -n 's/.*createdb -U mendix -T \([a-z0-9_]*\) .*_frozen\$/\1/p'; done | sort -u | wc -l | tr -d ' '" +ctl pipeline-db-name-kept-pass 0 '^upg_keep$' -- bash -c "mkdir -p $V/dbn3/run/app $V/dbn3/run/work/state; touch $V/dbn3/run/app/App.mpr; echo upg_keep > $V/dbn3/run/work/state/db-prefix; bash \${PIPE:-$B/cloud/upgrade-pipeline.sh} --app-dir $V/dbn3/run/app --source 10.24.26.1 --target 11.15.0.1 --work $V/dbn3/run/work --only baseline --dry-run 2>&1 | sed -n 's/.*createdb -U mendix -T \([a-z0-9_]*\) .*_frozen\$/\1/p'" +mkdir -p $V/fdk; cat > $V/fdk/docker <<'FAKE' +#!/bin/bash +case "$*" in *shobj_description*) cat "$FAKE_OWNER" 2>/dev/null;; *COMMENT*) echo "stamped: $*" >> "$FAKE_LOG";; esac +FAKE +chmod +x $V/fdk/docker +cat > $V/fdk/fo.sh <<'FO' +# fo.sh WORK: frozen_owner from the pipeline (or $PIPE), run folder /r/cmm, database upg_run_frozen +set -u; PGC=mig-pg; X=upg_run; X_FROZEN=upg_run_frozen; RUNP=/r/cmm; WORK=$1 +must() { "$@"; }; die4() { echo "DIE: $*"; exit 4; } +eval "$(sed -n '/^frozen_stamp()/,/^}/p' "$PIPE")" +frozen_owner && echo owner-ok +FO +FO="env PIPE=${PIPE:-$B/cloud/upgrade-pipeline.sh} bash $V/fdk/fo.sh" +echo 'upgrade-toolkit run /r/wm1' > $V/fdk/other; echo 'upgrade-toolkit run /r/cmm' > $V/fdk/own; : > $V/fdk/none +mkdir -p $V/fdk/w-legacy $V/fdk/w-new; echo 'upg_run_base snapshot before 2026-11-15T23:33:59' > $V/fdk/w-legacy/db-ledger.txt +ctl pipeline-frozen-other-run-catch 4 '^DIE: frozen database upg_run_frozen exists but this run did not make it \(upgrade-toolkit run /r/wm1' '!owner-ok' -- env PATH=$V/fdk:"$PATH" FAKE_OWNER=$V/fdk/other FAKE_LOG=$V/fdk/log $FO $V/fdk/w-new +ctl pipeline-frozen-no-stamp-no-ledger-catch 4 'no owner stamp' -- env PATH=$V/fdk:"$PATH" FAKE_OWNER=$V/fdk/none FAKE_LOG=$V/fdk/log $FO $V/fdk/w-new +ctl pipeline-frozen-own-pass 0 '^owner-ok$' -- env PATH=$V/fdk:"$PATH" FAKE_OWNER=$V/fdk/own FAKE_LOG=$V/fdk/log $FO $V/fdk/w-new +ctl pipeline-frozen-legacy-adopted-pass 0 '^owner-ok$' -- bash -c "rm -f $V/fdk/log2; env PATH=$V/fdk:\"\$PATH\" FAKE_OWNER=$V/fdk/none FAKE_LOG=$V/fdk/log2 $FO $V/fdk/w-legacy && grep -q 'COMMENT ON DATABASE upg_run_frozen IS .upgrade-toolkit run /r/cmm' $V/fdk/log2" # F1: force-ui.txt reaches stepupdate as --force-ui (else --unattended keeps every edited module), replay/.mdl runs after the updates mkdir -p $V/pw3/replay; printf '# update anyway, replay\nMendixSSO\nAtlas_Core\n' > $V/pw3/force-ui.txt; echo 'alter entity X.Y drop validation rule;' > $V/pw3/replay/MendixSSO.mdl; : > $V/pw3/replay/Atlas_Core.patch ctl pipeline-dry-forceui-replay-pass 0 'stepupdate\.sh .* --unattended --force-ui MendixSSO.?,Atlas_Core --saved-edits .*/pw3/saved-edits' 'exec .*/pw3/replay/MendixSSO\.mdl -p ' 'git -C .*/papp apply --check .*/pw3/replay/Atlas_Core\.patch' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pw3 --only update --dry-run From 5fd444b6a034dd8f886b3e67062c64461cef27f4 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 12:31:38 +0800 Subject: [PATCH 34/59] baseline: load the package's demo data (data-snapshot.zip) with seed_snapshot.py, not 5 synthetic rows per entity Cold run 2026-11-17 (Marketplace template, 9.24.2): the package carried 9,235 demo rows in 139 tables and bin/seed_snapshot.py already existed (steps/baseline.md item 1 prescribes it), yet fetch said "hsql data present, not replayed (no tool)" and baseline seeded seed_pg.py rows. Walks and journeys then judge near-empty pages. - seed_demo: replays data-snapshot.zip into X after the first start; nothing loaded (UNMAPPED) -> seed_pg.py as before, with a note; the file contents go to WORK/demo-files - runapp.py copies UPG_DATA_FILES into data/files at every start (runapp wipes the run folder) - fetch no longer prints the TODO when a data-snapshot.zip is there Controls: 4 new; 3 fail on the old pipeline (1 guard, no snapshot -> seed_pg.py, holds on both). Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 27 ++++++++++++++++--- upgrade-toolkit/bin/runapp.py | 3 +++ upgrade-toolkit/controls/run-controls.sh | 21 +++++++++++++++ upgrade-toolkit/steps/baseline.md | 4 ++- 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index d4763e4..0cf0249 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -349,6 +349,7 @@ psql_str() { echo "docker exec -i $PGC psql -U mendix -d $1"; } login_path() { local p="${UPG_LOGIN_PATH:-}"; [ -n "$p" ] || p=$(head -n1 "$WORK/login-path.txt" 2> /dev/null || true); echo "${p:-/login.html}"; } runenv() { # env words for runapp.py RUNENV=("WORKDIR=$WORK" "UPG_TOOLS=$UPG_TOOLS" "UPG_SESSION_TIMEOUT=30000" "PORT_OFF=$PORT_OFF") + [ ! -d "$WORK/demo-files" ] || RUNENV+=("UPG_DATA_FILES=$WORK/demo-files") if [ -s "$WORK/env-values.json" ]; then RUNENV+=("CONST_OVERRIDES=$(cat "$WORK/env-values.json")"); fi # heal reruns: the apimock stand-in's and the NATS broker's constant overrides, merged under the person's env-values.json (theirs win) local src=() @@ -601,12 +602,30 @@ step_fetch() { fi } -# TODO (no script exists): demo data inside the package (HSQLDB data-snapshot.zip) is NOT replayed into Postgres. -# The journals (T1) seeded synthetic rows with seed_pg.py instead. A person decides whether real data is needed. +# Demo data inside the package (HSQLDB data-snapshot.zip): baseline replays it into Postgres with seed_snapshot.py +# (seed_demo). A package with only loose *.script files has no route yet: those runs get seed_pg.py rows. todo_hsql_replay() { - note "TODO todo_hsql_replay: the package carries HSQLDB demo data; it was NOT replayed into Postgres (no tool), the data baseline uses seed_pg.py rows" + if [ -n "$(demo_snapshot)" ]; then say " hsql demo data present: baseline loads it (seed_snapshot.py)"; return 0; fi + note "TODO todo_hsql_replay: the package carries HSQLDB *.script data but no data-snapshot.zip; it was NOT replayed, the data baseline uses seed_pg.py rows" say "TODO hsql data present, not replayed (see $WORK/state/notes.txt)" } +demo_snapshot() { [ -n "${APP:-}" ] && find "$APP" -maxdepth 2 -name 'data-snapshot.zip' -print -quit 2> /dev/null || true; } +# seed_demo: load the package's demo rows into $X (the old app made the schema). 1 = no snapshot or nothing loaded: +# the caller seeds synthetic rows. The file contents go to $WORK/demo-files, which runapp.py copies into every start. +seed_demo() { + local snap rc=0 + snap=$(demo_snapshot); [ -n "$snap" ] || return 1 + xo "$WORK/logs/seed-snapshot.txt" python3 "$B/seed_snapshot.py" "$snap" --psql "$(psql_str "$X")" || rc=$? + if [ "$rc" != 0 ] && ! grep -q 'the load was applied' "$WORK/logs/seed-snapshot.txt" 2> /dev/null; then + note "demo data NOT loaded (seed_snapshot.py exit $rc, UNMAPPED in logs/seed-snapshot.txt): synthetic seed_pg.py rows instead" + return 1 + fi + [ "$rc" = 0 ] || note "demo data loaded, but some table counts differ (logs/seed-snapshot.txt)" + [ "$DRY" = 1 ] || { rm -rf "$WORK/demo-files"; mkdir -p "$WORK/demo-files" + unzip -qo "$snap" 'data/files/*' -d "$WORK/demo-files/x" 2> /dev/null && mv "$WORK/demo-files/x/data/files"/* "$WORK/demo-files/" 2> /dev/null; rm -rf "$WORK/demo-files/x"; } + note "demo data loaded from $(basename "$snap") ($(grep -o 'applied: [0-9]* rows in [0-9]* tables' "$WORK/logs/seed-snapshot.txt" 2> /dev/null || echo 'counts in logs/seed-snapshot.txt'))" + return 0 +} # triage: versions, catalog, localedits, xmod, scanner, inventory, jarclash, impact map, release impact. # Judgment: the module plan ($WORK/plan.txt, steps/triage.md item 3) has no script: exit 13 when absent. @@ -701,7 +720,7 @@ step_baseline() { app_up old "$old" "$SOURCE" "$X" || die4 "old app did not start on $X (log: $LOG, $WORK/logs/runtime-old.log)" app_down old if [ -z "$SEED_FROM" ]; then - must python3 "$B/seed_pg.py" --psql "$(psql_str "$X")" --per-entity 5 + seed_demo || must python3 "$B/seed_pg.py" --psql "$(psql_str "$X")" --per-entity 5 app_up old "$old" "$SOURCE" "$X" || die4 "old app did not start after the seed (log: $LOG)" app_down old fi diff --git a/upgrade-toolkit/bin/runapp.py b/upgrade-toolkit/bin/runapp.py index 472f51c..e62d5b2 100755 --- a/upgrade-toolkit/bin/runapp.py +++ b/upgrade-toolkit/bin/runapp.py @@ -378,6 +378,9 @@ def start(name, mda, rtv, db, port_off=0, login_override=None, harness=None): zipfile.ZipFile(mdap).extractall(base) for d in ("data/files", "data/tmp", "data/model-upload", "data/database"): (base / d).mkdir(parents=True, exist_ok=True) + df = os.environ.get("UPG_DATA_FILES") # the package's demo file contents (data-snapshot.zip data/files), back after the wipe + if df and Path(df).is_dir(): + shutil.copytree(df, base / "data/files", dirs_exist_ok=True) note = apply_login_override(base, login_override or os.environ.get("UPG_LOGIN_OVERRIDE", "auto"), name) if note: print("runapp: " + note) diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 383403a..82e2ea3 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -1629,6 +1629,27 @@ ctl pipeline-frozen-other-run-catch 4 '^DIE: frozen database upg_run_frozen exis ctl pipeline-frozen-no-stamp-no-ledger-catch 4 'no owner stamp' -- env PATH=$V/fdk:"$PATH" FAKE_OWNER=$V/fdk/none FAKE_LOG=$V/fdk/log $FO $V/fdk/w-new ctl pipeline-frozen-own-pass 0 '^owner-ok$' -- env PATH=$V/fdk:"$PATH" FAKE_OWNER=$V/fdk/own FAKE_LOG=$V/fdk/log $FO $V/fdk/w-new ctl pipeline-frozen-legacy-adopted-pass 0 '^owner-ok$' -- bash -c "rm -f $V/fdk/log2; env PATH=$V/fdk:\"\$PATH\" FAKE_OWNER=$V/fdk/none FAKE_LOG=$V/fdk/log2 $FO $V/fdk/w-legacy && grep -q 'COMMENT ON DATABASE upg_run_frozen IS .upgrade-toolkit run /r/cmm' $V/fdk/log2" +# cold CMM 2026-11-17: the package carried 9,235 demo rows (data-snapshot.zip) and seed_snapshot.py existed, yet baseline +# seeded 5 synthetic rows per entity ("hsql data present, not replayed (no tool)"): walks then judge near-empty pages. +mkdir -p $V/demo/run/app $V/nodemo/run/app; touch $V/demo/run/app/App.mpr $V/nodemo/run/app/App.mpr +( cd $V/demo && mkdir -p data/database/hsqldb/default && touch data/database/hsqldb/default/default.script && zip -qr run/app/data-snapshot.zip data && rm -rf data ) +ctl pipeline-demo-data-seeded-catch 0 'seed_snapshot\.py .*data-snapshot\.zip --psql' '!seed_pg\.py' -- bash -c "bash \${PIPE:-$B/cloud/upgrade-pipeline.sh} --app-dir $V/demo/run/app --source 10.24.26.1 --target 11.15.0.1 --work $V/demo/run/work --only baseline --dry-run 2>&1" +ctl pipeline-demo-data-none-seed-pg-pass 0 'seed_pg\.py' '!seed_snapshot' -- bash -c "bash \${PIPE:-$B/cloud/upgrade-pipeline.sh} --app-dir $V/nodemo/run/app --source 10.24.26.1 --target 11.15.0.1 --work $V/nodemo/run/work --only baseline --dry-run 2>&1" +# seed_demo: UNMAPPED with nothing loaded -> 1 (the caller seeds synthetic rows) and a note; a clean load -> 0 and the files +mkdir -p $V/sdm/bin $V/sdm/w/logs $V/sdm/w/state +printf '#!/bin/sh\necho "$SD_OUT"; exit $SD_RC\n' > $V/sdm/bin/seed_snapshot.py +cat > $V/sdm/sd.sh <<'SDEOF' +set -Eeuo pipefail +eval "$(sed -n '/^demo_snapshot()/p; /^seed_demo()/,/^}/p; /^x()/,/^}/p; /^xo()/,/^}/p; /^note()/p; /^psql_str()/p' "$PIPE")" +quoted() { printf ' %q' "$@"; } +python3() { sh "$@"; } +B=$SDB; WORK=$SDW; APP=$SDA; X=upg_t; PGC=pg; DRY=0; STEP=baseline; LOG=$SDW/logs/b.log +rc=0; seed_demo || rc=$?; echo "rc=$rc"; cat "$WORK/state/notes.txt" 2> /dev/null; ls "$WORK/demo-files" 2> /dev/null || true +SDEOF +( cd $V/sdm && mkdir -p data/files/00 && echo img > data/files/00/u1 && zip -qr app.zip data && rm -rf data && mkdir app && mv app.zip app/data-snapshot.zip ) +SD="env PIPE=${PIPE:-$B/cloud/upgrade-pipeline.sh} SDB=$V/sdm/bin SDW=$V/sdm/w SDA=$V/sdm/app" +ctl pipeline-seed-demo-unmapped-falls-back-catch 0 '^rc=1$' 'demo data NOT loaded' -- $SD SD_RC=1 SD_OUT='nothing was loaded' bash $V/sdm/sd.sh +ctl pipeline-seed-demo-loaded-pass 0 '^rc=0$' 'demo data loaded from data-snapshot.zip \(applied: 9 rows in 2 tables\)' '^00$' -- bash -c "rm -f $V/sdm/w/state/notes.txt; $SD SD_RC=0 SD_OUT='applied: 9 rows in 2 tables; every table' bash $V/sdm/sd.sh" # F1: force-ui.txt reaches stepupdate as --force-ui (else --unattended keeps every edited module), replay/.mdl runs after the updates mkdir -p $V/pw3/replay; printf '# update anyway, replay\nMendixSSO\nAtlas_Core\n' > $V/pw3/force-ui.txt; echo 'alter entity X.Y drop validation rule;' > $V/pw3/replay/MendixSSO.mdl; : > $V/pw3/replay/Atlas_Core.patch ctl pipeline-dry-forceui-replay-pass 0 'stepupdate\.sh .* --unattended --force-ui MendixSSO.?,Atlas_Core --saved-edits .*/pw3/saved-edits' 'exec .*/pw3/replay/MendixSSO\.mdl -p ' 'git -C .*/papp apply --check .*/pw3/replay/Atlas_Core\.patch' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pw3 --only update --dry-run diff --git a/upgrade-toolkit/steps/baseline.md b/upgrade-toolkit/steps/baseline.md index 09d1e9d..dbef732 100644 --- a/upgrade-toolkit/steps/baseline.md +++ b/upgrade-toolkit/steps/baseline.md @@ -37,7 +37,9 @@ copy's converted model. the script and log in order, remaps ids by entity GUID, matches System.User / UserRole by name, and lists every table or column it cannot map under UNMAPPED with exit 1 (`--skip TABLE` waives one, reported). Start the app again, then freeze. File contents (`data/files/*` in the zip) are copied into the app's `data/files` after EVERY start (runapp.py - wipes `run//`, T5-5). Open: autonumber counters are not advanced. + wipes `run//`, T5-5). Open: autonumber counters are not advanced. `upgrade-pipeline.sh` baseline does all of + this itself (seed_demo, log in `logs/seed-snapshot.txt`; the file contents go to `$W/demo-files`, which runapp.py + copies in at every start through UPG_DATA_FILES); only when nothing loads does it fall back to seed_pg.py, with a note. 2. Start the OLD app on X: `WORKDIR=$W UPG_TOOLS=/tools python3 $B/runapp.py start old X` (Java image by major: 9 to temurin 11, 10 and every 11.x to temurin 21, also for an 11.x to 11.y run, N1-14; `CONST_OVERRIDES` = the environment's values, the SAME for old and new: JSON text or the path of a JSON file, B4T2-12). Slow start (E5b-13): a start above a few minutes is usually a module calling home at From 3604c252214d1c64e48324bdadbb9126e5eea835 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 12:51:15 +0800 Subject: [PATCH 35/59] smoke walk: a popup opened from the menu counts as landed, and microflow steps are walked whenever the plan has them Cold run 2026-11-17 (Marketplace template, 9.24.2, Atlas 3), baseline walk on the OLD app: - "Switch user" FAILed in all 5 roles ("still on Dashboard", 15 s each) while the screenshot shows the Select Demo User popup open: Atlas styles .mx-window position:fixed, whose offsetParent is always null. landedWhere now also accepts a window with client rects - 13 of 23 steps were microflows and SKIPPED; two roles were judged on no page at all. The "SKIPPED microflow steps" item fired only when the walk was INVALID (exit > 1), so with exit 1 nothing said so. smoke_mf now passes --microflows whenever the plan has a microflow step (each walk runs on its own DB copy); UPG_SMOKE_MICROFLOWS=0 turns it off and leaves a note Controls: 3 new, 2 rewritten (awk extraction of the multi-line smoke_mf); 3 fail on the old code (popup, plan microflow, opt-out), the guards hold on both. Group: 31/31. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 17 +++++++++++------ upgrade-toolkit/bin/smokewalk.js | 3 ++- upgrade-toolkit/controls/run-controls.sh | 11 ++++++++--- .../controls/smokewalk_predicates.js | 6 ++++++ upgrade-toolkit/steps/baseline.md | 3 ++- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 0cf0249..3157017 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -18,9 +18,10 @@ # heal (steps/heal.md): classifies every failing row, heals what belongs to the test (start noise, selection-gated buttons, a runner # that did not run, outside calls via apimock, known harness log lines) and reruns only the affected journeys, at most 2 rounds. # UPG_HEAL=0 turns it off (journeys then stops on a failure as before). Resume it with --from heal. -# UPG_SMOKE_MICROFLOWS=1: the smoke walks (basewalk, afterwalk) also follow microflow homes and menu items (smokewalk.js -# --microflows). Safe: each walk runs on its own copy of the frozen DB. Needed when a role reaches every page through a -# microflow (WM1: the User home and both menu items), else that role's walk is all SKIPPED and INVALID. +# UPG_SMOKE_MICROFLOWS: the smoke walks (basewalk, afterwalk) follow microflow homes and menu items (smokewalk.js +# --microflows) whenever the plan has them; UPG_SMOKE_MICROFLOWS=0 turns that off. Safe: each walk runs on its own copy of +# the frozen DB. Needed when a role reaches its pages through microflows (WM1: the User home and both menu items; cold +# CMM: 13 of 23 steps, two roles judged on no page at all, and the skip was reported only when the walk was INVALID). # Each finished step writes $WORK/state/.done and is skipped on the next run. Logs: $WORK/logs/.log. # Judgment stops write $WORK/NEEDS-JUDGMENT.md. Exit codes: # 0 gate PASS, nothing left for a person @@ -484,7 +485,11 @@ json.dump({r: {"user": "smoke_" + r.lower(), "pass": pw} for r in roles.split(", } # UPG_SMOKE_MICROFLOWS (header); the after walk also follows the baseline: a baseline that ran microflow steps compared # with a walk that skipped them gave 4 false REGRESSIONs on WM1 (rerun 7, env not set on the rerun) -smoke_mf() { if [ "${UPG_SMOKE_MICROFLOWS:-0}" = 1 ] || grep -q '"landed": *"microflow ran"' "$WORK/smoke-before.json" 2> /dev/null; then echo --microflows; fi; } +smoke_mf() { + [ "${UPG_SMOKE_MICROFLOWS:-}" != 0 ] || return 0 + if [ "${UPG_SMOKE_MICROFLOWS:-}" = 1 ] || grep -q '"landed": *"microflow ran"' "$WORK/smoke-before.json" 2> /dev/null || + grep -q '"kind": *"microflow"' "$WORK/plan.json" 2> /dev/null; then echo --microflows; fi +} user_args() { # user_args ROLES -> sets UARGS for smokewalk.js UARGS=(); local r IFS=',' read -ra _rl <<< "$1" @@ -788,8 +793,8 @@ step_basewalk() { app_down oldwalk mark_used "$db" walk [ "$rc" -le 1 ] || { nj_file 11 "Smoke walk on the OLD app is INVALID (exit $rc)" "$LOG" 30 'INVALID|FAIL|credential'; } - [ "$rc" -le 1 ] || [ "${UPG_SMOKE_MICROFLOWS:-0}" = 1 ] || ! grep -q 'microflow (use --microflows)' "$LOG" 2> /dev/null || - nj_add 11 "The smoke walk SKIPPED microflow steps" "A role that reaches its pages only through microflows (home, menu items) proves nothing without them. Rerun with UPG_SMOKE_MICROFLOWS=1: the walk runs on its own copy of the frozen DB, so their side effects stay there." + ! grep -q 'microflow (use --microflows)' "$LOG" 2> /dev/null || + note "the smoke walk SKIPPED microflow steps (UPG_SMOKE_MICROFLOWS=0): roles that reach pages only through them are judged on less; unset it to walk them on the walk's own DB copy" [ "$rc" != 1 ] || note "smoke walk on the old app has FAIL steps (baseline defects, listed apart): $WORK/smoke-before.json" # always-on journeys # BT-4: baseline-evidence.txt (a NO-SIGNOUT line for an app with no sign-out UI) is read here as well as at the gate diff --git a/upgrade-toolkit/bin/smokewalk.js b/upgrade-toolkit/bin/smokewalk.js index 3c60203..ac6e0d7 100755 --- a/upgrade-toolkit/bin/smokewalk.js +++ b/upgrade-toolkit/bin/smokewalk.js @@ -131,7 +131,8 @@ function landedWhere({ want }) { else { const h = history.state, i = h && h.pageName ? h.pageName.indexOf('.') : -1; if (i > 0 && h.params && h.params.location === 'content') p = h.pageName.slice(0, i) + '/' + h.pageName.slice(i + 1) + '.page.xml'; } if (p === want) return 'content'; - const win = [...document.querySelectorAll('.mx-window')].find(w => w.offsetParent !== null); + // cold CMM: Atlas styles .mx-window position:fixed, whose offsetParent is always null: every popup menu item FAILed + const win = [...document.querySelectorAll('.mx-window')].find(w => w.offsetParent !== null || (w.getClientRects && w.getClientRects().length > 0)); if (win) return 'popup'; return false; } catch (e) { return false; } diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 82e2ea3..88c9ca4 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -535,8 +535,8 @@ ctl datamove-lost-catch 1 '^LOST .*: 5 before, 3 after$' -- $DM lost ctl datamove-count-fail-catch 1 '^UNCOUNTED .*count failed after$' -- $DM fail ctl datamove-bad-plan-catch 2 "not 'Src.Entity.Attr -> Dst.Entity.Attr'" -- $DM bad # WM1 rerun 7: baseline ran microflow steps, the after walk skipped them: 4 false REGRESSIONs -ctl pipeline-smoke-mf-follows-baseline-pass 0 '^--microflows$' -- bash -c "WORK=\$(mktemp -d); echo '{\"landed\": \"microflow ran\"}' > \$WORK/smoke-before.json; eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf" -ctl pipeline-smoke-mf-off-without-baseline-pass 0 '!microflows' -- bash -c "WORK=\$(mktemp -d); eval \"\$(grep '^smoke_mf()' $B/cloud/upgrade-pipeline.sh)\"; smoke_mf; echo done" +ctl pipeline-smoke-mf-follows-baseline-pass 0 '^--microflows$' -- bash -c "WORK=\$(mktemp -d); echo '{\"landed\": \"microflow ran\"}' > \$WORK/smoke-before.json; eval \"\$(awk '/^smoke_mf\(\)/{f=1} f{print} f&&/^}|; }\$/{exit}' \${PIPE:-$B/cloud/upgrade-pipeline.sh})\"; smoke_mf" +ctl pipeline-smoke-mf-off-without-baseline-pass 0 '!microflows' -- bash -c "WORK=\$(mktemp -d); eval \"\$(awk '/^smoke_mf\(\)/{f=1} f{print} f&&/^}|; }\$/{exit}' \${PIPE:-$B/cloud/upgrade-pipeline.sh})\"; smoke_mf; echo done" # cold WM1 #12 and cold CMM: without ~/.config/upgrade-toolkit/local.env the scanner lookup's sed failed inside $(...) and the ERR # trap printed "unexpected failure (exit 1) at line 624 in step 'triage'" on every machine but the author's; the run went on ctl pipeline-scanner-lookup-no-local-env-pass 0 '^EA=\[\]$' '!unexpected failure' -- bash -c "set -Eeuo pipefail; trap 'echo unexpected failure \$? >&2; exit 4' ERR; HOME=\$(mktemp -d); unset EA UPG_SCANNER; eval \"f() { \$(grep 'upgrade-toolkit/local.env\" 2>' \${PIPE:-$B/cloud/upgrade-pipeline.sh} | grep -v '^ *#'); echo \\\"EA=[\\\$EA]\\\"; }\"; f" @@ -976,6 +976,8 @@ if command -v node > /dev/null; then ctl smokewalk-predicates 0 '^ok +loggedIn w # R1: the React client has no mx.session.getUserName / mx.ui.getContentForm; an anonymous session is not logged in, a content page maps to its Dojo path ctl smokewalk-react-catch 0 '^ok +react loggedIn when anonymous' '^ok +react contentPath without page' '^ok +react landedWhere on a popup state' -- node $here/smokewalk_predicates.js ctl smokewalk-react-pass 0 '^ok +react loggedIn when ready' '^ok +react homeReady when ready' '^ok +react contentPath maps to the page path' '^ok +react landedWhere on the wanted page' '^ok +react landedWhere on another page' -- node $here/smokewalk_predicates.js +# cold CMM 2026-11-17: Atlas styles .mx-window position:fixed (offsetParent null): every "Switch user" popup menu item FAILed +ctl smokewalk-popup-fixed-catch 0 '^ok +landedWhere fixed popup window' '^ok +landedWhere hidden popup window' -- node $here/smokewalk_predicates.js # T3-6: a step is judged by what the browser shows: the login form and the "An error occurred" dialog FAIL, a clean page passes ctl smokewalk-landedok-pass 0 '^ok +landedOk login form page .*FAIL wanted' '^ok +landedOk error dialog .*FAIL wanted' '^ok +landedOk clean page .*PASS wanted' \ '^ok +landedOk hidden login form .*PASS wanted' '^ok +landedOk when inspection throws is not a pass' '!^FAIL' -- node $here/smokewalk_predicates.js @@ -1657,9 +1659,12 @@ ctl pipeline-dry-forceui-replay-pass 0 'stepupdate\.sh .* --unattended --force-u mkdir -p $V/ltools/mxbuild-10.24.9.1 $V/ltools/mxbuild-10.24.27.1 ctl pipeline-ladder-9-goes-through-10-24-pass 0 'version ladder 9\.24\.2\.1 -> 10\.24\.27\.1 -> 11\.12\.7$' -- env UPG_TOOLS=$V/ltools bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 9.24.2.1 --to 11.12.7 --work $V/lw --dry-run ctl pipeline-ladder-10-24-goes-straight-pass 0 'version ladder 10\.24\.3\.1 -> 11\.12\.7$' -- env UPG_TOOLS=$V/ltools bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.3.1 --to 11.12.7 --work $V/lw --dry-run +# cold CMM 2026-11-17: 13 of 23 walk steps were microflows, SKIPPED without a word (the notice fired only on an INVALID walk) +ctl pipeline-smoke-mf-plan-microflow-catch 0 '^--microflows$' -- bash -c "WORK=\$(mktemp -d); echo '{\"kind\": \"microflow\"}' > \$WORK/plan.json; eval \"\$(awk '/^smoke_mf\(\)/{f=1} f{print} f&&/^}|; }\$/{exit}' \${PIPE:-$B/cloud/upgrade-pipeline.sh})\"; smoke_mf" +ctl pipeline-smoke-mf-opt-out-pass 0 '!microflows' '^done$' -- bash -c "WORK=\$(mktemp -d); echo '{\"kind\": \"microflow\"}' > \$WORK/plan.json; echo '{\"landed\": \"microflow ran\"}' > \$WORK/smoke-before.json; UPG_SMOKE_MICROFLOWS=0; eval \"\$(awk '/^smoke_mf\(\)/{f=1} f{print} f&&/^}|; }\$/{exit}' \${PIPE:-$B/cloud/upgrade-pipeline.sh})\"; smoke_mf; echo done" ctl pipeline-ladder-no-10-24-toolset-catch 2 'goes to 11\.12\.7 through 10\.24: no mxbuild-10\.24\.\* toolset found, give --via' '!unexpected failure' -- env UPG_TOOLS=$V/none bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 9.24.2.1 --to 11.12.7 --work $V/lw --dry-run ctl pipeline-ladder-refuses-to-with-target-catch 2 'give one or the other' -- env UPG_TOOLS=$V/ltools bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 9.24.2.1 --to 11.12.7 --target 10.24.1 --work $V/lw --dry-run -# UPG_SMOKE_MICROFLOWS=1 passes --microflows to the smoke walk (WM1: a role whose home and menu are all microflows); off by default +# UPG_SMOKE_MICROFLOWS=1 passes --microflows to the smoke walk (WM1: a role whose home and menu are all microflows); off when the plan has no microflow step ctl pipeline-dry-smoke-microflows-pass 0 'smokewalk\.js .* --microflows --login-path' -- env UPG_SMOKE_MICROFLOWS=1 bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pmf1 --only basewalk --dry-run ctl pipeline-dry-smoke-microflows-off-catch 0 'smokewalk\.js .* --login-path' '!--microflows' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pmf0 --only basewalk --dry-run ctl pipeline-dry-noforceui-catch 0 '!--force-ui' '!replay/' 'stepupdate\.sh .* --unattended' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pw --only update --dry-run diff --git a/upgrade-toolkit/controls/smokewalk_predicates.js b/upgrade-toolkit/controls/smokewalk_predicates.js index 293d3af..2a91da8 100644 --- a/upgrade-toolkit/controls/smokewalk_predicates.js +++ b/upgrade-toolkit/controls/smokewalk_predicates.js @@ -23,6 +23,12 @@ global.mx = { session: { getUserName: () => 'smoke_user' }, ui: { getContentForm check('loggedIn when ready', loggedIn(), true); check('homeReady when ready', homeReady(), true); check('landedWhere on the wanted page', landedWhere({ want: 'P/Home.page.xml' }), 'content'); +// cold CMM: a popup opened from the menu (Atlas: .mx-window position:fixed, offsetParent null, but it has client rects) +global.document = { querySelectorAll: s => (s === '.mx-window' ? [{ offsetParent: null, getClientRects: () => [1] }] : []) }; +check('landedWhere fixed popup window', landedWhere({ want: 'P/Other.page.xml' }), 'popup'); +global.document = { querySelectorAll: s => (s === '.mx-window' ? [{ offsetParent: null, getClientRects: () => [] }] : []) }; +check('landedWhere hidden popup window', landedWhere({ want: 'P/Other.page.xml' }), false); +global.document = { querySelectorAll: () => [] }; // R1: React client (no getUserName / getContentForm). catch case: anonymous session on the login page global.mx = { session: { sessionData: { user: null } }, ui: {} }; global.history = { state: null }; diff --git a/upgrade-toolkit/steps/baseline.md b/upgrade-toolkit/steps/baseline.md index dbef732..abb4835 100644 --- a/upgrade-toolkit/steps/baseline.md +++ b/upgrade-toolkit/steps/baseline.md @@ -93,7 +93,8 @@ copy's converted model. the baseline commit; in a parallel run `app/` is owned by triage and is converted later, NR1-6): `python3 $B/smokewalk_gen.py -o $W/plan.json` `node $B/smokewalk.js --plan $W/plan.json --url http://localhost: --user =smoke_: ... --out $W/smoke-before.json` - The runner skips microflow steps unless it gets `--microflows`. When smokewalk_gen prints `WARN : ...` (the + The runner skips microflow steps unless it gets `--microflows`; upgrade-pipeline.sh passes it whenever the plan has + microflow steps (UPG_SMOKE_MICROFLOWS=0 turns it off, with a note). When smokewalk_gen prints `WARN : ...` (the home is a microflow, or over half the menu items are; PV-6: every page of Product View 360 was one), add `--microflows` to the baseline AND the gate walk, on the X_smoke copy only: they run for real. Add `--shots $W/smoke-before-shots` (and `--shots $W/smoke-after-shots` after the upgrade) so the gate can compare From 073e5aaab6e7065124a6d15ff71433b83fd6515e Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 12:51:16 +0800 Subject: [PATCH 36/59] pipeline: the step scripts get the pipeline's git identity when the machine has none Cold run 2026-11-17: on a machine with no git user.email, stepupdate.sh's plain `git commit` stopped the run in update (post-convert) with "Author identity unknown" (exit 4). The pipeline's own commits already passed -c user.*; 13 step scripts commit with plain git. git_identity exports GIT_AUTHOR_*/GIT_COMMITTER_* (upgrade-pipeline) only when no identity is configured; a configured one is kept. Controls: 2 new; the no-identity one fails on the old pipeline, the kept-identity guard holds on both. Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 8 ++++++++ upgrade-toolkit/controls/run-controls.sh | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 3157017..e1a2d76 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -438,6 +438,14 @@ heavy() { # heavy LABEL VER tool args...: a mx/mxbuild call under the heavy lock } mxcli_run() { if [ -n "${MXCLI:-}" ] && [ -x "$MXCLI" ]; then "$MXCLI" "$@"; else "$B/mxcli-container.sh" "$@"; fi; } git_app() { git -C "$APP" -c user.name="upgrade-pipeline" -c user.email="upgrade-pipeline@localhost" "$@"; } +# The step scripts (stepupdate.sh, convertpre.py, jardedupe.py, ...) commit with plain git: on a machine with no git identity +# that failed ("Author identity unknown", cold CMM: stepupdate post-convert, exit 4). They get the pipeline's identity then. +git_identity() { + [ -z "$(git config user.email 2> /dev/null || true)" ] || return 0 + export GIT_AUTHOR_NAME="${GIT_AUTHOR_NAME:-upgrade-pipeline}" GIT_AUTHOR_EMAIL="${GIT_AUTHOR_EMAIL:-upgrade-pipeline@localhost}" + export GIT_COMMITTER_NAME="${GIT_COMMITTER_NAME:-upgrade-pipeline}" GIT_COMMITTER_EMAIL="${GIT_COMMITTER_EMAIL:-upgrade-pipeline@localhost}" +} +git_identity ensure_gitignore() { local f="$APP/.gitignore" l for l in '.claude/' '.mxcli/' '.stepupdate/' 'deployment/'; do diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 88c9ca4..c13d8b6 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -1662,6 +1662,10 @@ ctl pipeline-ladder-10-24-goes-straight-pass 0 'version ladder 10\.24\.3\.1 -> 1 # cold CMM 2026-11-17: 13 of 23 walk steps were microflows, SKIPPED without a word (the notice fired only on an INVALID walk) ctl pipeline-smoke-mf-plan-microflow-catch 0 '^--microflows$' -- bash -c "WORK=\$(mktemp -d); echo '{\"kind\": \"microflow\"}' > \$WORK/plan.json; eval \"\$(awk '/^smoke_mf\(\)/{f=1} f{print} f&&/^}|; }\$/{exit}' \${PIPE:-$B/cloud/upgrade-pipeline.sh})\"; smoke_mf" ctl pipeline-smoke-mf-opt-out-pass 0 '!microflows' '^done$' -- bash -c "WORK=\$(mktemp -d); echo '{\"kind\": \"microflow\"}' > \$WORK/plan.json; echo '{\"landed\": \"microflow ran\"}' > \$WORK/smoke-before.json; UPG_SMOKE_MICROFLOWS=0; eval \"\$(awk '/^smoke_mf\(\)/{f=1} f{print} f&&/^}|; }\$/{exit}' \${PIPE:-$B/cloud/upgrade-pipeline.sh})\"; smoke_mf; echo done" +# cold CMM 2026-11-17: no git identity on the machine: stepupdate's plain `git commit` stopped the run (exit 4) +GI="eval \"\$(sed -n '/^git_identity()/,/^}/p' \${PIPE:-$B/cloud/upgrade-pipeline.sh})\"; git_identity 2> /dev/null || true" +ctl pipeline-git-identity-none-catch 0 '^committed$' -- bash -c "export HOME=\$(mktemp -d) GIT_CONFIG_NOSYSTEM=1; unset GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL EMAIL; $GI; r=\$(mktemp -d); git -C \$r init -q && touch \$r/f && git -C \$r add f && git -C \$r commit -qm t && echo committed" +ctl pipeline-git-identity-kept-pass 0 '^Person $' -- bash -c "export HOME=\$(mktemp -d) GIT_CONFIG_NOSYSTEM=1; unset GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL EMAIL; git config --global user.name Person; git config --global user.email p@x; $GI; r=\$(mktemp -d); git -C \$r init -q && touch \$r/f && git -C \$r add f && git -C \$r commit -qm t && git -C \$r log -1 --format='%an <%ae>'" ctl pipeline-ladder-no-10-24-toolset-catch 2 'goes to 11\.12\.7 through 10\.24: no mxbuild-10\.24\.\* toolset found, give --via' '!unexpected failure' -- env UPG_TOOLS=$V/none bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 9.24.2.1 --to 11.12.7 --work $V/lw --dry-run ctl pipeline-ladder-refuses-to-with-target-catch 2 'give one or the other' -- env UPG_TOOLS=$V/ltools bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 9.24.2.1 --to 11.12.7 --target 10.24.1 --work $V/lw --dry-run # UPG_SMOKE_MICROFLOWS=1 passes --microflows to the smoke walk (WM1: a role whose home and menu are all microflows); off when the plan has no microflow step From 81ae926b523a40820967c98500f0b354fed027c8 Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 13:22:05 +0800 Subject: [PATCH 37/59] pipeline: an explicit --db replaces the database name kept in state/db-prefix The kept name was written only when no db-prefix existed. A rerun with --db worked on the new database, but the next resume without --db silently dropped back to the old one. Now an explicit --db is written over the kept name; without --db the kept name stays. Rung 2 has its own work folder, so its --db _r2 does not touch rung 1's prefix. Controls: pipeline-db-arg-persisted-catch (fails on the old code: prefix stays upg_old) and pipeline-db-kept-no-arg-pass (holds on both). Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 5 ++++- upgrade-toolkit/controls/run-controls.sh | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index e1a2d76..b67c85e 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -519,7 +519,10 @@ else fi X_FROZEN="${X}_frozen" [[ "$X" =~ ^[a-z0-9_]+$ ]] || usage_die "database name '$X' must be lower case letters, digits, underscore (use --db)" -[ "$DRY" = 1 ] || [ -s "$WORK/state/db-prefix" ] || { mkdir -p "$WORK/state" && echo "$X" > "$WORK/state/db-prefix"; } +# An explicit --db replaces the kept name: a later --from without --db must not drop back to the old database (cold +# CMM: rerun --from baseline --db upg_cmm_demo, then a resume without --db would have run update on upg_run_*). +[ "$DRY" = 1 ] || { [ -s "$WORK/state/db-prefix" ] && [ "$(head -n1 "$WORK/state/db-prefix")" = "$X" ]; } || + { mkdir -p "$WORK/state" && echo "$X" > "$WORK/state/db-prefix"; } frozen_stamp() { must docker exec "$PGC" psql -U mendix -qc "COMMENT ON DATABASE $X_FROZEN IS 'upgrade-toolkit run $RUNP'"; } frozen_owner() { # refuse a frozen database that another run made: its comment names another run folder, or it has none and # this run's ledger never used a copy of it (a run from before the stamp adopts its own one here) diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index c13d8b6..eff14d3 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -1612,6 +1612,10 @@ ctl pipeline-appdir-outside-catch 3 'is outside the run folder' '!step ' -- bash mkdir -p $V/dbn1/run/app $V/dbn2/run/app; touch $V/dbn1/run/app/App.mpr $V/dbn2/run/app/App.mpr ctl pipeline-db-name-per-run-catch 0 '^2$' -- bash -c "for d in dbn1 dbn2; do bash \${PIPE:-$B/cloud/upgrade-pipeline.sh} --app-dir $V/\$d/run/app --source 10.24.26.1 --target 11.15.0.1 --work $V/\$d/run/work --only baseline --dry-run 2>&1 | sed -n 's/.*createdb -U mendix -T \([a-z0-9_]*\) .*_frozen\$/\1/p'; done | sort -u | wc -l | tr -d ' '" ctl pipeline-db-name-kept-pass 0 '^upg_keep$' -- bash -c "mkdir -p $V/dbn3/run/app $V/dbn3/run/work/state; touch $V/dbn3/run/app/App.mpr; echo upg_keep > $V/dbn3/run/work/state/db-prefix; bash \${PIPE:-$B/cloud/upgrade-pipeline.sh} --app-dir $V/dbn3/run/app --source 10.24.26.1 --target 11.15.0.1 --work $V/dbn3/run/work --only baseline --dry-run 2>&1 | sed -n 's/.*createdb -U mendix -T \([a-z0-9_]*\) .*_frozen\$/\1/p'" +# an explicit --db is kept for the next resume: the prefix block is evaluated with an older name already kept +awk '/^RUNP=/{f=1} /^frozen_stamp\(\)/{exit} f' "${PIPE:-$B/cloud/upgrade-pipeline.sh}" > $V/dbblock.sh +ctl pipeline-db-arg-persisted-catch 0 '^upg_new$' -- bash -c "mkdir -p $V/dba/run/work/state; echo upg_old > $V/dba/run/work/state/db-prefix; RUN=$V/dba/run WORK=$V/dba/run/work DB_ARG=upg_new DRY=0; usage_die() { echo \"\$*\"; exit 2; }; . $V/dbblock.sh; cat $V/dba/run/work/state/db-prefix" +ctl pipeline-db-kept-no-arg-pass 0 '^upg_old$' -- bash -c "mkdir -p $V/dbb/run/work/state; echo upg_old > $V/dbb/run/work/state/db-prefix; RUN=$V/dbb/run WORK=$V/dbb/run/work DB_ARG= DRY=0; usage_die() { echo \"\$*\"; exit 2; }; . $V/dbblock.sh; cat $V/dbb/run/work/state/db-prefix" mkdir -p $V/fdk; cat > $V/fdk/docker <<'FAKE' #!/bin/bash case "$*" in *shobj_description*) cat "$FAKE_OWNER" 2>/dev/null;; *COMMENT*) echo "stamped: $*" >> "$FAKE_LOG";; esac From ba76a0e5ec8a9d8f3819ae79cdae68ccfcdc6e0f Mon Sep 17 00:00:00 2001 From: MendixMau Date: Tue, 17 Nov 2026 13:59:56 +0800 Subject: [PATCH 38/59] Cockpit: per-action meaning, test runs old vs new, one sign-off for routine work; journeys skip old-app failures - cockpit drawer at half the screen; each Triage action shows "What it means" and the findings item by item (outside calls, module edits, local-edit decisions, React moves, convert, release rows), from the run's own files - Test stage: what ran on the old and the new app, in plain words, with the per-test list - proposal --unattended: routine work (section 2) is approved by the agent (source routine); only real decisions wait - run_journeys --skip-baseline-fail (pipeline default): a journey that failed on the old app is SKIPPED, not walked again - heal: both-fail is test setup; toolkit-gaps.md names the rule owner; diff/logdiff accept known platform differences - controls: found_means, test_runs, skip_baseline and the heal gaps control Co-Authored-By: Claude Opus 5.5 --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 2 + upgrade-toolkit/bin/cockpitmatrix.py | 231 +++++++++++++++++- upgrade-toolkit/bin/cockpitpage.py | 19 ++ upgrade-toolkit/bin/diff.py | 6 + upgrade-toolkit/bin/gate.sh | 23 ++ upgrade-toolkit/bin/heal.py | 40 ++- upgrade-toolkit/bin/logdiff.py | 9 +- upgrade-toolkit/bin/proposal.py | 5 +- upgrade-toolkit/bin/run_journeys.py | 36 ++- .../fixtures/cockpit/run/work/data.out | 6 + .../controls/found_means_control.py | 33 +++ upgrade-toolkit/controls/run-controls.sh | 51 +++- .../controls/skip_baseline_control.py | 34 +++ upgrade-toolkit/controls/test_runs_control.py | 38 +++ upgrade-toolkit/steps/heal.md | 30 +++ upgrade-toolkit/steps/journeys.md | 10 + 16 files changed, 548 insertions(+), 25 deletions(-) create mode 100644 upgrade-toolkit/controls/fixtures/cockpit/run/work/data.out create mode 100644 upgrade-toolkit/controls/found_means_control.py create mode 100644 upgrade-toolkit/controls/skip_baseline_control.py create mode 100644 upgrade-toolkit/controls/test_runs_control.py diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index b67c85e..929547a 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -1376,6 +1376,8 @@ step_journeys() { local sa=() ev=() [ ! -f "$WORK/shot-accept.txt" ] || sa=(--shot-accept "$WORK/shot-accept.txt") [ ! -f "$WORK/baseline-evidence.txt" ] || ev=(--baseline-evidence "$WORK/baseline-evidence.txt") # BT-4: same file the gate reads + # WM1 2026-11-17: a journey that failed on the old app and feeds no later one is listed SKIPPED, not walked again (UPG_JOURNEYS_ALL=1 walks all) + [ "${UPG_JOURNEYS_ALL:-0}" = 1 ] || ev+=(--skip-baseline-fail) local ja=(python3 "$B/run_journeys.py" "$WORK/journeys" --label after --port "$APP_PORT" --psql "$(psql_str "$db")" --admin-port "$ADMIN_PORT" --admin-pass "$M2EE_PASS" --users "$WORK/users.json" --login-path "$(login_path)" --work "$WORK/e2e-after" --pause 50 ${act[@]+"${act[@]}"} --compare "$WORK/e2e-before-picked" ${sa[@]+"${sa[@]}"} ${ev[@]+"${ev[@]}"}) diff --git a/upgrade-toolkit/bin/cockpitmatrix.py b/upgrade-toolkit/bin/cockpitmatrix.py index f7ca568..471b6a2 100644 --- a/upgrade-toolkit/bin/cockpitmatrix.py +++ b/upgrade-toolkit/bin/cockpitmatrix.py @@ -886,6 +886,129 @@ def _stage_ok(res): return res["verdict"] in ("PASS", "PASS*") +# ---- what a finding MEANS, per item (the owner, 2026-11-17: "not just what, but what it means"); built from the run's own files ---- +def _json(work, name): + try: + with open(os.path.join(work or "", name), encoding="utf-8") as f: + return json.load(f) + except (OSError, ValueError): + return None + + +def _edit_means(x): + """What one localedits.json row means for the upgrade, in plain words.""" + k, ch, det = x.get("kind", ""), x.get("change", ""), x.get("detail") or "" + if k == "layout" and ch == "CHANGED": + return "The app restyled this shared layout. Every page on it, Marketplace pages too, takes its look from it. The update puts the plain layout back, so those pages lose the app's look unless they move to the app's own layout." + if "removed by the update" in det: + return "The app's own document inside a Marketplace module. The update deletes it; whatever uses it breaks unless it moves to a module of the app first." + if k == "snippet": + return "The update puts the package version of this snippet back. Screens that show it look like the package again." + if k == "javascript": + return "This JavaScript action differs from the package. The update overwrites it, so a local fix in it is lost." + if k in ("entity", "attribute", "association") and ch == "ADDED" and x.get("data"): + return "The app stores its own data here. A Marketplace update removes it, and the data stored in it with it." + if "access rules" in det: + return "The app changed who may see or edit this. The update puts the package rules back, so the app's access changes." + if k in ("microflow", "nanoflow"): + return "The app changed this logic. The update overwrites it, so the app behaves like the package again." + return "The update overwrites this local change." + + +def _edit_name(x): + return x.get("name") or x.get("path") or "?" + + +def found_detail(rec, work): + """{"means": [sentences], "items": [[what, tag, tone, what it means]]} for the triage of one step, or {} when nothing is known. + tone: ok (nothing lost), wait (a person looks), bad (lost or broken unless someone acts).""" + t = rec.get("topic") or "" + ev = [str(e) for e in (rec["found"].get("evidence") or [])] + if t == "standins" or t.startswith("person:apis"): + inv = _json(work, "apimock-inventory.json") or {} + eps = inv.get("endpoints") or [] + if not eps: + return {} + items = [] + for e in eps: + why = e.get("why") or "" + if e.get("status") == "MOCKED": + items.append([e["doc"], "stand-in ready", "ok", "The tests answer this call with a stand-in, so the screens that use it can be checked."]) + elif why.startswith("location from data"): + fld = why.split(":")[0].replace("location from data", "").strip() + items.append([e["doc"], "address from data", "wait", f"The address is read from the database ({fld}) while the app runs. The tests cannot know where it goes; a person names the real test system."]) + elif "computed by Java action" in why: + items.append([e["doc"], "address built in Java", "wait", "A Java action builds the address while the app runs. The toolkit cannot see it, so a person says which system this is."]) + elif e.get("kind") == "const": + items.append([e["doc"], "unused address?", "wait", "A web address setting that nothing in the model reads. Either it is unused, or a Java action reads it. A person checks."]) + else: + items.append([e["doc"], e.get("status", "").lower() or "open", "wait", why]) + ok = sum(1 for i in items if i[2] == "ok") + mods = sorted({e["doc"].split(".")[0] for e in eps if e.get("status") != "MOCKED"}) + return {"means": [f"{ok} of {len(items)} outside calls are covered in the tests.", + f"For the other {len(items) - ok}, a passing test only shows the screen works without the outside system. It says nothing about the real call.", + f"Before go-live, a person tests these against real test systems ({', '.join(mods)})."], "items": items} + if t == "modules": + edits = {m["name"]: m for m in (_json(work, "localedits.json") or []) if isinstance(m, dict)} + items, clean, touched, unknown = [], 0, [], 0 + for e in ev: + m = re.match(r"module update (\S+) (\S+) -> (\S+)", e) + if not m: + continue + le = edits.get(m.group(1)) or {} + rows = [x for k in ("domain", "ui", "logic", "files") for x in (le.get(k) or [])] + if le.get("domain_error"): + unknown += 1 + items.append([f"{m.group(1)} {m.group(2)} → {m.group(3)}", "not compared", "wait", "The toolkit could not compare this module with its Marketplace version, so local edits in it are not known."]) + elif rows: + touched.append(m.group(1)) + items.append([f"{m.group(1)} {m.group(2)} → {m.group(3)}", f"{len(rows)} local edit" + ("" if len(rows) == 1 else "s"), "wait", + f"The app changed {', '.join(os.path.splitext(os.path.basename(x['path']))[0] if x.get('path') else _edit_name(x).split('.')[-1] for x in rows[:4])}{', ...' if len(rows) > 4 else ''}. " + + _edit_means(rows[0]) + " The toolkit re-applies screen edits after the update; check these screens in the Test step."]) + else: + clean += 1 + items.append([f"{m.group(1)} {m.group(2)} → {m.group(3)}", "no local edits", "ok", "The update swaps the whole module. Nothing of the app is lost."]) + if not items: + return {} + means = [f"{clean} of {len(items)} modules {'has' if clean == 1 else 'have'} no local edits: a clean swap."] + if touched: + means.append(f"{len(touched)} {'carries' if len(touched) == 1 else 'carry'} edits by the app ({', '.join(touched)}). They count as routine because the toolkit re-applies them, but this is where a screen can change. Look at them first when a test fails.") + if unknown: + means.append(f"{unknown} could not be compared, so nobody knows yet whether the app changed {'it' if unknown == 1 else 'them'}.") + return {"means": means, "items": items} + if t.startswith("module:"): + mod = t.split(":", 1)[1] + le = next((m for m in (_json(work, "localedits.json") or []) if isinstance(m, dict) and m.get("name") == mod), {}) or {} + rows = [x for k in ("domain", "ui", "logic", "files") for x in (le.get(k) or [])] + items = [[_edit_name(x), x.get("change", "").lower(), "bad" if x.get("data") else "wait", _edit_means(x)] for x in rows] + if not items: + items = [[e[:120], "to review", "wait", "The update step stopped at this module because stored data could be dropped or changed. A person accepts that with a reason, or keeps the old version."] for e in ev] + data = sum(1 for x in rows if x.get("data")) + means = [f"The app changed this Marketplace module in {len(rows)} place" + ("" if len(rows) == 1 else "s") + "." if rows else "The update would touch data this module stores."] + if data: + means.append(f"{data} of them {'holds' if data == 1 else 'hold'} the app's own data. A plain update deletes that data, so this cannot be routine.") + means.append("A person picks the way: carry the edits over (recommended), move them out by hand, or keep the old version for now.") + return {"means": means, "items": items} + if t == "react": + items = [] + for e in ev: + m = re.search(r"(\d+) x (.+?) -> (.+)$", e) + if m: + items.append([f"{m.group(1)} × {m.group(2)}", "→ " + re.sub(r"\s*\(.*$", "", m.group(3)), "ok", + f"{m.group(2)} does not exist in the React client. The toolkit swaps each of the {m.group(1)} for {re.sub(r'\s*[(:].*$', '', m.group(3))}. The screen can look a bit different; the Test step checks it still works."]) + n = sum(int(i[0].split(" ")[0]) for i in items) + return {"means": [f"{n} widgets in {len(items)} kind{'' if len(items) == 1 else 's'} move to their React successor.", "Nothing is lost: each swap keeps the data the widget showed. Screens can look a little different, which the screenshots in the Test step show."], "items": items} if items else {} + if t == "convert": + return {"means": ["Studio Pro's converter runs once. It moves the project to the new format and marks what it cannot convert.", + "Each mark becomes a build error that the toolkit fixes, and every fix is a change you can see in the Upgrade step."], + "items": [[e, "convert", "ok", "A major version step; only the Mendix converter can make it."] for e in ev]} + if t == "person:release-rows": + return {"means": ["Mendix release notes between the two versions list changes that can affect an app.", + "The toolkit matched them to this app. A person marks each row handled or not relevant; until then, nobody has judged them."], + "items": [[e, "open", "wait", "Each open row is a platform change nobody has checked against this app yet."] for e in ev]} + return {} + + def step_cells(rec): """The four cells and the drawer action of one step record. Every sentence is a template or the record's own plain text.""" ap, pr, fd, gaps = rec.get("approval"), rec["proposed"], rec["found"], rec["gaps"] @@ -972,13 +1095,15 @@ def step_cells(rec): return {"triage": triage, "plan": plan, "upgrade": upgrade, "test": test}, act -def step_row(rec): +def step_row(rec, work=None): """One overview row for one step record.""" cells, act = step_cells(rec) + fd = found_detail(rec, work) if work else {} + act["stages"]["triage"].update(means=fd.get("means") or [], items=fd.get("items") or []) r = row(rec["id"], rec["title"] or rec["id"], f"{rec['id']} · {rec['topic']}", STEP_GROUP[0], cells, rec["found"].get("evidence") or []) r["trail"] = [act] r["fromSteps"] = True - r["story"] = {"found": [x for x in [rec["found"].get("say")] if x], "means": [], + r["story"] = {"found": [x for x in [rec["found"].get("say")] if x], "means": fd.get("means") or [], "plan": [x for x in [rec["proposed"].get("say"), rec["proposed"].get("why")] if x]} if rec.get("phase"): # a two-version upgrade: the version this step belongs to (steps.py UPG_PHASE) r["phase"] = rec["phase"] @@ -1048,12 +1173,12 @@ def phases(rows): return sorted({r["phase"] for r in rows if r.get("phase")}, key=key) -def merge_steps(rows, recs, item_keys=()): +def merge_steps(rows, recs, item_keys=(), work=None): """Add one row per step after the derived rows of the group "app"; the rows keep step order. The same topic is the same step: a row that cockpit-items.json declared for a topic gives way to the step with that topic, so a topic never shows twice.""" topics = {r["topic"] for r in recs} rows[:] = [r for r in rows if not (r["id"] in topics and r["id"] in item_keys and not r.get("fromSteps"))] - rows.extend(step_row(r) for r in recs) + rows.extend(step_row(r, work) for r in recs) order = {k: i for i, (k, _) in enumerate([GROUPS[0], STEP_GROUP] + GROUPS[1:])} rows.sort(key=lambda r: order[r["group"]]) @@ -1069,7 +1194,7 @@ def build(work, model): groups = list(GROUPS) if recs is not None: try: - merge_steps(rows, recs, set(items.get("items") or {})) + merge_steps(rows, recs, set(items.get("items") or {}), work) groups.insert(1, STEP_GROUP) except (KeyError, TypeError, AttributeError, ValueError): recs, err = None, "a step record is incomplete" @@ -1089,8 +1214,74 @@ def build(work, model): "generated": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "phases": phases(rows), "ladder": ladder(work), "health": health(work), "itemsFile": os.path.exists(os.path.join(work, "cockpit-items.json")), "stepsFile": recs is not None, - "plan": plan_doc(cp.rd(os.path.join(work, "PROPOSAL.md")))} + "plan": plan_doc(cp.rd(os.path.join(work, "PROPOSAL.md"))), "ran": test_runs(work)} + + +def _count_line(txt): + """'FAIL 0 | REVIEW 2 | PASS* 1 | PASS 47' -> {'FAIL': 0, ...} from the first line that has it.""" + for ln in (txt or "").splitlines(): + m = re.findall(r"(PASS\*|[A-Z_]+) (\d+)(?: \||$)", ln) + if len(m) >= 2 and ("PASS" in dict(m) or "FAIL" in dict(m)): + return {k: int(v) for k, v in m} + return None + +# the page speaks plain words, never the gate's verdict names (control cockpit-words-no-plural-marker-pass) +RUN_WORD = {"PASS": "passed", "PASS*": "accepted with a reason", "FAIL": "failed", "INVALID": "could not be judged", + "REVIEW": "to look at", "INFO": "listed only", "SKIPPED": "skipped, failed on the old app", + "NOT CHECKED": "not checked", "NEW_ENDPOINT": "new endpoint", "UNREACHABLE": "unreachable", + "CONTAMINATED": "could not be judged"} + + +def _word(v): + return RUN_WORD.get(v, v.lower()) + + +def _vc(vs): + order = ("PASS", "PASS*", "FAIL", "INVALID", "SKIPPED", "INFO", "NOT CHECKED") + c = {} + for v in vs: + c[v] = c.get(v, 0) + 1 + return ", ".join(f"{c[k]} {_word(k)}" for k in sorted(c, key=lambda k: (order.index(k) if k in order else 99, k))) + + +def test_runs(work): + """What actually ran in the Test stage, on the old app and on the new one (the owner, 2026-11-17: "see which tests + actually ran, on old and new"). rows: [check, old app, new app, compared]; lists: per check every test with both + verdicts. Read from the stored results only; a side without a file says "not run".""" + rows, lists = [], {} + sm = {sd: cp.rj(os.path.join(work, f"smoke-{sd}.json")) for sd in ("before", "after")} + if any(sm.values()): + per = {} + for i, sd in enumerate(("before", "after")): + for r in (sm[sd] or {}).get("roles", []): + for st in r.get("steps", []): + per.setdefault(f"{r['role']}: {st.get('caption')}", ["not run", "not run"])[i] = st.get("verdict", "?") + side = lambda i: (cp.pl(sum(1 for v in per.values() if v[i] != 'not run'), "page") + ": " + _vc(v[i] for v in per.values() if v[i] != "not run")) if sm[("before", "after")[i]] else "not run" + scr = _count_line(cp.rd(os.path.join(work, "screens.out"))) + rows.append(["Page walk", side(0), side(1), (f"{sum(scr.values())} screenshot pairs: " + ", ".join(f"{v} {_word(k)}" for k, v in scr.items() if v)) if scr else "screenshots not compared"]) + lists["Page walk"] = [[k] + [_word(x) if x != "not run" else x for x in v] for k, v in sorted(per.items())] + js = {sd: cp.rj(os.path.join(work, f)) for sd, f in (("old", "e2e-before-picked/before-picked/summary.json"), ("new", "e2e-after/after/summary.json"))} + if any(js.values()): + ids = sorted(set().union(*[set((d or {}).get("journeys", {})) for d in js.values()])) + vv = {sd: {j: x.get("verdict", "?") for j, x in ((d or {}).get("journeys") or {}).items()} for sd, d in js.items()} + jf = {sd: os.path.join(work, f) for sd, f in (("old", "e2e-before-picked/before-picked/summary.json"), ("new", "e2e-after/after/summary.json"))} + # WM1 2026-11-17: a rerun walks the old app first; the new app's file was still the one of the run before + stale = js["old"] and js["new"] and os.path.getmtime(jf["new"]) < os.path.getmtime(jf["old"]) + if stale: + js["new"], vv["new"] = None, {} + side = lambda sd: (cp.pl(len(vv[sd]), "journey") + ": " + _vc(vv[sd].values())) if js[sd] else ("not run yet in this run" if sd == "new" and stale else "not run") + both = sum(1 for j in ids if vv["old"].get(j) == "PASS" and vv["new"].get(j) == "PASS") + lost = sum(1 for j in ids if vv["old"].get(j) == "PASS" and vv["new"].get(j) not in (None, "PASS")) + rows.append(["Journeys", side("old"), side("new"), f"{both} pass on both, {lost} passed on the old app and not on the new one" + if js["old"] and js["new"] else "needs both apps"]) + lists["Journeys"] = [[j, _word(vv["old"][j]) if j in vv["old"] else "not run", _word(vv["new"][j]) if j in vv["new"] else "not run"] for j in ids] + for f, nm in (("data.out", "Data compare"), ("logdiff.out", "Start log"), ("const.out", "Constants")): + c = _count_line(cp.rd(os.path.join(work, f))) + if c: + rows.append([nm, "snapshot taken" if nm != "Start log" else "start log read", "snapshot taken" if nm != "Start log" else "start log read", + f"{sum(c.values())} checks: " + ", ".join(f"{v} {_word(k)}" for k, v in c.items() if v)]) + return {"rows": rows, "lists": lists} # ---- the top band: what is closed, what is still to validate (generic, from rows and gate rungs) ---- def _rung_detail(g, name): @@ -1328,7 +1519,7 @@ def band(model, rows, chips, g, tl, kpi=None): .feed .new{font-weight:700} .note{font-size:12px;color:var(--muted)} .shade{position:fixed;inset:0;background:var(--shade);z-index:20} -.drawer{position:fixed;top:0;right:0;bottom:0;width:min(520px,100%);background:var(--panel);border-left:1px solid var(--line);z-index:21;overflow-y:auto; +.drawer{position:fixed;top:0;right:0;bottom:0;width:min(max(520px,50vw),100%);background:var(--panel);border-left:1px solid var(--line);z-index:21;overflow-y:auto; padding:calc(20px + env(safe-area-inset-top,0px)) max(20px,env(safe-area-inset-right,0px)) calc(28px + env(safe-area-inset-bottom,0px)) 22px;display:grid;gap:16px;align-content:start} details.kpi summary{cursor:pointer;list-style:none}details.kpi summary b{display:block}details.kpi summary span{font-size:13px;color:var(--muted)}.agentnote{margin:14px 2px 0;font-size:13px;color:var(--muted)}.drawer>*,.drawer *{min-width:0;overflow-wrap:anywhere}.drawer pre,.drawer code{white-space:pre-wrap} .drawer .x{all:unset;cursor:pointer;position:absolute;top:calc(14px + env(safe-area-inset-top,0px));right:max(16px,env(safe-area-inset-right,0px));font-size:22px;color:var(--muted);padding:4px 8px;border-radius:6px} @@ -1379,6 +1570,10 @@ def band(model, rows, chips, g, tl, kpi=None): .pcard .tag{font-size:12px;color:var(--muted)} .pcard .asm{color:var(--wait);font-weight:600} .trail>p{margin:0;color:var(--muted);font-size:13.5px} .act{border:1px solid var(--line);border-radius:10px;background:var(--panel)} +.means{border-left:4px solid var(--accent);background:var(--bg);padding:8px 12px;margin:0 0 12px;border-radius:0 8px 8px 0}.means h5{margin:0 0 4px}.means p{margin:4px 0;font-size:15px} +table.fnd{width:100%;border-collapse:collapse;margin-bottom:10px}table.fnd td{padding:8px 6px;border-top:1px solid var(--line);vertical-align:top}table.fnd td:first-child{width:14px} +table.fnd i{display:block;width:9px;height:9px;border-radius:50%;margin-top:6px;background:var(--ok)}table.fnd tr.wait i{background:var(--wait)}table.fnd tr.bad i{background:var(--bad)} +table.fnd b{overflow-wrap:anywhere}table.fnd .tg{margin-left:8px;font-size:12px;padding:1px 7px;border-radius:9px;border:1px solid var(--line);color:var(--muted);white-space:nowrap}table.fnd div{color:var(--ink,inherit);margin-top:3px;font-size:14px} .act>summary{list-style:none;cursor:pointer;display:grid;grid-template-columns:10px minmax(0,1fr) auto;gap:10px;align-items:start;padding:10px 12px} .act>summary::-webkit-details-marker{display:none} .act>summary:focus-visible{outline:2px solid var(--accent);outline-offset:2px} @@ -1394,6 +1589,8 @@ def band(model, rows, chips, g, tl, kpi=None): .actb dd{margin:0} .actb ul{margin:0;padding-left:18px} .res{width:100%;border-collapse:collapse;font-size:13.5px} +.ran{width:100%;border-collapse:collapse;font-size:13px;margin:4px 0 8px}.ran th{text-align:left;font-weight:600;color:var(--muted);padding:5px 8px}.ran td{padding:6px 8px;border-top:1px solid var(--line);vertical-align:top;overflow-wrap:normal;word-break:normal;hyphens:none}.ran td:first-child,.ran th{white-space:nowrap}.ran tr.lost td{background:color-mix(in srgb,var(--bad) 10%,transparent)} +details.ctx>summary{cursor:pointer;font-weight:600;color:var(--muted);margin-bottom:6px} .res td{padding:7px 8px;border-top:1px solid var(--line);vertical-align:top} .res td:first-child{width:22px;padding-right:0} .res .v{font-weight:700;white-space:nowrap} @@ -1571,6 +1768,7 @@ def band(model, rows, chips, g, tl, kpi=None):

Step by step

${ol(s.steps)}

How we run it

${esc(s.run)}

Your part

${esc(s.you)}
+ ${k==="test"?ranBox():""}

Every item in this stage

${ROWS.map((it,i)=>cellChip(it,i,k)).join("")}
`); } function decBox(di){ @@ -1593,14 +1791,26 @@ def band(model, rows, chips, g, tl, kpi=None): if(k==="plan"){b=`
${t.why?`
Why
${esc(t.why)}
`:""}${t.who?`
Who decided
${esc(t.who)}
`:""}${(t.options||[]).length?`
Other options
    ${t.options.map(o=>`
  • ${esc(o)}
  • `).join("")}
`:""}
`;} if(k==="upgrade"){const cs=(t.proof||[]).filter(p=>p.kind==="commit"); b=`${(t.facts||[]).length?`
${t.facts.map(f=>`${esc(f)}`).join("")}
`:""}${cs.length?`
Changes in the app (${cs.length})
    ${cs.map(c=>`
  1. ${esc(c.s.replace(/^[\w ./-]+?: /,""))}${esc(c.h)}${c.d?" · "+esc(c.d):""}
  2. `).join("")}
`:""}`;} + if(k==="triage"&&((t.means||[]).length||(t.items||[]).length)){const ni=(t.items||[]).filter(x=>x[2]!=="ok").length; + b=`${(t.means||[]).length?`
What it means
${t.means.map(x=>`

${esc(x)}

`).join("")}
`:""}${(t.items||[]).length?`
What we found, item by item (${t.items.length}${ni?`, ${ni} to look at`:""})
${t.items.slice().sort((p,q)=>(p[2]==="ok")-(q[2]==="ok")).map(x=>``).join("")}
${esc(x[0])}${esc(x[1])}
${esc(x[3])}
`:""}`;} if(k==="test"&&(t.results||[]).length)b=`${t.results.map(r=>``).join("")}
${esc(r[0])}
${esc(r[2])}
${VW[r[1]]||esc(r[1])}
`; const pf=(t.proof||[]).length?`
Show the proof (${t.proof.length})${t.proof.map(pfItem).join("")}
`:`

No file in the run backs this up.

`; return `
${esc(a.name)}${esc(a.sub)}${esc(t.say||"")}${t.gap?' not linked':""}${TLBL[a.status]||""}
${b}${pf}
`;}).join(""); return `

${TSAY[k]}, per action (${it.trail.length})

Open an action for the reasons and the result. The proof is the record from the run itself.

${acts}
`; } const STORY=[["found","What we found"],["means","What it means for this app"],["plan","What the plan does, and why"]]; -const storyBox=it=>{const s=it.story||{};const b=STORY.filter(([k])=>(s[k]||[]).length); - return b.length?`
${b.map(([k,h])=>`

${h}

${s[k].map(t=>`

${esc(t)}

`).join("")}
`).join("")}
`:"";}; +// the same context shows on every stage tab: one click hides it everywhere, and the page remembers it (the owner, 2026-11-17) +let ctxOpen=null;try{const v=localStorage.getItem("cockpit-ctx");if(v!==null)ctxOpen=v==="1";}catch(e){} +const storyBox=(it,k)=>{const s=it.story||{};const b=STORY.filter(([k])=>(s[k]||[]).length); + const op=ctxOpen===null?k==="triage":ctxOpen; + return b.length?`
Context: ${b.map(([,h])=>h.toLowerCase()).join(", ")}
${b.map(([k,h])=>`

${h}

${s[k].map(t=>`

${esc(t)}

`).join("")}
`).join("")}
`:"";}; +document.addEventListener("toggle",e=>{if(e.target.matches&&e.target.matches("details.ctx")){ctxOpen=e.target.open;try{localStorage.setItem("cockpit-ctx",ctxOpen?"1":"0");}catch(x){}}},true); +// what actually ran in the Test stage, on the old and on the new app (test_runs in cockpitmatrix.py) +function ranBox(){const R=M.ran||{rows:[],lists:{}};if(!R.rows.length)return ""; + const lost=r=>r[1]==="passed"&&r[2]!=="passed"&&r[2]!=="not run"; + const lst=Object.entries(R.lists).map(([k,l])=>{const nl=l.filter(lost).length; + return `
Every ${esc(k.toLowerCase())} test, old and new (${l.length}${nl?`, ${nl} passed on the old app only`:""})${l.slice().sort((a,b)=>lost(b)-lost(a)).map(r=>``).join("")}
TestOld appNew app
${esc(r[0])}${esc(r[1])}${esc(r[2])}
`;}).join(""); + return `

What ran, on the old and the new app

${R.rows.map(r=>``).join("")}
CheckOld appNew appCompared
${esc(r[0])}${esc(r[1])}${esc(r[2])}${esc(r[3])}
${lst}
`;} const planBtn=i=>(M.plan||[]).length?``:""; function topicPlan(it){ const acts=(it.trail||[]).map(a=>{const t=a.stages.plan||{}; @@ -1652,11 +1862,12 @@ def band(model, rows, chips, g, tl, kpi=None): open(`${navBar(esc(it.name)+" › "+esc(s.name))}
${tabs}

${esc(c.l)}

- ${storyBox(it)} + ${storyBox(it,k)} ${k==="plan"&&!it.fromSteps?propBox(it)+planBtn(i):""} ${k==="plan"&&it.fromSteps&&it.ask?`
Needs your OK. The agent chose this on its own while it worked. Nothing is final until a person approves it.
`:""} ${k==="plan"&&it.fromSteps&&it.ans?`
${it.ans.state==="approved"?"Approved":"Rejected"} by ${esc(who(it.ans.by))}${it.ans.day?" on "+esc(it.ans.day):""}.${it.ans.reason?" Reason: "+esc(it.ans.reason):""}
`:""}

This stage, step by step

${ol(c.steps.length?c.steps:["Nothing recorded for this cell."])}
+ ${k==="test"?ranBox():""} ${trailBox(it,k)} ${decBox(di)} ${it.trail&&it.trail.length||(it.story&&(it.story.plan||[]).length&&!it.fix)?"":fixBox(it.fix,NONGREEN.includes(c.s)&&!di)} diff --git a/upgrade-toolkit/bin/cockpitpage.py b/upgrade-toolkit/bin/cockpitpage.py index 44a33f4..280484c 100644 --- a/upgrade-toolkit/bin/cockpitpage.py +++ b/upgrade-toolkit/bin/cockpitpage.py @@ -237,6 +237,20 @@ def n(pat): return {"FAIL": "Failed.", "REVIEW": "Needs a look.", "INVALID": "Could not be measured yet."}.get(v, "Not measured yet.") + cause +def accepted_reasons(work, rung, detail=""): + # WM1 2026-11-17: the card said "All pass, 1 with accepted differences" or "Failed" and nobody could tell what was + # accepted. Each PASS* row carries its reason "(accepted: ...)" / "(noted: ...)" in the rung's output: group them by reason + if detail.startswith("noted, not a failure"): + return [detail.split(";")[0]] + got = {} + for ln in rd(os.path.join(work, f"{rung}.out")).splitlines(): + m = re.match(r"^\|\s*PASS\*\s*\|[^|]*\|.*\((accepted|noted): (.+)\)\s*\|?\s*$", ln) + if m: + why = re.split(r"[:(]", m.group(2))[0].strip()[:90] + got[why] = got.get(why, 0) + 1 + return [f"{why} ({n})" if n > 1 else why for why, n in sorted(got.items(), key=lambda kv: -kv[1])] + + def status_of(v): return {"PASS": "ok", "PASS*": "ok", "EXCEPT": "ok", "FAIL": "bad", "REVIEW": "warn", "INVALID": "warn", "NOT RUN": "idle"}.get(v, "idle") @@ -703,6 +717,11 @@ def phase_time(steps): if okr: acc = sum(1 for r in okr if r["verdict"] == "PASS*") rows.append([", ".join(RUNG_NAMES.get(r["rung"], r["rung"]) for r in okr)[:70], f"All pass{f', {acc} with accepted differences' if acc else ''}.", "N", "ok"]) + for r in okr: + why = accepted_reasons(work, r["rung"], r["detail"]) if r["verdict"] == "PASS*" else [] + if why: + rows.append([RUNG_NAMES.get(r["rung"], r["rung"]), "Accepted, no loss: " + "; ".join(why[:3]) + + (f"; {len(why) - 3} more in {r['rung']}.out" if len(why) > 3 else "") + ".", "N", "ok"]) for r in gate["rungs"]: v = r["verdict"] if status_of(v) == "ok" or v == "NOT RUN": diff --git a/upgrade-toolkit/bin/diff.py b/upgrade-toolkit/bin/diff.py index aab721f..e4e0707 100755 --- a/upgrade-toolkit/bin/diff.py +++ b/upgrade-toolkit/bin/diff.py @@ -217,6 +217,12 @@ def add(check, verdict, detail): ok_a = ra.get("status") and 200 <= ra["status"] < 300 if not ok_b: if rb.get("status") == 404 and ok_a: + # WM1 2026-11-17: 3 NEW_ENDPOINT rows were the same services' docs in the 10.x format (9.24 serves + # swagger.json, 10.x openapi.json too); the playbook had people allow-list them by hand every run + sw = hb.get(p[:-len("openapi.json")] + "swagger.json") or {} if p.endswith("/openapi.json") else {} + if sw.get("status") and 200 <= sw["status"] < 300: + add(cid, "PASS*", f"status 404 -> {ra['status']} (noted: the same service's docs in the OpenAPI 3 format " + "10.x serves next to swagger.json, not a new endpoint)"); continue add(cid, "NEW_ENDPOINT", f"status 404 -> {ra['status']}: endpoint did not exist before"); continue add(cid, "INFO" if ra.get("status") == rb.get("status") else "INVALID", f"baseline status {rb.get('status')}, after {ra.get('status')}"); continue if not ok_a: add(cid, "FAIL", f"status {rb['status']} -> {ra.get('status')} {ra.get('error', '')}"); continue diff --git a/upgrade-toolkit/bin/gate.sh b/upgrade-toolkit/bin/gate.sh index 6075619..c9cfcf7 100755 --- a/upgrade-toolkit/bin/gate.sh +++ b/upgrade-toolkit/bin/gate.sh @@ -328,9 +328,29 @@ oneline() { # the rung's own summary line(s): its counts, else the last line [ -n "$l" ] || l=$(grep -v '^[[:space:]]*$' "$f" 2>/dev/null | tail -1) printf '%s' "${l:-no output}" | cut -c1-200 } +# journeys_noted FILE: the journeys rung judged a person's way (the owner, 2026-11-17: "if it fails on the new one and the +# old one, it's not a fail; mention it, don't create issues which are not there"). FSM rung 2 read FAIL with 277 PASS, +# 0 FAIL, 243 failing on the old app too. True when nothing regressed (no REGRESSION / REVIEW, 0 FAIL outside the +# BASELINE DEFECT lines) and the instrument measures: at least JOURNEY_MEASURE_MIN% (default 20) of the judged journeys +# pass on both. The counts stay on the row; the rung is PASS*, never PASS. +journeys_noted() { + local f=$1 + has_pos REGRESSION "$f" && return 1 + has_pos REVIEW "$f" && return 1 + if has_pos FAIL "$f"; then fail_all_baseline "$f" || return 1; fi + # only "fails on the old app" coverage lines are a note; a changed journey set, no hash, an unfinished run or an + # unknown denominator is an instrument fault and stays INVALID + grep '^ coverage INVALID: ' "$f" 2>/dev/null | grep -Evq 'fail on the baseline|baseline defects without evidence' && return 1 + awk -v min="${JOURNEY_MEASURE_MIN:-20}" '/^journeys: / { l = $0 + p = match(l, /[0-9]+ PASS,/) ? substr(l, RSTART, RLENGTH) + 0 : 0 + t = p; s = l; while (match(s, /[0-9]+ (FAIL|INVALID|PASS\*|CONTAMINATED)/)) { t += substr(s, RSTART, RLENGTH) + 0; s = substr(s, RSTART + RLENGTH) } + found = 1 } + END { exit !(found && t > 0 && p * 100 >= min * t) }' "$f" 2>/dev/null +} verdict_of() { # RUNG RC OUTFILE -> prints verdict word local r=$1 rc=$2 f=$3 if [ "$rc" = 124 ]; then echo INVALID; return; fi + if [ "$r" = journeys ] && { [ "$rc" = 1 ] || [ "$rc" = 3 ]; } && journeys_noted "$f"; then echo 'PASS*'; return; fi case $rc in 0) if grep -Eq "$PASSSTAR" "$f" 2>/dev/null; then echo 'PASS*'; else echo PASS; fi;; 1) if [ "$r" = release ]; then echo INVALID; else echo FAIL; fi;; @@ -462,6 +482,9 @@ run_pre() { # RUNG ; returns 0 when absent or ok ln="$ln$(grep '^ coverage INVALID: ' "$out" | sed 's/^ */; /' | tr -d '\n')$(grep -m1 '^actions [0-9?]' "$out" | sed 's/^/; /')" # E2E-11 [ -n "$ln" ] || ln=$(oneline "$out") fi + if [ $r = journeys ] && [ "$v" = 'PASS*' ] && [ "$rc" != 0 ]; then + ln="noted, not a failure: nothing that passed on the old app fails on the new one; journeys that fail on the old app too are listed for the owner; $(oneline "$out")" + fi [ "$rc" = 124 ] && { ln="timed out after ${t}s (raise GATE_TIMEOUT); $ln"; echo "gate: rung $r timed out after ${t}s: raise GATE_TIMEOUT (smoke: steps x roles x repeat)" >&2; } if [ "$v" = EXCEPT ]; then nci=$(grep -c '^ *NOT CHECKED (excluded: ' "$out") # item lines only, not a count line diff --git a/upgrade-toolkit/bin/heal.py b/upgrade-toolkit/bin/heal.py index e960ea5..3cb592f 100644 --- a/upgrade-toolkit/bin/heal.py +++ b/upgrade-toolkit/bin/heal.py @@ -42,6 +42,11 @@ "pattern": "WARNING Configuration: Using derived value for SessionKeepAliveUpdatesInterval*", "needs": "the harness starts every app with SessionTimeout {timeout} ms (runapp.py UPG_SESSION_TIMEOUT) and the line itself names SessionTimeout", "reason": "test harness: runapp.py starts every app with SessionTimeout {timeout} ms, so the runtime derives the keep-alive interval; production sets its own"}, + # WM1 2026-11-17: 10.24 derives ClusterManagerActionInterval from the same short SessionTimeout (the line names it) + {"match": re.compile(r"ClusterManagerActionInterval"), "must_name": "SessionTimeout", + "pattern": "WARNING Configuration: Using derived value for ClusterManagerActionInterval*", + "needs": "the harness starts every app with SessionTimeout {timeout} ms (runapp.py UPG_SESSION_TIMEOUT) and the line itself names SessionTimeout", + "reason": "test harness: runapp.py starts every app with SessionTimeout {timeout} ms, so the runtime derives the cluster action interval; production sets its own"}, ] SELECT_GATED = re.compile(r"\$currentObject/[\w.]+\s*!=\s*empty") EMPTY_LIST = re.compile(r"(no (items|data|results|rows|records)( found)?|showing 0|0 of 0|nothing to show)", re.I) @@ -524,10 +529,12 @@ def finalize(W): x["result"] = f"passes on the new app (old app {bv})" elif bv == "PASS": x["result"], x["class"] = "REGRESSION: passes on the old app, fails on the new one", "real-regression" - else: + elif bv == "absent": x["result"] = f"still {av} on the new app, old app {bv}" if x.get("heal_kind") in JOURNEY_HEALS: x["class"] = "likely-real" + else: # 2026-11-17: failing on both apps is a note for the owner, never an issue; the test that does not run is a toolkit gap + x["result"], x["class"] = f"fails on both apps after the heal (new {av}, old {bv}): listed for the owner, not an issue", "test-setup" for x in res["rows"]: # a rerun on the healed base that passes closes an earlier open class (a probe on a plain base had put it there) if x["id"].startswith("journey:") and x["class"] in OPEN_CLASSES and x.get("heal_kind") in JOURNEY_HEALS \ and x["result"].startswith("passes"): @@ -536,9 +543,37 @@ def finalize(W): res["gate_rungs"] = {m.group(1): m.group(2).strip() for m in re.finditer(r"^\| \d+ \| (\w+) \| ([A-Z* ]+?) \|", gs, re.M)} write_text(W / "heal.json", json.dumps(res, indent=1) + "\n") write_md(W, res) + write_gaps(W, res) return 0 +# where a rule that closes a row belongs (steps/heal.md, "The learning loop") +GAP_OWNER = (("data:http.", "bin/diff.py (http rows)"), ("data:", "bin/diff.py, bin/snapshot.py (data rows)"), + ("log:", "bin/logdiff.py, or HARNESS_LOG in bin/heal.py for a line the harness causes"), + ("journey:", "the heals in bin/heal.py, bin/run_journeys.py, bin/gen_journeys.py"), ("", "bin/gate.sh")) + + +def gaps_of(res): + """The rows no rule of the toolkit explains: likely-real (an unknown difference) and tests that run on neither app.""" + return [x for x in res["rows"] if x["class"] == "likely-real" or x["result"].startswith("fails on both apps")] + + +def write_gaps(W, res): + """toolkit-gaps.md: the learning loop's input. Every row here is either a real defect of the app (it goes to + NEEDS-JUDGMENT and stays there) or a rule the toolkit lacks. A rule is closed in the toolkit, not in this run.""" + gs = gaps_of(res) + out = [f"# Toolkit gaps: {len(gs)} row(s) no rule of the toolkit explains", "", + "Close each with the learning loop (steps/heal.md): 1. prove it on both apps (twin-old.md, the old run, the screenshots); " + "2. a real defect of the app stays in NEEDS-JUDGMENT; 3. otherwise write the rule in the owner script, with its evidence condition, " + "never a bare allow line; 4. a control that fails on the old toolkit and passes on the new one, plus a guard that still catches the " + "real case; 5. rejudge the stored run (`--from heal`), no new walk; 6. write the Mendix fact to the brain, the rule to the step doc.", ""] + for x in gs: + own = next(o for p, o in GAP_OWNER if x["id"].startswith(p)) + kind = "test does not run on either app" if x["result"].startswith("fails on both apps") else "unexplained difference" + out.append(f"- `{x['id']}` ({kind}): {x['evidence'][:200]} | result: {x['result'][:120]} | rule goes in: {own}") + write_text(W / "toolkit-gaps.md", "\n".join(out) + "\n") + + def summary(W): res = jload(Path(W) / "heal.json") if not res: @@ -549,7 +584,8 @@ def summary(W): op = [x for x in res["rows"] if x["class"] in OPEN_CLASSES] print(f"heal: {len(res['rows'])} failing row(s) classified, {acc} accepted with a written reason, {ok} journey(s) pass after the heal, " f"{len(op)} stay open ({sum(1 for x in op if x['class'] == 'real-regression')} real-regression, " - f"{sum(1 for x in op if x['class'] == 'likely-real')} likely-real); {class_line(res)}") + f"{sum(1 for x in op if x['class'] == 'likely-real')} likely-real); {class_line(res)}; " + f"toolkit gaps: {len(gaps_of(res))} (toolkit-gaps.md)") OPTIONS = { diff --git a/upgrade-toolkit/bin/logdiff.py b/upgrade-toolkit/bin/logdiff.py index 9d50f80..382a457 100755 --- a/upgrade-toolkit/bin/logdiff.py +++ b/upgrade-toolkit/bin/logdiff.py @@ -113,8 +113,15 @@ def add(verdict, kind, s, detail=""): add("REVIEW", "NEW", x["sig"], "after-startup line not in the old log") for x in gone: add("REVIEW", "GONE", x["sig"], "after-startup line of the old log is missing") + # WM1 2026-11-17: 9.24 logs "started using a trial license" as Core, 10.x as LicenseService: the same line under a + # renamed logger is noted, not a new warning + omsg = {x["sig"].split(": ", 1)[-1]: x["node"] for x in o if x["level"] == "WARNING"} for s in sorted({x["sig"] for x in n if x["level"] == "WARNING" and x["phase"] != "after"} - osig): - add("REVIEW", "NEW_WARNING", s) + was = omsg.get(s.split(": ", 1)[-1]) + if was: + rows.append(("PASS*", "NEW_WARNING", s, f"(noted: the old app logs the same line as {was}; only the logger name changed)")) + else: + add("REVIEW", "NEW_WARNING", s) info_new = len({x["sig"] for x in n if x["level"] == "INFO" and x["phase"] != "after"} - osig) return rows, info_new diff --git a/upgrade-toolkit/bin/proposal.py b/upgrade-toolkit/bin/proposal.py index 195f302..7fb9dff 100644 --- a/upgrade-toolkit/bin/proposal.py +++ b/upgrade-toolkit/bin/proposal.py @@ -810,7 +810,10 @@ def accept_answers(signer): rows = [("P-0", "proposal", "proposal", "continue without a sign-off", "Interview mode: unattended signs nothing; the run goes on at the recommendations")] rows += [(it["id"], it["name"], it["kind"], it["rec"], "unattended: recommendation taken, nobody asked") for it in F.items3] write_decisions(F, rows, "-", "ASSUMED (unattended)", "a person signs PROPOSAL.md section 6") - approve_steps(a.work, sids, "assumed", "agent", "unattended") + # routine work with a known result (section 2) goes ahead in auto mode; only a real decision waits for a person (the owner, 2026-11-17) + routine = {s: k for s, k in sids.items() if k.startswith("2:")} + approve_steps(a.work, routine, "approved", "agent", steps.ROUTINE) + approve_steps(a.work, {s: k for s, k in sids.items() if s not in routine}, "assumed", "agent", "unattended") print("proposal: not signed (unattended): ASSUMED (unattended) lines in DECISIONS.md; every report says 'proposal not signed'") return 0 print(f"proposal awaits sign-off: {out_path} ({len(F.items2)} done by the toolkit, {len(F.items3)} to decide, {len(F.items4)} for a person)") diff --git a/upgrade-toolkit/bin/run_journeys.py b/upgrade-toolkit/bin/run_journeys.py index f6422e6..5238383 100644 --- a/upgrade-toolkit/bin/run_journeys.py +++ b/upgrade-toolkit/bin/run_journeys.py @@ -1239,7 +1239,24 @@ def destructive_scope(d): return d.get("_scope") if d.get("_scope") in DESTRUCTIVE_SCOPES else None -def data_waves(jdir, gen_dir, only=None): +def baseline_fail_skips(jdir, cmp_summary): + """--skip-baseline-fail (WM1 2026-11-17: 136 of 160 journeys that failed on the old app were walked on the new app + again, about half the journeys time, to land in "BASELINE DEFECT"): {id: (before verdict, + role folder)} of the journeys the new app need not walk. Only a FAIL on the old app, and only a journey that feeds no later one: one whose path needs data + (second and later passes) or any journey of a set without data passes. A first-pass journey may create the rows a + later one needs, so it still runs. heal.py's old-app recheck runs --only the failing ids, without these, the same way.""" + before = json.load(open(cmp_summary)).get("journeys", {}) + files = sorted(jdir.glob("*/*.journey.json")) if jdir.is_dir() else [] + waves = any(read_journey(f).get("_path_needs") for f in files) + out = {} + for f in files: + j = read_journey(f) + if before.get(j["id"], {}).get("verdict") == "FAIL" and (j.get("_path_needs") or not waves): + out[j["id"]] = ("FAIL", f.parent.name) + return out + + +def data_waves(jdir, gen_dir, only=None, skip=()): """[(key, role, role_dir, files)] in the order they run. First pass: every role (role_dirs order) with its journeys that need no data on the way. Second pass, same role order: the journeys whose path needs data ("_path_needs", gen_journeys.py: a row in a list, a workflow instance, an open user task), so the rows the first pass created @@ -1258,6 +1275,7 @@ def data_waves(jdir, gen_dir, only=None): files = sorted(role_dir.glob("*.journey.json")) if only: files = [f for f in files if read_journey(f)["id"] in only] + files = [f for f in files if read_journey(f)["id"] not in skip] tier = [f for f in files if destructive_scope(read_journey(f))] files = [f for f in files if f not in tier] destr += [(f"{role_dir.name}{DESTR_WAVE}{k}", role_dir.name, role_dir, [f]) for k, f in enumerate(tier, 1)] @@ -1389,6 +1407,9 @@ def main(): ap.add_argument("--work", required=True) ap.add_argument("--positive-control", action="store_true") ap.add_argument("--only", help="comma list of journey ids") + ap.add_argument("--skip-baseline-fail", action="store_true", + help="with --compare: do not walk a journey that FAILed on the old app and feeds no later journey; " + "it is listed as SKIPPED (failed on the old app too), never a FAIL of the new app") ap.add_argument("--resume", action="store_true", help="AD-1: run again only the journeys the last run of this label could not run because the app went " "down (summary.json app_down), and merge them into that summary.json (exit 75 = still some left)") @@ -1504,7 +1525,13 @@ def main(): parked = None dirty = None # PR-P4: the destructive journey that last ran on this database, " ()" sfx = f".resume{summary['resumed']}" if prev else "" - for key, role, role_dir, files in ([] if prev and not only else data_waves(jdir, a.gen_dir, only)): + skip = baseline_fail_skips(jdir, cmp_summary) if a.skip_baseline_fail and cmp_summary and not prev else {} + for jid, (bv, rl) in skip.items(): + summary["journeys"][jid] = {"role": rl, "verdict": "SKIPPED", "compare": f"BASELINE DEFECT (before {bv})", + "why": f"not run on the new app: it {bv}ed on the old app too (--skip-baseline-fail)"} + if skip: + print(f"skip: {len(skip)} journeys failed on the old app and feed no later one: not run on the new app", file=sys.stderr) + for key, role, role_dir, files in ([] if prev and not only else data_waves(jdir, a.gen_dir, only, skip)): prior_dirty = dirty if DESTR_WAVE in key and files: dirty = f"{read_journey(files[0])['id']} ({destructive_scope(read_journey(files[0]))})" @@ -1811,7 +1838,7 @@ def main(): for s in summary["journeys"].values()) # SCREEN / EFFECT INVALID is INVALID, never FAIL (N1R2-8) else: fail = "FAIL" in vs - counts = {v: vs.count(v) for v in ("PASS", "FAIL", "INVALID", "INFO", "NOT CHECKED", "UNREACHABLE", "CONTAMINATED")} + counts = {v: vs.count(v) for v in ("PASS", "FAIL", "INVALID", "INFO", "NOT CHECKED", "UNREACHABLE", "CONTAMINATED", "SKIPPED")} # E2E-11: a baseline defect with checked evidence is PASS* (listed), not INVALID; without evidence it stays INVALID. # B4N1-13: the headline counts it as PASS* (baseline defect), not as FAIL next to "= N PASS*" two lines down; # an unevidenced FAIL stays FAIL @@ -1823,6 +1850,7 @@ def main(): print(f"journeys: {counts['PASS']} PASS, {counts['FAIL']} FAIL, {counts['INVALID']} INVALID" + (f", {len(evid)} PASS* (baseline defect)" if evid else "") + (f", {counts['UNREACHABLE']} PASS* (unreachable in the OLD UI, evidence checked)" if counts["UNREACHABLE"] else "") + + (f", {counts['SKIPPED']} SKIPPED (failed on the old app too, not run on the new one)" if counts["SKIPPED"] else "") + (f", {counts['NOT CHECKED']} NOT CHECKED (excluded: {a.exclude_systems})" if counts["NOT CHECKED"] else "") + (f", {counts['CONTAMINATED']} CONTAMINATED (login failed after a destructive journey on the same database)" if counts["CONTAMINATED"] else "") + (f", {counts['INFO']} INFO (covered, unreachable or out of scope, listed only)" if counts["INFO"] else "")) @@ -1835,7 +1863,7 @@ def main(): if anon: print(anon["line"]) veff = ["PASS*" if j in evid or s["verdict"] == "UNREACHABLE" else s["verdict"] for j, s in summary["journeys"].items()] - ex = (1 if fail or ctl_bad else INVALID_ONLY if any(v not in ("PASS", "INFO", "PASS*", "NOT CHECKED") for v in veff) + ex = (1 if fail or ctl_bad else INVALID_ONLY if any(v not in ("PASS", "INFO", "PASS*", "NOT CHECKED", "SKIPPED") for v in veff) else EXCLUDED_ONLY if counts["NOT CHECKED"] else 0) if cov and a.compare: if a.no_actions: diff --git a/upgrade-toolkit/controls/fixtures/cockpit/run/work/data.out b/upgrade-toolkit/controls/fixtures/cockpit/run/work/data.out new file mode 100644 index 0000000..b15ec44 --- /dev/null +++ b/upgrade-toolkit/controls/fixtures/cockpit/run/work/data.out @@ -0,0 +1,6 @@ +| verdict | check | detail | +|---|---|---| +| PASS* | db.attr.Shop.Order.Total | values changed (non-null 4 -> 4) (accepted: start noise: also differs between two starts of the SAME old app) | +| PASS* | db.attr.Vendmod.Ent.Field | column gone (noted: the Vendmod update 1.0.0 -> 2.0.0 changed its own attribute, not the app's data) | +| PASS | db.entity.Shop.Order | rows 4 -> 4 | +INVALID 0 | FAIL 0 | REVIEW 0 | PASS* 2 | PASS 40 diff --git a/upgrade-toolkit/controls/found_means_control.py b/upgrade-toolkit/controls/found_means_control.py new file mode 100644 index 0000000..26ee239 --- /dev/null +++ b/upgrade-toolkit/controls/found_means_control.py @@ -0,0 +1,33 @@ +"""Control for cockpitmatrix.found_detail: what a finding MEANS, per item, built from the run's own files. +Usage: found_means_control.py . Prints "means : ..." and "item : what | tag | tone | meaning" lines.""" +import json, os, sys + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bin")) +import cockpitmatrix as cm + +w = sys.argv[1] +os.makedirs(w, exist_ok=True) +json.dump({"endpoints": [ + {"doc": "Shop.GetRates", "kind": "rest", "status": "MOCKED", "why": "every location is an overridden constant"}, + {"doc": "Mail.GetToken", "kind": "rest", "status": "UNKNOWN", "why": "location from data Mail.Provider.TokenUrl: set it to the printed URL"}, + {"doc": "Sso.Refresh", "kind": "rest", "status": "UNKNOWN", "why": "$Uri is computed by Java action Sso.GetUri at runtime"}]}, + open(os.path.join(w, "apimock-inventory.json"), "w")) +json.dump([{"name": "Theme", "ui": [{"change": "CHANGED", "kind": "layout", "name": "Theme.Default", "detail": "content differs from the package"}]}, + {"name": "Plain", "ui": []}, {"name": "Odd", "domain_error": ["not found"]}, + {"name": "Admin", "domain": [{"change": "ADDED", "kind": "entity", "name": "Admin.Extra", "data": True, "detail": ""}], + "logic": [{"change": "ADDED", "kind": "microflow", "name": "Admin.Save", "detail": "local document, removed by the update"}]}], + open(os.path.join(w, "localedits.json"), "w")) +recs = [{"topic": "standins", "found": {}}, + {"topic": "modules", "found": {"evidence": ["module update Theme 1.0.0 -> 2.0.0 (after the convert)", "module update Plain 1.0.0 -> 1.1.0 (after the convert)", + "module update Odd 3.0.0 -> 3.1.0 (after the convert)"]}}, + {"topic": "module:Admin", "found": {}}, + {"topic": "react", "found": {"evidence": ["Dojo to React on the 11.12.7 rung: 4 x Drop-down (core) -> Combo box"]}}, + {"topic": "nothing:known", "found": {}}] +for r in recs: + d = cm.found_detail(r, w) + if not d: + print(f"empty {r['topic']}") + for m in d.get("means", []): + print(f"means {r['topic']}: {m}") + for i in d.get("items", []): + print(f"item {r['topic']}: " + " | ".join(i)) diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index eff14d3..b707ddf 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -265,6 +265,11 @@ ctl constcheck-restore-config-file-pass 0 "^ALTER SETTINGS CONSTANT 'MFAmodule\. # --- gate: start log beyond ERROR lines (E1 18, E1 24) ------------------------------------------------------------- ctl logdiff-mfa-off-catch 1 'REVIEW +CHANGED.*MFA module login NOT active' 'was: .*MFA module login active' -- python3 $B/logdiff.py $FX/logs/old.log $FX/logs/new-mfa-off.log ctl logdiff-same-pass 0 'FAIL 0 \| REVIEW 0' -- python3 $B/logdiff.py $FX/logs/old.log $FX/logs/new-same.log +# WM1 2026-11-17: 9.24 logs the trial-license line as Core, 10.x as LicenseService: same line, renamed logger, noted +sed 's/WARNING - Core: Cache size/WARNING - CacheService: Cache size/' $FX/logs/old.log > $T/new-logger.log +sed 's/WARNING - Core: Cache size for entity/WARNING - CacheService: Cache limit for entity/' $FX/logs/old.log > $T/new-logger-text.log +ctl logdiff-renamed-logger-noted-pass 0 'PASS\* +NEW_WARNING +WARNING CacheService: Cache size' 'noted: the old app logs the same line as Core; only the logger name changed' 'FAIL 0 \| REVIEW 0' -- python3 $B/logdiff.py $FX/logs/old.log $T/new-logger.log +ctl logdiff-renamed-logger-other-text-catch 1 'REVIEW +NEW_WARNING +WARNING CacheService: Cache limit' -- python3 $B/logdiff.py $FX/logs/old.log $T/new-logger-text.log ctl logdiff-start-failed-catch 1 'FAIL +START_FAILED' 'deprecated' -- python3 $B/logdiff.py $FX/logs/old.log $FX/logs/new-start-failed.log # an ERROR logged after the start window before and inside it now is MOVED (REVIEW), not NEW_ERROR (E4 24) ctl logdiff-moved-pass 1 'REVIEW +MOVED +ERROR SAML_SSO' '!NEW_ERROR' -- python3 $B/logdiff.py $FX/logs/old-lazy-error.log $FX/logs/new-early-error.log @@ -345,14 +350,18 @@ ctl diff-admin-row-noted-pass 0 'PASS\* \| db\.attr\.System\.User\.changedDate \ ctl diff-admin-other-user-catch 1 'FAIL \| db\.attr\.System\.User\.changedDate \| values changed' '!local admin' -- python3 $B/diff.py $T/admin-other-b.json $T/admin-other-a.json --allow $FX/allow/narrow.txt ctl diff-owner-catch 1 'FAIL \| db\.identity\.assoc:System\.owner' -- python3 $B/diff.py $T/sb.json $T/s-owner-lost.json --allow $FX/allow/narrow.txt ctl diff-owner-pass 0 '!System\.owner' 'FAIL 0 \| REVIEW 0' -- python3 $B/diff.py $T/sb.json $T/s-owner-order.json --allow $FX/allow/narrow.txt -ctl diff-new-endpoint-catch 2 'NEW_ENDPOINT \| http\./rest-doc/notify/openapi\.json' -- python3 $B/diff.py $T/sb.json $T/sa.json --allow $FX/allow/email-only.txt +# WM1 2026-11-17: openapi.json of a service whose swagger.json the old app served is the same docs in the 10.x format: noted. +# A service the old app had no docs for at all stays NEW_ENDPOINT +ctl diff-openapi-next-to-swagger-noted-pass 0 'PASS\* \| http\./rest-doc/notify/openapi\.json \| .*noted: the same service.s docs in the OpenAPI 3 format' -- python3 $B/diff.py $T/sb.json $T/sa.json --allow $FX/allow/email-only.txt +python3 -c "import json,sys; d=json.load(open(sys.argv[1])); assert d['http'].pop('/rest-doc/notify/swagger.json'); json.dump(d, open(sys.argv[2], 'w'))" $T/sb.json $T/sb-nosw.json +ctl diff-new-endpoint-catch 2 'NEW_ENDPOINT \| http\./rest-doc/notify/openapi\.json' -- python3 $B/diff.py $T/sb-nosw.json $T/sa.json --allow $FX/allow/email-only.txt ctl diff-allowlist-pass 0 'PASS\* \| http\./rest-doc/notify/openapi\.json' 'PASS\* \| db\.attr\.Administration\.Account\.Email' -- python3 $B/diff.py $T/sb.json $T/sa.json --allow $FX/allow/narrow.txt ctl diff-allow-detail-catch 1 'FAIL \| db\.attr\.Administration\.Account\.Email \| values changed' -- python3 $B/diff.py $T/sb.json $T/sa.json --allow $FX/allow/wrong-detail.txt # PV-7: the restart control. diff --emit-allow turns the FAIL rows between two starts of the SAME old app into allow lines # (start noise); fed back as --allow they leave 0 FAIL, and a clean pair writes an empty file. ctl diff-emit-allow-catch 1 '^--emit-allow: [1-9][0-9]* check\(s\) written' -- python3 $B/diff.py $T/sb.json $T/sa.json --emit-allow $T/noise.txt ctl diff-emit-allow-lines-pass 0 'db\.attr\.Administration\.Account\.Email values changed[^#]* # start noise: differs between two starts of the same old app \(' -- cat $T/noise.txt -ctl diff-emit-allow-roundtrip-pass 2 'FAIL 0 \| REVIEW 0' -- python3 $B/diff.py $T/sb.json $T/sa.json --allow $T/noise.txt +ctl diff-emit-allow-roundtrip-pass 0 'FAIL 0 \| REVIEW 0 \| NEW_ENDPOINT 0' -- python3 $B/diff.py $T/sb.json $T/sa.json --allow $T/noise.txt ctl diff-emit-allow-clean-pass 0 '^--emit-allow: 0 check\(s\)' -- python3 $B/diff.py $T/sb.json $T/sb.json --db-only --emit-allow $T/noise-clean.txt # NMR-1: the emitted line carries the noise row's own detail, so one noise error accepts that error only, never every new error { cat $FX/logs/old.log; echo "2026-09-14 10:16:02.296 ERROR - Cache: NoiseError: cache warmup"; } > $T/old-noise.log @@ -2258,6 +2267,15 @@ cp -R $SH/old/shots-Administrator $J3/old/before/; cp -R $SH/empty/shots-Adminis for d in old/before new/after; do echo '{"journeys": {"login-Administrator": {"role": "Administrator", "verdict": "PASS"}}, "roles": {}}' > $J3/$d/summary.json; done ctl journeys-compare-screens-orphan-catch 1 '^screens: 1 PASS, 0 PASS\*, 1 REVIEW, 0 INVALID' -- python3 $B/run_journeys.py x --label after --work $J3/new --compare $J3/old --rejudge ctl journeys-screen-accept-pass 0 '!SCREEN REVIEW' -- python3 $B/run_journeys.py x --label after --work $J/new --compare $J/old/before/summary.json --rejudge --shot-accept $T/shot-accept.txt "${NOACT[@]}" +# --skip-baseline-fail (WM1 2026-11-17): the new app does not walk a journey that failed on the old app and feeds no later one +ctl journeys-skip-baseline-fail-pick-pass 0 '^waves skip: use-User=FAIL/User$' '^waves walked: make-User ok-User$' '^no-waves skip: make-User$' -- python3 $here/skip_baseline_control.py $(mkdir -p $T/skc && echo $T/skc) +# a SKIPPED journey is counted and named in the headline, is no REGRESSION, and its old-app failure stays the baseline rule's +# (exit 3, which gate.sh journeys_noted turns into a note) +J4=$T/rj4s; mkdir -p $J4; cp -R $J2/old $J2/new $J4/ +python3 -I -c "import json,sys +for f,v in ((sys.argv[1],{'role':'Administrator','verdict':'FAIL'}),(sys.argv[2],{'role':'Administrator','verdict':'SKIPPED','compare':'BASELINE DEFECT (before FAIL)','why':'not run on the new app'})): + d=json.load(open(f)); d['journeys']['report-Administrator']=v; json.dump(d,open(f,'w'))" $J4/old/before/summary.json $J4/new/after/summary.json +ctl journeys-skipped-counted-pass 3 '^journeys: 2 PASS, 0 FAIL, 0 INVALID, 1 SKIPPED \(failed on the old app too, not run on the new one\)' ' 0 regression ' '!^ *REGRESSION' -- python3 $B/run_journeys.py x --label after --work $J4/new --compare $J4/old --rejudge "${NOACT[@]}" ctl journeys-compare-path-catch 2 'no summary.json there.*nothing was run' -- python3 $B/run_journeys.py x --label after --work $J/new --compare $J/nothere --rejudge # N1R2-8 / T2R3-11: INVALID only (SCREEN INVALID, a BASELINE DEFECT) exits 3 with its own verdict line, never 1 (FAIL) J4=$T/rj4; mkdir -p $J4/old/before $J4/new/after; cp -R $SH/old/shots-Administrator $J4/old/before/ @@ -3773,7 +3791,13 @@ ctl gate-journey-coverage-rc3-catch 2 '\| 9 \| journeys \| INVALID \| 3 \| INVAL # a decision, not a fix; one untagged FAIL line keeps it a regression JOUTB=$'journeys: 4 PASS, 1 FAIL, 0 INVALID\n FAIL j3-User x [BASELINE DEFECT (before FAIL): no evidence]' JOUTU=$'journeys: 3 PASS, 2 FAIL, 0 INVALID\n FAIL j3-User x [BASELINE DEFECT (before FAIL): no evidence]\n FAIL j4-User y' -ctl gate-journey-baseline-only-decision-pass 1 'journeys: needs a decision' '!journeys: REGRESSION to fix' -- gcase jbd GATE_STUB_RC_run_journeys=1 GATE_STUB_OUT_run_journeys="$JOUTB" +# 2026-11-17 (owner: "if it fails on the new one and the old one, it's not a fail; mention it"): FSM rung 2 read FAIL with +# 277 PASS, 0 FAIL, 243 failing on the old app too. Failing on both is PASS* with a note; it still needs >= 20% passing on both +ctl gate-journey-baseline-only-noted-pass 0 '\| journeys \| PASS\* \| 1 \| noted, not a failure: nothing that passed on the old app fails' '!journeys: REGRESSION to fix' '!journeys: needs a decision' -- gcase jbd GATE_STUB_RC_run_journeys=1 GATE_STUB_OUT_run_journeys="$JOUTB" +JOUTF=$'journeys: 277 PASS, 0 FAIL, 2 INVALID, 243 PASS* (baseline defect), 1 CONTAMINATED (x)\n coverage INVALID: instrument not measuring: 233 of 457 actions fail on the baseline (50% > 30%)\n coverage INVALID: 3 baseline defects without evidence (FIXTURE / OLD-APP / FLAKY, --baseline-evidence): a, b, c\nverdict: INVALID only (exit 3)' +ctl gate-journey-fsm-shape-noted-pass 0 '\| journeys \| PASS\* \| 3 \| noted, not a failure' -- gcase jfsm GATE_STUB_RC_run_journeys=3 GATE_STUB_OUT_run_journeys="$JOUTF" +JOUTN=$'journeys: 5 PASS, 0 FAIL, 0 INVALID, 95 PASS* (baseline defect)\n coverage INVALID: instrument not measuring: 95 of 100 actions fail on the baseline (95% > 30%)\nverdict: INVALID only (exit 3)' +ctl gate-journey-not-measuring-stays-invalid-catch 2 '\| journeys \| INVALID \| 3 \|' '!noted, not a failure' -- gcase jnm GATE_STUB_RC_run_journeys=3 GATE_STUB_OUT_run_journeys="$JOUTN" ctl gate-journey-untagged-fail-regression-catch 1 'journeys: REGRESSION to fix' -- gcase jbu GATE_STUB_RC_run_journeys=1 GATE_STUB_OUT_run_journeys="$JOUTU" # the config file: relative paths resolve against the config's own folder, so a worker's cwd cannot break them printf 'WORK=w\nLABEL=cf\nBEFORE_JSON=before.json\nAFTER_JSON=after.json\n' > "$GI/gate.conf" @@ -5756,8 +5780,8 @@ ctl proposal-steps-rerun-no-duplicates-catch 0 '^same events: yes$' '^same steps printf '88 2.0.0 Newmod\n' >> $PR/rerun/plan.txt; bash $PR/prop.sh $PR/rerun > /dev/null [ \"\$b\" = \"\$(bash $PR/ssum.sh $PR/rerun | cut -d' ' -f1,2 | tr '\\n' ' ')\" ] && echo 'plan change keeps ids: yes'" ctl proposal-steps-plan-change-updates-found-pass 0 'plan holds 2 module updates' -- sh -c "grep -Eo 'plan holds [0-9]+ module updates?' $PR/rerun/steps.jsonl | tail -1" -# unattended: every step has an approval, assumed, by agent, source unattended; none is approved -ctl proposal-steps-unattended-assumed-catch 0 '^S-[0-9]+ modules .* approval=assumed by=agent source=unattended ' '^S-[0-9]+ module:DataWidgets .* approval=assumed by=agent source=unattended ' '!approval=none' '!approval=approved' -- bash $PR/ssum.sh $PR/unatt +# unattended (auto mode): routine work goes ahead (approved, source routine); a real decision stays assumed until a person answers (the owner, 2026-11-17) +ctl proposal-steps-unattended-assumed-catch 0 '^S-[0-9]+ modules .* approval=approved by=agent source=routine ' '^S-[0-9]+ module:DataWidgets .* approval=assumed by=agent source=unattended ' '!approval=none' '!module:DataWidgets .*approval=approved' -- bash $PR/ssum.sh $PR/unatt # signed: every step is approved with the signer; the item Jan answered by hand (P-1) is approved by Jan, not by the signer ctl proposal-steps-signed-approved-pass 0 '^S-[0-9]+ modules .* approval=approved by=Jan Jansen source=signoff ' '^S-[0-9]+ person:go-live .* approval=approved by=Jan Jansen source=signoff ' \ '^S-[0-9]+ module:Administration alias=P-1 approval=approved by=Jan source=decisions ' '!approval=none' '!approval=assumed' '!by=agent' -- bash $PR/ssum.sh $PR/signed @@ -6595,6 +6619,10 @@ ctl heal-harness-log-accepted-catch 0 'test-setup / logallow / known harness lin ctl heal-log-allow-line-pass 0 '^WARNING Configuration: Using derived value for SessionKeepAliveUpdatesInterval\* # test harness: ' '!Something else' -- cat $HL/h1/heal-log-allow.txt ctl heal-new-error-never-accepted-catch 0 'log:log\.start\.NEW_ERROR:.* => real-regression / None' -- hcls 'log:log.start.NEW_ERROR:ERROR_Shop_failed_to_start_the_queue' ctl heal-unknown-log-line-stays-open-catch 0 'Something_else_changed => likely-real' 'not a known harness line' -- hcls 'log:log.start.NEW_WARNING:WARNING_Something_else_changed' +# WM1 2026-11-17: 10.24 also derives ClusterManagerActionInterval from the harness's short SessionTimeout (a REVIEW every run) +python3 $here/lib/mkheal.py $HL/h3 && sed -i.bak 's/^- PASS Shop.Other: same$/- REVIEW log.start.NEW_WARNING: WARNING Configuration: Using derived value for ClusterManagerActionInterval (# ms) based on SessionTimeout\n&/' $HL/h3/diff.md \ + && python3 $B/heal.py plan --work $HL/h3 --round 1 > $HL/plan3.out 2>&1 +ctl heal-cluster-interval-harness-line-pass 0 '^WARNING Configuration: Using derived value for ClusterManagerActionInterval\* # test harness: .*SessionTimeout' -- cat $HL/h3/heal-log-allow.txt ctl heal-effective-allow-has-both-pass 0 '^# written by heal.py' 'Shop\.Item\.changedDate' -- cat $HL/h1/allow-effective.txt ctl heal-no-test-data-seed-catch 0 'journey:j-empty => no-test-data / seed / the page Shop\.Empty shows an empty list' -- hcls journey:j-empty ctl heal-seed-entity-in-plan-pass 0 '"Shop\.Thing"' '!Shop\.Item"' -- sh -c "sed -n '/\"seed\"/,/\]/p' $HL/h1/heal-plan.json" @@ -6637,13 +6665,15 @@ ctl heal-round2-changes-nothing-catch 0 'CHANGED 0' -- cat $HL/plan2.out ctl heal-finalize-pass-result-pass 0 'passes on the old and the new app' -- sh -c "python3 -I -c \"import json;print([r['result'] for r in json.load(open('$HL/h1/heal.json'))['rows'] if r['id']=='journey:j-flake'])\"" ctl heal-open-rows-nj-text-catch 0 'real-regression: 3 row\(s\)' 'Options: fix the app' 'Recommendation: ' 'likely-real: ' -- python3 $B/heal.py nj --work $HL/h1 ctl heal-summary-line-pass 0 '^heal: [0-9]+ failing row\(s\) classified, [0-9]+ accepted with a written reason' 'stay open \(3 real-regression' -- python3 $B/heal.py summary --work $HL/h1 -# a rerun that still fails turns the row into likely-real (the heal did not work), never into a pass +# a rerun that still fails on both apps is a note for the owner and a toolkit gap (2026-11-17), never a pass and never likely-real python3 $here/lib/mkheal.py $HL/h3 && python3 $B/heal.py plan --work $HL/h3 --round 1 > /dev/null 2>&1 mkdir -p $HL/ro3/before-picked $HL/rn3/after echo '{"journeys": {"j-flake": {"verdict": "FAIL"}}}' > $HL/ro3/before-picked/summary.json echo '{"journeys": {"j-flake": {"verdict": "FAIL", "not_pass": ["x"]}}}' > $HL/rn3/after/summary.json python3 $B/heal.py merge --work $HL/h3 --old $HL/ro3 --new $HL/rn3 > /dev/null 2>&1; python3 $B/heal.py finalize --work $HL/h3 > /dev/null 2>&1 -ctl heal-rerun-still-fails-likely-real-catch 0 "\['still FAIL on the new app, old app FAIL'\]" 'likely-real' -- sh -c "python3 -I -c \"import json;r=[r for r in json.load(open('$HL/h3/heal.json'))['rows'] if r['id']=='journey:j-flake'][0];print([r['result']], r['class'])\"" +ctl heal-rerun-fails-both-noted-pass 0 "\['fails on both apps after the heal \(new FAIL, old FAIL\): listed for the owner, not an issue'\] test-setup" -- sh -c "python3 -I -c \"import json;r=[r for r in json.load(open('$HL/h3/heal.json'))['rows'] if r['id']=='journey:j-flake'][0];print([r['result']], r['class'])\"" +# the learning loop's input (steps/heal.md): rows no rule explains, and tests that run on neither app; a real regression is not a gap +ctl heal-toolkit-gaps-pass 0 '^# Toolkit gaps: ' '`data:Shop\.Customer\.name` \(unexplained difference\).*rule goes in: bin/diff\.py' '`journey:j-flake` \(test does not run on either app\)' '!Shop\.Order\.total' -- cat $HL/h3/toolkit-gaps.md # a rerun that now FAILs on the new app while the old app passes is a regression echo '{"journeys": {"j-flake": {"verdict": "PASS"}}}' > $HL/ro3/before-picked/summary.json python3 $B/heal.py merge --work $HL/h3 --old $HL/ro3 --new $HL/rn3 > /dev/null 2>&1; python3 $B/heal.py finalize --work $HL/h3 > /dev/null 2>&1 @@ -6767,6 +6797,13 @@ ctl cockpit-render-needs-you-from-run-catch 0 '"id": "regression:overlay"' '"id" ctl cockpit-render-effort-from-proposal-catch 0 '"Gridmod", "kept at its version|"Gridmod", "local edits refuse the update[^"]*", "M"' '"Themepack styles", "[^"]*", "S"' -- $CK render ctl cockpit-render-upgrade-from-git-log-catch 0 '"Widgetpack 2\.0\.0", "Automatic module update' '"Repair D2"' '"Rebuilds"' -- $CK render ctl cockpit-render-gate-and-heal-catch 0 'Click through pages|"Smoke"' '[0-9] real problem' '"1 check not run"' '1 of 2 page checks fails\. Cause: Gridmod\.' 'Replace Gridmod' 'Overlay blocks 2 clicks' -- $CK render +# WM1 2026-11-17: "All pass, 1 with accepted differences" said nothing about what was accepted; each reason is on the card +ctl cockpit-render-accepted-reasons-pass 0 'Accepted, no loss: start noise; the Vendmod update 1\.0\.0 -> 2\.0\.0 changed its own attribute' -- $CK render +# the Test stage shows what ran on the old and the new app; a new-app file older than the old one is not this run's +ctl cockpit-test-runs-both-pass 0 '^row: Journeys \| 2 journeys: 2 passed \| 2 journeys: 1 passed, 1 failed \| 1 pass on both, 1 passed on the old app and not on the new one$' '^list Journeys: b-User \| passed \| failed$' '^row: Page walk \| 1 page: 1 passed' -- python3 $here/test_runs_control.py $T/trb both +ctl cockpit-test-runs-stale-catch 0 '^row: Journeys \| 2 journeys: 2 passed \| not run yet in this run \| needs both apps$' '!1 passed, 1 failed' -- python3 $here/test_runs_control.py $T/trs stale +# what a finding MEANS, per item, from the run's own files (the owner, 2026-11-17: "not just what, but what it means") +ctl cockpit-found-means-pass 0 '^means standins: 1 of 3 outside calls are covered in the tests\.$' '^item standins: Mail\.GetToken \| address from data \| wait \| .*Mail\.Provider\.TokenUrl' '^item modules: Theme 1\.0\.0 → 2\.0\.0 \| 1 local edit \| wait \| .*restyled this shared layout' '^item modules: Plain .* \| no local edits \| ok \|' '^means modules: 1 could not be compared' '^item module:Admin: Admin\.Extra \| added \| bad \| .*data stored in it' '^means module:Admin: 1 of them holds the app' '^means react: 4 widgets in 1 kind move' '^empty nothing:known$' -- python3 $here/found_means_control.py $T/fm ctl cockpit-render-time-from-timing-pass 0 'Shop|App upgrade' '1 h 40|1 h 30' -- $CK render ctl cockpit-page-content-only-pass 0 'no page skeleton tags' '^' -- $CK nodoc ctl cockpit-no-hardcoded-run-pass 0 'no hard-coded run names' -- $CK nohard diff --git a/upgrade-toolkit/controls/skip_baseline_control.py b/upgrade-toolkit/controls/skip_baseline_control.py new file mode 100644 index 0000000..f7309b9 --- /dev/null +++ b/upgrade-toolkit/controls/skip_baseline_control.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""skip_baseline_control.py DIR: run_journeys --skip-baseline-fail picks, on a fake journey set. Prints the skipped and the +walked ids. A set with data passes: the failing consumer (a _path_needs journey) is skipped, the failing creator still runs, +a passing one runs. A set without data passes: every journey that failed on the old app is skipped.""" +import json +import os +import sys +from pathlib import Path + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bin")) +import run_journeys as rj # noqa: E402 + +t = Path(sys.argv[1]) + + +def mk(d, jid, needs): + p = t / d / "journeys" / "User" + p.mkdir(parents=True, exist_ok=True) + j = {"id": jid, "role": "User", "steps": []} + if needs: + j["_path_needs"] = ["a row in a list"] + (p / f"{jid}.journey.json").write_text(json.dumps(j)) + + +(t / "b.json").write_text(json.dumps({"journeys": {"make-User": {"verdict": "FAIL"}, "use-User": {"verdict": "FAIL"}, + "ok-User": {"verdict": "PASS"}}})) +mk("w", "make-User", False); mk("w", "use-User", True); mk("w", "ok-User", True) +mk("n", "make-User", False); mk("n", "ok-User", False) +sw = rj.baseline_fail_skips(t / "w" / "journeys", t / "b.json") +sn = rj.baseline_fail_skips(t / "n" / "journeys", t / "b.json") +walked = [rj.read_journey(f)["id"] for _, _, _, fs in rj.data_waves(t / "w" / "journeys", str(t / "w"), None, sw) for f in fs] +print("waves skip:", " ".join(f"{k}={v[0]}/{v[1]}" for k, v in sorted(sw.items()))) +print("waves walked:", " ".join(sorted(walked))) +print("no-waves skip:", " ".join(sorted(sn))) diff --git a/upgrade-toolkit/controls/test_runs_control.py b/upgrade-toolkit/controls/test_runs_control.py new file mode 100644 index 0000000..96721bb --- /dev/null +++ b/upgrade-toolkit/controls/test_runs_control.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""test_runs_control.py DIR CASE: cockpitmatrix.test_runs on a fake work folder, the Test stage's "what ran, on the old and +the new app" table. Prints one line per row and per listed test. Cases: + both both apps walked: a journey that passed on the old app only is counted and listed + stale the new app's journeys file is older than the old app's (a rerun that has not walked the new app yet)""" +import json +import os +import sys +import time + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bin")) +import cockpitmatrix as cm # noqa: E402 + +w, case = sys.argv[1], sys.argv[2] + + +def put(rel, obj): + p = os.path.join(w, rel) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "w") as f: + json.dump(obj, f) + return p + + +for sd, v in (("before", "PASS"), ("after", "PASS")): + put(f"smoke-{sd}.json", {"roles": [{"role": "User", "steps": [{"caption": "Orders", "verdict": v}]}]}) +old = put("e2e-before-picked/before-picked/summary.json", {"journeys": {"a-User": {"verdict": "PASS"}, "b-User": {"verdict": "PASS"}}}) +new = put("e2e-after/after/summary.json", {"journeys": {"a-User": {"verdict": "PASS"}, "b-User": {"verdict": "FAIL"}}}) +t0 = os.path.getmtime(old) +os.utime(new, (t0 - 3600, t0 - 3600) if case == "stale" else (t0 + 60, t0 + 60)) +with open(os.path.join(w, "data.out"), "w") as f: + f.write("INVALID 0 | FAIL 0 | REVIEW 0 | PASS* 1 | PASS 9 | INFO 2\n") +r = cm.test_runs(w) +for row in r["rows"]: + print("row: " + " | ".join(row)) +for k, l in r["lists"].items(): + for t in l: + print(f"list {k}: " + " | ".join(t)) diff --git a/upgrade-toolkit/steps/heal.md b/upgrade-toolkit/steps/heal.md index 5e9c21e..3ad061b 100644 --- a/upgrade-toolkit/steps/heal.md +++ b/upgrade-toolkit/steps/heal.md @@ -45,6 +45,36 @@ listed as "needs a decision": check production configurations with an empty or w - The merge keeps the stored summaries as `summary.pre-heal.json` and lists the healed ids under `healed`. - `heal-pre-diff.md` is the data diff the classifier reads (the heal step runs `diff.py` itself, before the gate). +## The learning loop (toolkit-gaps.md) + +A FAIL is only "worked on the old app, broken on the new one". Everything else that turns a run red is a rule the toolkit +lacks, and the rule is closed in the toolkit, so the next app gets it for free. `heal.py finalize` writes +`work/toolkit-gaps.md`: every likely-real row (a difference no rule explains) and every test that runs on neither app, +each with the script its rule belongs in. The `heal:` summary line counts them. Work the list, one row at a time: + +1. **Prove it on both apps.** The old run, `twin-old.md` (two starts of the old app), the screenshots, the log lines. + A guess is not evidence. +2. **A real defect of the app stays open.** It is in NEEDS-JUDGMENT; nothing in the loop accepts it. +3. **Otherwise write the rule in the owner script, with its evidence condition.** Never a bare allow line for one app: + the rule names what makes the row safe, so the same row without that evidence is still caught. +4. **Two controls:** one that fails on the old toolkit and passes on the new one, and a guard that still catches the real + case. Run the group, then check the first control against the old code. +5. **Rejudge the stored run** (`--from heal`, or the script's `--rejudge`); no new walk. +6. **Write it down.** The rule and its why in this step doc or the playbook; a Mendix fact (not a toolkit fact) also as a + note in the company brain (`internal/upgrade/`). + +The rules closed this way on a 9.24 -> 10.24 -> 11.12 app (2026-11-17), each printed as PASS* with its reason: + +| row | rule | where | +|---|---|---| +| data: a Marketplace module's own attributes changed (MendixSSO, MxModelReflection) | that module was updated in this run (`--module-jumps`) and no rows of other entities are lost | `diff.py` | +| data: the admin's password and `changedDate` | the toolkit signs in as `UPG_ADMIN_USER`; only that one row may differ (`noadmin` figures) | `snapshot.py`, `diff.py` | +| http: `.../openapi.json` 404 -> 200 | the same service's `swagger.json` answered 2xx on the old app (10.x serves both formats) | `diff.py` | +| log: a new WARNING whose text the old app logs under another logger | same text, only the logger name changed (trial license: `Core` -> `LicenseService`) | `logdiff.py` | +| log: `Using derived value for ClusterManagerActionInterval` | `runapp.py` starts every app with a short `SessionTimeout` | `HARNESS_LOG` in `heal.py` | +| journeys: failures on the old app too | nothing that passed on the old app fails on the new one, and at least 20% pass on both (otherwise the run is not measuring) | `gate.sh journeys_noted` | +| journeys: a heal that leaves a journey failing on both apps | listed for the owner, `test-setup`, and a toolkit gap; not likely-real | `heal.py finalize` | + ## What the gate reads from it `gate.sh` keys `HEAL_JSON` (prints the one-line `heal:` summary), `GATE_NOTES` (L56: every non-empty line that is not a diff --git a/upgrade-toolkit/steps/journeys.md b/upgrade-toolkit/steps/journeys.md index 718d2ba..679a85d 100644 --- a/upgrade-toolkit/steps/journeys.md +++ b/upgrade-toolkit/steps/journeys.md @@ -224,6 +224,16 @@ with the `defects:` line; a defect without evidence stays in FAIL, B4N1-13): arguments, or a journey file that is empty or not a journey: the file is named, nothing runs, B4T2-18). The gate shows 3 and 2 as INVALID. +### Journeys that failed on the old app are not walked again (`--skip-baseline-fail`) + +A failure is only "worked on the old app, broken on the new one". On a 9.24 app 160 of 421 journeys failed on the old app; +walking them on the new app took about half the journeys time and could only end in BASELINE DEFECT. The pipeline passes +`--skip-baseline-fail` to the new-app run: a journey that FAILed on the old app and feeds no later one (a second-pass +journey with `_path_needs`, or any journey of a set without data passes) is listed as `SKIPPED` ("failed on the old app +too, not run on the new one"), counted in the headline, never a FAIL. A first-pass journey still runs, because it may +create the rows a later one needs. `UPG_JOURNEYS_ALL=1` walks every journey. The cockpit's Test tab lists every journey +with its old and its new verdict. + ## Action inventory (E2E-8): the denominator, from the old model ``` From 775ddae8961236273fc87ee9917536b42d0c49a5 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 14:20:49 +0800 Subject: [PATCH 39/59] Cockpit: a decision opens its case (headline, tiles, old and new screenshot, why, options with the catch) - work/cases/<id>.json, written by the step that raises the decision; screenshots embedded, so the cockpit stays one file - Resolve opens a wide dialog with the case; each option card says what it does and its catch; no case file: the plain question - steps/heal.md: ask a person only on a deviation, and how to write the case - control cockpit-case-file-pass Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/cockpitmatrix.py | 77 +++++++++++++++++-- .../controls/found_means_control.py | 16 ++++ upgrade-toolkit/controls/run-controls.sh | 2 + upgrade-toolkit/steps/heal.md | 19 +++++ 4 files changed, 107 insertions(+), 7 deletions(-) diff --git a/upgrade-toolkit/bin/cockpitmatrix.py b/upgrade-toolkit/bin/cockpitmatrix.py index 471b6a2..b969229 100644 --- a/upgrade-toolkit/bin/cockpitmatrix.py +++ b/upgrade-toolkit/bin/cockpitmatrix.py @@ -1212,7 +1212,7 @@ def build(work, model): "tiles": tl, "feed": feed(work, model, ledger, items, g.get("closed_q") or []), "loop": list(LOOP), "band": band(model, rows, chips, g, tl, kpi), "generated": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - "phases": phases(rows), "ladder": ladder(work), "health": health(work), + "phases": phases(rows), "ladder": ladder(work), "health": health(work), "cases": cases(work), "itemsFile": os.path.exists(os.path.join(work, "cockpit-items.json")), "stepsFile": recs is not None, "plan": plan_doc(cp.rd(os.path.join(work, "PROPOSAL.md"))), "ran": test_runs(work)} @@ -1636,6 +1636,15 @@ def band(model, rows, chips, g, tl, kpi=None): .qk{font:12px var(--mono);color:var(--muted)}.qcard h3{margin:4px 0 10px;font-size:19px}.qcard p{margin:6px 0;font-size:14px;line-height:1.45} .qev{margin:10px 0;padding:10px 12px;border:1px solid var(--line,#ddd);border-radius:8px}.qev ul{margin:6px 0 4px 18px;padding:0}.qev li{margin:2px 0}.qev small{opacity:.7} .qrec{padding:8px 10px;border-radius:10px;background:var(--accentbg)} +.qdlg.wide{max-width:1100px;width:92vw}.chead{font-size:17px!important;font-weight:600} +.ctiles{display:flex;gap:10px;margin:10px 0}.ct{flex:1;border:1px solid var(--line);border-radius:10px;padding:8px 12px;text-align:center}.ct b{display:block;font-size:28px}.ct span{font-size:13px;color:var(--muted)} +.ct.ok b{color:var(--ok)}.ct.bad b{color:var(--bad)}.ct.wait b{color:var(--wait)} +.cshot{margin:10px 0;display:grid;grid-template-columns:1fr 1fr;gap:10px}.cshot div{border:1px solid var(--line);border-radius:10px;overflow:hidden}.cshot span{display:block;font:12px var(--mono);padding:4px 8px;color:var(--muted);border-bottom:1px solid var(--line)} +.cshot img{display:block;width:100%}.cshot i{display:block;padding:30px;color:var(--muted);text-align:center}.cshot figcaption{grid-column:1/-1;font-size:13px;color:var(--muted)} +.cwh{margin:12px 0 6px}.cwhy{counter-reset:w;list-style:none;padding:0;margin:0;display:flex;gap:8px;flex-wrap:wrap}.cwhy li{counter-increment:w;flex:1 1 200px;border:1px solid var(--line);border-radius:10px;padding:8px 10px 8px 34px;position:relative;font-size:14px} +.cwhy li::before{content:counter(w);position:absolute;left:8px;top:8px;width:20px;height:20px;border-radius:50%;background:var(--accent);color:#fff;font-size:12px;text-align:center;line-height:20px} +.cdet{margin-top:8px;font-size:13px}.qopts small.catch{color:var(--wait)} +@media (max-width:700px){.cshot{grid-template-columns:1fr}.ctiles{flex-wrap:wrap}} .qopts{display:grid;gap:8px;margin-top:12px} .qopts button{font:inherit;font-size:14px;text-align:left;padding:10px 12px;border-radius:10px;border:1px solid var(--line);background:var(--bg);color:var(--fg);cursor:pointer} .qopts button.rec{border:2px solid var(--accent);background:var(--accentbg)}.qopts button em{margin-left:8px;font-size:12px;font-style:normal;color:var(--accent);font-weight:600} @@ -2195,7 +2204,55 @@ def context_of(i): return [plain(i.get("problem") or "")] if i.get("problem") else [] -def h_questions(model, matrix=None): +CASE_DIR = "cases" + + +def case_id(item_id): + """The file name of the case of one decision: work/cases/<id>.json, with every character outside [A-Za-z0-9._-] as _.""" + return re.sub(r"[^A-Za-z0-9._-]", "_", item_id) + ".json" + + +def cases(work): + """The case files of the open decisions (the owner, 2026-11-17: "such a report per issue that needs a decision"). A case is what the + agent that raised the decision found: {"headline", "tiles": [[number, label, tone]], "shots": [{"old", "new", "cap"}], "why": [steps], + "options": {option key: {"does", "catch"}}, "details": [lines]}. Screenshots are paths in the work folder, embedded as data URIs, + so the cockpit stays one file. A missing or broken case file leaves the plain question.""" + import base64 + out, d = {}, os.path.join(work, CASE_DIR) + for f in sorted(os.listdir(d)) if os.path.isdir(d) else []: + c = cp.rj(os.path.join(d, f)) if f.endswith(".json") else None + if not isinstance(c, dict): + continue + for sh in c.get("shots") or []: + for k in ("old", "new"): + pth = os.path.join(work, sh.get(k) or "") + if sh.get(k) and os.path.isfile(pth) and pth.lower().endswith((".png", ".jpg", ".jpeg")): + with open(pth, "rb") as fh: + sh[k] = ("data:image/png;base64," if pth.lower().endswith(".png") else "data:image/jpeg;base64,") + base64.b64encode(fh.read()).decode() + else: + sh[k] = "" + out[f] = c + return out + + +def h_case(c): + """The case block inside a question dialog: tiles, old and new screenshot, why in steps.""" + if not c: + return "" + tl = "".join(f'<div class="ct {E(str(t[2]) if len(t) > 2 else "")}"><b>{E(str(t[0]))}</b><span>{E(str(t[1]))}</span></div>' for t in c.get("tiles") or [] if len(t) > 1) + def pane(src, lab): + img = f'<img src="{src}" alt="{lab}">' if src else "<i>no screenshot</i>" + return f"<div><span>{lab}</span>{img}</div>" + sh = "".join(f'<figure class="cshot">{pane(x.get("old"), "Old app")}{pane(x.get("new"), "New app")}<figcaption>{E(x.get("cap") or "")}</figcaption></figure>' + for x in c.get("shots") or []) + wy = "".join(f"<li>{E(w)}</li>" for w in c.get("why") or []) + dt = "".join(f"<li>{E(x)}</li>" for x in c.get("details") or []) + return ((f'<p class="chead">{E(c["headline"])}</p>' if c.get("headline") else "") + (f'<div class="ctiles">{tl}</div>' if tl else "") + sh + + (f'<h4 class="cwh">Why it happened</h4><ol class="cwhy">{wy}</ol>' if wy else "") + + (f'<details class="cdet"><summary>The evidence</summary><ul>{dt}</ul></details>' if dt else "")) + + +def h_questions(model, matrix=None, cases_=None): """The open questions: a "Blocked on you" list in the sidebar, each with a Resolve button that opens one question in a dialog with its context, one button per option and what each option does. The buttons post to the local server (cockpitserve.py /decide, live page only); on a static copy they stay off and the answer goes through chat.""" @@ -2211,9 +2268,11 @@ def h_questions(model, matrix=None): for n, i in enumerate(qs, 1): rows.append(f'<li><span>{E(question_of(i))}</span><button type="button" class="resolve" data-q="q{n}">Resolve →</button></li>') btns = [] + cs = (cases_ or {}).get(case_id(i["id"])) or {} for o in i["options"]: lab = plain(o["label"]) or o["label"] - say = o.get("causes") or next((t for k, t in OPTION_SAYS if k in lab.lower()), "") if i["kind"] == "plan-choice" else "" + co = (cs.get("options") or {}).get(o["key"]) or {} + say = co.get("does") or (o.get("causes") or next((t for k, t in OPTION_SAYS if k in lab.lower()), "") if i["kind"] == "plan-choice" else "") if not say and ": " in lab: lab, say = lab.split(": ", 1) say = say[:1].upper() + say[1:] + "." @@ -2221,7 +2280,8 @@ def h_questions(model, matrix=None): lab = "Approve and continue" rec = o["key"] == i.get("rec") and len(i["options"]) > 1 btns.append(f'<button type="button" disabled data-choice="{E(o["key"])}" class="{"rec" if rec else ""}"><b>{E(lab)}</b>' - f'{"<em>Recommended</em>" if rec else ""}{"<small>" + E(say[:220]) + "</small>" if say else ""}</button>') + f'{"<em>Recommended</em>" if rec else ""}{"<small>" + E(say[:320]) + "</small>" if say else ""}' + f'{"<small class=" + chr(34) + "catch" + chr(34) + ">The catch: " + E(co["catch"]) + "</small>" if co.get("catch") else ""}</button>') ctx = "".join(f"<p>{E(t)}</p>" for t in context_of(i) if t) # what exactly is behind it, from the run's own files (cold test WM1: "how can a person ever decide without those details?") ev = i.get("evidence") or {} @@ -2234,8 +2294,10 @@ def h_questions(model, matrix=None): if rl and why.startswith(rl) and why[len(rl):].startswith(";"): # the option itself is the button; keep effort and risk why = why[len(rl) + 1:].strip() why = why[:1].upper() + why[1:] - rec = f'<p class="qrec"><b>Why we recommend it:</b> {E(why)}</p>' if why and len(i["options"]) > 1 else "" - dlgs.append(f'<dialog class="qdlg" id="q{n}"><div class="qcard" data-item="{E(i["id"])}"><div class="qk">Question {n} of {len(qs)} · ' + rec = f'<p class="qrec"><b>Why we recommend it:</b> {E(why)}</p>' if why and len(i["options"]) > 1 and not cs.get("options") else "" + if cs: # a case written by the agent replaces the generic context: the case says what is wrong, shows it, and why + ctx = h_case(cs) + dlgs.append(f'<dialog class="qdlg{" wide" if cs else ""}" id="q{n}"><div class="qcard" data-item="{E(i["id"])}"><div class="qk">Question {n} of {len(qs)} · ' f'blocks step {E(i.get("step") or "")}</div><h3>{E(question_of(i))}</h3>{ctx}{rec}<div class="qopts">{btns and "".join(btns)}</div>' '<div class="qmsg" data-qmsg aria-live="polite"></div><p class="qhint">The buttons work in the live cockpit. On a copy, answer in chat.</p>' '<button type="button" class="qclose">Close</button></div></dialog>') @@ -2249,7 +2311,8 @@ def render(model, matrix): data = data.replace("</", "<\\/").replace("<!--", "<\\!--") e = html.escape head_, tiles_ = h_tiles(matrix) - qside, qdlgs = h_questions(model, matrix) + cases_ = matrix.pop("cases", None) or {} # kept out of the page data: the screenshots are embedded once, in the dialog + qside, qdlgs = h_questions(model, matrix, cases_) return (f"<title>{e(model['title'])}\n" '\n' '\n' diff --git a/upgrade-toolkit/controls/found_means_control.py b/upgrade-toolkit/controls/found_means_control.py index 26ee239..b4618fa 100644 --- a/upgrade-toolkit/controls/found_means_control.py +++ b/upgrade-toolkit/controls/found_means_control.py @@ -31,3 +31,19 @@ print(f"means {r['topic']}: {m}") for i in d.get("items", []): print(f"item {r['topic']}: " + " | ".join(i)) + +# a case file per decision: loaded by id, screenshots embedded, a broken file ignored +if len(sys.argv) > 2 and sys.argv[2] == "cases": + import base64 + os.makedirs(os.path.join(w, "cases"), exist_ok=True) + open(os.path.join(w, "old.png"), "wb").write(base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==")) + json.dump({"headline": "Seven break", "tiles": [[7, "fail on the new", "bad"]], "shots": [{"old": "old.png", "new": "gone.png", "cap": "look"}], + "why": ["one", "two"], "options": {"fix": {"does": "We fix it.", "catch": "One rerun."}}}, + open(os.path.join(w, "cases", cm.case_id("regression:module-X")), "w")) + open(os.path.join(w, "cases", "broken.json"), "w").write("{not json") + cs = cm.cases(w) + print("case files:", sorted(cs)) + c = cs[cm.case_id("regression:module-X")] + print("old shot:", c["shots"][0]["old"][:22], "| new shot:", repr(c["shots"][0]["new"])) + h = cm.h_case(c) + print("html:", "tiles" if 'class="ct bad"' in h else "-", "why=" + str(h.count("
  • ")), "no-shot" if "no screenshot" in h else "-") diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index b707ddf..472d67d 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6804,6 +6804,8 @@ ctl cockpit-test-runs-both-pass 0 '^row: Journeys \| 2 journeys: 2 passed \| 2 j ctl cockpit-test-runs-stale-catch 0 '^row: Journeys \| 2 journeys: 2 passed \| not run yet in this run \| needs both apps$' '!1 passed, 1 failed' -- python3 $here/test_runs_control.py $T/trs stale # what a finding MEANS, per item, from the run's own files (the owner, 2026-11-17: "not just what, but what it means") ctl cockpit-found-means-pass 0 '^means standins: 1 of 3 outside calls are covered in the tests\.$' '^item standins: Mail\.GetToken \| address from data \| wait \| .*Mail\.Provider\.TokenUrl' '^item modules: Theme 1\.0\.0 → 2\.0\.0 \| 1 local edit \| wait \| .*restyled this shared layout' '^item modules: Plain .* \| no local edits \| ok \|' '^means modules: 1 could not be compared' '^item module:Admin: Admin\.Extra \| added \| bad \| .*data stored in it' '^means module:Admin: 1 of them holds the app' '^means react: 4 widgets in 1 kind move' '^empty nothing:known$' -- python3 $here/found_means_control.py $T/fm +# a decision's case file (work/cases/.json): found by id, screenshots embedded, a missing shot shown as such, a broken file ignored +ctl cockpit-case-file-pass 0 "^case files: \['regression_module-X\.json'\]$" '^old shot: data:image/png;base64, \| new shot: .{2}$' '^html: tiles why=2 no-shot$' -- python3 $here/found_means_control.py $T/fmc cases ctl cockpit-render-time-from-timing-pass 0 'Shop|App upgrade' '1 h 40|1 h 30' -- $CK render ctl cockpit-page-content-only-pass 0 'no page skeleton tags' '^' -- $CK nodoc ctl cockpit-no-hardcoded-run-pass 0 'no hard-coded run names' -- $CK nohard diff --git a/upgrade-toolkit/steps/heal.md b/upgrade-toolkit/steps/heal.md index 3ad061b..c828a8b 100644 --- a/upgrade-toolkit/steps/heal.md +++ b/upgrade-toolkit/steps/heal.md @@ -75,6 +75,25 @@ The rules closed this way on a 9.24 -> 10.24 -> 11.12 app (2026-11-17), each pri | journeys: failures on the old app too | nothing that passed on the old app fails on the new one, and at least 20% pass on both (otherwise the run is not measuring) | `gate.sh journeys_noted` | | journeys: a heal that leaves a journey failing on both apps | listed for the owner, `test-setup`, and a toolkit gap; not likely-real | `heal.py finalize` | +## A decision for a person comes with its case (`work/cases/<id>.json`) + +Ask a person only when there is a deviation: an edit of the app, a check that worked on the old app and fails on the new +one, or no clear path. Routine work goes ahead (auto mode) or under the one plan sign-off. When the loop raises a decision, +the agent writes its case next to it, so the person decides on one screen (the cockpit's Resolve opens it): + +```json +{"headline": "what is wrong, in one sentence", + "tiles": [[30, "pass on both apps", "ok"], [7, "pass on the old app, fail on the new", "bad"], [1, "cause", "wait"]], + "shots": [{"old": "path/in/work/old.png", "new": "path/in/work/new.png", "cap": "what to look at"}], + "why": ["step 1 of the cause", "step 2", "step 3"], + "options": {"<option key>": {"does": "what happens when you pick it", "catch": "what it costs or breaks"}}, + "details": ["the evidence: tests, files, what was checked with which tool"]} +``` + +The file name is the decision id with every character outside `[A-Za-z0-9._-]` as `_` (`regression:module-Administration` +-> `regression_module-Administration.json`). Every sentence comes from evidence on both apps; the screenshots are embedded, +so the cockpit stays one file. Without a case file the dialog shows the plain question. + ## What the gate reads from it `gate.sh` keys `HEAL_JSON` (prints the one-line `heal:` summary), `GATE_NOTES` (L56: every non-empty line that is not a From 8aa073dcaca3df53d711353fbee13b056d71ad2a Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 14:31:59 +0800 Subject: [PATCH 40/59] editroute: a Marketplace page the menu opens gets the app's layout in place when a copy is not safe - No app module that holds copies, or a copy that would drop a widget mxcli cannot write (WM1 Accounts: the user grid): ALTER PAGE <page> { SET Layout = <company> } instead of holding it for a person. Only the layout line changes, every widget stays, the menu keeps the page. Checked on WM1's model: mxcli check passes, describe differs in one line - mxcli check refusing the in-place change: still held for a person - controls: no-home and unwritable-widget now set in place; a refused in-place change is held Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/editroute.py | 14 ++++++++------ upgrade-toolkit/controls/editroute_control.sh | 5 +++-- upgrade-toolkit/controls/run-controls.sh | 6 ++++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/upgrade-toolkit/bin/editroute.py b/upgrade-toolkit/bin/editroute.py index d2bac92..a59232d 100755 --- a/upgrade-toolkit/bin/editroute.py +++ b/upgrade-toolkit/bin/editroute.py @@ -326,7 +326,7 @@ def nav_copies(qn, company, orig, app_mpr): sidebar and header gone, the mx logo over the title (screens REVIEW, 2026-11-17). Copy each into the module where the app already keeps its copies of that vendor module's pages (a page with the same title: WM1's DISW_DesignSystem_Extention.MyAccount_Edit), on the company layout, grants mapped by role name, the menu pointed at - the copy. No such module: not copied, named in the label for a person.""" + the copy. No such module, or a copy would drop a widget: the page's own layout set in place (ALTER PAGE), menu unchanged.""" rc, out = mx(["-p", orig, "-c", "SHOW MODULES"]) mods = table(out) if rc == 0 else [] market = {m["Module"] for m in mods if m.get("Source", "").startswith("Marketplace")} @@ -346,9 +346,12 @@ def nav_copies(qn, company, orig, app_mpr): vmod, name = page.split(".", 1) titles = {p["Title"] for p in allp if p["Module"] == vmod and p.get("Title")} homes = [p for p in allp if p["Module"] not in market | {vmod, "System"} and p.get("Title") in titles] + # WM1 2026-11-17: ALTER PAGE sets one Marketplace page's layout in place (ALTER PAGES skips them); only that line + # changes, every widget stays, the menu keeps the page. The way when a copy has no home or would lose a widget + inplace = (f"mdl 1;\nALTER PAGE {page} {{ SET Layout = {company}; }};\n") if not homes: - res.append((f"page {page}: opened from the navigation, keeps the vendor layout; no app module holds copies of " - f"{vmod} pages, copy it by hand", "")) + res.append((f"page {page}: opened from the navigation, no app module holds copies of {vmod} pages: its layout " + f"set to {company} in place, the menu keeps it", inplace)) continue tmod, folder = homes[0]["Module"], homes[0]["Folder"] src = describe(app_mpr, "page", page) or describe(orig, "page", page) @@ -358,9 +361,8 @@ def nav_copies(qn, company, orig, app_mpr): # and the copy came out without its user list (menu back, page empty). A copy that would drop a widget is no copy if "NOT re-executable" in src: w = re.search(r"--\s*(\S+ \(\w+\))\s*-- NOT re-executable", src) - res.append((f"page {page}: opened from the navigation, keeps the vendor layout; a copy would drop " - f"{w.group(1) if w else 'a widget'} (mxcli cannot write it): copy it by hand in Studio Pro onto " - f"{company} and point the menu at the copy", "")) + res.append((f"page {page}: opened from the navigation, a copy would drop {w.group(1) if w else 'a widget'} " + f"(mxcli cannot write it): its layout set to {company} in place, the menu keeps it", inplace)) continue copy = f"{tmod}.{name}" rc, out = mx(["-p", app_mpr, "-c", f"SHOW MODULE ROLES IN {tmod}"]) diff --git a/upgrade-toolkit/controls/editroute_control.sh b/upgrade-toolkit/controls/editroute_control.sh index 527fc02..35d2ee4 100644 --- a/upgrade-toolkit/controls/editroute_control.sh +++ b/upgrade-toolkit/controls/editroute_control.sh @@ -36,7 +36,7 @@ if nav and a[0] == "refs" and a[3] in ("Uimod.Redir", "Uimod.Acct"): else "| NAVIGATION | Navigation.Responsive | menu_item |")) sys.exit(0) if nav and a[0] == "describe" and a[3] in ("page", "navigation"): - grid = " -- Forms$DataGrid (grid12) -- NOT re-executable: mxcli cannot author this widget\n" if nav == "grid" else "" + grid = " -- Forms$DataGrid (grid12) -- NOT re-executable: mxcli cannot author this widget\n" if nav in ("grid", "refuse") else "" print("mdl 1;\ncreate or modify page Uimod.Acct folder 'Admin' (\n Title: 'Accounts',\n Layout: Uimod.Redir\n) {\n" + grid + "};\n" "grant view on page Uimod.Acct to Uimod.Administrator, Uimod.Auditor;" if a[3] == "page" else "mdl 1;\ncreate or modify navigation Responsive\n app title 'Uimod App'\n app icon Brand.Icons.Logo\n{\n menu item 'Accounts' ( OnClick: show page Uimod.Acct with (DisabledDuringExecution: false) )\n}") @@ -67,7 +67,8 @@ if a[0] == "describe": body = "logic A" if pkg or "Cosmetic" in qn else "logic B" print(("@position(1, 1)\n" if pkg else "@position(9, 9)\n@anchor(from: right, to: left)\n") + body) elif a[0] == "check": - bad = "BROKEN" in open(a[1]).read() + t = open(a[1]).read() + bad = "BROKEN" in t or (nav == "refuse" and "ALTER PAGE" in t) # refuse: mxcli check says no to the in-place layout print(" ✗ class BROKEN is not a class [MDL000]" if bad else "ok") sys.exit(1 if bad else 0) elif a[0] == "refs": diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 472d67d..2ac6621 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6838,9 +6838,11 @@ ctl editroute-collect-layout-repoint-pass 0 '^Uimod.mdl: ALTER PAGES SET LAYOUT # WM1 gate 2026-11-17: ALTER PAGES skips Marketplace pages, so the Accounts menu opened Administration.Account_Overview on plain # Atlas 4 (screens REVIEW); the page is copied to the app's module for copies, on the company layout, the menu pointed at it ctl editroute-collect-nav-page-copied-pass 0 "^Uimod.mdl: create or modify page Brand.Acct folder 'Pages' \\($" '^Uimod.mdl: Layout: Brand.Main$' '^Uimod.mdl: grant view on page Brand.Acct to Brand.Administrator;$' '^Uimod.mdl: menu item .Accounts. \( OnClick: show page Brand.Acct ' 'no Brand role for Uimod.Auditor: not granted' '!^Uimod.mdl: app (title|icon)' -- env NAVCOPY=home $ER collect -ctl editroute-collect-nav-page-no-home-held-catch 0 '^HELD Uimod page Uimod.Acct: opened from the navigation, keeps the vendor layout; no app module holds copies' '!Brand.Acct' -- env NAVCOPY=nohome $ER collect +ctl editroute-collect-nav-page-no-home-inplace-pass 0 '^Uimod.mdl: ALTER PAGE Uimod.Acct \{ SET Layout = Brand.Main; \};$' 'no app module holds copies of Uimod pages: its layout set to Brand.Main in place, the menu keeps it' '!Brand.Acct' '!^HELD Uimod page' -- env NAVCOPY=nohome $ER collect # WM1 2026-11-17: the copy of a page with a widget mxcli cannot write came out without it (Account_Overview, no user list) -ctl editroute-collect-nav-page-unwritable-widget-held-catch 0 '^HELD Uimod page Uimod.Acct: opened from the navigation, keeps the vendor layout; a copy would drop Forms\$DataGrid \(grid12\)' '!create or modify page Brand.Acct' '!show page Brand.Acct' -- env NAVCOPY=grid $ER collect +# WM1 2026-11-17: the page's own layout is set in place (ALTER PAGE): every widget stays, the menu keeps the page +ctl editroute-collect-nav-page-unwritable-widget-inplace-pass 0 '^Uimod.mdl: ALTER PAGE Uimod.Acct \{ SET Layout = Brand.Main; \};$' 'a copy would drop Forms\$DataGrid \(grid12\) \(mxcli cannot write it\): its layout set to Brand.Main in place' '!create or modify page Brand.Acct' '!show page Brand.Acct' '!^HELD Uimod page' -- env NAVCOPY=grid $ER collect +ctl editroute-collect-nav-page-inplace-refused-held-catch 0 '^HELD Uimod page Uimod.Acct: opened from the navigation, a copy would drop .*: mxcli check refuses the result' '!ALTER PAGE Uimod.Acct' -- env NAVCOPY=refuse $ER collect # WM1 rerun 5: a dropped association with no data (non-persistent owner) is an info row, and the app still used it (CE1613) ctl editroute-xassoc-no-data-assoc-pass 0 '^ROUTE Target XASSOC: the update drops 2 association' '^approved-step.txt: Other.Np_B ' -- $ER plan ctl editroute-collect-check-held-catch 0 '^HELD Uimod snippet Bad: mxcli check refuses' '!BROKEN.*Uimod.mdl|Uimod.mdl:.*BROKEN' -- $ER collect From 91b1098580a81935640c926e9f5c9581e823f7a9 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 13:39:35 +0800 Subject: [PATCH 41/59] controls: the throwaway Postgres also starts from binaries on PATH, so the seed_snapshot controls run on a Mac pgtmp.sh looked only under /usr/lib/postgresql/*/bin. On a Mac with Homebrew Postgres every seed-snapshot Postgres control SKIPped, and a suite reading 0 failed said nothing about the snapshot loader. It now falls back to the initdb on PATH. Control: pgtmp-path-binaries-catch (exit 3 on the old script, a cluster starts and stops on the new one). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/controls/lib/pgtmp.sh | 2 ++ upgrade-toolkit/controls/run-controls.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/upgrade-toolkit/controls/lib/pgtmp.sh b/upgrade-toolkit/controls/lib/pgtmp.sh index e3424fe..8c26dad 100755 --- a/upgrade-toolkit/controls/lib/pgtmp.sh +++ b/upgrade-toolkit/controls/lib/pgtmp.sh @@ -2,6 +2,8 @@ # pgtmp.sh start | stop DIR: a throwaway Postgres (unix socket in a fresh /var/tmp/upg-pg.* folder the postgres user can # reach, no network) for the seed_snapshot controls. start prints the psql command line (it names DIR) on stdout; exit 3 when no postgres binaries are installed (the control SKIPs). bin=$(ls -d /usr/lib/postgresql/*/bin 2>/dev/null | sort -V | tail -1) +# cold CMM 2026-11-17: on a Mac (Homebrew) the binaries are on PATH, not under /usr/lib, and every Postgres control SKIPped +[ -n "$bin" ] || { i=$(command -v initdb 2>/dev/null) && bin=${i%/*}; } [ -n "$bin" ] && [ -x "$bin/initdb" ] || { echo "no postgres binaries"; exit 3; } d=${2:-}; run() { if [ "$(id -u)" = 0 ]; then runuser -u postgres -- "$@"; else "$@"; fi; } case "$1" in diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 2ac6621..ba10469 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -960,6 +960,10 @@ printf '%s\n' '/*C1*/INSERT INTO BLOCKS VALUES(0,2147483647,0)' > $T/snapblocks/ ctl seed-snapshot-blocks-bookkeeping-catch 0 '!UNMAPPED' '!statement not understood' 'to load 18' -- $SS $T/snapblocks --dry-run printf '%s\n' '/*C1*/INSERT INTO BLOCKS VALUES(0,2147483647,0)' '/*C1*/INSERT INTO BLOCKSX VALUES(1)' > $T/snapblocks/d/default.log ctl seed-snapshot-blocks-not-blanket-pass 1 'UNMAPPED \(1\)' 'statement not understood: INSERT INTO BLOCKSX' -- $SS $T/snapblocks --dry-run +# the throwaway cluster also starts from binaries on PATH (Homebrew), not only /usr/lib/postgresql (cold CMM: every Postgres control SKIPped on a Mac) +if command -v initdb > /dev/null && ! ls -d /usr/lib/postgresql/*/bin > /dev/null 2>&1; then + ctl pgtmp-path-binaries-catch 0 '^psql -h /var/tmp/upg-pg\.' '^stopped$' -- bash -c "c=\$(bash $here/lib/pgtmp.sh start) && echo \"\$c\" && bash $here/lib/pgtmp.sh stop \$(echo \"\$c\" | awk '{print \$3}') && echo stopped" +else skip pgtmp-path-binaries "no initdb on PATH, or the /usr/lib/postgresql layout is there"; fi if PGC=$(bash $here/lib/pgtmp.sh start 2>/dev/null); then PGD=$(echo "$PGC" | awk '{print $3}'); pgx() { psql -h "$PGD" -U mendix -qtA "$@"; } pgx -d fx -f $SNAP/pg-schema.sql > /dev/null && for n in ok colgone noguid skipt; do pgx -d fx -c "create database t_$n template fx" > /dev/null; done From 68156c58eade42bec6992dfbf9892007c1bd6d02 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 13:39:35 +0800 Subject: [PATCH 42/59] seed_snapshot: an external entity's remote key column is kept, not remapped as an object id A Mendix 9.24 demo package's snapshot holds <entity>$pk tables for external (OData) entities; their "_id" is the remote key, a Long whose high bits look like an entity short id. seed_snapshot tried to remap it, found no entity for those bits, listed 5 tables as UNMAPPED and loaded nothing, so the baseline fell back to 5 synthetic rows per entity. Columns other than id of a table whose entity has remote_primary_key are now kept as they are. Controls: seed-snapshot-remote-key-catch and seed-snapshot-remote-key-kept-pass (both FAIL on the old code: exit 1 UNMAPPED, no row loaded; the id is remapped to the database's short id and _id is kept on the new code). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/seed_snapshot.py | 6 ++++++ upgrade-toolkit/controls/run-controls.sh | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/upgrade-toolkit/bin/seed_snapshot.py b/upgrade-toolkit/bin/seed_snapshot.py index e3db6a8..065980a 100644 --- a/upgrade-toolkit/bin/seed_snapshot.py +++ b/upgrade-toolkit/bin/seed_snapshot.py @@ -248,6 +248,11 @@ def rows_of(t): return list(tables[t].rows.values()) s_ei = {r[0].v: (int(r[1].v), int(r[2].v)) for r in rows_of("mendixsystem$entityidentifier")} # GUID -> (short, seq) ec = {c: col("mendixsystem$entity", c) for c in ("id", "entity_name", "table_name", "superentity_id")} s_ent = {r[ec["id"]].v: (r[ec["entity_name"]].v, r[ec["table_name"]].v, r[ec["superentity_id"]].v) for r in rows_of("mendixsystem$entity")} + # cold CMM 2026-11-17: an external (OData) entity keeps a <entity>$pk table whose "_id" is the REMOTE key, a Long that + # can look like an object id; remapping it made 5 such tables UNMAPPED and nothing loaded. Columns other than id of a + # table whose entity has remote_primary_key are kept as they are. + rk = col("mendixsystem$entity", "remote_primary_key") if "remote_primary_key" in [x[0] for x in tables["mendixsystem$entity"].cols] else None + remote_pk = {r[ec["table_name"]].v for r in rows_of("mendixsystem$entity") if rk is not None and r[rk].k == "t"} skip_re = [(re.compile(p), why) for p, why in SKIP_RULES] waived = set(a.skip) @@ -321,6 +326,7 @@ def pg_table(t): maxseq = collections.defaultdict(int) def remap(t, cname, ctype, v): if v.k != "n" or not ctype.upper().startswith("BIGINT") or f"{t}.{cname}" in a.keep_column: return v + if t in remote_pk and cname != "id": return v x = int(v.v) if x in idmap: return V("n", str(idmap[x])) if x <= (1 << 48): return v diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index ba10469..320fa95 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -980,6 +980,21 @@ if PGC=$(bash $here/lib/pgtmp.sh start 2>/dev/null); then ctl seed-snapshot-column-missing-loaded-nothing-pass 0 '^0$' -- pgx -d t_colgone -c 'select count(*) from "demo$product"' ctl seed-snapshot-skip-waiver-pass 1 'waived \(--skip\)' 'column demo\$product.price' '!has no counterpart in the database \(' -- $SS $SNAP --psql "$(mypsql colgone)" --dry-run --skip demo\$order_product ctl seed-snapshot-entity-guid-catch 1 'GUID b2222222-0000-4000-8000-000000000001\) is not in the database' 'carry entity short id 3, which' -- $SS $SNAP --psql "$(mypsql noguid)" --dry-run + # cold CMM 2026-11-17: an external entity's <entity>$pk table holds the REMOTE key in "_id" (remote_primary_key in + # mendixsystem$entity); remapping it as an object id made 5 such tables UNMAPPED and nothing loaded. id is remapped, _id kept + rm -rf $T/snapremote; cp -R $SNAP $T/snapremote; RS=$T/snapremote/data/database/hsqldb/default.script + sed -i.bak -e 's/"superentity_id" VARCHAR(36))$/"superentity_id" VARCHAR(36),"remote" BOOLEAN,"remote_primary_key" BOOLEAN)/' \ + -e '/^INSERT INTO "mendixsystem\$entity" VALUES/s/)$/,FALSE,FALSE)/' $RS && rm -f $RS.bak + printf '%s\n' "INSERT INTO \"mendixsystem\$entityidentifier\" VALUES('c3333333-0000-4000-8000-000000000001',6,2)" \ + "INSERT INTO \"mendixsystem\$entity\" VALUES('c3333333-0000-4000-8000-000000000001','Demo.Ext','demo\$ext\$pk',NULL,TRUE,TRUE)" \ + 'CREATE MEMORY TABLE PUBLIC."demo$ext$pk"("id" BIGINT NOT NULL PRIMARY KEY,"_id" BIGINT)' \ + 'INSERT INTO "demo$ext$pk" VALUES(1688849860263937,27866022694354949)' >> $RS + pgx -d fx -c "create database t_remote template fx" > /dev/null + pgx -d t_remote -c 'create table "demo$ext$pk"(id bigint primary key, _id bigint)' \ + -c "insert into \"mendixsystem\$entityidentifier\" values ('c3333333-0000-4000-8000-000000000001',16,0)" \ + -c "insert into \"mendixsystem\$entity\" values ('c3333333-0000-4000-8000-000000000001','Demo.Ext','demo\$ext\$pk',NULL)" > /dev/null + ctl seed-snapshot-remote-key-catch 0 'applied: 15 rows' '!UNMAPPED' -- $SS $T/snapremote --psql "$(mypsql remote)" + ctl seed-snapshot-remote-key-kept-pass 0 '^16\|27866022694354949$' -- pgx -d t_remote -F'|' -c 'select id>>48, _id from "demo$ext$pk"' ctl seed-snapshot-psql-error-catch 2 'psql failed' -- $SS $SNAP --psql "psql -h $PGD -U mendix -d nope" --dry-run bash $here/lib/pgtmp.sh stop "$PGD" else skip seed-snapshot-postgres "no postgres binaries or the throwaway cluster did not start"; fi From 1fcd5243d8b299f6531de394b862d9a6e505fc08 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 14:14:35 +0800 Subject: [PATCH 43/59] stepupdate: a module forced for another route that has no UI edits is committed, not kept as UNSAVED The edit router forced Administration through --force-ui for its association replay (XASSOC). Its update had to take the --no-baseline route, which saved 0 files, and stepupdate KEPT it as UNSAVED. But localedits listed no UI edit in Administration, so nothing was lost. With Administration kept, MendixSSO 10.6.2 moved on without it and mx check stopped the build on 2 CE0117 (repair row 18). The pipeline also never passed its localedits report to stepupdate. Now it does, and a forced module whose localedits entry has ui [] is committed with a note. A module whose localedits list UI edits is still kept, as before. Controls: stepupdate-unsaved-no-ui-edits-catch and pipeline-dry-stepupdate-localedits-catch (both FAIL on the old code), stepupdate-unsaved-ui-edits-listed-pass (holds on both). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 5 ++++- upgrade-toolkit/bin/stepupdate.sh | 14 +++++++++++++- upgrade-toolkit/controls/run-controls.sh | 10 ++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 929547a..1d37575 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -995,8 +995,11 @@ run_stepupdate() { # run_stepupdate PLAN label apf="$WORK/.approved-with-step.txt" [ "$DRY" = 1 ] || cat "$WORK/approved.txt" "$WORK/approved-step.txt" > "$apf" fi + # the pre-convert localedits report tells stepupdate which modules have UI edits at all (UNSAVED, convert-made rows) + local le=() + [ ! -s "$WORK/localedits.json" ] || le=(--localedits "$WORK/localedits.json") xo "$WORK/stepupdate-$2.out" env "${envs[@]}" bash "$B/stepupdate.sh" "$APP" "$plan" --approved "$apf" \ - --baseline "$WORK/original.mpr" --unattended ${fu[@]+"${fu[@]}"} ${au[@]+"${au[@]}"} || rc=$? + --baseline "$WORK/original.mpr" --unattended ${fu[@]+"${fu[@]}"} ${au[@]+"${au[@]}"} ${le[@]+"${le[@]}"} || rc=$? [ "$DRY" = 1 ] || cat "$WORK/stepupdate-$2.out" >> "$LOG" if [ "$rc" -gt 1 ]; then die4 "stepupdate ($2) failed (exit $rc), output: $WORK/stepupdate-$2.out"; fi return 0 diff --git a/upgrade-toolkit/bin/stepupdate.sh b/upgrade-toolkit/bin/stepupdate.sh index a2e4971..5a8e9d7 100755 --- a/upgrade-toolkit/bin/stepupdate.sh +++ b/upgrade-toolkit/bin/stepupdate.sh @@ -444,7 +444,19 @@ PY fi case "$forceui" in *",$mod,"*) nsaved=$(find "$saved/$mod" -type f 2>/dev/null | wc -l | tr -d ' ') - if [ $nobase = 1 ] && [ "$nsaved" = 0 ]; then + if [ $nobase = 1 ] && [ "$nsaved" = 0 ] && [ -n "$ledits" ] && python3 - "$ledits" "$mod" <<'PY' +import json, sys +try: + e = next((x for x in json.load(open(sys.argv[1])) if isinstance(x, dict) and x.get("name") == sys.argv[2]), None) +except Exception: + e = None +sys.exit(0 if e and e.get("ui") == [] else 1) +PY + then + # cold CMM 2026-11-17: Administration was forced for the XASSOC route only; localedits listed no UI edit in it, so + # saving 0 files lost nothing. It was KEPT as UNSAVED, MendixSSO moved on without it and mx check had 2 CE0117 + echo " $mod: the --no-baseline route saved 0 files, and localedits ($ledits) lists no UI edits in $mod: nothing to save" + elif [ $nobase = 1 ] && [ "$nsaved" = 0 ]; then # E7-1: the --no-baseline route has no baseline to tell an edit from the package: it saved nothing [ $fstate = edited ] && echo " (its theme/JavaScript FILE edits ARE saved: $st/saved/$mod)" echo " UNSAVED: $mod updated through --no-baseline and saved 0 files; its UI edits (localedits, --force-ui) are NOT saved" diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 320fa95..d4a4f32 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -1760,6 +1760,16 @@ newapp nbu '1 1.0 NoBaseUi\n' ctl stepupdate-unsaved-unattended-catch 1 'KEPT: NoBaseUi .*UNSAVED' 'kept: NoBaseUi UNSAVED' -- bash $B/stepupdate.sh $S/nbu $S/nbu.plan --force-ui NoBaseUi --unattended newapp nbu '1 1.0 NoBaseUi\n' ctl stepupdate-unsaved-accept-pass 0 'accepted \(--accept-unsaved NoBaseUi\)' 'committed' 'UNSAVED \(replay by hand\): NoBaseUi 1.0' -- bash $B/stepupdate.sh $S/nbu $S/nbu.plan --force-ui NoBaseUi --accept-unsaved NoBaseUi +# cold CMM 2026-11-17: a module forced for another route (XASSOC) whose localedits list no UI edit loses nothing when the +# --no-baseline route saves 0 files: committed, not KEPT as UNSAVED; one whose localedits list UI edits is still kept +printf '[{"name": "NoBaseUi", "domain": [], "ui": [], "logic": [], "files": []}]' > $V/le-noui.json +printf '[{"name": "NoBaseUi", "domain": [], "ui": [{"change": "CHANGED", "kind": "page", "name": "NoBaseUi.P"}], "logic": [], "files": []}]' > $V/le-ui.json +newapp nbu '1 1.0 NoBaseUi\n' +ctl stepupdate-unsaved-no-ui-edits-catch 0 'lists no UI edits in NoBaseUi: nothing to save' 'committed' '!KEPT' '!UNSAVED' -- bash $B/stepupdate.sh $S/nbu $S/nbu.plan --force-ui NoBaseUi --unattended --localedits $V/le-noui.json +newapp nbu '1 1.0 NoBaseUi\n' +ctl stepupdate-unsaved-ui-edits-listed-pass 1 'KEPT: NoBaseUi .*UNSAVED' '!nothing to save' -- bash $B/stepupdate.sh $S/nbu $S/nbu.plan --force-ui NoBaseUi --unattended --localedits $V/le-ui.json +mkdir -p $V/pwle; echo '[]' > $V/pwle/localedits.json +ctl pipeline-dry-stepupdate-localedits-catch 0 'stepupdate\.sh .* --unattended.* --localedits .*/pwle/localedits\.json' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pwle --only update --dry-run # v3.1 E6-7: a removal (idcheck verdict REVIEW) is named REVIEW in the KEPT line, a GUID change stays a plain finding mkdir -p $S/base; cp $M/base/App.mpr $S/base/ newapp rv '1 1.0 Removes\n'; cp $M/step1-adds/App.mpr $S/rv/App.mpr; git -C $S/rv commit -qam 'start at step1-adds' From 28990240c8d7f30138003ddb77e1613426496337 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 14:31:11 +0800 Subject: [PATCH 44/59] build: mx check -j names the document of each error; the stop report lists them The text output of mx check names only the element: cold CMM stopped at build with four CE0117 "at Action button 'actionButton1'" (and 2, 4, 7), names that occur on many pages, so NEEDS-JUDGMENT gave no page to repair. mxcheck now passes -j <work>/check-N.json after the mpr on mx 10 and later (mx 9 has no -j; before the mpr mx prints its usage, repair row 15), and the new bin/checkwhere.py appends one "[where] CODE Module / Document: element" line per error location to the check log. The [error] lines and the summary are unchanged, so checkdiff.py and the error count read the log as before. The stop report gets "Where they are". In CMM it named Page 'Mix_New_WizardStep1' (desktop and phone): the buttons' dynamic classes expressions. repair.md row 27 records that case (an association test that 10.24 rejects in dynamic classes only; remedy not verified). Controls: checkwhere-document-catch, pipeline-mxcheck-where-catch and pipeline-dry-check-json-catch fail on the old code; checkwhere-checkdiff-count-pass and pipeline-mxcheck-9-no-json-pass guard the count and the mx 9 call. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/checkwhere.py | 36 +++++++++++++++++++ upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 12 +++++-- upgrade-toolkit/controls/run-controls.sh | 17 +++++++++ upgrade-toolkit/steps/repair.md | 3 +- 4 files changed, 65 insertions(+), 3 deletions(-) create mode 100755 upgrade-toolkit/bin/checkwhere.py diff --git a/upgrade-toolkit/bin/checkwhere.py b/upgrade-toolkit/bin/checkwhere.py new file mode 100755 index 0000000..72e5362 --- /dev/null +++ b/upgrade-toolkit/bin/checkwhere.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""checkwhere.py: add the document of each error to an `mx check` log, from the JSON that `mx check -j` writes. + + checkwhere.py CHECK.json CHECK.log + +The text output of mx check names only the element ("at Action button 'actionButton1'"), and a widget name like that +occurs on many pages (cold CMM: 4 CE0117 on actionButton1/2/4/7, which sat in two Mix_New_WizardStep1 pages, not in the +login templates that also have such buttons). The JSON has the module, document and property. Appends one line per +error location to the log: + + [where] CE0117 MaterialManagement / Page 'Mix_New_WizardStep1': Property 'Dynamic classes expression' of action button 'actionButton1' + +The [error] lines and the summary stay as they are, so checkdiff.py and the error count read the log as before. +Exit: 0 = lines added (or none to add), 2 = the JSON is unreadable (the log is left alone). +""" +import json, sys + + +def where(d): + for e in d.get("errors") or []: + for loc in e.get("locations") or [{}]: + yield (f"[where] {e.get('code', '?')} {loc.get('module-name', '?')} / {loc.get('document-name', '?')}: " + f"{loc.get('element-name', '?')}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit(__doc__) + try: + d = json.load(open(sys.argv[1])) + except (OSError, ValueError) as e: + print(f"checkwhere: {sys.argv[1]}: {e}", file=sys.stderr); sys.exit(2) + lines = list(where(d)) + if lines: + with open(sys.argv[2], "a") as f: + f.write("Where (mx check -j):\n" + "\n".join(lines) + "\n") diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 1d37575..30209c3 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -1152,8 +1152,15 @@ step_update() { CHECK_N=0 mxcheck() { # writes $WORK/check-N.log, sets CHECK_LOG CHECK_N=$((CHECK_N + 1)); CHECK_LOG="$WORK/check-$CHECK_N.log" - if [ "$DRY" = 1 ]; then printf '+ %s check %s > %s\n' "$B/heavy-lock.sh check $(mxw "$TARGET") $TARGET mx" "$MPR" "$CHECK_LOG"; return 0; fi - heavy check "$TARGET" mx check "$MPR" > "$CHECK_LOG" 2>&1 || true + # -j (mx 10 and later): the JSON names the document of each error, which the text output leaves out (cold CMM: four + # CE0117 "at Action button 'actionButton1'" with no page); checkwhere.py appends those as [where] lines + local cj="$WORK/check-$CHECK_N.json" j=() + [ "$(major "$TARGET")" -le 9 ] || j=(-j "$cj") + # -j goes AFTER the mpr: before it, mx prints its usage (steps/repair.md row 15, T8R2 R-6) + if [ "$DRY" = 1 ]; then printf '+ %s check %s%s > %s\n' "$B/heavy-lock.sh check $(mxw "$TARGET") $TARGET mx" "$MPR" "${j[@]+" ${j[*]}"}" "$CHECK_LOG"; return 0; fi + rm -f "$cj" + heavy check "$TARGET" mx check "$MPR" ${j[@]+"${j[@]}"} > "$CHECK_LOG" 2>&1 || true + [ ! -s "$cj" ] || python3 "$B/checkwhere.py" "$cj" "$CHECK_LOG" || true } has_code() { grep -q "\[$1\]" "$CHECK_LOG" 2> /dev/null; } try_fix() { # try_fix "subject" RULE CODES cmd...: run a fixer, re-check, keep when checkdiff says not worse, else undo @@ -1239,6 +1246,7 @@ step_build() { Codes: $(grep -oE '^\[error\] *\[C[EW][0-9]+\]' "$CHECK_LOG" | grep -oE 'C[EW][0-9]+' | sort | uniq -c | sort -rn | head -n 15 | tr '\n' ';') Work the list with steps/repair.md (rows 4, 7-10 and own-document fixes; one commit per fix; checkdiff.py after each)." nj_file 10 "First 20 error lines" "$CHECK_LOG" 20 '^\[error\]' + grep -q '^\[where\]' "$CHECK_LOG" 2> /dev/null && nj_file 10 "Where they are (first 20)" "$CHECK_LOG" 20 '^\[where\]' nj_flush fi # mxbuild diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index d4a4f32..965b8f6 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -1770,6 +1770,23 @@ newapp nbu '1 1.0 NoBaseUi\n' ctl stepupdate-unsaved-ui-edits-listed-pass 1 'KEPT: NoBaseUi .*UNSAVED' '!nothing to save' -- bash $B/stepupdate.sh $S/nbu $S/nbu.plan --force-ui NoBaseUi --unattended --localedits $V/le-ui.json mkdir -p $V/pwle; echo '[]' > $V/pwle/localedits.json ctl pipeline-dry-stepupdate-localedits-catch 0 'stepupdate\.sh .* --unattended.* --localedits .*/pwle/localedits\.json' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pwle --only update --dry-run +# cold CMM 2026-11-17: the mx check text names only the element ("at Action button 'actionButton1'", on many pages); the +# build check passes -j (mx 10 and later) and checkwhere.py appends the module and document of each error as [where] lines +mkdir -p $V/cw +printf '%s\n' '{"serialization_version": 1, "errors": [{"code": "CE0117", "message": "Error(s) in expression.", "locations": [{"module-name": "Shop", "document-name": "Page '"'"'Order_Step1'"'"'", "element-name": "Property '"'"'Dynamic classes expression'"'"' of action button '"'"'actionButton1'"'"'"}]}]}' > $V/cw/check.json +printf '%s\n' '[error] [CE0117] "Error(s) in expression." at Action button '"'"'actionButton1'"'"'' 'The app contains: 1 errors.' > $V/cw/check0.log +cp $V/cw/check0.log $V/cw/check.log +ctl checkwhere-document-catch 0 '^\[where\] CE0117 Shop / Page .Order_Step1.: Property .Dynamic classes expression. of action button .actionButton1.$' '^\[error\] \[CE0117\]' -- bash -c "python3 $B/checkwhere.py $V/cw/check.json $V/cw/check.log && cat $V/cw/check.log" +ctl checkwhere-checkdiff-count-pass 0 '^OK: errors 1 -> 1' -- python3 $B/checkdiff.py $V/cw/check0.log $V/cw/check.log +awk '/^CHECK_N=0/{p=1} /^has_code\(\)/{p=0} p' $B/cloud/upgrade-pipeline.sh > $V/cw/mxcheck.sh +cat > $V/cw/stub.sh <<STUB +major() { echo "\${1%%.*}"; } +heavy() { while [ \$# -gt 0 ]; do if [ "\$1" = -j ]; then if [ "\$2" = x.mpr ]; then echo "usage: -j before the mpr"; else cp $V/cw/check.json "\$2"; fi; fi; shift; done; cat $V/cw/check0.log; } +STUB +mkdir -p $V/cw/w10 $V/cw/w9 +ctl pipeline-mxcheck-where-catch 0 '^\[where\] CE0117 Shop / Page .Order_Step1.' '^The app contains: 1 errors' '!usage' -- bash -c "B=$B WORK=$V/cw/w10 TARGET=11.12.7 MPR=x.mpr DRY=0; . $V/cw/stub.sh; . $V/cw/mxcheck.sh; mxcheck; cat \$CHECK_LOG" +ctl pipeline-mxcheck-9-no-json-pass 0 '^\[error\] \[CE0117\]' '!\[where\]' -- bash -c "B=$B WORK=$V/cw/w9 TARGET=9.24.2 MPR=x.mpr DRY=0; . $V/cw/stub.sh; . $V/cw/mxcheck.sh; mxcheck; cat \$CHECK_LOG; ls $V/cw/w9" +ctl pipeline-dry-check-json-catch 0 'mx10\.sh 11\.15\.0\.1 mx check .*/papp/App\.mpr -j .*/pwle/check-1\.json > ' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pwle --only build --dry-run # v3.1 E6-7: a removal (idcheck verdict REVIEW) is named REVIEW in the KEPT line, a GUID change stays a plain finding mkdir -p $S/base; cp $M/base/App.mpr $S/base/ newapp rv '1 1.0 Removes\n'; cp $M/step1-adds/App.mpr $S/rv/App.mpr; git -C $S/rv commit -qam 'start at step1-adds' diff --git a/upgrade-toolkit/steps/repair.md b/upgrade-toolkit/steps/repair.md index 7b49ef4..e462dc7 100644 --- a/upgrade-toolkit/steps/repair.md +++ b/upgrade-toolkit/steps/repair.md @@ -25,7 +25,7 @@ keep the fix and continue with the new class. Without `--fixed` a rise is always | 12 | CE1571 "No argument has been selected" after a 9 to 10 convert, at a call to a flow in a protected add-on module (`modules/*.mxmodule`, not in app.mpr): convert could not read the flow's parameters, so the mapping stayed empty (Opcenter ClientConfiguration_NewEdit.dataView1, T2-4, T2R3-7) | `python3 $B/repair/copy_mappings.py app/ --scan` lists the empty calls to flows outside the model; per page `python3 $B/repair/copy_mappings.py app/ --page <Module.Page> --ref <publisher's newer app dir or .mpr>` (dry run: COPY / SAME / DIFFERS / NOREF), then `--apply` | do NOT fix it with `mxcli alter page ... set DataSource = microflow M(P = $x)`: for an add-on flow it prints "Altered page" and saves nothing (twice in Opcenter). Run any `alter page` through `python3 $B/repair/alter_guard.py app/ -- mxcli ...`: REFUSED (exit 2, not run) for a page in an add-on, WARN for an add-on flow called with arguments, NO-OP or MAPPING NOT SAVED (exit 1) when mxcli exited 0 and the unit did not change. A call to an own flow can be set in MDL (`$currentObject`, T2R3 HANDWORK 10). NOREF (no reference maps it): call the add-on flow from an own wrapper microflow that sets the arguments (T3). Then mx check (CE1571 gone) and `convertcheck.py --mappings` (the copied mapping must map the data view's object; `$currentObject` counts as that object, T2R3-8; a REVIEW you checked and keep is answered with `--answer FILE`, `<Page>: <site> @maps:<hash> # what you checked`, B4T2-8) | | 13 | CE1613 "<Module>.<Layout> no longer exists" on own pages: an updated module dropped a layout (Logging 2.3.0 dropped Logging.PopupLayout: 6 errors on 3 pages, T8R-5; 7 on own pages in T3) | When only the layout differs (also the replay of pages a no-baseline update moved to another layout): `python3 $B/repair/alter_guard.py app/ -- mxcli exec -p app/<app>.mpr $W/replay.mdl` with one line per page, `ALTER PAGE <Module.Page> { SET Layout = <Module.Layout>; };` (MAP for other placeholders); alter_guard says OK or REFUSED (NR1-15: 4 replays, one line each). When more than the layout differs: `mxcli -p app/<app>.mpr -c "describe page <Module.Page>" > $W/<page>.mdl`, set `Layout:` to the sibling layout of the same type (a popup: `Atlas_Core.PopupLayout`), then `python3 $B/repair/alter_guard.py app/ -- mxcli exec -p app/<app>.mpr $W/<page>.mdl` | `mxcli -c "exec file"` is a parse error, use the `exec` subcommand (T8R2 R-3). One commit per module. CE1613 on menu items (an icon or a page that is gone) after an add-on swap: `describe menu`, drop the dead targets and run it back as `create or modify menu ...` (T2R3-9) | | 14 | CE6911 "index.html missing the cachebust template tag(s)" after a 9 to 10 convert (9.x hard-coded `?637643641483639052`, 34 places in FSM) | `python3 $B/repair/cachebust.py app/`, then `--apply` | also rewrites the dojoConfig `cacheBust: "<number>"` that CE6911 does not flag and that otherwise stays until the gate (T8R-5, T8R2 G-5). Commit as "theme: cachebust tags" | -| 15 | CE0161 "Error(s) in XPath constraint" at a widget (`Data grid 'grid2'`) with no document named | `mx check app/<app>.mpr -j $W/check-<n>.json` (`-j` AFTER the mpr; before it, mx prints usage, T8R2 R-6) names the document and unit id | the 9 to 10 cause seen twice (FSM batch 1 and T8R2): `[%CurrentObject%]` in a classic data grid inside a data view now means the data view's object. mxcli cannot author the classic DataGrid, so the constraint was rewritten by a BSON patch of that unit (T8R2 R-5). A behaviour change: list it in the report and have a human look at the page. No tool yet | +| 15 | CE0161 "Error(s) in XPath constraint" at a widget (`Data grid 'grid2'`) with no document named | `mx check app/<app>.mpr -j $W/check-<n>.json` (`-j` AFTER the mpr; before it, mx prints usage, T8R2 R-6) names the document and unit id; the build step now does this itself and lists `[where]` lines in check-N.log and in the stop report (checkwhere.py, cold CMM) | the 9 to 10 cause seen twice (FSM batch 1 and T8R2): `[%CurrentObject%]` in a classic data grid inside a data view now means the data view's object. mxcli cannot author the classic DataGrid, so the constraint was rewritten by a BSON patch of that unit (T8R2 R-5). A behaviour change: list it in the report and have a human look at the page. No tool yet | | 16 | CE1707 "layout ... not found" (and CE1613 on menu items) after replacing an add-on by the publisher's newer build (playbooks/10-to-11.md row 9; Opcenter `EXFN_CrossNavigation_InCanvas`, T2R3-9) | `mxcli describe layout` returns an empty body, so the layout cannot be re-created in MDL: copy the unit by name from the publisher's MPR with fresh `$ID`s (T2R3 used a BSON script, runs/t2r3-opcenter-ef evidence/rung2-upgrade/portunits.py; no toolkit tool yet) | then the menu as in row 13, mx check, and `idcheck.py` old against new before the convert | | 17 | CE0114 at an own microflow that does not apply entity access and calls one that does: an updated module turned it on (DeepLink 10.0.10 `DeepLink.DeepLinkHome`, called first by the role home microflows; T6R and batch 1 T6-12: Masterdata `NAV_ApplicationSettings_Admin`, `NAV_Dashboard_DepartmentManager`, `NAV_Dashboard_Inspector`) | mx check does not name the calling microflow (T6-12), so list the callers: `mxcli -p app/<app>.mpr -c "show callers of DeepLink.DeepLinkHome"`. Per own caller: `mxcli -p app/<app>.mpr -c "describe microflow <Module.Flow>" > $W/rep-<Flow>.mdl`, put `@applyentityaccess` on the line above `create or modify microflow`, `mxcli exec -p app/<app>.mpr $W/rep-<Flow>.mdl` (the T6R files: runs/t6r2-checklist evidence/rep-NAV_*.mdl) | a decision, not a mechanical fix (no tool on purpose): applying entity access in the caller changes what that role may read in the rest of the flow. The alternative the publisher took (Checklist 2.0.0) is an own copy of the callee without entity access (`Mobile_Inspector.DeepLinkHome_custom`). Say which in the report; then mx check (T6R: 3 CE0114 to 0) and the role's login journey | | 18 | CE0117 in a KEPT Administration (2.1.x) after MendixSSO 4.2.0 or newer: `Administration.MendixSSO_CreateUser` / `MendixSSO_UpdateUser` still read `$UserProfile/EmailAddress`, which MendixSSO removed; its `UserMapper` now passes an `EmailAddress` parameter (T6R and batch 1 T6-13: 2 errors). triage's `crosscalls.py` predicts it (T5F-4) | `describe microflow` each one, add the parameter `$EmailAddress: String` and use it where `$UserProfile/EmailAddress` was (`Email = $EmailAddress`), then run the file. `create or modify microflow` refuses this change when the free annotations differ ("cannot be spliced"): write `drop microflow <Module.Flow>;` above the `create` in the SAME file so the flow is never missing between two commands (T6R: evidence/rep-Administration.MendixSSO_*.mdl) | update Administration with MendixSSO instead when the stop rules allow it (the publisher moved to Administration 4.1.0). A drop + create gives the flow a new id: name both flows in the report. Then mx check (T6R: 2 CE0117 to 0) and the SSO login journey | @@ -37,6 +37,7 @@ keep the fix and continue with the new class. Without `--fixed` a rise is always | 24 | CE0463 after a hand-set value on a pluggable widget property; or `mx update-widgets` puts a property value back (FM-5) | read the widget's `getProperties()` in its editorConfig (inside the .mpk): a property that the config hides unless another property is on (for SprintrFeedback `targetContainerSelector` needs `showAdvancedSettings` = true) is hidden in the model, and a hidden property with a non-default value is a CE0463 and is reset by update-widgets | set the controlling property first (`showAdvancedSettings` true), then the value, in one commit; run `mx update-widgets` AFTER and check the value survived (diff the unit), then mx check. Seen: SprintrFeedback 12.0.4 (bundled with Atlas_Core 4.5.1) portals its button into `targetContainerSelector` and renders nothing when the default `.mx-scrollcontainer-center` is absent (an app layout without a centre region): the Feedback tab vanished from every page and shotdiff said "lost feedback". The remedy card `sprintr-feedback-target` (playbooks/remedies.tsv) carries it | | 25 | `mx update-widgets` moves a widget across a technology jump (Slider 1.3.2 Dojo to 3.0.4 pluggable) and then CE7006 on every attribute of it, or the widget lost its min/max/step attributes and its on-change microflow (QI-6) | a Dojo-to-pluggable version jump is a widget SWAP, not an update: before it, list the widget's properties in the old model (BSON of the page or snippet unit) and after it again, and write every dropped binding down; `bin/widgetdiff.py` names the package versions only | re-bind the dropped properties by BSON (not through mxcli describe, which keeps only the value attribute); a pluggable widget gets no entity context from a snippet parameter, so wrap it in a data view on the snippet parameter (the CE7006 goes); a microflow action on a pluggable widget takes empty ParameterMappings. Seen: Slider on a Quality Inspection snippet: only valueAttribute survived, 4 CE7006 until the data view wrap, 0 after. Then mx check and a by-eye look at the page | | 26 | screens rung REVIEW on looks that changed by design: the Charts 6 palette, the Data grid 2 column selector (eye) icon size, the MxAdmin changedDate stamped at runtime start on the 10.24 rung (data rung accept) (QI-10) | look at the pair first; these three are known and each has a fix or an accept | Charts 6 (6.x) comes without the old series palette: re-apply it in the theme (a `.widget-charts` series colour rule) when the dashboard look is wanted; the Data grid 2 column selector icon grows: add a theme rule for its size; the 10.24 runtime writes MxAdmin changedDate at start: accept it in the data rung accept file with that reason. Never accept an unseen pair | +| 27 | CE0117 at the "Dynamic classes expression" of a button after the 9 to 10 convert, where the expression follows an association (`if $currentObject/Module.A_B = empty then 'disabled' else ''`) | the `[where]` lines of check-N.log name the page and property | cold CMM (10.24.28): the same association test passed in 8 conditional-visibility expressions on the same pages and failed only in the 4 dynamic-classes expressions (Mix_New_WizardStep1, desktop and phone; the association leads to an external OData entity). The 9.24 build accepted it. Remedy NOT verified: move the test into conditional visibility, or compute a boolean attribute on the helper object and test that. A judgment item: the button's look changes, so a human looks at the page. No tool yet | mxcli v0.28.0 (#1408, #1474), what it changes for the rows above: - `mxcli exec` is all-or-nothing. A script whose statement N fails now changes NOTHING and exits non-zero; before, statements 1 to N-1 From 51665bd52fd627cc47cef73869490bda43cd694b Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 14:31:12 +0800 Subject: [PATCH 45/59] journeys: a scroll container is no landing guard Cold CMM: login-MaterialHandler and login-MaterialManager FAILed on the healthy 9.24 app: the home page Dashboard_MaterialHandler rendered in full, but its guard ".mx-name-scrollContainer1" never appeared. landing_widget took the scroll container (inside the page's data view; the data view itself was skipped because the layout also has a dataView1) as the outermost content widget. A Forms$ScrollContainer now counts as NON_RENDERING: it is never a guard, so the layout grid inside it is taken, a page with nothing else stays a todo, and a kept impact.json that landed on one is re-picked because guard_names leaves it out. Controls (fixture models/guard6): gen-journeys-guard-scrollcontainer-catch, -only-catch and -names-catch fail on the old code. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/impact_map.py | 4 +- .../controls/fixtures/models/guard6.json | 62 +++++++++++++++++++ upgrade-toolkit/controls/run-controls.sh | 8 +++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 upgrade-toolkit/controls/fixtures/models/guard6.json diff --git a/upgrade-toolkit/bin/impact_map.py b/upgrade-toolkit/bin/impact_map.py index 208e9af..ee5db11 100644 --- a/upgrade-toolkit/bin/impact_map.py +++ b/upgrade-toolkit/bin/impact_map.py @@ -377,7 +377,9 @@ def sign_out_why(m, page_qn, limit=6): # T7: widgets that draw no element of their own (Set URL, timers, listen-to, trigger-only): the field's "setURL1" was # 0 x 0 and the engine's first match, so every login journey FAILed on a healthy app. Matched by type and, for # pluggable widgets whose type is generic, by the default name Studio Pro gives them. -NON_RENDERING = re.compile(r"\$(SetURL|SetUrl|Timer|ListenTo|(Microflow|Nanoflow)Trigger)$") +# ScrollContainer: no .mx-name-<name> in the client's DOM (cold CMM: the MaterialHandler home's scrollContainer1, inside a +# data view, never appeared on the healthy 9.24 page and the login journey FAILed); the layout grid inside it is the guard +NON_RENDERING = re.compile(r"\$(SetURL|SetUrl|Timer|ListenTo|(Microflow|Nanoflow)Trigger|ScrollContainer)$") NON_RENDERING_NAME = re.compile(r"^(setURL|setUrl|timer|listenTo|microflowTrigger|nanoflowTrigger|trigger)\d*$") REPEATING = re.compile(r"\$(ListView|DataGrid|TemplateGrid|Gallery|ReferenceSetSelector|TreeView)$") TITLE = re.compile(r"\$(Title|PageTitle)$") diff --git a/upgrade-toolkit/controls/fixtures/models/guard6.json b/upgrade-toolkit/controls/fixtures/models/guard6.json new file mode 100644 index 0000000..348d1f0 --- /dev/null +++ b/upgrade-toolkit/controls/fixtures/models/guard6.json @@ -0,0 +1,62 @@ +{ + "version": "9.24.2.0", + "_why": "Cold CMM 2026-11-17: the MaterialHandler home (a data view holding a scroll container, the layout also has a dataView1) got the guard .mx-name-scrollContainer1, which never appeared on the healthy 9.24 page: the login journey FAILed on the OLD app. A scroll container is no guard; the layout grid inside it is.", + "modules": [ + { + "name": "Shop", + "layouts": [ + { + "name": "Layout_Plain", + "named": [ + { + "type": "DataView", + "name": "dataView1" + } + ] + } + ], + "pages": [ + { + "name": "Dash", + "layout": "Shop.Layout_Plain", + "named": [ + { + "type": "DataView", + "name": "dataView1", + "children": [ + { + "type": "ScrollContainer", + "name": "scrollContainer1", + "children": [ + { + "type": "LayoutGrid", + "name": "layoutGrid5", + "children": [ + { + "type": "Label", + "name": "label1", + "text": "Dashboard" + } + ] + } + ] + } + ] + } + ] + }, + { + "name": "Scroll_Only", + "layout": "Shop.Layout_Plain", + "named": [ + { + "type": "ScrollContainer", + "name": "scrollContainer2" + } + ] + } + ], + "docs": [] + } + ] +} diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 965b8f6..1e69db4 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -3087,6 +3087,14 @@ ctl gen-journeys-guard-dataview-sources-catch 0 '^Home_Nf label4$' '^Home_Path l ctl gen-journeys-guard-dataview-page-param-pass 0 '^Order_Edit dataView7$' -- python3 -c "$G3PY; print('Order_Edit', w('Order_Edit'))" ctl gen-journeys-guard-dataview-around-last-pass 0 '^Grid_Only layoutGrid3$' -- python3 -c "$G3PY; print('Grid_Only', w('Grid_Only'))" ctl gen-journeys-guard-dataview-only-catch 0 '^Only_Dependent None$' -- python3 -c "$G3PY; print('Only_Dependent', w('Only_Dependent'))" +# cold CMM 2026-11-17: a scroll container has no .mx-name-<name> in the DOM (the MaterialHandler home's scrollContainer1 +# never appeared on the healthy 9.24 page): no guard, not even as the last resort; the layout grid inside it is taken, and +# a kept impact.json landing on one is re-picked (guard_names leaves it out) +model guard6 > /dev/null +G6PY="import sys; sys.path.insert(0,'$B'); import impact_map as I, mprmodel; m=mprmodel.Model.load('$M/guard6'); w=lambda p: (I.landing_widget(m, 'Shop.' + p) or {}).get('widget')" +ctl gen-journeys-guard-scrollcontainer-catch 0 '^Dash layoutGrid5$' '!scrollContainer' -- python3 -c "$G6PY; print('Dash', w('Dash'))" +ctl gen-journeys-guard-scrollcontainer-only-catch 0 '^Scroll_Only None$' -- python3 -c "$G6PY; print('Scroll_Only', w('Scroll_Only'))" +ctl gen-journeys-guard-scrollcontainer-names-catch 0 '^kept False$' -- python3 -c "$G6PY; print('kept', 'scrollContainer1' in I.guard_names(m, 'Shop.Dash'))" cp $FX/journeys/impact-guard3.json $T/impact-guard3.json G3=$T/gen-guard3; (cd / && python3 $B/gen_journeys.py $T/impact-guard3.json --out-dir $G3 --roles-with-users Inspector,Planner > /dev/null 2>&1) ctl gen-journeys-guard-dataview-home-closes-pass 0 "^ready .mx-name-actionButton2 closes \['home:Shop.NAV_Inspector'\]$" -- python3 -c "import json; j=json.load(open('$G3/journeys/Inspector/login-Inspector.journey.json')); print('ready', j['steps'][0]['ui']['ready'], 'closes', j.get('closes'))" From 33d8c2fee374133bf329c93d4ebea838c2f0a4b9 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 14:50:24 +0800 Subject: [PATCH 46/59] docs: feedback ledger, every piece of the owner's feedback with its conclusion and where it landed Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/docs/feedback-ledger.md | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 upgrade-toolkit/docs/feedback-ledger.md diff --git a/upgrade-toolkit/docs/feedback-ledger.md b/upgrade-toolkit/docs/feedback-ledger.md new file mode 100644 index 0000000..52d0bff --- /dev/null +++ b/upgrade-toolkit/docs/feedback-ledger.md @@ -0,0 +1,37 @@ +# Feedback ledger + +Every piece of the owner's feedback on the toolkit, the conclusion we reached, and where it landed. One row per point. +Status: **done** (merged to main), **branch** (committed on `fix/signed-cards`, no PR yet), **PR** (open pull request), +**open** (agreed, not built), **owner** (waits for the owner's call). Newest first. Update this file in the same commit +that closes a row. + +| date | feedback (owner's words, short) | conclusion | status | where | +|---|---|---|---|---| +| 11-17 | "Make the sidebar 50% of screen ... show what was found per action, and what it MEANS" | drawer is 50% wide; per action a "What it means" box and an item-by-item table, worst first | branch | ba76a0e | +| 11-17 | "If we could write such a thing per issue which needs a decision ... embed this" | a decision opens its case in the cockpit: headline, tiles, old vs new screenshot, why, options with the catch | branch | 775ddae | +| 11-17 | the step writes its case on its own, not by hand | case files written by the step that raises the decision | open | | +| 11-17 | "All routine decisions just have one sign off ... in auto mode they just upgrade" | routine work: one plan sign-off, or approved by the agent in auto mode; only a deviation asks a person | branch | ba76a0e | +| 11-17 | modules with screen edits counted as routine (6 of 10 carry local edits) | stop counting a module with local edits as routine | owner | | +| 11-17 | "No manual copying" (Accounts page) | a Marketplace page the menu opens: copied onto the app layout, or its layout set in place when a copy is not safe | branch | 132323a, 183d044, 8aa073d | +| 11-17 | 7 Accounts actions work on the old app, fail on the new | in-place layout keeps the user grid; rerun WM1 from build to prove the 7 pass | open | | +| 11-17 | "we still don't understand what fails means" | a FAIL is only "worked on old, broken on new"; fails on both = a note; known platform differences PASS* with reason | branch | 44d9111, ba76a0e | +| 11-17 | "see which tests actually ran? On old & new" | test step shows the runs on old and new; repeated context hideable | branch | ba76a0e | +| 11-17 | "build the SQL standin skill into toolkit" | 9.24 data reads go through the toolkit's own Postgres route | branch | e1b677f, 186c0f1 | +| 11-17 | CLI testing does not work on the old app: skill + bug report for mxcli | playbook entry in the toolkit; mxcli issue filed | branch + issue | e1b677f, mendixlabs/mxcli#1879 | +| 11-17 | "we don't spend 50 hours of testing when it's not needed" | e2e strategy: journeys once at the end, skip what failed on the old app | open (partly) | ba76a0e | +| 11-17 | proper e2e journeys incl. unhappy paths | rendered widget check | PR | #8 | +| 11-16 | "Is our advice at all correct to keep it on the old version?" | update instead of keep: refused modules routed to an update that carries the edits | done | #10 | +| 11-16 | "we give people 000 information on what the local edits are" | every plan card shows the evidence behind it; cockpitcheck fails a card without it | done | #10 | +| 11-16 | "Is our brain and script so far routing as it should? Are we measuring that?" | route score per run and a ledger; brain notes hit@k | branch | 6d73740, 081799b | +| 11-16 | rows "not approved yet" with nothing to approve; "create a gate" | cockpitcheck gate; grey when unmeasured | done | #10 | +| 11-16 | "Dojo React will always be a topic" | Dojo to React report on every upgrade | done | #10 | +| 11-16 | "Don't we want more than click paths?" | one test row per kind of app part | done | #10 | +| 11-16 | "Cockpit not up to date? Talk to your agent" | sidebar note | done | #10 | +| 11-16 | "how do I now let this continue? ... a mode for upgrades" | judgment stop as a red bar with the rerun command; interview mode auto | done + branch | #9, acdabb2 | +| 11-16 | "I signed off ... but it still said blocked" | a sign-off counts for the plan it was given on; signed item leaves Blocked | done | #10 | +| 11-16 | decisions hard to find; buttons like ask-user-question, read every 15 s | Decide now cards, one click per question, saved by the local server | done | #10 | +| 11-16 | end-to-end testing was poor | test resilience | done | #6 | +| 11-15 | cockpit: show when stuff is running | live cockpit shows the running step | done | #4 | +| 11-15 | open the cockpit as localhost after triage | live cockpit opens after triage | done | #4 | +| 11-15 | harvest learnings and submit them back | harvest and submit learnings | PR | #3 | +| 11-15 | version chip label | says what is open | PR | #2 | From 09cd6c484bd472c7dfc57fe11e59afde672444bc Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 15:22:41 +0800 Subject: [PATCH 47/59] Module updates that carry the app's edits are not routine; regressions get a case automatically - proposal.py: a module with local edits in localedits.json moves from the routine "modules" step to its own "module-edits" step; auto mode approves it with source carry-over (steps.CARRY), not routine - cockpitmatrix.py: found detail for module-edits; cases() builds a case for a regression without a case file (auto_case: counts, old and new shot of the failing step, carried screen edits, evidence) - controls: proposal-steps-module-edits-carried-pass, cockpit-auto-case-pass - heal.md: the three cases; feedback ledger rows updated Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/cockpitmatrix.py | 107 ++++++++++++++---- upgrade-toolkit/bin/proposal.py | 30 ++++- upgrade-toolkit/bin/steps.py | 3 +- .../controls/found_means_control.py | 16 +++ upgrade-toolkit/controls/run-controls.sh | 10 ++ upgrade-toolkit/docs/feedback-ledger.md | 6 +- upgrade-toolkit/steps/heal.md | 17 ++- 7 files changed, 164 insertions(+), 25 deletions(-) diff --git a/upgrade-toolkit/bin/cockpitmatrix.py b/upgrade-toolkit/bin/cockpitmatrix.py index b969229..8a133dc 100644 --- a/upgrade-toolkit/bin/cockpitmatrix.py +++ b/upgrade-toolkit/bin/cockpitmatrix.py @@ -948,7 +948,7 @@ def found_detail(rec, work): return {"means": [f"{ok} of {len(items)} outside calls are covered in the tests.", f"For the other {len(items) - ok}, a passing test only shows the screen works without the outside system. It says nothing about the real call.", f"Before go-live, a person tests these against real test systems ({', '.join(mods)})."], "items": items} - if t == "modules": + if t in ("modules", "module-edits"): edits = {m["name"]: m for m in (_json(work, "localedits.json") or []) if isinstance(m, dict)} items, clean, touched, unknown = [], 0, [], 0 for e in ev: @@ -964,15 +964,16 @@ def found_detail(rec, work): touched.append(m.group(1)) items.append([f"{m.group(1)} {m.group(2)} → {m.group(3)}", f"{len(rows)} local edit" + ("" if len(rows) == 1 else "s"), "wait", f"The app changed {', '.join(os.path.splitext(os.path.basename(x['path']))[0] if x.get('path') else _edit_name(x).split('.')[-1] for x in rows[:4])}{', ...' if len(rows) > 4 else ''}. " - + _edit_means(rows[0]) + " The toolkit re-applies screen edits after the update; check these screens in the Test step."]) + + _edit_means(rows[0]) + " The toolkit carries the edit over; the Test step compares its screens on both apps."]) else: clean += 1 items.append([f"{m.group(1)} {m.group(2)} → {m.group(3)}", "no local edits", "ok", "The update swaps the whole module. Nothing of the app is lost."]) if not items: return {} - means = [f"{clean} of {len(items)} modules {'has' if clean == 1 else 'have'} no local edits: a clean swap."] + means = [f"{clean} of {len(items)} modules {'has' if clean == 1 else 'have'} no local edits: a clean swap."] if clean else [] if touched: - means.append(f"{len(touched)} {'carries' if len(touched) == 1 else 'carry'} edits by the app ({', '.join(touched)}). They count as routine because the toolkit re-applies them, but this is where a screen can change. Look at them first when a test fails.") + means.append(f"{len(touched)} {'carries' if len(touched) == 1 else 'carry'} edits by the app ({', '.join(touched)}). Not routine: the update replaces the whole module and the toolkit carries each edit back.") + means.append("After the build, the screens that use these edits are compared on the old and the new app. The same: it goes ahead. A difference: a decision for you, with its case.") if unknown: means.append(f"{unknown} could not be compared, so nobody knows yet whether the app changed {'it' if unknown == 1 else 'them'}.") return {"means": means, "items": items} @@ -1212,7 +1213,7 @@ def build(work, model): "tiles": tl, "feed": feed(work, model, ledger, items, g.get("closed_q") or []), "loop": list(LOOP), "band": band(model, rows, chips, g, tl, kpi), "generated": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - "phases": phases(rows), "ladder": ladder(work), "health": health(work), "cases": cases(work), + "phases": phases(rows), "ladder": ladder(work), "health": health(work), "cases": cases(work, model), "itemsFile": os.path.exists(os.path.join(work, "cockpit-items.json")), "stepsFile": recs is not None, "plan": plan_doc(cp.rd(os.path.join(work, "PROPOSAL.md"))), "ran": test_runs(work)} @@ -2212,29 +2213,97 @@ def case_id(item_id): return re.sub(r"[^A-Za-z0-9._-]", "_", item_id) + ".json" -def cases(work): +def _embed(work, c): + """The case's screenshots (paths in the work folder) as data URIs, "" when missing.""" + import base64 + for sh in c.get("shots") or []: + for k in ("old", "new"): + pth = os.path.join(work, sh.get(k) or "") + if sh.get(k) and os.path.isfile(pth) and pth.lower().endswith((".png", ".jpg", ".jpeg")): + with open(pth, "rb") as fh: + sh[k] = ("data:image/png;base64," if pth.lower().endswith(".png") else "data:image/jpeg;base64,") + base64.b64encode(fh.read()).decode() + else: + sh[k] = "" + return c + + +def cases(work, model=None): """The case files of the open decisions (the owner, 2026-11-17: "such a report per issue that needs a decision"). A case is what the agent that raised the decision found: {"headline", "tiles": [[number, label, tone]], "shots": [{"old", "new", "cap"}], "why": [steps], "options": {option key: {"does", "catch"}}, "details": [lines]}. Screenshots are paths in the work folder, embedded as data URIs, - so the cockpit stays one file. A missing or broken case file leaves the plain question.""" - import base64 + so the cockpit stays one file. A regression without a case file gets one built from the run's own files (auto_case); a broken + case file is skipped.""" out, d = {}, os.path.join(work, CASE_DIR) for f in sorted(os.listdir(d)) if os.path.isdir(d) else []: c = cp.rj(os.path.join(d, f)) if f.endswith(".json") else None - if not isinstance(c, dict): - continue - for sh in c.get("shots") or []: - for k in ("old", "new"): - pth = os.path.join(work, sh.get(k) or "") - if sh.get(k) and os.path.isfile(pth) and pth.lower().endswith((".png", ".jpg", ".jpeg")): - with open(pth, "rb") as fh: - sh[k] = ("data:image/png;base64," if pth.lower().endswith(".png") else "data:image/jpeg;base64,") + base64.b64encode(fh.read()).decode() - else: - sh[k] = "" - out[f] = c + if isinstance(c, dict): + out[f] = _embed(work, c) + for i in (model or {}).get("items") or []: + if i.get("kind") == "regression" and case_id(i["id"]) not in out: + c = auto_case(work, i) + if c: + out[case_id(i["id"])] = _embed(work, c) return out +def _shot_paths(work): + """{file name: [paths relative to work]} of the walk screenshots.""" + got = {} + for root, dirs, files in os.walk(work): + dirs[:] = [x for x in dirs if x not in (CASE_DIR, "node_modules", ".git")] + for f in files: + if f.endswith(".png"): + got.setdefault(f, []).append(os.path.relpath(os.path.join(root, f), work)) + return got + + +def auto_case(work, item): + """The case of a regression nobody wrote one for, from the run's own files: how many actions broke, the first old and new + screenshot of the same step, the module edits the update carried over (the first place to look), and the evidence lines.""" + heal = _json(work, "heal.json") or {} + ids = item.get("tech") or [] + ev = {r.get("id"): r.get("evidence", "") for r in heal.get("rows", []) if r.get("class") == "real-regression"} + lines = [ev[x] for x in ids if ev.get(x)] + if not ids: + return None + pages = sorted({re.sub(r"^act-", "", x.split(":", 1)[-1]).rsplit(".", 1)[0] for x in ids}) + n = len(ids) + shots, names = [], None + for ln in lines: + m = re.search(r"screenshot (\S+\.png)", ln) + if m: + names = names or _shot_paths(work) + new = (names.get(m.group(1)) or [""])[0] + oldn = re.sub(r"-[A-Z][A-Z-]*\.png$", ".png", m.group(1)) + olds = names.get(oldn) or [] + old = next((x for x in olds if "before" in x or "old" in x), olds[0] if olds else "") + if new or old: + step = re.search(r"-(step\d+)", m.group(1)) + shots.append({"old": old, "new": new, "cap": f"{pages[0]}, {step.group(1) if step else 'the step'} that fails: " + "the old app on the left, where the new app stopped on the right."}) + break + # a vendor screen the app changed (not a page the app added to the module): the update carries it over + carried = sorted({m["name"] for m in (_json(work, "localedits.json") or []) if isinstance(m, dict) + and any(x.get("change") == "CHANGED" for x in m.get("ui") or [])}) + reason = next((r.group(1) for ln in lines for r in [re.search(r"FAIL (.+?)(?: \u2014| \||$)", ln)] if r), "") + why = [f"{n} action{'s' if n != 1 else ''} on {len(pages)} page{'s' if len(pages) != 1 else ''} pass on the old app and fail on the new one" + + (f": {reason}." if reason else ".")] + if carried: + why.append(f"The update carried over the app's own screen edits in {', '.join(carried)}. A carried screen edit is the first place to look.") + tiles = [[n, "pass on the old app, fail on the new", "bad"], [len(pages), "page" + ("s" if len(pages) != 1 else ""), "wait"]] + if carried: + tiles.append([len(carried), "modules with screen edits carried over", "wait"]) + return {"headline": f"{n} action{'s' if n != 1 else ''} that work on the old app fail on the new one ({', '.join(pages[:3])}{', ...' if len(pages) > 3 else ''}).", + "tiles": tiles, "shots": shots, "why": why, + "options": {"fix": {"does": "The agent finds the cause, fixes it in the toolkit or the app, and runs again from the step that broke. These actions must pass.", + "catch": "One more run of the tests."}, + "accept": {"does": "Record these changes as intended, with your name and reason.", + "catch": "Users can no longer do on the new app what these actions did on the old one."}, + "old-wrong": {"does": "Record that the old app was wrong here.", + "catch": "Only true when the old behaviour was a bug; the screenshots show what it did."}}, + "details": [cp.short(x, 300) for x in lines[:6]]} + + def h_case(c): """The case block inside a question dialog: tiles, old and new screenshot, why in steps.""" if not c: diff --git a/upgrade-toolkit/bin/proposal.py b/upgrade-toolkit/bin/proposal.py index 7fb9dff..d2cb9a9 100644 --- a/upgrade-toolkit/bin/proposal.py +++ b/upgrade-toolkit/bin/proposal.py @@ -499,6 +499,17 @@ def major(v): return int(m.group(1)) if m else None +def edited_modules(work): + """Modules whose comparison with their Marketplace package found the app's own edits, or could not be made (localedits.json).""" + try: + with open(os.path.join(work, "localedits.json"), encoding="utf-8") as f: + le = json.load(f) + except (OSError, ValueError): + return set() + return {m["name"] for m in le if isinstance(m, dict) and m.get("name") and + (m.get("domain_error") or any(m.get(k) for k in ("domain", "ui", "logic", "files")))} + + def write_steps(F, work): """Open (or find) one step per triage item and set found and proposed (STEP-RECORDS-PLAN.md section 3). A rerun with the same input adds no event; the same topic and title is the same step. Returns {step id: key} in item order. @@ -512,8 +523,24 @@ def write_steps(F, work): "Make these repairs during the upgrade, as the repair table describes."), ("react", "Dojo to React client", "react", ("move to the React client that the toolkit makes", "moves to the React client that the toolkit makes"), "Move the app to the React client on the last rung: every legacy widget to its React successor, then test (playbooks/dojo-to-react.md).")] + carry = edited_modules(work) for topic, title, key, what, prop in groups: its = [x for x in F.items2 if x["key"].startswith(key + ":")] + if topic == "modules": + # a module with the app's own edits is no routine swap (the owner, 2026-11-17): it goes ahead and the edits are carried + # over, but the screens that use them are checked on both apps after the build; only a difference asks a person + ed = [x for x in its if x["key"].split(":", 1)[1] in carry] + its = [x for x in its if x not in ed] + if ed: + names = [x["key"].split(":", 1)[1] for x in ed] + sid = steps.ensure(work, "module-edits", "Module updates that carry the app's edits", { + "found_say": f"{len(ed)} module {'update holds' if len(ed) == 1 else 'updates hold'} the app's own edits ({', '.join(names)}).", + "stage": "triage", "evidence": [x["text"] for x in ed], + "proposed_say": "Update these modules and carry every edit over to the new version. After the build, compare the screens " + "that use them on the old and the new app. A difference becomes a decision with its case.", + "why": "Not routine: the update replaces the whole module, so each edit has to come back. It goes ahead with the plan " + "(in auto mode at once) and asks a person only when a screen differs.", "closes_by": ["build", "journeys"]}) + out[sid] = "2:module-edits" if not its: continue sid = steps.ensure(work, topic, title, { @@ -812,7 +839,8 @@ def accept_answers(signer): write_decisions(F, rows, "-", "ASSUMED (unattended)", "a person signs PROPOSAL.md section 6") # routine work with a known result (section 2) goes ahead in auto mode; only a real decision waits for a person (the owner, 2026-11-17) routine = {s: k for s, k in sids.items() if k.startswith("2:")} - approve_steps(a.work, routine, "approved", "agent", steps.ROUTINE) + approve_steps(a.work, {s: k for s, k in routine.items() if k != "2:module-edits"}, "approved", "agent", steps.ROUTINE) + approve_steps(a.work, {s: k for s, k in routine.items() if k == "2:module-edits"}, "approved", "agent", steps.CARRY) approve_steps(a.work, {s: k for s, k in sids.items() if s not in routine}, "assumed", "agent", "unattended") print("proposal: not signed (unattended): ASSUMED (unattended) lines in DECISIONS.md; every report says 'proposal not signed'") return 0 diff --git a/upgrade-toolkit/bin/steps.py b/upgrade-toolkit/bin/steps.py index 3be9c88..58e9568 100755 --- a/upgrade-toolkit/bin/steps.py +++ b/upgrade-toolkit/bin/steps.py @@ -949,6 +949,7 @@ def topic_title(work, sid): ROUTINE = "routine" +CARRY = "carry-over" # goes ahead like routine work; the screens that use the carried edits are checked on both apps (the owner, 2026-11-17) GENERIC = {"build", "builds", "widget", "widgets", "module", "modules", "error", "errors", "later", "after", "before", "which", "their", "there", "check", "checks", "rows", "journeys", "journey", "pages", "page", "client", "app", "apps", "script", "scripts", "update", "updates", "files", "agent", "notes", "edits", "using", "other", "kept", "cannot", "without", "should"} @@ -1292,7 +1293,7 @@ def routine_ok(sid): "decisions": merge_list(W, host, cur["proposed"].get("decisions") or [], [d]), "evidence": merge_list(W, host, cur["found"].get("evidence") or [], row["rows"])}) keep(host) - if choice and (not cur.get("approval") or cur["approval"].get("source") == ROUTINE): + if choice and (not cur.get("approval") or cur["approval"].get("source") in (ROUTINE, CARRY)): append(W, host, "approval", {"state": "assumed", "source": LEGACY, "needs": "the owner's OK"}, "agent") elif not choice and not cur.get("approval"): routine_ok(host) diff --git a/upgrade-toolkit/controls/found_means_control.py b/upgrade-toolkit/controls/found_means_control.py index b4618fa..1743842 100644 --- a/upgrade-toolkit/controls/found_means_control.py +++ b/upgrade-toolkit/controls/found_means_control.py @@ -47,3 +47,19 @@ print("old shot:", c["shots"][0]["old"][:22], "| new shot:", repr(c["shots"][0]["new"])) h = cm.h_case(c) print("html:", "tiles" if 'class="ct bad"' in h else "-", "why=" + str(h.count("<li>")), "no-shot" if "no screenshot" in h else "-") + + # a regression nobody wrote a case for: built from heal.json, the walk screenshots and the carried screen edits + for sub in ("e2e-before/shots-A", "e2e-after/shots-A"): + os.makedirs(os.path.join(w, sub), exist_ok=True) + png = base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==") + open(os.path.join(w, "e2e-before/shots-A/act-M.P.b1-step2.png"), "wb").write(png) + open(os.path.join(w, "e2e-after/shots-A/act-M.P.b1-step2-NOT-REACHED.png"), "wb").write(png) + json.dump({"rows": [{"id": "act:act-M.P.b1", "class": "real-regression", + "evidence": "passes on the old app, fails on the new one: FAIL open M.P: button never appeared, screenshot act-M.P.b1-step2-NOT-REACHED.png"}, + {"id": "act:act-M.Q.b2", "class": "real-regression", "evidence": "passes on the old app, fails on the new one: FAIL open M.Q"}]}, + open(os.path.join(w, "heal.json"), "w")) + model = {"items": [{"id": "regression:module-M", "kind": "regression", "tech": ["act:act-M.P.b1", "act:act-M.Q.b2"]}]} + a = cm.cases(w, model)[cm.case_id("regression:module-M")] + print("auto:", a["headline"], "| tiles:", [t[0] for t in a["tiles"]], "| shots:", bool(a["shots"][0]["old"]), bool(a["shots"][0]["new"]), + "| carried:", a["why"][1] if len(a["why"]) > 1 else "-", "| options:", sorted(a["options"])) + print("written case kept:", cm.cases(w, {"items": [{"id": "regression:module-X", "kind": "regression", "tech": ["act:act-M.P.b1"]}]})[cm.case_id("regression:module-X")]["headline"]) diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 1e69db4..624b576 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -5836,6 +5836,13 @@ ctl proposal-steps-rerun-no-duplicates-catch 0 '^same events: yes$' '^same steps ctl proposal-steps-plan-change-updates-found-pass 0 'plan holds 2 module updates' -- sh -c "grep -Eo 'plan holds [0-9]+ module updates?' $PR/rerun/steps.jsonl | tail -1" # unattended (auto mode): routine work goes ahead (approved, source routine); a real decision stays assumed until a person answers (the owner, 2026-11-17) ctl proposal-steps-unattended-assumed-catch 0 '^S-[0-9]+ modules .* approval=approved by=agent source=routine ' '^S-[0-9]+ module:DataWidgets .* approval=assumed by=agent source=unattended ' '!approval=none' '!module:DataWidgets .*approval=approved' -- bash $PR/ssum.sh $PR/unatt +# a routine update of a module with the app's own edits is not routine (the owner, 2026-11-17): its own step, carried over in auto mode +# (source carry-over), and the plain routine step no longer holds it +mkw carry kept; python3 -c "import json,sys; p=sys.argv[1]; d=json.load(open(p)); d.append({'name': 'Plainmod', 'domain': [], 'ui': [{'kind': 'snippet', 'change': 'CHANGED', 'name': 'Plainmod.Card'}]}); json.dump(d, open(p, 'w'))" $PR/carry/localedits.json +bash $PR/prop.sh $PR/carry --unattended > $PR/carry.out 2>&1 || cat $PR/carry.out >&2 +ctl proposal-steps-module-edits-carried-pass 0 '^S-[0-9]+ module-edits .* approval=approved by=agent source=carry-over ' '!^S-[0-9]+ module:Plainmod ' '^evidence module-edits: module update Plainmod ' '!^evidence modules: module update Plainmod ' -- sh -c "bash $PR/ssum.sh $PR/carry; python3 $B/steps.py show --work $PR/carry --json | python3 -c 'import json,sys +for r in json.load(sys.stdin): + [print(\"evidence\", r[\"topic\"] + \":\", e) for e in r[\"found\"].get(\"evidence\") or [] if r[\"topic\"] in (\"modules\", \"module-edits\")]'" # signed: every step is approved with the signer; the item Jan answered by hand (P-1) is approved by Jan, not by the signer ctl proposal-steps-signed-approved-pass 0 '^S-[0-9]+ modules .* approval=approved by=Jan Jansen source=signoff ' '^S-[0-9]+ person:go-live .* approval=approved by=Jan Jansen source=signoff ' \ '^S-[0-9]+ module:Administration alias=P-1 approval=approved by=Jan source=decisions ' '!approval=none' '!approval=assumed' '!by=agent' -- bash $PR/ssum.sh $PR/signed @@ -6860,6 +6867,9 @@ ctl cockpit-test-runs-stale-catch 0 '^row: Journeys \| 2 journeys: 2 passed \| n ctl cockpit-found-means-pass 0 '^means standins: 1 of 3 outside calls are covered in the tests\.$' '^item standins: Mail\.GetToken \| address from data \| wait \| .*Mail\.Provider\.TokenUrl' '^item modules: Theme 1\.0\.0 → 2\.0\.0 \| 1 local edit \| wait \| .*restyled this shared layout' '^item modules: Plain .* \| no local edits \| ok \|' '^means modules: 1 could not be compared' '^item module:Admin: Admin\.Extra \| added \| bad \| .*data stored in it' '^means module:Admin: 1 of them holds the app' '^means react: 4 widgets in 1 kind move' '^empty nothing:known$' -- python3 $here/found_means_control.py $T/fm # a decision's case file (work/cases/<id>.json): found by id, screenshots embedded, a missing shot shown as such, a broken file ignored ctl cockpit-case-file-pass 0 "^case files: \['regression_module-X\.json'\]$" '^old shot: data:image/png;base64, \| new shot: .{2}$' '^html: tiles why=2 no-shot$' -- python3 $here/found_means_control.py $T/fmc cases +# a regression without a case file gets one from the run's own files (no manual case writing): count, old and new shot of the failing +# step, the carried screen edits as the first place to look; a written case file wins +ctl cockpit-auto-case-pass 0 '^auto: 2 actions that work on the old app fail on the new one \(M\.P, M\.Q\)\. \| tiles: \[2, 2, 1\] \| shots: True True \| carried: .*screen edits in Theme\. .*options: \[.accept., .fix., .old-wrong.\]$' '^written case kept: Seven break$' -- python3 $here/found_means_control.py $T/fmc2 cases ctl cockpit-render-time-from-timing-pass 0 'Shop|App upgrade' '1 h 40|1 h 30' -- $CK render ctl cockpit-page-content-only-pass 0 'no page skeleton tags' '^<title>' -- $CK nodoc ctl cockpit-no-hardcoded-run-pass 0 'no hard-coded run names' -- $CK nohard diff --git a/upgrade-toolkit/docs/feedback-ledger.md b/upgrade-toolkit/docs/feedback-ledger.md index 52d0bff..c82c418 100644 --- a/upgrade-toolkit/docs/feedback-ledger.md +++ b/upgrade-toolkit/docs/feedback-ledger.md @@ -2,16 +2,16 @@ Every piece of the owner's feedback on the toolkit, the conclusion we reached, and where it landed. One row per point. Status: **done** (merged to main), **branch** (committed on `fix/signed-cards`, no PR yet), **PR** (open pull request), -**open** (agreed, not built), **owner** (waits for the owner's call). Newest first. Update this file in the same commit +**open** (agreed, not built), **owner** (waits for the owner's call). "this PR" rows close with the PR that adds them. Newest first. Update this file in the same commit that closes a row. | date | feedback (owner's words, short) | conclusion | status | where | |---|---|---|---|---| | 11-17 | "Make the sidebar 50% of screen ... show what was found per action, and what it MEANS" | drawer is 50% wide; per action a "What it means" box and an item-by-item table, worst first | branch | ba76a0e | | 11-17 | "If we could write such a thing per issue which needs a decision ... embed this" | a decision opens its case in the cockpit: headline, tiles, old vs new screenshot, why, options with the catch | branch | 775ddae | -| 11-17 | the step writes its case on its own, not by hand | case files written by the step that raises the decision | open | | +| 11-17 | the step writes its case on its own, not by hand | a regression without a case file gets one built from the run's own files | PR | this PR | | 11-17 | "All routine decisions just have one sign off ... in auto mode they just upgrade" | routine work: one plan sign-off, or approved by the agent in auto mode; only a deviation asks a person | branch | ba76a0e | -| 11-17 | modules with screen edits counted as routine (6 of 10 carry local edits) | stop counting a module with local edits as routine | owner | | +| 11-17 | modules with screen edits counted as routine (5 of 10 carry local edits) | a module with the app's edits gets its own step: carried over, screens compared, a difference is a decision with its case | PR | this PR | | 11-17 | "No manual copying" (Accounts page) | a Marketplace page the menu opens: copied onto the app layout, or its layout set in place when a copy is not safe | branch | 132323a, 183d044, 8aa073d | | 11-17 | 7 Accounts actions work on the old app, fail on the new | in-place layout keeps the user grid; rerun WM1 from build to prove the 7 pass | open | | | 11-17 | "we still don't understand what fails means" | a FAIL is only "worked on old, broken on new"; fails on both = a note; known platform differences PASS* with reason | branch | 44d9111, ba76a0e | diff --git a/upgrade-toolkit/steps/heal.md b/upgrade-toolkit/steps/heal.md index c828a8b..6dfe7d6 100644 --- a/upgrade-toolkit/steps/heal.md +++ b/upgrade-toolkit/steps/heal.md @@ -92,7 +92,22 @@ the agent writes its case next to it, so the person decides on one screen (the c The file name is the decision id with every character outside `[A-Za-z0-9._-]` as `_` (`regression:module-Administration` -> `regression_module-Administration.json`). Every sentence comes from evidence on both apps; the screenshots are embedded, -so the cockpit stays one file. Without a case file the dialog shows the plain question. +so the cockpit stays one file. A regression without a case file gets one built by the cockpit from the run's own files +(`cockpitmatrix.auto_case`): how many actions pass on the old app and fail on the new one, the old and new screenshot of +the first failing step, the modules whose own screen edits the update carried over (the first place to look), and the +heal evidence lines. A written case file wins over the built one. + +## A module update that carries the app's own edits is not routine + +A Marketplace module the app edited (a changed layout, snippet, page, JavaScript action or entity in `localedits.json`) +is never counted as a routine update: the update replaces the whole module, and the toolkit carries each edit back. +`proposal.py` puts these modules in their own step, `module-edits` ("Module updates that carry the app's edits"), +apart from the routine `modules` step. Three cases: + +1. No edits: routine, goes ahead under the plan sign-off (auto mode: approved with source `routine`). +2. Edits carried over and the screens that use them look and work the same on both apps: goes ahead. In auto mode the + step is approved with source `carry-over`, so the cockpit says which edits were carried, with the found detail. +3. An edit could not be carried, or a screen differs on the new app: a decision for a person, with its case (above). ## What the gate reads from it From 4fe34940513a4d13fbe5239a88ca3df0a1ee59f5 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 15:11:20 +0800 Subject: [PATCH 48/59] localedits: files the OS writes into a folder are no theme edit Cold CMM: DataWidgets was kept at its version and got a plan card "themesource edits" asking a person to restyle the hero image, for one change: the package holds themesource/datawidgets/.DS_Store and the app does not. themefiles.py now skips .DS_Store, ._* (AppleDouble), Thumbs.db and desktop.ini on both sides; a deleted real package file is still FILES-EDITED. Controls: localedits-os-junk-files-pass, localedits-os-junk-real-removed-catch (red on the old code: 2 failed; green: 2 passed). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/themefiles.py | 8 ++++++-- upgrade-toolkit/controls/run-controls.sh | 9 +++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/upgrade-toolkit/bin/themefiles.py b/upgrade-toolkit/bin/themefiles.py index 1070ca2..c12f9e2 100644 --- a/upgrade-toolkit/bin/themefiles.py +++ b/upgrade-toolkit/bin/themefiles.py @@ -43,6 +43,9 @@ # nobody edited, and the module read as FILES-EDITED. A file whose first line says it is auto-generated and must not be # modified is the tool's output, not a person's edit: it is not compared. WHOLEGEN = re.compile(r"auto-?generated.*do not (modify|edit)", re.I) +# files the OS writes into any folder it shows (Finder, Explorer): no edit when the app has one or lacks the package's +# (cold CMM: DataWidgets got a plan card "themesource edits" for one .DS_Store that the package held and the app did not) +OSJUNK = re.compile(r"(^|/)(\.DS_Store|\._[^/]*|Thumbs\.db|desktop\.ini)$", re.I) VAR = re.compile(r"^\s*(\$[\w-]+)\s*:\s*(.+?)\s*;\s*(?://.*)?$") @@ -113,7 +116,8 @@ def app_files(app, mod): for base, _, files in os.walk(os.path.join(top, d)): for f in files: p = os.path.join(base, f) - out[os.path.relpath(p, app).replace(os.sep, "/")] = open(p, "rb").read() + if not OSJUNK.search(f): + out[os.path.relpath(p, app).replace(os.sep, "/")] = open(p, "rb").read() return out @@ -122,7 +126,7 @@ def pkg_files(mpk, mod): with zipfile.ZipFile(mpk) as z: for n in z.namelist(): parts = n.split("/") - if len(parts) < 3 or parts[0] not in dict(ROOTS) or parts[1].lower() != mod.lower() or n.endswith("/"): + if len(parts) < 3 or parts[0] not in dict(ROOTS) or parts[1].lower() != mod.lower() or n.endswith("/") or OSJUNK.search(n): continue out[n] = z.read(n) return out diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 624b576..c88f3a2 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -1934,6 +1934,15 @@ cp $TH/gen-pkg.scss $TH/fg-genlike/themesource/administration/web/_generated-exc printf '// generated from the brand sheet\n$brand-x: #222;\n' > $TH/fg-genlike/themesource/administration/web/_brand.scss ctl localedits-autogenerated-file-pass 0 'files unchanged' '!FILES-EDITED' -- env UPG_CACHE=$TH/cache-gen python3 $B/localedits.py $TH/fg-gen/App.mpr --module Administration --files-only ctl localedits-generated-comment-edit-catch 1 'FILES-EDITED themesource/administration/web/_brand\.scss' '!_generated-exclusion' -- env UPG_CACHE=$TH/cache-gen python3 $B/localedits.py $TH/fg-genlike/App.mpr --module Administration --files-only +# cold CMM: DataWidgets got a plan card "themesource edits" for one package .DS_Store the app did not have; files the OS +# writes into a folder (.DS_Store, ._x, Thumbs.db, desktop.ini) are no edit either way, a deleted real file still is +mkdir -p $TH/cache-junk; cp $TH/cache/23513-4.1.0.mpk $TH/cache-junk/; : > $TH/junk.bin +python3 -c "import zipfile; z=zipfile.ZipFile('$TH/cache-junk/23513-4.1.0.mpk','a'); z.write('$TH/junk.bin','themesource/administration/.DS_Store'); z.write('$TH/junk.bin','javascriptsource/administration/actions/._Decode.js'); z.close()" +for k in junk real; do rm -rf $TH/fj-$k; cp -R $TH/same $TH/fj-$k; done +: > $TH/fj-junk/themesource/administration/web/Thumbs.db; : > $TH/fj-junk/themesource/administration/desktop.ini +rm $TH/fj-real/themesource/administration/web/_variables.scss +ctl localedits-os-junk-files-pass 0 'files unchanged' '!FILES-EDITED' '!DS_Store' -- env UPG_CACHE=$TH/cache-junk python3 $B/localedits.py $TH/fj-junk/App.mpr --module Administration --files-only +ctl localedits-os-junk-real-removed-catch 1 'FILES-EDITED themesource/administration/web/_variables\.scss' '!DS_Store' -- env UPG_CACHE=$TH/cache-junk python3 $B/localedits.py $TH/fj-real/App.mpr --module Administration --files-only # after the update (themefiles report): a regenerated action is not FILES-OVERWRITTEN, a replaced USER CODE line is rm -rf $TH/js-pkgform; cp -R $TH/js-regen $TH/js-pkgform; cp $TH/jsact-pkg.js $TH/js-pkgform/javascriptsource/administration/actions/Decode.js python3 -c "import sys; sys.path.insert(0,'$B'); import themefiles as t; t.snapshot('$TH/js-regen','Administration','$TH/snap-regen'); t.snapshot('$TH/js-user','Administration','$TH/snap-user')" From ddbc1fb15b62fe36860c644d94b997655db0837c Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 15:11:29 +0800 Subject: [PATCH 49/59] cockpit: check and theme plan cards show their evidence Cold CMM: every cockpit render said "the page contradicts the run" for the "Release notes (check)" and "<M> themesource edits (theme)" cards (ask-no-evidence), because evidence.py only looked up module names. Each kind now has its own source: release-impact.out for the release-note check, the file rows of the module in localedits.json for a theme card. cockpitpage passes the plan item's kind. Controls: evidence-check-release-notes-catch, evidence-theme-files-catch, evidence-theme-other-module-pass, cockpitpage-evidence-passes-kind-pass (red on the old code: 4 failed; green: 4 passed). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/cockpitpage.py | 2 +- upgrade-toolkit/bin/evidence.py | 34 +++++++++++++++++++++--- upgrade-toolkit/controls/run-controls.sh | 8 ++++++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/upgrade-toolkit/bin/cockpitpage.py b/upgrade-toolkit/bin/cockpitpage.py index 280484c..a4df701 100644 --- a/upgrade-toolkit/bin/cockpitpage.py +++ b/upgrade-toolkit/bin/cockpitpage.py @@ -590,7 +590,7 @@ def add(it): items.append({"id": pid, "kind": "plan-choice", "hidden": True, "dot": "warn", "title": f"{it['name']} ({it['kind']})", "step": "propose", "meta": "Plan choice", "problem": short(it["sub"].get("evidence", ""), 260), "rec": "rec", "rec_text": short(rec_txt, 320), "options": opts, "decided": last.get(pid), "assumed": is_assumed, "effort": effort_of(rec_txt), - "evidence": evidence.for_item(work, it["name"])}) + "evidence": evidence.for_item(work, it["name"], it["kind"])}) plan_signed = bool(signed) if P["present"] and not plan_signed: # Source of the count: every decision the agent recorded in AGENT-DECISIONS.md (len(AD), one per Dn) plus the plan choices diff --git a/upgrade-toolkit/bin/evidence.py b/upgrade-toolkit/bin/evidence.py index 0bf7aea..f7f2337 100644 --- a/upgrade-toolkit/bin/evidence.py +++ b/upgrade-toolkit/bin/evidence.py @@ -6,8 +6,13 @@ the owner asked how anyone can decide that. The run had the answer in its logs: Atlas_Core had 7 layouts and snippets edited, Encryption only its EncryptionKey constant, Email_Connector one microflow, MendixSSO an attribute the new version drops. - for_item(work, name) -> {"what": [plain lines], "files": [paths relative to the run folder]} - evidence.py --work W NAME prints the lines and the files (a check by hand) + for_item(work, name, kind=None) -> {"what": [plain lines], "files": [paths relative to the run folder]} + evidence.py --work W [--kind K] NAME prints the lines and the files (a check by hand) + +A plan item that is no module has its own source (cold CMM, 2026-11-17: the "Release notes" and "DataWidgets themesource +edits" cards were flagged ask-no-evidence on every run, because only module names were looked up): + kind check, Release notes W/release-impact.out (why the release-note check did not run) + kind theme, "<M> themesource edits" the theme and JavaScript file rows of <M> in W/localedits.json Sources, per module (app = state/app-dir, S = app/.stepupdate): S/step*-<M>.log the elements mxcli refused to overwrite, or why it could not read the module @@ -49,8 +54,28 @@ def _refused(text): return out -def for_item(work, name): +def _other(work, name, kind): + if kind == "check" and name == "Release notes": + p = os.path.join(work, "release-impact.out") + what = [ln.strip() for ln in (_rd(p) or "").splitlines() if ln.strip()][:5] + return {"what": what, "files": [_rel(work, p)] if what else []} + if kind == "theme" and name.endswith(" themesource edits"): + mod = name[:-len(" themesource edits")] + try: + rep = json.loads(_rd(os.path.join(work, "localedits.json")) or "[]") + except ValueError: + rep = [] + what = [f"{f.get('path')}: {(f.get('change') or '').lower()} ({f.get('detail')})" + for e in rep if isinstance(e, dict) and e.get("name") == mod for f in e.get("files") or []] + return {"what": what, "files": [_rel(work, os.path.join(work, "localedits.json"))] if what else []} + return None + + +def for_item(work, name, kind=None): work = os.path.abspath(work) + other = _other(work, name, kind) + if other is not None: + return other app = (_rd(os.path.join(work, "state", "app-dir")) or "").strip() S = os.path.join(app, ".stepupdate") if app else "" what, files = [], [] @@ -115,9 +140,10 @@ def for_item(work, name): def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) ap.add_argument("--work", required=True) + ap.add_argument("--kind") ap.add_argument("name") a = ap.parse_args() - ev = for_item(a.work, a.name) + ev = for_item(a.work, a.name, a.kind) print("\n".join("- " + w for w in ev["what"]) or "(no evidence found)") print("files: " + ", ".join(ev["files"])) return 0 if ev["what"] else 1 diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index c88f3a2..051232b 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6885,6 +6885,14 @@ ctl cockpit-no-hardcoded-run-pass 0 'no hard-coded run names' -- $CK nohard ctl cockpit-check-ghost-catch 0 '^ghost-step S-4 \(P-99, Keep or update module Oldmod\)' 'rc=1' -- $CK check-ghost ctl cockpit-check-withdraw-pass 0 "withdrawn \['S-4'\]" '!ghost-step' 'rows Oldmod: 0' -- $CK check-withdraw ctl cockpit-check-running-no-stopbar-pass 0 'stopbar: 0' '^0$' -- $CK check-running +# cold CMM: the "Release notes (check)" and "<M> themesource edits (theme)" cards were ask-no-evidence on every run, as only +# module names were looked up; each kind has its own source now, and a name that has none still shows nothing +mkdir -p $T/evw/work; printf 'release_impact: no brain version tables under /x/baseline/versions\n' > $T/evw/work/release-impact.out +printf '[{"name": "Shopmod", "files": [{"change": "CHANGED", "kind": "theme", "path": "themesource/shopmod/web/main.scss", "detail": "1 variable(s) changed: $brand"}]}]' > $T/evw/work/localedits.json +ctl evidence-check-release-notes-catch 0 '^- release_impact: no brain version tables' '^files: work/release-impact\.out$' -- python3 $B/evidence.py --work $T/evw/work --kind check "Release notes" +ctl evidence-theme-files-catch 0 '^- themesource/shopmod/web/main\.scss: changed \(1 variable' '^files: work/localedits\.json$' -- python3 $B/evidence.py --work $T/evw/work --kind theme "Shopmod themesource edits" +ctl evidence-theme-other-module-pass 1 '\(no evidence found\)' '!main\.scss' -- python3 $B/evidence.py --work $T/evw/work --kind theme "Othermod themesource edits" +ctl cockpitpage-evidence-passes-kind-pass 0 'evidence\.for_item\(work, it\["name"\], it\["kind"\]\)' -- grep -F 'evidence.for_item(work, it["name"], it["kind"])' $B/cockpitpage.py # editroute.py (cold test WM1): a module the update refused is routed to an update; only a real logic edit, Java code, a removal # something still uses or a tool fault stays kept. A stub mxcli answers describe and refs. ER="bash $here/editroute_control.sh" From 6d59cc94fdb9bf8e4e93ba3bf725a563c73b89d7 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 15:11:29 +0800 Subject: [PATCH 50/59] repair row 27: CE1613 errors hide the CE0117 Cold CMM: the 4 CE0117 at the dynamic classes of Mix_New_WizardStep1 vanished in run 5 while 3 CE1613 were open, and came back unchanged in run 7 once those were repaired (the pages are byte-identical in both runs). So mx check reports no CE0117 while CE1613 errors stand: a check that lost its CE0117 next to new CE1613 has not fixed them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/steps/repair.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upgrade-toolkit/steps/repair.md b/upgrade-toolkit/steps/repair.md index e462dc7..2ad39d9 100644 --- a/upgrade-toolkit/steps/repair.md +++ b/upgrade-toolkit/steps/repair.md @@ -37,7 +37,7 @@ keep the fix and continue with the new class. Without `--fixed` a rise is always | 24 | CE0463 after a hand-set value on a pluggable widget property; or `mx update-widgets` puts a property value back (FM-5) | read the widget's `getProperties()` in its editorConfig (inside the .mpk): a property that the config hides unless another property is on (for SprintrFeedback `targetContainerSelector` needs `showAdvancedSettings` = true) is hidden in the model, and a hidden property with a non-default value is a CE0463 and is reset by update-widgets | set the controlling property first (`showAdvancedSettings` true), then the value, in one commit; run `mx update-widgets` AFTER and check the value survived (diff the unit), then mx check. Seen: SprintrFeedback 12.0.4 (bundled with Atlas_Core 4.5.1) portals its button into `targetContainerSelector` and renders nothing when the default `.mx-scrollcontainer-center` is absent (an app layout without a centre region): the Feedback tab vanished from every page and shotdiff said "lost feedback". The remedy card `sprintr-feedback-target` (playbooks/remedies.tsv) carries it | | 25 | `mx update-widgets` moves a widget across a technology jump (Slider 1.3.2 Dojo to 3.0.4 pluggable) and then CE7006 on every attribute of it, or the widget lost its min/max/step attributes and its on-change microflow (QI-6) | a Dojo-to-pluggable version jump is a widget SWAP, not an update: before it, list the widget's properties in the old model (BSON of the page or snippet unit) and after it again, and write every dropped binding down; `bin/widgetdiff.py` names the package versions only | re-bind the dropped properties by BSON (not through mxcli describe, which keeps only the value attribute); a pluggable widget gets no entity context from a snippet parameter, so wrap it in a data view on the snippet parameter (the CE7006 goes); a microflow action on a pluggable widget takes empty ParameterMappings. Seen: Slider on a Quality Inspection snippet: only valueAttribute survived, 4 CE7006 until the data view wrap, 0 after. Then mx check and a by-eye look at the page | | 26 | screens rung REVIEW on looks that changed by design: the Charts 6 palette, the Data grid 2 column selector (eye) icon size, the MxAdmin changedDate stamped at runtime start on the 10.24 rung (data rung accept) (QI-10) | look at the pair first; these three are known and each has a fix or an accept | Charts 6 (6.x) comes without the old series palette: re-apply it in the theme (a `.widget-charts` series colour rule) when the dashboard look is wanted; the Data grid 2 column selector icon grows: add a theme rule for its size; the 10.24 runtime writes MxAdmin changedDate at start: accept it in the data rung accept file with that reason. Never accept an unseen pair | -| 27 | CE0117 at the "Dynamic classes expression" of a button after the 9 to 10 convert, where the expression follows an association (`if $currentObject/Module.A_B = empty then 'disabled' else ''`) | the `[where]` lines of check-N.log name the page and property | cold CMM (10.24.28): the same association test passed in 8 conditional-visibility expressions on the same pages and failed only in the 4 dynamic-classes expressions (Mix_New_WizardStep1, desktop and phone; the association leads to an external OData entity). The 9.24 build accepted it. Remedy NOT verified: move the test into conditional visibility, or compute a boolean attribute on the helper object and test that. A judgment item: the button's look changes, so a human looks at the page. No tool yet | +| 27 | CE0117 at the "Dynamic classes expression" of a button after the 9 to 10 convert, where the expression follows an association (`if $currentObject/Module.A_B = empty then 'disabled' else ''`) | the `[where]` lines of check-N.log name the page and property | cold CMM (10.24.28): the same association test passed in 8 conditional-visibility expressions on the same pages and failed only in the 4 dynamic-classes expressions (Mix_New_WizardStep1, desktop and phone; the association leads to an external OData entity). The 9.24 build accepted it. They can be HIDDEN: in cold CMM run 5 the 4 vanished while 3 CE1613 (row 28) were open, and came back unchanged once those were repaired (run 7), so mx check reports no CE0117 while CE1613 errors stand; a check that lost its CE0117 next to new CE1613 has not fixed them. (The 2 CE0117 in the kept Administration pages did go for good with its update.) Remedy (NOT verified): move the test into conditional visibility, or compute a boolean attribute on the helper object and test that. A judgment item: the button's look changes, so a human looks at the page. No tool yet | mxcli v0.28.0 (#1408, #1474), what it changes for the rows above: - `mxcli exec` is all-or-nothing. A script whose statement N fails now changes NOTHING and exits non-zero; before, statements 1 to N-1 From 047506d0ec5ab3406e8f94c6ee1bdcd84f87b9bb Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 15:21:59 +0800 Subject: [PATCH 51/59] crosscalls --own: name own documents a module update drops, before the update Cold CMM: Administration 2.1.2 -> 4.5.0 dropped UserRole_Select and RetrieveTimeZones, which two own employee pages used. Nothing said so until mx check showed 3 CE1613 at the build. crosscalls only judged kept callers, and the pipeline never ran it. - crosscalls.py --own: the app's own modules are callers too, judged against the plan version package at any major (OWN / GONE lines, exit 1) - the update step writes owngone.out before the updates and notes it; the build stop quotes it when mx check has CE1613 - steps/repair.md row 28: the remedy (copy the old document, or keep the module); a judgment item, mxcli cannot author selectors - 7 controls: red 5 failed on the old code, green 21/21 with the group Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 16 ++++++++ upgrade-toolkit/bin/crosscalls.py | 41 +++++++++++++++++-- .../controls/fixtures/models/xcall-own.json | 3 ++ upgrade-toolkit/controls/run-controls.sh | 19 +++++++++ upgrade-toolkit/steps/repair.md | 1 + 5 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 upgrade-toolkit/controls/fixtures/models/xcall-own.json diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 30209c3..62099e0 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -1120,6 +1120,19 @@ step_update() { [ -f "$WORK/plan.txt" ] || [ "$DRY" = 1 ] || { nj_add 13 "plan.txt missing" "$WORK/plan.txt does not exist"; nj_flush; } # a rerun starts as the first run did: an earlier routing's force-ui lines would force modules before they are routed [ "$DRY" = 1 ] || { python3 "$B/editroute.py" reset --work "$WORK" | tee -a "$LOG" > /dev/null || true; } + # own modules that use a document the plan version drops get CE1613 after the update; say which before it (cold CMM: + # Administration 4.5.0 has no UserRole_Select or RetrieveTimeZones, two own employee pages used them) + if [ "$DRY" != 1 ]; then + local orc=0 + xo "$WORK/owngone.out" python3 "$B/crosscalls.py" "$WORK/original.mpr" --kept '' --own --plan-file "$WORK/plan.txt" || orc=$? + if grep -q '^ GONE' "$WORK/owngone.out" 2> /dev/null; then + note "own modules use $(grep -c '^ GONE' "$WORK/owngone.out") target(s) the plan versions drop: CE1613 after the update, see $WORK/owngone.out (steps/repair.md row 28)" + elif [ "$orc" = 2 ]; then + note "crosscalls --own exit 2: own modules not checked against the plan versions, see $WORK/owngone.out" + fi + else + x python3 "$B/crosscalls.py" "$WORK/original.mpr" --kept '' --own --plan-file "$WORK/plan.txt" + fi run_stepupdate "$WORK/plan.txt" post-convert route_kept replay_edits @@ -1247,6 +1260,9 @@ Codes: $(grep -oE '^\[error\] *\[C[EW][0-9]+\]' "$CHECK_LOG" | grep -oE 'C[EW][0 Work the list with steps/repair.md (rows 4, 7-10 and own-document fixes; one commit per fix; checkdiff.py after each)." nj_file 10 "First 20 error lines" "$CHECK_LOG" 20 '^\[error\]' grep -q '^\[where\]' "$CHECK_LOG" 2> /dev/null && nj_file 10 "Where they are (first 20)" "$CHECK_LOG" 20 '^\[where\]' + if has_code CE1613 && grep -q '^ GONE' "$WORK/owngone.out" 2> /dev/null; then + nj_file 10 "Own documents use targets the module update dropped (CE1613; steps/repair.md row 28)" "$WORK/owngone.out" 20 '^(OWN| GONE)' + fi nj_flush fi # mxbuild diff --git a/upgrade-toolkit/bin/crosscalls.py b/upgrade-toolkit/bin/crosscalls.py index 740310b..5ac2570 100755 --- a/upgrade-toolkit/bin/crosscalls.py +++ b/upgrade-toolkit/bin/crosscalls.py @@ -38,11 +38,19 @@ --newest names the newest version of the current major per capped module (`mxcli marketplace versions <cid>`); without it the line says "newest <major>.x" and the installed version, to be looked up. +--own adds the app's own modules (not from the Marketplace, not on either list) as callers, judged on one thing only: +a target the plan version package no longer has, whatever the major (own code is changed, not held back). Cold CMM: +Administration 2.1.2 -> 4.5.0 dropped Administration.UserRole_Select and RetrieveTimeZones, used by two own employee +pages; nothing said so before mx check showed 3 CE1613. Per planned module with own callers: + OWN <P> <installed> -> <plan>: <n> target(s) own modules use are gone from the <plan> package + GONE <kind> <target> (used by own <document>) +or a NOTE when the package is not in the cache (targets not checked). + Output: a header with the denominators (kept modules, planned modules, documents and Java files scanned), then per capped planned module CAP <P> at <newest version of current major>: called by kept <K> (<n> refs, e.g. ...) Exit: 0 = no cap, 1 = at least one planned module is called by a kept module (hold it at its current major, or -update the kept module too), 2 = input error (a named module is not in the model, no model, no kept option). +update the kept module too), or (--own) an own module uses a target that is gone, 2 = input error (a named module is not in the model, no model, no kept option). """ import argparse, json, os, re, sys, zipfile @@ -193,6 +201,7 @@ def main(): ap.add_argument("--newest", default="") ap.add_argument("--javasource") ap.add_argument("--cache", default=os.environ.get("UPG_CACHE") or os.path.expanduser("~/.cache/upgrade-toolkit/mpk")) + ap.add_argument("--own", action="store_true", help="also check the app's own modules against the plan version packages") ap.add_argument("--json") a = ap.parse_args() @@ -276,9 +285,35 @@ def main(): print(f"crosscalls: {caps} planned module(s) capped, {len(notes)} noted, {len(planned) - caps - len(notes)} not called by a kept module") if caps: print(" hold each CAP module at its current major in the plan (or update the kept caller too), re-run after the plan changes") + gone_own, ores = 0, {} + if a.own: + own = sorted(n for n, v in model.modules.items() if not v.get("from_appstore") and n not in kept and n not in planned) + orefs, odocs = model_refs(model, set(own), set(planned)) + print(f"crosscalls: own modules {len(own)}; scanned {odocs} of their documents") + for pm in planned: + callers = {k: orefs[(k, pm)] for k in own if (k, pm) in orefs} + pv = planv.get(pm, "") + if not callers or not pv: + continue + inst = model.modules[pm].get("version") or "" + pkg = package_model(a.cache, model.modules[pm].get("package_id"), pv) + if pkg is None: + print(f"NOTE {pm} {inst} -> {pv}: used by own {', '.join(callers)}; targets NOT checked, the {pv} package is not in the cache") + continue + gone = gone_targets(pkg, callers) + if gone: + print(f"OWN {pm} {inst} -> {pv}: {len(gone)} target(s) own modules use are gone from the {pv} package") + for kind, src, tgt in gone: + print(f" GONE {kind} {tgt} (used by own {src})") + ores[pm] = [list(x) for x in gone] + gone_own += len(gone) + print(f"crosscalls: {gone_own} target(s) used by own modules are gone in the plan versions") + if gone_own: + print(" each one is CE1613 after the update: copy the old document into an own module and point the users at it, " + "or keep the module (steps/repair.md row 28)") if a.json: - json.dump({"kept": kept, "planned": planned, "caps": res}, open(a.json, "w"), indent=1) - return 1 if caps else 0 + json.dump({"kept": kept, "planned": planned, "caps": res, "own_gone": ores}, open(a.json, "w"), indent=1) + return 1 if caps or gone_own else 0 if __name__ == "__main__": diff --git a/upgrade-toolkit/controls/fixtures/models/xcall-own.json b/upgrade-toolkit/controls/fixtures/models/xcall-own.json new file mode 100644 index 0000000..28a4c75 --- /dev/null +++ b/upgrade-toolkit/controls/fixtures/models/xcall-own.json @@ -0,0 +1,3 @@ +{"from": "xcall-page.json", "ops": [["append", "modules", {"name": "Shop", + "docs": [{"type": "Microflows$Microflow", "name": "ACT_Open", "data": {"Call": {"Page": "AuthMod.Login_Page"}}}, + {"type": "Microflows$Microflow", "name": "ACT_Pay", "data": {"Call": {"Action": "AuthMod.ACT_Sign"}}}]}]]} diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 051232b..366a0d3 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -226,6 +226,23 @@ ctl crosscalls-page-listed-first-catch 1 'e\.g\. page KeptConn\.ACT_Show -> Auth ctl crosscalls-no-kept-invalid 2 'give --kept' -- python3 $B/crosscalls.py $M/xcall --planned AuthMod ctl crosscalls-unknown-module-invalid 2 'not a module of the model: Typo' -- python3 $B/crosscalls.py $M/xcall --kept KeptConn --planned Typo ctl crosscalls-empty-kept-pass 0 'kept 0' '!CAP ' -- python3 $B/crosscalls.py $M/xcall --kept '' --planned AuthMod +# cold CMM: Administration 2.1.2 -> 4.5.0 dropped a page and a microflow two OWN employee pages used: 3 CE1613 at the build, +# nothing said so before. --own judges the app's own modules against the plan version package, any major (own code is +# changed, not held back); the target still in the package passes, no package in the cache is a NOTE +model xcall-own > /dev/null +mkdir -p $T/cache-xc-own-gone $T/cache-xc-own-here $T/cache-xc-own-none +cp $T/cache-xc-gone/2222-3.9.0.mpk $T/cache-xc-own-gone/2222-4.1.0.mpk; cp $T/cache-xc-here/2222-3.9.0.mpk $T/cache-xc-own-here/2222-4.1.0.mpk +ctl crosscalls-own-gone-catch 1 '^OWN AuthMod 3\.2\.0 -> 4\.1\.0: 1 target\(s\) own modules use are gone from the 4\.1\.0 package' '^ GONE page AuthMod\.Login_Page \(used by own Shop\.ACT_Open\)' '!GONE .*ACT_Sign' '!GONE .*KeptConn' 'repair\.md row 28' -- python3 $B/crosscalls.py $M/xcall-own --kept '' --own --plan-file $T/xc-plan-major.txt --cache $T/cache-xc-own-gone +ctl crosscalls-own-present-pass 0 '^crosscalls: 0 target\(s\) used by own modules are gone' '!GONE' -- python3 $B/crosscalls.py $M/xcall-own --kept '' --own --plan-file $T/xc-plan-major.txt --cache $T/cache-xc-own-here +ctl crosscalls-own-unchecked-pass 0 '^NOTE AuthMod 3\.2\.0 -> 4\.1\.0: used by own Shop; targets NOT checked' '!GONE' -- python3 $B/crosscalls.py $M/xcall-own --kept '' --own --plan-file $T/xc-plan-major.txt --cache $T/cache-xc-own-none +ctl crosscalls-own-off-pass 0 '!OWN|GONE' -- python3 $B/crosscalls.py $M/xcall-own --kept '' --plan-file $T/xc-plan-major.txt --cache $T/cache-xc-own-gone +# the build stop quotes owngone.out when mx check has CE1613 (and only then) +mkdir -p $T/og; printf '%s\n' 'crosscalls: own modules 1' 'OWN AuthMod 3.2.0 -> 4.1.0: 1 target(s) own modules use are gone from the 4.1.0 package' ' GONE page AuthMod.Login_Page (used by own Shop.ACT_Open)' > $T/og/owngone.out +printf '%s\n' '[error] [CE1613] "The selected page no longer exists." at Microflow ACT_Open' > $T/og/check-1613.log; printf '%s\n' '[error] [CE0117] "x"' > $T/og/check-0117.log +awk '/if has_code CE1613 && grep/{p=1} p{print} p&&/^ fi$/{exit}' $B/cloud/upgrade-pipeline.sh > $T/og/nj.sh +printf '%s\n' 'has_code() { grep -q "\[$1\]" "$CHECK_LOG"; }' 'nj_file() { echo "NJ $1 $2"; grep -E "$5" "$3"; }' > $T/og/stub.sh +ctl pipeline-build-owngone-catch 0 '^NJ 10 Own documents use targets the module update dropped \(CE1613; steps/repair\.md row 28\)' '^ GONE page AuthMod\.Login_Page \(used by own Shop\.ACT_Open\)' '!crosscalls: own' -- bash -c ". $T/og/stub.sh; WORK=$T/og CHECK_LOG=$T/og/check-1613.log; . $T/og/nj.sh; echo done" +ctl pipeline-build-owngone-other-code-pass 0 '^done$' '!NJ' -- bash -c ". $T/og/stub.sh; WORK=$T/og CHECK_LOG=$T/og/check-0117.log; . $T/og/nj.sh; echo done" # T6R-7: the other direction. An UPDATED module that needs what a KEPT module lacks (a name, a parameter of a flow, a Java # class) is FAIL with the pair named; all names present is REVIEW (signatures unproven), never PASS; a kept module calling @@ -1704,6 +1721,8 @@ ctl pipeline-ladder-refuses-to-with-target-catch 2 'give one or the other' -- en ctl pipeline-dry-smoke-microflows-pass 0 'smokewalk\.js .* --microflows --login-path' -- env UPG_SMOKE_MICROFLOWS=1 bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pmf1 --only basewalk --dry-run ctl pipeline-dry-smoke-microflows-off-catch 0 'smokewalk\.js .* --login-path' '!--microflows' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pmf0 --only basewalk --dry-run ctl pipeline-dry-noforceui-catch 0 '!--force-ui' '!replay/' 'stepupdate\.sh .* --unattended' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pw --only update --dry-run +# the update step runs crosscalls --own on the original model BEFORE the module updates (cold CMM: 3 CE1613 found only at the build) +ctl pipeline-dry-update-owngone-catch 0 '^own-before-update$' -- bash -c "bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pw --only update --dry-run | awk '/crosscalls\.py .*\/pw\/original\.mpr --kept .*--own --plan-file .*\/pw\/plan\.txt/{if(!c)c=NR} /stepupdate\.sh/{if(!s)s=NR} END{print (c && c<s) ? \"own-before-update\" : \"no c=\" c \" s=\" s}'" # F14: replay/<Module>.guids restores the original GUIDs after the MDL replay; approved-step.txt reaches the step idcheck only mkdir -p $V/pw4/replay; printf 'MendixSSO\n' > $V/pw4/force-ui.txt; echo 'create or modify association A.R_X from A.R to B.U;' > $V/pw4/replay/MendixSSO.mdl printf '# original GUIDs back\nA.R_X\nA.R_Y\n' > $V/pw4/replay/MendixSSO.guids; printf 'A.R_X # restored by replay/MendixSSO.guids\n' > $V/pw4/approved-step.txt; printf '# accepted\n' > $V/pw4/approved.txt diff --git a/upgrade-toolkit/steps/repair.md b/upgrade-toolkit/steps/repair.md index 2ad39d9..aeb7333 100644 --- a/upgrade-toolkit/steps/repair.md +++ b/upgrade-toolkit/steps/repair.md @@ -38,6 +38,7 @@ keep the fix and continue with the new class. Without `--fixed` a rise is always | 25 | `mx update-widgets` moves a widget across a technology jump (Slider 1.3.2 Dojo to 3.0.4 pluggable) and then CE7006 on every attribute of it, or the widget lost its min/max/step attributes and its on-change microflow (QI-6) | a Dojo-to-pluggable version jump is a widget SWAP, not an update: before it, list the widget's properties in the old model (BSON of the page or snippet unit) and after it again, and write every dropped binding down; `bin/widgetdiff.py` names the package versions only | re-bind the dropped properties by BSON (not through mxcli describe, which keeps only the value attribute); a pluggable widget gets no entity context from a snippet parameter, so wrap it in a data view on the snippet parameter (the CE7006 goes); a microflow action on a pluggable widget takes empty ParameterMappings. Seen: Slider on a Quality Inspection snippet: only valueAttribute survived, 4 CE7006 until the data view wrap, 0 after. Then mx check and a by-eye look at the page | | 26 | screens rung REVIEW on looks that changed by design: the Charts 6 palette, the Data grid 2 column selector (eye) icon size, the MxAdmin changedDate stamped at runtime start on the 10.24 rung (data rung accept) (QI-10) | look at the pair first; these three are known and each has a fix or an accept | Charts 6 (6.x) comes without the old series palette: re-apply it in the theme (a `.widget-charts` series colour rule) when the dashboard look is wanted; the Data grid 2 column selector icon grows: add a theme rule for its size; the 10.24 runtime writes MxAdmin changedDate at start: accept it in the data rung accept file with that reason. Never accept an unseen pair | | 27 | CE0117 at the "Dynamic classes expression" of a button after the 9 to 10 convert, where the expression follows an association (`if $currentObject/Module.A_B = empty then 'disabled' else ''`) | the `[where]` lines of check-N.log name the page and property | cold CMM (10.24.28): the same association test passed in 8 conditional-visibility expressions on the same pages and failed only in the 4 dynamic-classes expressions (Mix_New_WizardStep1, desktop and phone; the association leads to an external OData entity). The 9.24 build accepted it. They can be HIDDEN: in cold CMM run 5 the 4 vanished while 3 CE1613 (row 28) were open, and came back unchanged once those were repaired (run 7), so mx check reports no CE0117 while CE1613 errors stand; a check that lost its CE0117 next to new CE1613 has not fixed them. (The 2 CE0117 in the kept Administration pages did go for good with its update.) Remedy (NOT verified): move the test into conditional visibility, or compute a boolean attribute on the helper object and test that. A judgment item: the button's look changes, so a human looks at the page. No tool yet | +| 28 | CE1613 "the selected page/microflow no longer exists" in an OWN module after a Marketplace module update: an own reference selector's select page, a microflow data source or a Call microflow that points into the updated module | `owngone.out` (crosscalls.py --own, written by the update step before the update) lists each `GONE <kind> <target> (used by own <document>)`; the build stop quotes it | cold CMM: Administration 2.1.2 -> 4.5.0 dropped `Administration.UserRole_Select` (the select page of the user role selector on two own Employee pages) and `Administration.RetrieveTimeZones` (the time zone data source on Employee_New): 3 CE1613. Remedy (NOT verified): copy the old document out of `original.mpr` into an own module and point the users at the copy, or switch the selector's source to a database/XPath source on the same entity; if neither fits, keep the module at its installed version (kept, a held update). mxcli cannot author reference selectors (not re-executable), so this is a judgment item done in Studio Pro; check the selector on the page as a real user afterwards | mxcli v0.28.0 (#1408, #1474), what it changes for the rows above: - `mxcli exec` is all-or-nothing. A script whose statement N fails now changes NOTHING and exits non-zero; before, statements 1 to N-1 From f3afb78d68814a912756531117b2aec0eb5694a2 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 15:37:12 +0800 Subject: [PATCH 52/59] build: a crashed mx check is never 0 errors Cold CMM: mx check died loading the model (KeyNotFoundException from a copied page with a stale pointer) and printed no summary. open_errors counted 0, so the run went on to mxbuild, which failed on the same load. All 227 finished checks in the lab logs print "The app contains: N errors."; the 6 without it are all crashes. - mxcheck appends "[error] [MXCHECK] mx check ended without a result" with the first ERROR line when the summary is missing; the build stops on it - steps/repair.md row 29: what it means and how to undo the last change - 2 controls: red 1 failed on the old code, green with the group Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 7 +++++++ upgrade-toolkit/controls/run-controls.sh | 8 ++++++++ upgrade-toolkit/steps/repair.md | 1 + 3 files changed, 16 insertions(+) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 62099e0..8d1aaf0 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -1174,6 +1174,13 @@ mxcheck() { # writes $WORK/check-N.log, sets CHECK_LOG rm -f "$cj" heavy check "$TARGET" mx check "$MPR" ${j[@]+"${j[@]}"} > "$CHECK_LOG" 2>&1 || true [ ! -s "$cj" ] || python3 "$B/checkwhere.py" "$cj" "$CHECK_LOG" || true + # no summary line = mx check died (it prints "The app contains: N errors." on every finished check, 0 included); a + # crash is never 0 errors (cold CMM: a copied page with a stale pointer, KeyNotFoundException on load, and the run went on + # to mxbuild with "0 errors") + if ! grep -q 'The app contains:' "$CHECK_LOG" 2> /dev/null; then + printf '[error] [MXCHECK] mx check ended without a result (it died loading or checking the model): %s\n' \ + "$(grep -m1 -E '^ERROR' "$CHECK_LOG" 2> /dev/null | cut -c1-200 || true)" >> "$CHECK_LOG" + fi } has_code() { grep -q "\[$1\]" "$CHECK_LOG" 2> /dev/null; } try_fix() { # try_fix "subject" RULE CODES cmd...: run a fixer, re-check, keep when checkdiff says not worse, else undo diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 366a0d3..3bfac37 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -1805,6 +1805,14 @@ STUB mkdir -p $V/cw/w10 $V/cw/w9 ctl pipeline-mxcheck-where-catch 0 '^\[where\] CE0117 Shop / Page .Order_Step1.' '^The app contains: 1 errors' '!usage' -- bash -c "B=$B WORK=$V/cw/w10 TARGET=11.12.7 MPR=x.mpr DRY=0; . $V/cw/stub.sh; . $V/cw/mxcheck.sh; mxcheck; cat \$CHECK_LOG" ctl pipeline-mxcheck-9-no-json-pass 0 '^\[error\] \[CE0117\]' '!\[where\]' -- bash -c "B=$B WORK=$V/cw/w9 TARGET=9.24.2 MPR=x.mpr DRY=0; . $V/cw/stub.sh; . $V/cw/mxcheck.sh; mxcheck; cat \$CHECK_LOG; ls $V/cw/w9" +# cold CMM: mx check died loading the model (KeyNotFoundException, no summary line) and the run counted 0 errors and went +# on to mxbuild. A check without "The app contains" is an error line; a finished clean check stays 0 +mkdir -p $V/cw/crash $V/cw/clean +awk '/^open_errors\(\)/,/^}/' $B/cloud/upgrade-pipeline.sh > $V/cw/openerr.sh +printf '%s\n' 'Loading the mpr file.' "ERROR: System.Collections.Generic.KeyNotFoundException: The given key 'cb8a' was not present in the dictionary." ' at Mendix.Modeler.Storage.Operations.UnitContentsLoader.ConvertFromStorage' > $V/cw/crash.log +printf '%s\n' 'Checking app for errors...' 'The app contains: 0 errors.' > $V/cw/clean.log +ctl pipeline-mxcheck-crash-catch 0 '^\[error\] \[MXCHECK\] mx check ended without a result .*KeyNotFoundException' '^errors 1$' -- bash -c "B=$B WORK=$V/cw/crash TARGET=11.12.7 MPR=x.mpr DRY=0; . $V/cw/stub.sh; heavy() { cat $V/cw/crash.log; return 1; }; . $V/cw/mxcheck.sh; . $V/cw/openerr.sh; mxcheck; cat \$CHECK_LOG; echo errors \$(open_errors \$CHECK_LOG)" +ctl pipeline-mxcheck-clean-pass 0 '^errors 0$' '!MXCHECK' -- bash -c "B=$B WORK=$V/cw/clean TARGET=11.12.7 MPR=x.mpr DRY=0; . $V/cw/stub.sh; heavy() { cat $V/cw/clean.log; }; . $V/cw/mxcheck.sh; . $V/cw/openerr.sh; mxcheck; cat \$CHECK_LOG; echo errors \$(open_errors \$CHECK_LOG)" ctl pipeline-dry-check-json-catch 0 'mx10\.sh 11\.15\.0\.1 mx check .*/papp/App\.mpr -j .*/pwle/check-1\.json > ' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pwle --only build --dry-run # v3.1 E6-7: a removal (idcheck verdict REVIEW) is named REVIEW in the KEPT line, a GUID change stays a plain finding mkdir -p $S/base; cp $M/base/App.mpr $S/base/ diff --git a/upgrade-toolkit/steps/repair.md b/upgrade-toolkit/steps/repair.md index aeb7333..e76a08e 100644 --- a/upgrade-toolkit/steps/repair.md +++ b/upgrade-toolkit/steps/repair.md @@ -39,6 +39,7 @@ keep the fix and continue with the new class. Without `--fixed` a rise is always | 26 | screens rung REVIEW on looks that changed by design: the Charts 6 palette, the Data grid 2 column selector (eye) icon size, the MxAdmin changedDate stamped at runtime start on the 10.24 rung (data rung accept) (QI-10) | look at the pair first; these three are known and each has a fix or an accept | Charts 6 (6.x) comes without the old series palette: re-apply it in the theme (a `.widget-charts` series colour rule) when the dashboard look is wanted; the Data grid 2 column selector icon grows: add a theme rule for its size; the 10.24 runtime writes MxAdmin changedDate at start: accept it in the data rung accept file with that reason. Never accept an unseen pair | | 27 | CE0117 at the "Dynamic classes expression" of a button after the 9 to 10 convert, where the expression follows an association (`if $currentObject/Module.A_B = empty then 'disabled' else ''`) | the `[where]` lines of check-N.log name the page and property | cold CMM (10.24.28): the same association test passed in 8 conditional-visibility expressions on the same pages and failed only in the 4 dynamic-classes expressions (Mix_New_WizardStep1, desktop and phone; the association leads to an external OData entity). The 9.24 build accepted it. They can be HIDDEN: in cold CMM run 5 the 4 vanished while 3 CE1613 (row 28) were open, and came back unchanged once those were repaired (run 7), so mx check reports no CE0117 while CE1613 errors stand; a check that lost its CE0117 next to new CE1613 has not fixed them. (The 2 CE0117 in the kept Administration pages did go for good with its update.) Remedy (NOT verified): move the test into conditional visibility, or compute a boolean attribute on the helper object and test that. A judgment item: the button's look changes, so a human looks at the page. No tool yet | | 28 | CE1613 "the selected page/microflow no longer exists" in an OWN module after a Marketplace module update: an own reference selector's select page, a microflow data source or a Call microflow that points into the updated module | `owngone.out` (crosscalls.py --own, written by the update step before the update) lists each `GONE <kind> <target> (used by own <document>)`; the build stop quotes it | cold CMM: Administration 2.1.2 -> 4.5.0 dropped `Administration.UserRole_Select` (the select page of the user role selector on two own Employee pages) and `Administration.RetrieveTimeZones` (the time zone data source on Employee_New): 3 CE1613. Remedy (NOT verified): copy the old document out of `original.mpr` into an own module and point the users at the copy, or switch the selector's source to a database/XPath source on the same entity; if neither fits, keep the module at its installed version (kept, a held update). mxcli cannot author reference selectors (not re-executable), so this is a judgment item done in Studio Pro; check the selector on the page as a real user afterwards | +| 29 | `[MXCHECK] mx check ended without a result` in check-N.log: mx check died (an `ERROR: System.…Exception` and a stack, no "The app contains" line) | the ERROR line it quotes; `git -C app log -1` names the last change | mx check cannot load the model, so it reports nothing: the pipeline counts it as an error and stops (before, it counted 0 and ran mxbuild). The last commit broke the MPR: a raw unit edit (copydoc.py, copy_mappings.py, restoreunit.py) or a tool crash mid-write. KeyNotFoundException with a GUID = a pointer to an element id that is not in the unit (cold CMM: a page copied with fresh ids kept its old `DefaultButtonPointer`; copydoc.py now remaps them). Tag, `git reset --hard` to the commit before, fix the tool, redo. EntryPointNotFound or NullReferenceException were seen too (an earlier run, twice), cause not established: find the last change first | mxcli v0.28.0 (#1408, #1474), what it changes for the rows above: - `mxcli exec` is all-or-nothing. A script whose statement N fails now changes NOTHING and exits non-zero; before, statements 1 to N-1 From 9f3331f60ddef9eb4d709ee4f2ef6d0401a566d0 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 15:37:21 +0800 Subject: [PATCH 53/59] repair: copydoc.py copies a document a module update dropped into an own module Repair row 28 as a tool. Cold CMM: Administration 4.5.0 dropped UserRole_Select and RetrieveTimeZones, used by two own Employee pages (3 CE1613). mxcli cannot author or repoint those selectors. copydoc.py copies the document from the same-version model before the update into an own module, with fresh ids (pointers follow), its module roles mapped (--roles), and repoints only own users (--retarget). Run 7: the 3 CE1613 gone. The first version renewed the ids but not ControlBar.DefaultButtonPointer; mx check died on load. The apply control checks the pointer: it fails on that version. - refusals: different Mendix versions, document or module missing, a Marketplace target, a dependency the app lacks, roles not given, MPR v2 - 7 controls: red 7 failed without the tool, 1 failed on the first version, green with the group; row 28 names the tool and the verified run Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/repair/copydoc.py | 242 ++++++++++++++++++ .../fixtures/models/copydoc-base.json | 10 + .../controls/fixtures/models/copydoc-new.json | 1 + .../controls/fixtures/models/copydoc-v11.json | 1 + upgrade-toolkit/controls/run-controls.sh | 45 ++++ upgrade-toolkit/steps/repair.md | 2 +- 6 files changed, 300 insertions(+), 1 deletion(-) create mode 100755 upgrade-toolkit/bin/repair/copydoc.py create mode 100644 upgrade-toolkit/controls/fixtures/models/copydoc-base.json create mode 100644 upgrade-toolkit/controls/fixtures/models/copydoc-new.json create mode 100644 upgrade-toolkit/controls/fixtures/models/copydoc-v11.json diff --git a/upgrade-toolkit/bin/repair/copydoc.py b/upgrade-toolkit/bin/repair/copydoc.py new file mode 100755 index 0000000..4ce0a1f --- /dev/null +++ b/upgrade-toolkit/bin/repair/copydoc.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""copydoc.py: copy a document a module update dropped into an own module, and point its users at the copy (repair row 28). + + copydoc.py <app dir|.mpr> --from <baseline .mpr> --doc Module.Doc --to Dest [--roles Dest.R1,Dest.R2] [--retarget] [--apply] + +Why (cold CMM): Administration 2.1.2 -> 4.5.0 dropped the page UserRole_Select and the microflow RetrieveTimeZones; two +own Employee pages used them as the select page of a reference set selector and the source of a reference selector: +3 CE1613. mxcli cannot author those selectors (describe says NOT re-executable), so it cannot repoint them either. + +The baseline is the same app at the same Mendix version BEFORE the update (a git commit of the converted tree, written +out with `git show <commit>:<app>.mpr > base.mpr`). The document's BSON is copied with fresh ids into module Dest (at +its top level, same name). Its allowed module roles belong to the old module: --roles names the Dest roles that +replace them (required when it had any). --retarget replaces every string that is exactly Module.Doc with Dest.Doc in +the units of the app's own (non-Marketplace) modules and lists each unit it changed. + +Refusals (exit 2): different Mendix versions; the document missing in the baseline; Dest missing or not an own +module; Dest.Doc already present; a document reference inside the copy that the app does not have (copy that first); +roles needed and not given; an MPR v2 project (mprcontents: not supported yet). Dry run unless --apply; the changes +are one new unit plus one per retargeted user, so commit the app before --apply and run mx check after. +Exit: 0 = done (or the dry run shows it), 2 = refused. +""" +import argparse, base64, copy, hashlib, os, re, sqlite3, sys, uuid + +import bson +from bson.binary import Binary + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from copy_mappings import Mpr # noqa: E402 + +QN = re.compile(r"^[A-Za-z_]\w*\.[A-Za-z_]\w*$") +APPSTORE = ("FromAppStore", "IsThemeModule") + + +def die(msg): + print(f"copydoc: REFUSED: {msg}") + sys.exit(2) + + +def version(m): + try: + return m.con.execute("select _ProductVersion from _MetaData").fetchone()[0] + except sqlite3.DatabaseError: + return None + + +def all_docs(m): + """qualified name -> uid for every named unit inside a module (pages, flows, snippets, enums, ...).""" + out = {} + for uid in m.rows: + d = m.decode(uid) + if d and d.get("Name") and d.get("$Type") not in ("Projects$Folder", "Projects$ModuleImpl", "Projects$Module"): + mod = m.module_of(uid) + if mod: + out[f"{mod}.{d['Name']}"] = uid + return out + + +def module_unit(m, name): + for uid in m.rows: + d = m.decode(uid) + if d and d.get("$Type") in ("Projects$ModuleImpl", "Projects$Module") and d.get("Name") == name: + return uid, d + return None, None + + +def strings(o): + if isinstance(o, dict): + for v in o.values(): + yield from strings(v) + elif isinstance(o, list): + for v in o: + yield from strings(v) + elif isinstance(o, str): + yield o + + +def replace_exact(o, old, new): + n = 0 + if isinstance(o, dict): + for k, v in o.items(): + if v == old: + o[k] = new; n += 1 + else: + n += replace_exact(v, old, new) + elif isinstance(o, list): + for i, v in enumerate(o): + if v == old: + o[i] = new; n += 1 + else: + n += replace_exact(v, old, new) + return n + + +def new_ids(doc): + """Fresh $IDs for the whole copy, and every pointer to one of them follows (a page's ControlBar.DefaultButtonPointer + holds the $ID of its default button: renewing the button but not the pointer makes mx check die on load with + KeyNotFoundException, cold CMM).""" + ids = {} + + def collect(o): + if isinstance(o, dict): + for k, v in o.items(): + if k == "$ID" and isinstance(v, (bytes, Binary)): + ids[bytes(v)] = uuid.uuid4().bytes + else: + collect(v) + elif isinstance(o, list): + for v in o: + collect(v) + + def swap(v): + return (Binary(ids[bytes(v)], v.subtype) if isinstance(v, Binary) else ids[bytes(v)]) \ + if isinstance(v, (bytes, Binary)) and bytes(v) in ids else v + + def remap(o): + if isinstance(o, dict): + for k, v in o.items(): + o[k] = swap(v) if isinstance(v, (bytes, Binary)) else remap(v) + elif isinstance(o, list): + for i, v in enumerate(o): + o[i] = swap(v) if isinstance(v, (bytes, Binary)) else remap(v) + return o + + collect(doc) + return remap(doc), ids + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("app") + ap.add_argument("--from", dest="base", required=True) + ap.add_argument("--doc", required=True) + ap.add_argument("--to", required=True) + ap.add_argument("--roles", default="") + ap.add_argument("--retarget", action="store_true") + ap.add_argument("--apply", action="store_true") + a = ap.parse_args() + try: + app, base = Mpr(a.app), Mpr(a.base) + except ValueError as e: + die(str(e)) + if "Contents" not in app.cols or any(f for _, _, f in app.rows.values()): + die(f"{app.path} is an MPR v2 project (mprcontents): not supported yet") + va, vb = version(app), version(base) + if va != vb: + die(f"Mendix versions differ: app {va}, baseline {vb} (write the baseline out of the converted tree's git history)") + if not QN.match(a.doc): + die(f"--doc {a.doc}: give Module.Doc") + src_mod, name = a.doc.split(".") + bdocs, adocs = all_docs(base), all_docs(app) + if a.doc not in bdocs: + die(f"{a.doc} is not in the baseline {base.path}") + new_qn = f"{a.to}.{name}" + if new_qn in adocs: + die(f"{new_qn} already exists in the app") + muid, mod = module_unit(app, a.to) + if muid is None: + die(f"module {a.to} is not in the app") + if any(mod.get(k) for k in APPSTORE) or mod.get("AppStoreVersion"): + die(f"module {a.to} is a Marketplace module: copy into one of the app's own modules") + doc = copy.deepcopy(base.decode(bdocs[a.doc])) + roles = [r for r in (doc.get("AllowedModuleRoles") or [])[1:] if isinstance(r, str)] + new_roles = [r.strip() for r in a.roles.split(",") if r.strip()] + if roles and not new_roles: + die(f"{a.doc} allows {', '.join(roles)}: name the {a.to} roles that replace them with --roles") + missing = sorted({s for s in strings(doc) if QN.match(s) and s.split(".")[0] not in ("System",) + and s in bdocs and s not in adocs and s != a.doc}) + if missing: + die(f"{a.doc} uses {', '.join(missing)}, which the app does not have: copy those first") + if roles: + doc["AllowedModuleRoles"] = [doc["AllowedModuleRoles"][0]] + new_roles + doc, ids = new_ids(doc) + nid = bytes(doc["$ID"]) + blob = bson.encode(doc) + users = [] + if a.retarget: + for uid in app.rows: + d = app.decode(uid) + if not d or not any(s == a.doc for s in strings(d)): + continue + mname = app.module_of(uid) + _, md = module_unit(app, mname) if mname else (None, None) + if md is None or any(md.get(k) for k in APPSTORE) or md.get("AppStoreVersion"): + continue + d2 = copy.deepcopy(d) + n = replace_exact(d2, a.doc, new_qn) + users.append((uid, f"{mname}.{d.get('Name')}", n, bson.encode(d2))) + print(f"copydoc: {a.doc} ({doc.get('$Type')}) -> {new_qn}" + (f", roles {', '.join(roles)} -> {', '.join(new_roles)}" if roles else "")) + for _, qn, n, _ in users: + print(f" RETARGET {qn}: {n} reference(s) {a.doc} -> {new_qn}") + if a.retarget and not users: + print(f" no own unit uses {a.doc}") + if not a.apply: + print("dry run: nothing written (--apply writes it)") + return 0 + h = lambda b: base64.b64encode(hashlib.sha256(b).digest()).decode() + cols = ["UnitID", "ContainerID", "ContainmentName", "TreeConflict", "ContentsHash", "ContentsConflicts", "Contents"] + have = [c for c in cols if c in app.cols] + vals = {"UnitID": nid, "ContainerID": muid, "ContainmentName": "Documents", "TreeConflict": 0, + "ContentsHash": h(blob), "ContentsConflicts": "", "Contents": blob} + app.con.execute(f"insert into Unit ({', '.join(have)}) values ({', '.join('?' * len(have))})", [vals[c] for c in have]) + app.con.commit() + for uid, qn, n, b in users: + app.write(uid, b) + back = Mpr(app.path) + copied = back.decode(nid) or {} + dangling = [] + + def ptrs(o, p=""): + if isinstance(o, dict): + for k, v in o.items(): + if k.endswith("Pointer") and isinstance(v, (bytes, Binary)) and bytes(v) not in seen: + dangling.append(p + "/" + k) + elif not isinstance(v, (bytes, Binary)): + ptrs(v, p + "/" + k) + elif isinstance(o, list): + for i, v in enumerate(o): + ptrs(v, f"{p}[{i}]") + + seen = set() + + def allids(o): + if isinstance(o, dict): + for k, v in o.items(): + if k == "$ID": + seen.add(bytes(v)) + else: + allids(v) + elif isinstance(o, list): + for v in o: + allids(v) + allids(copied); ptrs(copied) + for x in dangling: + print(f" DANGLING pointer {x}: points at no element of the copy") + ok = not dangling and new_qn in all_docs(back) and all(not any(s == a.doc for s in strings(back.decode(u))) for u, _, _, _ in users) + print(f"copydoc: written, re-read {'OK' if ok else 'MISMATCH'}; run mx check now") + return 0 if ok else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/controls/fixtures/models/copydoc-base.json b/upgrade-toolkit/controls/fixtures/models/copydoc-base.json new file mode 100644 index 0000000..d2cd6a0 --- /dev/null +++ b/upgrade-toolkit/controls/fixtures/models/copydoc-base.json @@ -0,0 +1,10 @@ +{"modules": [ + {"name": "Src", "appstore": [4444, "1.0.0"], "module_roles": ["User"], + "pages": [{"name": "Pick_Page", "allowed": ["Src.User"]}, {"name": "Pick_Deep", "refs": {"Form": "Src.Pick_Snippet"}}], + "docs": [{"type": "Forms$Snippet", "name": "Pick_Snippet"}, {"type": "Microflows$Microflow", "name": "Get_List"}]}, + {"name": "KeptConn", "appstore": [1111, "3.0.0"], + "docs": [{"type": "Microflows$Microflow", "name": "ACT_Pick", "data": {"Call": {"Page": "Src.Pick_Page"}}}]}, + {"name": "Shop", "module_roles": ["User"], + "pages": [{"name": "Order_Edit", "refs": {"Page": "Src.Pick_Page"}}], + "docs": [{"type": "Microflows$Microflow", "name": "ACT_Open", "data": {"Call": {"Page": "Src.Pick_Page"}}}]} + ]} diff --git a/upgrade-toolkit/controls/fixtures/models/copydoc-new.json b/upgrade-toolkit/controls/fixtures/models/copydoc-new.json new file mode 100644 index 0000000..2df058a --- /dev/null +++ b/upgrade-toolkit/controls/fixtures/models/copydoc-new.json @@ -0,0 +1 @@ +{"from": "copydoc-base.json", "ops": [["set", "modules/0/pages", []], ["set", "modules/0/docs", [{"type": "Microflows$Microflow", "name": "Get_List"}]]]} diff --git a/upgrade-toolkit/controls/fixtures/models/copydoc-v11.json b/upgrade-toolkit/controls/fixtures/models/copydoc-v11.json new file mode 100644 index 0000000..70e6cc3 --- /dev/null +++ b/upgrade-toolkit/controls/fixtures/models/copydoc-v11.json @@ -0,0 +1 @@ +{"from": "copydoc-new.json", "ops": [["set", "version", "11.12.7.1"]]} diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 3bfac37..4ddc5c2 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -236,6 +236,51 @@ ctl crosscalls-own-gone-catch 1 '^OWN AuthMod 3\.2\.0 -> 4\.1\.0: 1 target\(s\) ctl crosscalls-own-present-pass 0 '^crosscalls: 0 target\(s\) used by own modules are gone' '!GONE' -- python3 $B/crosscalls.py $M/xcall-own --kept '' --own --plan-file $T/xc-plan-major.txt --cache $T/cache-xc-own-here ctl crosscalls-own-unchecked-pass 0 '^NOTE AuthMod 3\.2\.0 -> 4\.1\.0: used by own Shop; targets NOT checked' '!GONE' -- python3 $B/crosscalls.py $M/xcall-own --kept '' --own --plan-file $T/xc-plan-major.txt --cache $T/cache-xc-own-none ctl crosscalls-own-off-pass 0 '!OWN|GONE' -- python3 $B/crosscalls.py $M/xcall-own --kept '' --plan-file $T/xc-plan-major.txt --cache $T/cache-xc-own-gone +# repair row 28 as a tool: copydoc.py copies the dropped document from the pre-update model into an own module, with +# fresh ids (pointers follow) and its roles mapped, and repoints the OWN users only (a Marketplace caller is left alone) +model copydoc-base > /dev/null; model copydoc-new > /dev/null; model copydoc-v11 > /dev/null +rm -rf $T/cd; mkdir -p $T/cd; cp $M/copydoc-new/App.mpr $T/cd/App.mpr +python3 - $M/copydoc-base/App.mpr <<'PY' +import sqlite3, sys, bson +con = sqlite3.connect(sys.argv[1]) +for uid, blob in con.execute("select UnitID, Contents from Unit").fetchall(): + d = bson.decode(blob) + if d.get("Name") == "Pick_Page": # a default button and the pointer to it, as a real popup page has + d["Widgets"].append({"$ID": b"\x01" * 16, "$Type": "Forms$ActionButton", "Name": "okButton"}) + d["ControlBar"] = {"$ID": b"\x02" * 16, "$Type": "Forms$ControlBar", "DefaultButtonPointer": b"\x01" * 16} + con.execute("update Unit set Contents=? where UnitID=?", (bson.encode(d), uid)) +con.commit() +PY +cat > $T/cd/look.py <<'PY' +import sqlite3, sys, bson +con = sqlite3.connect(sys.argv[1]) +units = {bytes(u): (bytes(c) if c else None, bson.decode(bytes(b))) for u, c, b in con.execute("select UnitID, ContainerID, Contents from Unit") if b} +mod = {u: d["Name"] for u, (c, d) in units.items() if d["$Type"] == "Projects$ModuleImpl"} +for u, (c, d) in units.items(): + if d.get("Name") == "Pick_Page" and c in mod: + ids = set() + def walk(o): + if isinstance(o, dict): + ids.add(bytes(o.get("$ID", b""))); [walk(v) for v in o.values()] + elif isinstance(o, list): + [walk(v) for v in o] + walk(d) + p = d["ControlBar"]["DefaultButtonPointer"] + print(f"page {mod[c]}.Pick_Page roles {d.get('AllowedModuleRoles')[1:]} pointer {'inside' if bytes(p) in ids else 'DANGLING'} " + f"fresh {bytes(p) != b'\x01' * 16}") +for u, (c, d) in units.items(): + s = bson.encode(d) + for q in (b"Src.Pick_Page", b"Shop.Pick_Page"): + if q in s and d.get("Name") != "Pick_Page": + print(f"ref {d['Name']} -> {q.decode()}") +PY +ctl copydoc-dry-catch 0 '^copydoc: Src\.Pick_Page \(Forms\$Page\) -> Shop\.Pick_Page, roles Src\.User -> Shop\.User$' '^ RETARGET Shop\.Order_Edit: 1 reference' '^ RETARGET Shop\.ACT_Open: 1 reference' '!RETARGET KeptConn' '^dry run' -- python3 $B/repair/copydoc.py $T/cd --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to Shop --roles Shop.User --retarget +ctl copydoc-apply-catch 0 '^page Shop\.Pick_Page roles \[.Shop\.User.\] pointer inside fresh True$' '^ref Order_Edit -> Shop\.Pick_Page$' '^ref ACT_Open -> Shop\.Pick_Page$' '^ref ACT_Pick -> Src\.Pick_Page$' '!ref (Order_Edit|ACT_Open) -> Src' -- bash -c "python3 $B/repair/copydoc.py $T/cd --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to Shop --roles Shop.User --retarget --apply | grep -q 're-read OK' && python3 $T/cd/look.py $T/cd/App.mpr" +ctl copydoc-again-invalid 2 'Shop\.Pick_Page already exists' -- python3 $B/repair/copydoc.py $T/cd --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to Shop --roles Shop.User +ctl copydoc-roles-needed-invalid 2 'allows Src\.User: name the Shop roles' -- python3 $B/repair/copydoc.py $M/copydoc-new --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to Shop +ctl copydoc-missing-dep-invalid 2 'uses Src\.Pick_Snippet, which the app does not have' -- python3 $B/repair/copydoc.py $M/copydoc-new --from $M/copydoc-base/App.mpr --doc Src.Pick_Deep --to Shop +ctl copydoc-into-marketplace-invalid 2 'KeptConn is a Marketplace module' -- python3 $B/repair/copydoc.py $M/copydoc-new --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to KeptConn --roles KeptConn.User +ctl copydoc-version-invalid 2 'Mendix versions differ: app 11\.12\.7\.1' -- python3 $B/repair/copydoc.py $M/copydoc-v11 --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to Shop --roles Shop.User # the build stop quotes owngone.out when mx check has CE1613 (and only then) mkdir -p $T/og; printf '%s\n' 'crosscalls: own modules 1' 'OWN AuthMod 3.2.0 -> 4.1.0: 1 target(s) own modules use are gone from the 4.1.0 package' ' GONE page AuthMod.Login_Page (used by own Shop.ACT_Open)' > $T/og/owngone.out printf '%s\n' '[error] [CE1613] "The selected page no longer exists." at Microflow ACT_Open' > $T/og/check-1613.log; printf '%s\n' '[error] [CE0117] "x"' > $T/og/check-0117.log diff --git a/upgrade-toolkit/steps/repair.md b/upgrade-toolkit/steps/repair.md index e76a08e..8c7a3a8 100644 --- a/upgrade-toolkit/steps/repair.md +++ b/upgrade-toolkit/steps/repair.md @@ -38,7 +38,7 @@ keep the fix and continue with the new class. Without `--fixed` a rise is always | 25 | `mx update-widgets` moves a widget across a technology jump (Slider 1.3.2 Dojo to 3.0.4 pluggable) and then CE7006 on every attribute of it, or the widget lost its min/max/step attributes and its on-change microflow (QI-6) | a Dojo-to-pluggable version jump is a widget SWAP, not an update: before it, list the widget's properties in the old model (BSON of the page or snippet unit) and after it again, and write every dropped binding down; `bin/widgetdiff.py` names the package versions only | re-bind the dropped properties by BSON (not through mxcli describe, which keeps only the value attribute); a pluggable widget gets no entity context from a snippet parameter, so wrap it in a data view on the snippet parameter (the CE7006 goes); a microflow action on a pluggable widget takes empty ParameterMappings. Seen: Slider on a Quality Inspection snippet: only valueAttribute survived, 4 CE7006 until the data view wrap, 0 after. Then mx check and a by-eye look at the page | | 26 | screens rung REVIEW on looks that changed by design: the Charts 6 palette, the Data grid 2 column selector (eye) icon size, the MxAdmin changedDate stamped at runtime start on the 10.24 rung (data rung accept) (QI-10) | look at the pair first; these three are known and each has a fix or an accept | Charts 6 (6.x) comes without the old series palette: re-apply it in the theme (a `.widget-charts` series colour rule) when the dashboard look is wanted; the Data grid 2 column selector icon grows: add a theme rule for its size; the 10.24 runtime writes MxAdmin changedDate at start: accept it in the data rung accept file with that reason. Never accept an unseen pair | | 27 | CE0117 at the "Dynamic classes expression" of a button after the 9 to 10 convert, where the expression follows an association (`if $currentObject/Module.A_B = empty then 'disabled' else ''`) | the `[where]` lines of check-N.log name the page and property | cold CMM (10.24.28): the same association test passed in 8 conditional-visibility expressions on the same pages and failed only in the 4 dynamic-classes expressions (Mix_New_WizardStep1, desktop and phone; the association leads to an external OData entity). The 9.24 build accepted it. They can be HIDDEN: in cold CMM run 5 the 4 vanished while 3 CE1613 (row 28) were open, and came back unchanged once those were repaired (run 7), so mx check reports no CE0117 while CE1613 errors stand; a check that lost its CE0117 next to new CE1613 has not fixed them. (The 2 CE0117 in the kept Administration pages did go for good with its update.) Remedy (NOT verified): move the test into conditional visibility, or compute a boolean attribute on the helper object and test that. A judgment item: the button's look changes, so a human looks at the page. No tool yet | -| 28 | CE1613 "the selected page/microflow no longer exists" in an OWN module after a Marketplace module update: an own reference selector's select page, a microflow data source or a Call microflow that points into the updated module | `owngone.out` (crosscalls.py --own, written by the update step before the update) lists each `GONE <kind> <target> (used by own <document>)`; the build stop quotes it | cold CMM: Administration 2.1.2 -> 4.5.0 dropped `Administration.UserRole_Select` (the select page of the user role selector on two own Employee pages) and `Administration.RetrieveTimeZones` (the time zone data source on Employee_New): 3 CE1613. Remedy (NOT verified): copy the old document out of `original.mpr` into an own module and point the users at the copy, or switch the selector's source to a database/XPath source on the same entity; if neither fits, keep the module at its installed version (kept, a held update). mxcli cannot author reference selectors (not re-executable), so this is a judgment item done in Studio Pro; check the selector on the page as a real user afterwards | +| 28 | CE1613 "the selected page/microflow no longer exists" in an OWN module after a Marketplace module update: an own reference selector's select page, a microflow data source or a Call microflow that points into the updated module | `owngone.out` (crosscalls.py --own, written by the update step before the update) lists each `GONE <kind> <target> (used by own <document>)`; the build stop quotes it | cold CMM: Administration 2.1.2 -> 4.5.0 dropped `Administration.UserRole_Select` (the select page of the user role selector on two own Employee pages) and `Administration.RetrieveTimeZones` (the time zone data source on Employee_New): 3 CE1613. Remedy (verified, cold CMM run 7: the 3 CE1613 gone): `bin/repair/copydoc.py <app> --from base.mpr --doc Administration.UserRole_Select --to EmployeeManagement --roles EmployeeManagement.Administrator --retarget` (dry run; then `--apply`, commit, mx check). base.mpr is the app at the SAME Mendix version before the update: `git -C app show <commit before the update>:'<App>.mpr' > base.mpr`. It copies with fresh ids (pointers follow), maps the roles, repoints only own users; old widget versions in the copy are CE0463 that the build loop's update-widgets clears (cold CMM: 11). mxcli cannot author reference selectors (not re-executable), so it cannot do this. Alternatives: switch the selector's source to a database/XPath source in Studio Pro, or keep the module (a held update). The copy is old code in an own module: a judgment item; check the selector on the page as a real user afterwards | | 29 | `[MXCHECK] mx check ended without a result` in check-N.log: mx check died (an `ERROR: System.…Exception` and a stack, no "The app contains" line) | the ERROR line it quotes; `git -C app log -1` names the last change | mx check cannot load the model, so it reports nothing: the pipeline counts it as an error and stops (before, it counted 0 and ran mxbuild). The last commit broke the MPR: a raw unit edit (copydoc.py, copy_mappings.py, restoreunit.py) or a tool crash mid-write. KeyNotFoundException with a GUID = a pointer to an element id that is not in the unit (cold CMM: a page copied with fresh ids kept its old `DefaultButtonPointer`; copydoc.py now remaps them). Tag, `git reset --hard` to the commit before, fix the tool, redo. EntryPointNotFound or NullReferenceException were seen too (an earlier run, twice), cause not established: find the last change first | mxcli v0.28.0 (#1408, #1474), what it changes for the rows above: From f6ebf4c23696dad9aad08c8d257474ede07249e4 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 15:46:50 +0800 Subject: [PATCH 54/59] editroute: a replayed association gets member access on its other entity too Cold CMM run 8: after repair rows 27 and 28 mx check showed 5 CE2729, "No read access to association 'Employee_Account' in entity 'Administration.Account'". The XASSOC replay re-created EmployeeManagement.Employee_Account (Employee to Administration.Account) and restored the owner's grants only; the original model gave Account ReadOnly on it for Administrator and User, after the update it was None. collect now also reads the other entity's grants from the original model and writes each one cut down to the association member (member_grant). mxcli's grant merges members into the role's rule, so the vendor's own rights stay. Control editroute-collect-assoc-other-side-catch: fails on the old code (1 failed, 11 passed), passes on the new. Repair row 27 rewritten with the real cause ($currentObject in a nested data view is the inner object; $dataView1, verified run 8); row 30 for CE2729 (verified run 9: 0 errors). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/editroute.py | 27 +++++++++++++++++-- upgrade-toolkit/controls/editroute_control.sh | 3 +++ upgrade-toolkit/controls/run-controls.sh | 3 +++ upgrade-toolkit/steps/repair.md | 3 ++- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/upgrade-toolkit/bin/editroute.py b/upgrade-toolkit/bin/editroute.py index a59232d..1a20acf 100755 --- a/upgrade-toolkit/bin/editroute.py +++ b/upgrade-toolkit/bin/editroute.py @@ -408,6 +408,20 @@ def checked(mdl, app_mpr, tmp, qn): return rc == 0, " ".join(ln.strip() for ln in out.splitlines() if "✗" in ln or "error" in ln.lower())[:300] +def member_grant(line, names): + """The grant line cut down to the given members: 'grant read (A_B), write (A_B) on entity E to R [where ...];', or None + when the line gives none of them.""" + m = re.match(r"\s*grant\s+(.*?)\s+on entity\s+(\S+\s+to\s+.*;)\s*$", line) + if not m: + return None + rights = [] + for kind, members in re.findall(r"\b(read|write)\s*\(([^)]*)\)", m.group(1)): + keep = [y.strip() for y in members.split(",") if y.strip().strip('"') in names] + if keep: + rights.append(f"{kind} ({', '.join(keep)})") + return f"grant {', '.join(rights)} on entity {m.group(2)}" if rights else None + + def cmd_collect(a): W = os.path.abspath(a.work) try: @@ -468,7 +482,7 @@ def cmd_collect(a): if "XASSOC" in routes: # from the ORIGINAL model, not save-edits: on WM1 the forced update wrote no association-*.mdl for the 4 # associations the app's Email module owns into Email_Connector (data test 2026-11-16), so nothing was replayed - owners = {} + owners, others = {}, {} for q in x.get("assocs", []): f = os.path.join(d, f"association-{q.replace('.', '_')}.mdl") mdl = describe(x["original"], "association", q) if x.get("original") else None @@ -477,9 +491,11 @@ def cmd_collect(a): held.append(f"association {q}: not in the original model or the saved edits") continue parts.append((f"association {q}: from the original model, original id restored after", mdl)) - m = re.search(r"\bfrom\s+([\w.]+)\s+to\b", mdl) + m = re.search(r"\bfrom\s+([\w.]+)\s+to\s+([\w.]+)", mdl) if m: owners.setdefault(m.group(1), []).append(q.split(".", 1)[1]) + if m.group(2) != m.group(1): + others.setdefault(m.group(2), []).append(q.split(".", 1)[1]) # a re-created association has no member access (ReadWrite became None; a build error on a non-persistent # owner, data test 2026-11-16): the owner's grant lines that name it go in again for ent, names in owners.items(): @@ -487,6 +503,13 @@ def cmd_collect(a): if ln.strip().startswith("grant ") and any(re.search(rf"\b{re.escape(n)}\b", ln) for n in names)] if g: parts.append((f"member access on {ent} for {len(names)} association(s)", "\n".join(g))) + # the other entity lost it too (cold CMM run 8: Employee_Account to Administration.Account, 5 CE2729 on the + # Employee pages); that one is often a vendor entity, so only the association member goes back, merged into + # each rule (mxcli's grant adds members to the role's rule, it keeps the rest) + for ent, names in others.items(): + g = [y for ln in (describe(x["original"], "entity", ent) or "").splitlines() if (y := member_grant(ln, names))] + if g: + parts.append((f"member access on {ent} for {len(names)} association(s), the other side", "\n".join(g))) with open(os.path.join(W, "replay", f"{mod}.guids"), "w") as g: g.write("".join(q + "\n" for q in x.get("assocs", []))) for h in held: diff --git a/upgrade-toolkit/controls/editroute_control.sh b/upgrade-toolkit/controls/editroute_control.sh index 35d2ee4..3bd1661 100644 --- a/upgrade-toolkit/controls/editroute_control.sh +++ b/upgrade-toolkit/controls/editroute_control.sh @@ -64,6 +64,9 @@ if a[0] == "describe": if kind == "entity" and qn == "Other.X": print("create or modify entity Other.X (\n);\ngrant read (A_B, Name), write (A_B) on entity Other.X to Other.User;\ngrant read (Name) on entity Other.X to Other.Guest;") sys.exit(0) + if kind == "entity" and qn == "Target.Y": + print("create or modify entity Target.Y (\n);\ngrant create, read (Code, A_B), write (Code) on entity Target.Y to Target.User where [Code='x'];\ngrant read (Code) on entity Target.Y to Target.Guest;") + sys.exit(0) body = "logic A" if pkg or "Cosmetic" in qn else "logic B" print(("@position(1, 1)\n" if pkg else "@position(9, 9)\n@anchor(from: right, to: left)\n") + body) elif a[0] == "check": diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 4ddc5c2..f911873 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -6985,6 +6985,9 @@ ctl editroute-collect-app-added-pass 0 "snippet Added: added by the app" "^Uimod # data test 2026-11-16: save-edits wrote no association file; a re-created association lost its member access; a saved # whole entity put widened vendor attributes back ctl editroute-collect-assoc-from-original-pass 0 'association Other.A_B: from the original model' '^Target.mdl: grant read \(A_B, Name\), write \(A_B\) on entity Other.X to Other.User;$' '!Other.Guest' -- $ER collect +# cold CMM run 8: the re-created Employee_Account (Employee -> Administration.Account) had member access only on the owner +# side; Account's rules said None, 5 CE2729 on the Employee pages. The other entity gets just the association member back +ctl editroute-collect-assoc-other-side-catch 0 "^Target.mdl: grant read \\(A_B\\) on entity Target.Y to Target.User where \\[Code='x'\\];$" '!Target.mdl: .*Target.Guest' '!Target.mdl: grant create' -- $ER collect ctl editroute-collect-no-whole-entity-catch 0 '!entity-Whole' '!saved whole element entity' -- $ER collect # WM1 rerun: mxcli never writes a layout into a Marketplace module; the app's redirect to its design module becomes a page repoint ctl editroute-collect-layout-repoint-pass 0 '^Uimod.mdl: ALTER PAGES SET LAYOUT = Brand.Main WHERE LAYOUT = Uimod.Redir;$' '^SKIP Uimod layout Spare: used by nothing' '^HELD Uimod layout UsedOwn: mxcli never writes a layout' '!create or modify layout' -- $ER collect diff --git a/upgrade-toolkit/steps/repair.md b/upgrade-toolkit/steps/repair.md index 8c7a3a8..8180c4e 100644 --- a/upgrade-toolkit/steps/repair.md +++ b/upgrade-toolkit/steps/repair.md @@ -37,9 +37,10 @@ keep the fix and continue with the new class. Without `--fixed` a rise is always | 24 | CE0463 after a hand-set value on a pluggable widget property; or `mx update-widgets` puts a property value back (FM-5) | read the widget's `getProperties()` in its editorConfig (inside the .mpk): a property that the config hides unless another property is on (for SprintrFeedback `targetContainerSelector` needs `showAdvancedSettings` = true) is hidden in the model, and a hidden property with a non-default value is a CE0463 and is reset by update-widgets | set the controlling property first (`showAdvancedSettings` true), then the value, in one commit; run `mx update-widgets` AFTER and check the value survived (diff the unit), then mx check. Seen: SprintrFeedback 12.0.4 (bundled with Atlas_Core 4.5.1) portals its button into `targetContainerSelector` and renders nothing when the default `.mx-scrollcontainer-center` is absent (an app layout without a centre region): the Feedback tab vanished from every page and shotdiff said "lost feedback". The remedy card `sprintr-feedback-target` (playbooks/remedies.tsv) carries it | | 25 | `mx update-widgets` moves a widget across a technology jump (Slider 1.3.2 Dojo to 3.0.4 pluggable) and then CE7006 on every attribute of it, or the widget lost its min/max/step attributes and its on-change microflow (QI-6) | a Dojo-to-pluggable version jump is a widget SWAP, not an update: before it, list the widget's properties in the old model (BSON of the page or snippet unit) and after it again, and write every dropped binding down; `bin/widgetdiff.py` names the package versions only | re-bind the dropped properties by BSON (not through mxcli describe, which keeps only the value attribute); a pluggable widget gets no entity context from a snippet parameter, so wrap it in a data view on the snippet parameter (the CE7006 goes); a microflow action on a pluggable widget takes empty ParameterMappings. Seen: Slider on a Quality Inspection snippet: only valueAttribute survived, 4 CE7006 until the data view wrap, 0 after. Then mx check and a by-eye look at the page | | 26 | screens rung REVIEW on looks that changed by design: the Charts 6 palette, the Data grid 2 column selector (eye) icon size, the MxAdmin changedDate stamped at runtime start on the 10.24 rung (data rung accept) (QI-10) | look at the pair first; these three are known and each has a fix or an accept | Charts 6 (6.x) comes without the old series palette: re-apply it in the theme (a `.widget-charts` series colour rule) when the dashboard look is wanted; the Data grid 2 column selector icon grows: add a theme rule for its size; the 10.24 runtime writes MxAdmin changedDate at start: accept it in the data rung accept file with that reason. Never accept an unseen pair | -| 27 | CE0117 at the "Dynamic classes expression" of a button after the 9 to 10 convert, where the expression follows an association (`if $currentObject/Module.A_B = empty then 'disabled' else ''`) | the `[where]` lines of check-N.log name the page and property | cold CMM (10.24.28): the same association test passed in 8 conditional-visibility expressions on the same pages and failed only in the 4 dynamic-classes expressions (Mix_New_WizardStep1, desktop and phone; the association leads to an external OData entity). The 9.24 build accepted it. They can be HIDDEN: in cold CMM run 5 the 4 vanished while 3 CE1613 (row 28) were open, and came back unchanged once those were repaired (run 7), so mx check reports no CE0117 while CE1613 errors stand; a check that lost its CE0117 next to new CE1613 has not fixed them. (The 2 CE0117 in the kept Administration pages did go for good with its update.) Remedy (NOT verified): move the test into conditional visibility, or compute a boolean attribute on the helper object and test that. A judgment item: the button's look changes, so a human looks at the page. No tool yet | +| 27 | CE0117 at the "Dynamic classes expression" of a button after the 9 to 10 convert, where the expression follows an association from `$currentObject` (`if $currentObject/Module.A_B = empty then 'disabled' else ''`) | the `[where]` lines of check-N.log name the page and property | Cause: the button sits in a data view nested in another one, and in Mendix 10 `$currentObject` there is the INNERMOST data view's object, which has no such association; the expression meant the outer object. Cold CMM (10.24.28): 4 buttons in the inner Recipes data view of Mix_New_WizardStep1 (desktop and phone) tested `$currentObject/MaterialManagement.RecipeHelper_Recipes`, where RecipeHelper is the outer data view. The 9.24 build accepted it. Remedy (verified, cold CMM run 8: the 4 CE0117 gone): name the outer data view instead, `$dataView1/Module.A_B` (the outer data view's widget name), in each expression; check the page renders the same. They can be HIDDEN: in cold CMM run 5 the 4 vanished while 3 CE1613 (row 28) were open, and came back unchanged once those were repaired (run 7), so mx check reports no CE0117 while CE1613 errors stand; a check that lost its CE0117 next to new CE1613 has not fixed them. Fixing them can reveal the next phase in turn (run 8: 5 CE2729, row 30). No tool yet | | 28 | CE1613 "the selected page/microflow no longer exists" in an OWN module after a Marketplace module update: an own reference selector's select page, a microflow data source or a Call microflow that points into the updated module | `owngone.out` (crosscalls.py --own, written by the update step before the update) lists each `GONE <kind> <target> (used by own <document>)`; the build stop quotes it | cold CMM: Administration 2.1.2 -> 4.5.0 dropped `Administration.UserRole_Select` (the select page of the user role selector on two own Employee pages) and `Administration.RetrieveTimeZones` (the time zone data source on Employee_New): 3 CE1613. Remedy (verified, cold CMM run 7: the 3 CE1613 gone): `bin/repair/copydoc.py <app> --from base.mpr --doc Administration.UserRole_Select --to EmployeeManagement --roles EmployeeManagement.Administrator --retarget` (dry run; then `--apply`, commit, mx check). base.mpr is the app at the SAME Mendix version before the update: `git -C app show <commit before the update>:'<App>.mpr' > base.mpr`. It copies with fresh ids (pointers follow), maps the roles, repoints only own users; old widget versions in the copy are CE0463 that the build loop's update-widgets clears (cold CMM: 11). mxcli cannot author reference selectors (not re-executable), so it cannot do this. Alternatives: switch the selector's source to a database/XPath source in Studio Pro, or keep the module (a held update). The copy is old code in an own module: a judgment item; check the selector on the page as a real user afterwards | | 29 | `[MXCHECK] mx check ended without a result` in check-N.log: mx check died (an `ERROR: System.…Exception` and a stack, no "The app contains" line) | the ERROR line it quotes; `git -C app log -1` names the last change | mx check cannot load the model, so it reports nothing: the pipeline counts it as an error and stops (before, it counted 0 and ran mxbuild). The last commit broke the MPR: a raw unit edit (copydoc.py, copy_mappings.py, restoreunit.py) or a tool crash mid-write. KeyNotFoundException with a GUID = a pointer to an element id that is not in the unit (cold CMM: a page copied with fresh ids kept its old `DefaultButtonPointer`; copydoc.py now remaps them). Tag, `git reset --hard` to the commit before, fix the tool, redo. EntryPointNotFound or NullReferenceException were seen too (an earlier run, twice), cause not established: find the last change first | +| 30 | CE2729 "No read access to association 'A_B' in entity 'Other.E' for user role ..." after an update that re-created an association to another module's entity (editroute XASSOC replay) | `work/replay/<Module>.mdl` has `member access on <owner>` but no `the other side` part; `mxcli describe entity Other.E` shows the member gone from its grants, the original model has it | The re-created association starts with no member access on BOTH entities. The replay restored only the owner side; since fix 15 it also restores the other side, as only that member merged into each rule (mxcli's grant adds to a role's rule and keeps the rest). Cold CMM run 8: EmployeeManagement.Employee_Account to Administration.Account, ReadOnly for Administrator and User became None, 5 CE2729 on the Employee text boxes. Remedy for a run replayed before the fix (verified, cold CMM run 9): per grant line of the entity in the ORIGINAL model, `grant read (A_B) on entity Other.E to <role> [where ...];` (editroute.member_grant prints these), mxcli exec, commit | mxcli v0.28.0 (#1408, #1474), what it changes for the rows above: - `mxcli exec` is all-or-nothing. A script whose statement N fails now changes NOTHING and exits non-zero; before, statements 1 to N-1 From 47b55b047d218a5d8311488816edd9193f1108b0 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 15:54:30 +0800 Subject: [PATCH 55/59] build: a javac missing symbol names the jar that hides the class (javacjar.py) Cold CMM run 9, 10.24.28: mxbuild stopped in javac, "cannot find symbol CookieHeaderNames.SameSite" in MendixSSO 10.6.2's OpenIDUtils. vendorlib has netty-codec-http 4.2.17 with it; userlib/server-examples-0.4.1-SNAPSHOT.jar, a fat jar, carries its own older Netty, and javac took that copy. jarclash listed the clash as a note; the stop named only javafinal.py. bin/repair/javacjar.py resolves each missing symbol's class through the source file's imports, looks it up in every userlib/vendorlib jar (nested too) and prints HAS / HIDDEN BY. --strip [--apply] writes the fat jar again without the classes a vendorlib jar also carries. The mxbuild stop runs it on "cannot find symbol" and quotes it when a class is hidden. Repair row 31. Controls javacjar-hidden-catch, -not-hidden-pass, -strip-dry-pass, -strip-apply-catch, pipeline-build-javacjar-catch, -not-hidden-pass: 5 failed on the old code, all pass on the new. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 7 + upgrade-toolkit/bin/repair/javacjar.py | 146 ++++++++++++++++++ upgrade-toolkit/controls/run-controls.sh | 25 +++ upgrade-toolkit/steps/repair.md | 1 + 4 files changed, 179 insertions(+) create mode 100644 upgrade-toolkit/bin/repair/javacjar.py diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 8d1aaf0..7fcf172 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -1280,6 +1280,13 @@ Work the list with steps/repair.md (rows 4, 7-10 and own-document fixes; one com disk_cause "$WORK/build-$TARGET.log" nj_add 10 "mxbuild failed (exit $rc, $n ERROR at lines)" "Summary file: $WORK/build-$TARGET.log (a 30 s theme-compile timeout is not a model error: retry when the machine is quiet; javac errors: javafinal.py, kept-module API removals)" nj_file 10 "First 20 error lines" "$WORK/build-$TARGET.log" 20 '^ERROR at|error:|ERROR' + # cold CMM: a userlib fat jar's older Netty hid the class MendixSSO 10.6.2 needs; name the jar, not only the error + if grep -q 'error: cannot find symbol' "$WORK/build-$TARGET.log" 2> /dev/null; then + xo "$WORK/javacjar.out" python3 "$B/repair/javacjar.py" "$APP" "$WORK/build-$TARGET.log" || true + if grep -q '^ HIDDEN BY' "$WORK/javacjar.out" 2> /dev/null; then + nj_file 10 "An older copy of a class inside another jar hides what the code needs (steps/repair.md row 31)" "$WORK/javacjar.out" 20 '^(SYMBOL| HAS| HIDDEN BY)' + fi + fi nj_flush fi commit_app "build: regenerated stubs" config build-stubs diff --git a/upgrade-toolkit/bin/repair/javacjar.py b/upgrade-toolkit/bin/repair/javacjar.py new file mode 100644 index 0000000..f6a9b63 --- /dev/null +++ b/upgrade-toolkit/bin/repair/javacjar.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""javacjar.py: for each javac "cannot find symbol" in an mxbuild log, name the jars that carry the class and the one that hides it. + + javacjar.py <project dir> <build log> + javacjar.py <project dir> --strip userlib/<fat>.jar [--apply] + +Why (cold CMM, 10.24.28): MendixSSO 10.6.2 calls CookieHeaderNames.SameSite, which netty-codec-http 4.2.17 in +vendorlib has. userlib/server-examples-0.4.1-SNAPSHOT.jar (an OPC UA library, built as one fat jar) carries its +own older Netty without it, and javac took that copy: "cannot find symbol ... location: class CookieHeaderNames". +jarclash.py listed the clash after the updates, as a note; the build stop named only javafinal.py. + +The class of each error is resolved through the imports of the source file, then looked up in every userlib/ and +vendorlib/ jar (and their nested jars). A jar that has the class but not the symbol (no nested class, no name in the +class file) HIDES the jar that has both: + + SYMBOL io.netty.handler.codec.http.cookie.CookieHeaderNames.SameSite (OpenIDUtils.java:237) + HAS vendorlib/netty-codec-http-4.2.17.Final.jar + HIDDEN BY userlib/server-examples-0.4.1-SNAPSHOT.jar (an older copy of the class) + +--strip writes the fat jar again without every class a vendorlib jar also carries (the libraries the modules manage +win), dry run unless --apply; its own classes and the classes only it has stay. The app's git keeps the old jar. +Exit: 0 = nothing hidden (or a strip done), 1 = a class is hidden by an older copy, 2 = input error. +""" +import argparse, io, os, re, shutil, sys, tempfile, zipfile + +ERR = re.compile(r"^(?P<file>\S+\.java):(?P<line>\d+): error: cannot find symbol\n(?:.*\n){0,3}?\s*symbol:\s+(?P<kind>\w+)\s+(?P<sym>[\w$]+)[^\n]*\n\s*location:\s+(?:\w+\s+)*?(?:of type\s+)?(?P<loc>[\w.$]+)\s*$", re.M) + + +def jars(project): + """[(label, ZipFile)] of userlib/ and vendorlib/ jars and the jars nested in them.""" + out = [] + for d in ("userlib", "vendorlib"): + p = os.path.join(project, d) + for f in sorted(os.listdir(p)) if os.path.isdir(p) else []: + if not f.endswith(".jar"): + continue + try: + z = zipfile.ZipFile(os.path.join(p, f)) + except zipfile.BadZipFile: + continue + out.append((f"{d}/{f}", z)) + for n in z.namelist(): + if n.endswith(".jar") and re.search(r"(^|/)(lib|BOOT-INF/lib)/[^/]+\.jar$", n): + try: + out.append((f"{d}/{f}!{n}", zipfile.ZipFile(io.BytesIO(z.read(n))))) + except zipfile.BadZipFile: + pass + return out + + +def fqcn(project, src, simple): + """The full class name of a simple name, from the imports of the source file.""" + if "." in simple: + return simple + path = src if os.path.isabs(src) else os.path.join(project, src) + try: + text = open(path, encoding="utf-8", errors="replace").read() + except OSError: + return None + m = re.search(rf"^import\s+([\w.]+\.{re.escape(simple)})\s*;", text, re.M) + if m: + return m.group(1) + m = re.search(r"^package\s+([\w.]+)\s*;", text, re.M) + return f"{m.group(1)}.{simple}" if m else None + + +def errors(log): + t = open(log, encoding="utf-8", errors="replace").read() + for m in ERR.finditer(t): + yield m.group("file"), m.group("line"), m.group("sym"), m.group("loc") + + +def strip(project, jar, apply): + vend = set() + for label, z in jars(project): + if label.startswith("vendorlib/"): + vend |= {n for n in z.namelist() if n.endswith(".class")} + path = os.path.join(project, jar) + try: + src = zipfile.ZipFile(path) + except (OSError, zipfile.BadZipFile) as e: + print(f"javacjar: {jar}: {e}", file=sys.stderr); return 2 + drop = [i for i in src.infolist() if i.filename in vend] + pk = sorted({i.filename.rsplit("/", 1)[0].replace("/", ".") for i in drop}) + print(f"STRIP {jar}: {len(drop)} class(es) a vendorlib jar also carries, in {len(pk)} package(s)" + + (f" ({', '.join(pk[:4])}{', ...' if len(pk) > 4 else ''})" if pk else "")) + print(f" KEEP {len(src.infolist()) - len(drop)} entries") + if not drop or not apply: + print("dry run (add --apply to write the jar)" if drop else "nothing to strip") + return 0 + gone = {i.filename for i in drop} + fd, tmp = tempfile.mkstemp(suffix=".jar", dir=os.path.dirname(path)); os.close(fd) + with zipfile.ZipFile(tmp, "w") as out: + for i in src.infolist(): + if i.filename not in gone: + out.writestr(i, src.read(i.filename)) + src.close() + shutil.move(tmp, path) + print(f"wrote {jar}; rerun mxbuild, then the quick start (the library now runs on the vendorlib copies)") + return 0 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("project"); ap.add_argument("log", nargs="?") + ap.add_argument("--strip"); ap.add_argument("--apply", action="store_true") + a = ap.parse_args() + if a.strip: + sys.exit(strip(a.project, a.strip, a.apply)) + if not a.log or not os.path.isfile(a.log): + sys.exit(f"javacjar: give the mxbuild log ({a.log}: not a file)") + js = jars(a.project) + seen, hidden, n = set(), 0, 0 + for src, line, sym, loc in errors(a.log): + cls = fqcn(a.project, src, loc.split("<")[0]) + key = (cls, sym) + if key in seen: + continue + seen.add(key); n += 1 + where = f"{os.path.basename(src)}:{line}" + if not cls: + print(f"SYMBOL {loc}.{sym} ({where}): the class name is not in the imports, not looked up"); continue + entry = cls.replace(".", "/") + ".class" + has, lacks = [], [] + for label, z in js: + names = set(z.namelist()) + if entry not in names: + continue + nested = cls.replace(".", "/") + "$" + sym + ".class" + (has if nested in names or sym.encode() in z.read(entry) else lacks).append(label) + print(f"SYMBOL {cls}.{sym} ({where})") + for h in has: + print(f" HAS {h}") + for l in lacks: + print(f" HIDDEN BY {l} (an older copy of the class)" if has else f" LACKS {l}") + if has and lacks: + hidden += 1 + if not has and not lacks: + print(" in no jar of userlib/ or vendorlib/") + print(f"javacjar: {n} missing symbol(s), {hidden} hidden by an older copy of the class in another jar" + + ("; repair: steps/repair.md row 31" if hidden else "")) + sys.exit(1 if hidden else 0) + + +if __name__ == "__main__": + main() diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index f911873..f30e227 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -281,6 +281,31 @@ ctl copydoc-roles-needed-invalid 2 'allows Src\.User: name the Shop roles' -- py ctl copydoc-missing-dep-invalid 2 'uses Src\.Pick_Snippet, which the app does not have' -- python3 $B/repair/copydoc.py $M/copydoc-new --from $M/copydoc-base/App.mpr --doc Src.Pick_Deep --to Shop ctl copydoc-into-marketplace-invalid 2 'KeptConn is a Marketplace module' -- python3 $B/repair/copydoc.py $M/copydoc-new --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to KeptConn --roles KeptConn.User ctl copydoc-version-invalid 2 'Mendix versions differ: app 11\.12\.7\.1' -- python3 $B/repair/copydoc.py $M/copydoc-v11 --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to Shop --roles Shop.User +# javacjar.py (cold CMM 10.24.28): MendixSSO 10.6.2 needs CookieHeaderNames.SameSite (vendorlib netty 4.2); a userlib fat +# jar's older Netty hid it from javac. The error names the jar that hides the class, --strip drops the vendorlib copies +JC=$T/jc; mkdir -p $JC/app/userlib $JC/app/vendorlib $JC/app/javasource/sso $JC/solo/vendorlib +python3 - "$JC" <<'PY' +import sys, zipfile +d = sys.argv[1]; c = "io/netty/handler/codec/http/cookie/CookieHeaderNames" +with zipfile.ZipFile(f"{d}/app/userlib/fat-0.4.1.jar", "w") as z: + z.writestr(c + ".class", b"\xca\xfe\xba\xbe old PATH DOMAIN"); z.writestr("org/opc/Server.class", b"\xca\xfe\xba\xbe") +for j in (f"{d}/app/vendorlib/netty-codec-http-4.2.17.Final.jar", f"{d}/solo/vendorlib/netty-codec-http-4.2.17.Final.jar"): + with zipfile.ZipFile(j, "w") as z: + z.writestr(c + ".class", b"\xca\xfe\xba\xbe new"); z.writestr(c + "$SameSite.class", b"\xca\xfe\xba\xbe") +PY +printf 'package sso;\nimport io.netty.handler.codec.http.cookie.CookieHeaderNames;\nclass U {}\n' > $JC/app/javasource/sso/U.java +cp -R $JC/app/javasource $JC/solo/ +printf '%s\n' 'ERROR: An error occurred while compiling Java actions' "$JC/app/javasource/sso/U.java:7: error: cannot find symbol" ' c.setSameSite(CookieHeaderNames.SameSite.Lax);' ' ^' ' symbol: variable SameSite' ' location: class CookieHeaderNames' > $JC/build.log +ctl javacjar-hidden-catch 1 '^SYMBOL io\.netty\.handler\.codec\.http\.cookie\.CookieHeaderNames\.SameSite \(U\.java:7\)$' '^ HAS vendorlib/netty-codec-http-4\.2\.17\.Final\.jar$' '^ HIDDEN BY userlib/fat-0\.4\.1\.jar' 'row 31' -- python3 $B/repair/javacjar.py $JC/app $JC/build.log +ctl javacjar-not-hidden-pass 0 '^ HAS vendorlib/' '!HIDDEN BY' '0 hidden' -- python3 $B/repair/javacjar.py $JC/solo $JC/build.log +ctl javacjar-strip-dry-pass 0 '^STRIP userlib/fat-0\.4\.1\.jar: 1 class\(es\) a vendorlib jar also carries' '^dry run' '^entries 2$' -- bash -c "python3 $B/repair/javacjar.py $JC/app --strip userlib/fat-0.4.1.jar && python3 -c 'import zipfile,sys; print(\"entries\", len(zipfile.ZipFile(sys.argv[1]).namelist()))' $JC/app/userlib/fat-0.4.1.jar" +ctl javacjar-strip-apply-catch 0 '^wrote userlib/fat-0\.4\.1\.jar' "^left \['org/opc/Server\.class'\]$" -- bash -c "cp -R $JC/app $JC/app2 && python3 $B/repair/javacjar.py $JC/app2 --strip userlib/fat-0.4.1.jar --apply && python3 -c 'import zipfile,sys; print(\"left\", zipfile.ZipFile(sys.argv[1]).namelist())' $JC/app2/userlib/fat-0.4.1.jar" +# the mxbuild stop runs it on a javac "cannot find symbol" and quotes it when a class is hidden +awk '/if grep -q .error: cannot find symbol/{p=1} p{print} p&&/^ fi$/{exit}' $B/cloud/upgrade-pipeline.sh > $JC/nj.sh +printf '%s\n' 'xo() { o=$1; shift; "$@" > "$o"; }' 'nj_file() { echo "NJ $1 $2"; grep -E "$5" "$3"; }' > $JC/stub.sh +cp $JC/build.log $JC/build-10.24.1.log +ctl pipeline-build-javacjar-catch 0 '^NJ 10 An older copy of a class inside another jar hides what the code needs \(steps/repair\.md row 31\)' '^ HIDDEN BY userlib/fat' -- bash -c ". $JC/stub.sh; WORK=$JC APP=$JC/app TARGET=10.24.1 B=$B; . $JC/nj.sh; echo done" +ctl pipeline-build-javacjar-not-hidden-pass 0 '!^NJ' '^done$' -- bash -c ". $JC/stub.sh; WORK=$JC APP=$JC/solo TARGET=10.24.1 B=$B; . $JC/nj.sh; echo done" # the build stop quotes owngone.out when mx check has CE1613 (and only then) mkdir -p $T/og; printf '%s\n' 'crosscalls: own modules 1' 'OWN AuthMod 3.2.0 -> 4.1.0: 1 target(s) own modules use are gone from the 4.1.0 package' ' GONE page AuthMod.Login_Page (used by own Shop.ACT_Open)' > $T/og/owngone.out printf '%s\n' '[error] [CE1613] "The selected page no longer exists." at Microflow ACT_Open' > $T/og/check-1613.log; printf '%s\n' '[error] [CE0117] "x"' > $T/og/check-0117.log diff --git a/upgrade-toolkit/steps/repair.md b/upgrade-toolkit/steps/repair.md index 8180c4e..00f02fa 100644 --- a/upgrade-toolkit/steps/repair.md +++ b/upgrade-toolkit/steps/repair.md @@ -41,6 +41,7 @@ keep the fix and continue with the new class. Without `--fixed` a rise is always | 28 | CE1613 "the selected page/microflow no longer exists" in an OWN module after a Marketplace module update: an own reference selector's select page, a microflow data source or a Call microflow that points into the updated module | `owngone.out` (crosscalls.py --own, written by the update step before the update) lists each `GONE <kind> <target> (used by own <document>)`; the build stop quotes it | cold CMM: Administration 2.1.2 -> 4.5.0 dropped `Administration.UserRole_Select` (the select page of the user role selector on two own Employee pages) and `Administration.RetrieveTimeZones` (the time zone data source on Employee_New): 3 CE1613. Remedy (verified, cold CMM run 7: the 3 CE1613 gone): `bin/repair/copydoc.py <app> --from base.mpr --doc Administration.UserRole_Select --to EmployeeManagement --roles EmployeeManagement.Administrator --retarget` (dry run; then `--apply`, commit, mx check). base.mpr is the app at the SAME Mendix version before the update: `git -C app show <commit before the update>:'<App>.mpr' > base.mpr`. It copies with fresh ids (pointers follow), maps the roles, repoints only own users; old widget versions in the copy are CE0463 that the build loop's update-widgets clears (cold CMM: 11). mxcli cannot author reference selectors (not re-executable), so it cannot do this. Alternatives: switch the selector's source to a database/XPath source in Studio Pro, or keep the module (a held update). The copy is old code in an own module: a judgment item; check the selector on the page as a real user afterwards | | 29 | `[MXCHECK] mx check ended without a result` in check-N.log: mx check died (an `ERROR: System.…Exception` and a stack, no "The app contains" line) | the ERROR line it quotes; `git -C app log -1` names the last change | mx check cannot load the model, so it reports nothing: the pipeline counts it as an error and stops (before, it counted 0 and ran mxbuild). The last commit broke the MPR: a raw unit edit (copydoc.py, copy_mappings.py, restoreunit.py) or a tool crash mid-write. KeyNotFoundException with a GUID = a pointer to an element id that is not in the unit (cold CMM: a page copied with fresh ids kept its old `DefaultButtonPointer`; copydoc.py now remaps them). Tag, `git reset --hard` to the commit before, fix the tool, redo. EntryPointNotFound or NullReferenceException were seen too (an earlier run, twice), cause not established: find the last change first | | 30 | CE2729 "No read access to association 'A_B' in entity 'Other.E' for user role ..." after an update that re-created an association to another module's entity (editroute XASSOC replay) | `work/replay/<Module>.mdl` has `member access on <owner>` but no `the other side` part; `mxcli describe entity Other.E` shows the member gone from its grants, the original model has it | The re-created association starts with no member access on BOTH entities. The replay restored only the owner side; since fix 15 it also restores the other side, as only that member merged into each rule (mxcli's grant adds to a role's rule and keeps the rest). Cold CMM run 8: EmployeeManagement.Employee_Account to Administration.Account, ReadOnly for Administrator and User became None, 5 CE2729 on the Employee text boxes. Remedy for a run replayed before the fix (verified, cold CMM run 9): per grant line of the entity in the ORIGINAL model, `grant read (A_B) on entity Other.E to <role> [where ...];` (editroute.member_grant prints these), mxcli exec, commit | +| 31 | javac `cannot find symbol` in an updated module's Java (MendixSSO, CommunityCommons) at mxbuild, where the class exists but the member does not | `javacjar.out` (the build stop runs `bin/repair/javacjar.py <app> <build log>` and quotes it): `SYMBOL <class>.<member>`, `HAS <jar>`, `HIDDEN BY <jar>` | An older copy of the class sits in another jar, often a fat (shaded, jar-with-dependencies) jar in userlib that bundles its libraries, and javac takes that copy. Cold CMM (10.24.28): MendixSSO 10.6.2 calls `CookieHeaderNames.SameSite` (vendorlib netty-codec-http 4.2.17); userlib/server-examples-0.4.1-SNAPSHOT.jar (OPC UA, 12,588 entries) carries its own older Netty, Guava and BouncyCastle. jardedupe keeps such a jar (only copy of some classes) and jarclash lists it as a note. Remedy (verified, cold CMM run 10: mxbuild passes): `python3 $B/repair/javacjar.py app/ --strip userlib/<fat>.jar` (dry run: how many classes a vendorlib jar also carries), then `--apply`, commit `jar:`; the library then runs on the vendorlib versions (newer majors: Netty 4.2, Guava 33): a judgment item, so test what uses the library at the gate. Alternatives: a thin jar plus its separate dependencies, or a newer version of the library | mxcli v0.28.0 (#1408, #1474), what it changes for the rows above: - `mxcli exec` is all-or-nothing. A script whose statement N fails now changes NOTHING and exits non-zero; before, statements 1 to N-1 From 4a37eb8847645b9956c3d98a69ab8900ba1edfc9 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 16:08:41 +0800 Subject: [PATCH 56/59] update: a widget replaced under a kept module's styles parks the update (widgetskew.py) Cold CMM (9.24.2 -> 10.24.28): Atlas_Web_Content 3.9.0 replaced Datagrid 2.2.2 with 2.30.6 in widgets/, while DataWidgets, whose themesource carries the grid styles, was kept (FILES-EDITED). Every grid page rendered unstyled rows; afterwalk passed 23/23 because the data was on the page, and the screens rung only said REVIEW. bin/widgetskew.py joins .stepupdate/widgets.txt with kept.txt and the kept module's cached target package: a widget that package ships and another step replaced is a WIDGET-SKEW (exit 1). step_update parks on it (exit 13) unless widgetskew-ack.txt accepts it. Repair row 32. Controls: widgetskew-catch, widgetskew-nothing-kept-pass, pipeline-update-widgetskew-catch, pipeline-update-widgetskew-ack-pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/bin/cloud/upgrade-pipeline.sh | 10 +++ upgrade-toolkit/bin/widgetskew.py | 73 +++++++++++++++++++ upgrade-toolkit/controls/run-controls.sh | 16 ++++ upgrade-toolkit/steps/repair.md | 1 + 4 files changed, 100 insertions(+) create mode 100644 upgrade-toolkit/bin/widgetskew.py diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index 7fcf172..c1cdbcc 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -1147,8 +1147,18 @@ step_update() { local rc=0 xo "$WORK/jarclash.out" python3 "$B/repair/jarclash.py" "$APP" --md "$WORK/jarclash.md" || rc=$? [ "$rc" = 0 ] || note "jarclash after the updates exit $rc: classes in two jars, see $WORK/jarclash.md (clashes listed in jarclash-before.md are older than the upgrade)" + # a widget another module's update replaced, styled by a kept module (cold CMM: Datagrid 2.30.6 under DataWidgets' old + # styles; afterwalk passed every grid page, the rows rendered unstyled) + rc=0 + xo "$WORK/widgetskew.out" python3 "$B/widgetskew.py" "$APP" --plan "$WORK/plan.txt" || rc=$? + if [ "$rc" = 1 ] && [ ! -f "$WORK/widgetskew-ack.txt" ]; then + nj_file 13 "A module update replaced a widget whose styles stay with a kept module (steps/repair.md row 32)" "$WORK/widgetskew.out" 10 '^WIDGET-SKEW' + nj_add 13 "Decision" "Update the kept module (stepupdate with a one-line plan), or put the old widget back; to accept the skew, create $WORK/widgetskew-ack.txt." + nj_park + fi else x python3 "$B/repair/jarclash.py" "$APP" --md "$WORK/jarclash.md" + x python3 "$B/widgetskew.py" "$APP" --plan "$WORK/plan.txt" fi local rc=0 ap=() [ ! -f "$WORK/approved.txt" ] || ap=(--approved "$WORK/approved.txt") diff --git a/upgrade-toolkit/bin/widgetskew.py b/upgrade-toolkit/bin/widgetskew.py new file mode 100644 index 0000000..956348b --- /dev/null +++ b/upgrade-toolkit/bin/widgetskew.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""widgetskew.py: did a module update replace a widget whose styles belong to a module that stayed at its old version? + + widgetskew.py <app dir> --plan <plan.txt> [--cache DIR] + +Why (cold CMM, 9.24.2 -> 10.24.28): Atlas_Web_Content 3.9.0 brought Datagrid 2.30.6 into widgets/ (was 2.2.2), while +DataWidgets, whose package ships that widget and whose themesource/datawidgets holds its styles, was kept (FILES-EDITED). +Every data grid page rendered as unstyled rows; afterwalk passed 23/23 (the data is on the page) and the screens rung +only said REVIEW on each one. + +Reads .stepupdate/widgets.txt (the widgets each module step changed, written by stepupdate) and .stepupdate/kept.txt. +For each kept module that has a plan line ("<content id> <version> <module>"), the cached package <id>-<version>.mpk +lists the widgets its target version ships; a widget another step replaced is a skew: + + WIDGET-SKEW Datagrid 2.2.2 -> 2.30.6 (by Atlas_Web_Content 3.9.0) ships with DataWidgets 3.11.8; DataWidgets stays at + the installed version, its theme styles do not match the new widget + +Exit: 0 = no skew, 1 = a skew (update the kept module, or put the old widget back), 2 = input error. +A kept module without a cached package is named on a NOTE line and not counted. +""" +import argparse, os, re, sys, zipfile + + +def steps(app): + """{widget file: (name, old, new, by)} from .stepupdate/widgets.txt (last step wins).""" + out, p = {}, os.path.join(app, ".stepupdate", "widgets.txt") + for line in open(p, encoding="utf-8") if os.path.isfile(p) else []: + by, _, rest = line.strip().partition(": ") + for part in rest.split("; "): + m = re.match(r"(\S+\.mpk) (.+?) (\S+) -> (\S+)$", part.strip()) + if m: + out[os.path.basename(m.group(1))] = (m.group(2), m.group(3), m.group(4), by) + return out + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("app"); ap.add_argument("--plan", required=True) + ap.add_argument("--cache", default=os.environ.get("UPG_CACHE") or os.path.expanduser("~/.cache/upgrade-toolkit/mpk")) + a = ap.parse_args() + if not os.path.isfile(a.plan): + sys.exit(f"widgetskew: {a.plan}: not a file") + kp = os.path.join(a.app, ".stepupdate", "kept.txt") + kept = [l.split()[0] for l in open(kp, encoding="utf-8") if l.strip()] if os.path.isfile(kp) else [] + plan = {} + for l in open(a.plan, encoding="utf-8"): + f = l.split() + if len(f) >= 3 and not l.lstrip().startswith("#"): + plan[f[2]] = (f[0], f[1]) + changed, skew = steps(a.app), 0 + for mod in kept: + if mod not in plan: + continue + cid, ver = plan[mod] + mpk = os.path.join(a.cache, f"{cid}-{ver}.mpk") + try: + ships = {os.path.basename(n) for n in zipfile.ZipFile(mpk).namelist() if re.match(r"widgets/[^/]+\.mpk$", n)} + except (OSError, zipfile.BadZipFile): + print(f"NOTE {mod} {ver}: no cached package ({mpk}), its widgets not checked"); continue + for w in sorted(ships & set(changed)): + name, old, new, by = changed[w] + if by.split()[0] == mod: + continue + skew += 1 + print(f"WIDGET-SKEW {name} {old} -> {new} (by {by}) ships with {mod} {ver}; {mod} stays at the installed version, " + f"its theme styles do not match the new widget") + print(f"widgetskew: {len(kept)} kept module(s), {skew} widget(s) replaced under a kept module" + + ("; repair: steps/repair.md row 32" if skew else "")) + sys.exit(1 if skew else 0) + + +if __name__ == "__main__": + main() diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index f30e227..e363325 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -306,6 +306,22 @@ printf '%s\n' 'xo() { o=$1; shift; "$@" > "$o"; }' 'nj_file() { echo "NJ $1 $2"; cp $JC/build.log $JC/build-10.24.1.log ctl pipeline-build-javacjar-catch 0 '^NJ 10 An older copy of a class inside another jar hides what the code needs \(steps/repair\.md row 31\)' '^ HIDDEN BY userlib/fat' -- bash -c ". $JC/stub.sh; WORK=$JC APP=$JC/app TARGET=10.24.1 B=$B; . $JC/nj.sh; echo done" ctl pipeline-build-javacjar-not-hidden-pass 0 '!^NJ' '^done$' -- bash -c ". $JC/stub.sh; WORK=$JC APP=$JC/solo TARGET=10.24.1 B=$B; . $JC/nj.sh; echo done" +# widgetskew.py (cold CMM 10.24.28): Atlas_Web_Content 3.9.0 replaced Datagrid 2.2.2 with 2.30.6 while DataWidgets (its styles) +# was kept; every grid page lost its layout and afterwalk passed. A widget replaced under a kept module's target package stops +WS=$T/ws; mkdir -p $WS/app/.stepupdate $WS/cache $WS/solo/.stepupdate +printf '%s\n' 'Atlas_Web_Content 3.9.0: Badge.mpk Badge 3.1.0 -> 3.2.2; com.mendix.widget.web.Datagrid.mpk Datagrid 2.2.2 -> 2.30.6' 'DataWidgets 3.11.8: com.mendix.widget.web.Gallery.mpk Gallery 1.0.0 -> 1.4.0' > $WS/app/.stepupdate/widgets.txt +printf '%s\n' 'DataWidgets local theme/javascript file edits refuse the update (FILES-EDITED)' 'NoCache diagram-only edits (UNKNOWN)' > $WS/app/.stepupdate/kept.txt +cp $WS/app/.stepupdate/widgets.txt $WS/solo/.stepupdate/ +printf '%s\n' '# plan' '117183 3.9.0 Atlas_Web_Content' '116540 3.11.8 DataWidgets' '999 1.0.0 NoCache' > $WS/plan.txt +python3 -c 'import zipfile,sys; z=zipfile.ZipFile(sys.argv[1],"w"); [z.writestr(n,b"x") for n in ("widgets/com.mendix.widget.web.Datagrid.mpk","widgets/com.mendix.widget.web.Gallery.mpk","themesource/datawidgets/web/datagrid.scss")]' $WS/cache/116540-3.11.8.mpk +ctl widgetskew-catch 1 '^WIDGET-SKEW Datagrid 2\.2\.2 -> 2\.30\.6 \(by Atlas_Web_Content 3\.9\.0\) ships with DataWidgets 3\.11\.8' '!Badge' '!Gallery' '^NOTE NoCache 1\.0\.0: no cached package' '1 widget\(s\) replaced under a kept module; repair: steps/repair\.md row 32' -- python3 $B/widgetskew.py $WS/app --plan $WS/plan.txt --cache $WS/cache +ctl widgetskew-nothing-kept-pass 0 '!WIDGET-SKEW' '0 kept module\(s\), 0 widget' -- python3 $B/widgetskew.py $WS/solo --plan $WS/plan.txt --cache $WS/cache +# step_update parks on it, unless widgetskew-ack.txt accepts it +awk '/xo "\$WORK\/widgetskew.out"/{p=1} p{print} p&&/^ fi$/{exit}' $B/cloud/upgrade-pipeline.sh > $WS/nj.sh +printf '%s\n' 'xo() { o=$1; shift; "$@" > "$o"; }' 'nj_file() { echo "NJ $1 $2"; grep -E "$5" "$3"; }' 'nj_add() { echo "NJ $1 $2: $3"; }' 'nj_park() { echo PARK; }' 'export UPG_CACHE='"$WS"'/cache; rc=0' > $WS/stub.sh +mkdir -p $WS/w1 $WS/w2; cp $WS/plan.txt $WS/w1/; cp $WS/plan.txt $WS/w2/; : > $WS/w2/widgetskew-ack.txt +ctl pipeline-update-widgetskew-catch 0 '^NJ 13 A module update replaced a widget whose styles stay with a kept module \(steps/repair\.md row 32\)$' '^WIDGET-SKEW Datagrid' '^NJ 13 Decision: Update the kept module' '^PARK$' -- bash -c ". $WS/stub.sh; WORK=$WS/w1 APP=$WS/app B=$B; . $WS/nj.sh; echo done" +ctl pipeline-update-widgetskew-ack-pass 0 '!^NJ' '!PARK' '^done$' -- bash -c ". $WS/stub.sh; WORK=$WS/w2 APP=$WS/app B=$B; . $WS/nj.sh; echo done" # the build stop quotes owngone.out when mx check has CE1613 (and only then) mkdir -p $T/og; printf '%s\n' 'crosscalls: own modules 1' 'OWN AuthMod 3.2.0 -> 4.1.0: 1 target(s) own modules use are gone from the 4.1.0 package' ' GONE page AuthMod.Login_Page (used by own Shop.ACT_Open)' > $T/og/owngone.out printf '%s\n' '[error] [CE1613] "The selected page no longer exists." at Microflow ACT_Open' > $T/og/check-1613.log; printf '%s\n' '[error] [CE0117] "x"' > $T/og/check-0117.log diff --git a/upgrade-toolkit/steps/repair.md b/upgrade-toolkit/steps/repair.md index 00f02fa..1afe51d 100644 --- a/upgrade-toolkit/steps/repair.md +++ b/upgrade-toolkit/steps/repair.md @@ -42,6 +42,7 @@ keep the fix and continue with the new class. Without `--fixed` a rise is always | 29 | `[MXCHECK] mx check ended without a result` in check-N.log: mx check died (an `ERROR: System.…Exception` and a stack, no "The app contains" line) | the ERROR line it quotes; `git -C app log -1` names the last change | mx check cannot load the model, so it reports nothing: the pipeline counts it as an error and stops (before, it counted 0 and ran mxbuild). The last commit broke the MPR: a raw unit edit (copydoc.py, copy_mappings.py, restoreunit.py) or a tool crash mid-write. KeyNotFoundException with a GUID = a pointer to an element id that is not in the unit (cold CMM: a page copied with fresh ids kept its old `DefaultButtonPointer`; copydoc.py now remaps them). Tag, `git reset --hard` to the commit before, fix the tool, redo. EntryPointNotFound or NullReferenceException were seen too (an earlier run, twice), cause not established: find the last change first | | 30 | CE2729 "No read access to association 'A_B' in entity 'Other.E' for user role ..." after an update that re-created an association to another module's entity (editroute XASSOC replay) | `work/replay/<Module>.mdl` has `member access on <owner>` but no `the other side` part; `mxcli describe entity Other.E` shows the member gone from its grants, the original model has it | The re-created association starts with no member access on BOTH entities. The replay restored only the owner side; since fix 15 it also restores the other side, as only that member merged into each rule (mxcli's grant adds to a role's rule and keeps the rest). Cold CMM run 8: EmployeeManagement.Employee_Account to Administration.Account, ReadOnly for Administrator and User became None, 5 CE2729 on the Employee text boxes. Remedy for a run replayed before the fix (verified, cold CMM run 9): per grant line of the entity in the ORIGINAL model, `grant read (A_B) on entity Other.E to <role> [where ...];` (editroute.member_grant prints these), mxcli exec, commit | | 31 | javac `cannot find symbol` in an updated module's Java (MendixSSO, CommunityCommons) at mxbuild, where the class exists but the member does not | `javacjar.out` (the build stop runs `bin/repair/javacjar.py <app> <build log>` and quotes it): `SYMBOL <class>.<member>`, `HAS <jar>`, `HIDDEN BY <jar>` | An older copy of the class sits in another jar, often a fat (shaded, jar-with-dependencies) jar in userlib that bundles its libraries, and javac takes that copy. Cold CMM (10.24.28): MendixSSO 10.6.2 calls `CookieHeaderNames.SameSite` (vendorlib netty-codec-http 4.2.17); userlib/server-examples-0.4.1-SNAPSHOT.jar (OPC UA, 12,588 entries) carries its own older Netty, Guava and BouncyCastle. jardedupe keeps such a jar (only copy of some classes) and jarclash lists it as a note. Remedy (verified, cold CMM run 10: mxbuild passes): `python3 $B/repair/javacjar.py app/ --strip userlib/<fat>.jar` (dry run: how many classes a vendorlib jar also carries), then `--apply`, commit `jar:`; the library then runs on the vendorlib versions (newer majors: Netty 4.2, Guava 33): a judgment item, so test what uses the library at the gate. Alternatives: a thin jar plus its separate dependencies, or a newer version of the library | +| 32 | After the module updates, data grid (or other widget) pages show their rows without layout: no columns, no header styling; afterwalk passes (the data is on the page), the screens rung says REVIEW | `widgetskew.out` (step_update runs `bin/widgetskew.py <app> --plan plan.txt` and parks): `WIDGET-SKEW <widget> <old> -> <new> (by <module>) ships with <kept module> <version>`. One module's package replaced a shared widget in widgets/ (`.stepupdate/widgets.txt`), while the module that carries its styles (themesource) was kept at its old version | Update the kept module too: a one-line plan (`<content id> <version> <module>`) through `bin/stepupdate.sh` (its local edits are often OS junk, `.DS_Store`); or put the old widget .mpk back from git. Accepting the skew: `widgetskew-ack.txt` in the work dir. Cold CMM: Atlas_Web_Content 3.9.0 brought Datagrid 2.30.6, DataWidgets stayed 3.x (FILES-EDITED on `.DS_Store`); 23 grid pages unstyled | mxcli v0.28.0 (#1408, #1474), what it changes for the rows above: - `mxcli exec` is all-or-nothing. A script whose statement N fails now changes NOTHING and exits non-zero; before, statements 1 to N-1 From ea991cfd70d6da3350fa3b6451f4f3b2d98d9219 Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 16:34:52 +0800 Subject: [PATCH 57/59] ci: install pymongo, the fixture model builder needs its bson module Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .github/workflows/controls.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/controls.yml b/.github/workflows/controls.yml index 93ddbe7..bc4fa91 100644 --- a/.github/workflows/controls.yml +++ b/.github/workflows/controls.yml @@ -17,6 +17,10 @@ jobs: with: python-version: '3.12' + - name: Python packages the controls use + # controls/lib/mkmodel.py writes fixture models as BSON (the bson module ships with pymongo) + run: pip install pymongo + - name: Toolkit controls (fast) run: bash upgrade-toolkit/controls/run-controls.sh --fast From 47c49633051492afb242dbb33dffc152d036b59a Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 16:44:31 +0800 Subject: [PATCH 58/59] ci: install zsh, the toolkit wrappers exec it Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- .github/workflows/controls.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/controls.yml b/.github/workflows/controls.yml index bc4fa91..44ad30f 100644 --- a/.github/workflows/controls.yml +++ b/.github/workflows/controls.yml @@ -21,6 +21,10 @@ jobs: # controls/lib/mkmodel.py writes fixture models as BSON (the bson module ships with pymongo) run: pip install pymongo + - name: Shell the toolkit scripts use + # bin/*.sh wrappers exec zsh (macOS default); the Ubuntu runner has none + run: sudo apt-get install -y zsh + - name: Toolkit controls (fast) run: bash upgrade-toolkit/controls/run-controls.sh --fast From f7ed981473aeeabb5c1d939f28578e061294ff9f Mon Sep 17 00:00:00 2001 From: MendixMau <mauritscgvisser@gmail.com> Date: Tue, 17 Nov 2026 16:53:00 +0800 Subject: [PATCH 59/59] docs: ledger row for the 10 controls that fail only on Linux CI Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --- upgrade-toolkit/docs/feedback-ledger.md | 1 + 1 file changed, 1 insertion(+) diff --git a/upgrade-toolkit/docs/feedback-ledger.md b/upgrade-toolkit/docs/feedback-ledger.md index c82c418..84fed9b 100644 --- a/upgrade-toolkit/docs/feedback-ledger.md +++ b/upgrade-toolkit/docs/feedback-ledger.md @@ -7,6 +7,7 @@ that closes a row. | date | feedback (owner's words, short) | conclusion | status | where | |---|---|---|---|---| +| 11-17 | CI on GitHub was red since #10 (setup: no bson, no zsh) | CI installs pymongo and zsh: 2624 passed, 10 failed on Linux (smokewalk, stepupdate x6, remedy, smoke-users, decide-cards; all pass on macOS); make those 10 run on Linux | open | #11 | | 11-17 | "Make the sidebar 50% of screen ... show what was found per action, and what it MEANS" | drawer is 50% wide; per action a "What it means" box and an item-by-item table, worst first | branch | ba76a0e | | 11-17 | "If we could write such a thing per issue which needs a decision ... embed this" | a decision opens its case in the cockpit: headline, tiles, old vs new screenshot, why, options with the catch | branch | 775ddae | | 11-17 | the step writes its case on its own, not by hand | a regression without a case file gets one built from the run's own files | PR | this PR |