diff --git a/upgrade-toolkit/SKILL.md b/upgrade-toolkit/SKILL.md index 8144c27..64fb9bd 100644 --- a/upgrade-toolkit/SKILL.md +++ b/upgrade-toolkit/SKILL.md @@ -110,7 +110,9 @@ rung whether the app can be judged, what the customer must supply, and the best associations that other modules own; all of them hold data. List the targets with `idcheck.py app.mpr --xmod` and update those modules LAST (or not headless). Step idcheck catches it (E1: 27 associations). - Updating an edited module replaces it with the package: the local edit is gone, and re-creating it gives new GUIDs - (data dropped, while mx check, mxbuild and the start all succeed). See the edited-modules playbook. + (data dropped, while mx check, mxbuild and the start all succeed). See the edited-modules playbook. `bin/editroute.py` + handles the safe cases (diagram-only, UI, constants, unused vendor drops, other modules' associations with their GUIDs). + A real logic edit belongs in `_Ext`, a module of the app's own: never edit a Marketplace module in place. - A module update resets the module's constants to package defaults (E1: `MFAmodule.EnabledMFA` True to False, MFA silently off) and can ADD a switch whose default hides data (E4 and E5b: a new boolean emptied the dashboard charts, every journey PASSed). constcheck.py lists an added boolean as REVIEW; shotdiff shows the empty page. @@ -172,8 +174,10 @@ plain PASS; a failure that is not a call into an excluded system stays FAIL, an - Proposal not signed: `$W/PROPOSAL.md` section 6 has no `Signed-off: ` (pipeline exit 14), or it says `Status: PAUSED` (exit 15), or a plan input changed after the sign-off (the sign-off is cleared). Do not convert, update or repair. An agent never writes the sign-off line. Unattended runs sign nothing: ASSUMED (unattended) in DECISIONS.md, "proposal not signed" in every report. -- RED line in localedits for a module on the plan: ask (steps/decide.md). UNATTENDED default: keep that module at its version, carry - on with the rest, and report "upgrade complete except kept modules: " (never PASS for the kept ones). +- RED line in localedits for a module on the plan: ask (steps/decide.md). The recommendation is the update with the edits carried + over (`bin/editroute.py`), never "keep" without a reason and an end date. UNATTENDED: editroute routes what it can; a module it + HOLDs (real logic edit, Java user code, a removal still used) stays at its version, the rest carries on, and the report says + "upgrade complete except held modules: " (never PASS for the held ones). - `bin/steps.py check --work $W` names a D row of AGENT-DECISIONS.md that has no step ("MISSING D71: ..."): open the step (`steps.py propose --new`), write its `S-n` in the row, then go on. Do not start the next change of that kind first. diff --git a/upgrade-toolkit/bin/brain_mine.py b/upgrade-toolkit/bin/brain_mine.py index 69b114f..a51743a 100644 --- a/upgrade-toolkit/bin/brain_mine.py +++ b/upgrade-toolkit/bin/brain_mine.py @@ -16,7 +16,7 @@ usage: brain_mine.py [--brain DIR] [--table replacements.tsv] [--to VER] [--min N] [--out brain-patterns.md] [--tsv candidates.tsv] - --brain default $UPG_BRAIN or ~/Mendix/-brain (read only, never written) + --brain default: bin/brainpath.py, a ~/Mendix/*-brain with version tables (read only, never written) --table default playbooks/replacements.tsv next to this bin/ --to drop rows of component versions whose min Mendix is above VER (not installable on the target) --min components a successor needs to count as recurring (default 2) @@ -107,7 +107,8 @@ def covered_by(names, cov): def main(): here = Path(__file__).resolve().parent ap = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) - ap.add_argument("--brain", default=os.environ.get("UPG_BRAIN") + import brainpath + ap.add_argument("--brain", default=brainpath.find()[0] or os.environ.get("UPG_BRAIN") or str(Path.home() / "Mendix/-brain")) ap.add_argument("--table", default=str(here.parent / "playbooks" / "replacements.tsv")) ap.add_argument("--to") diff --git a/upgrade-toolkit/bin/brainpath.py b/upgrade-toolkit/bin/brainpath.py new file mode 100644 index 0000000..deaf5a3 --- /dev/null +++ b/upgrade-toolkit/bin/brainpath.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Find the company brain: the folder whose baseline/versions holds the Studio Pro version tables. + +Order: $UPG_BRAIN, then UPG_BRAIN= in ~/.config/upgrade-toolkit/local.env, then the first ~/Mendix/*-brain folder with +version tables (a folder ending in -test only when nothing else has tables). Prints the path, or nothing and exit 1. + +Cold test WM1: the default was the literal text ~/Mendix/-brain, which never matches a folder, so the +release-note check never ran unless a person had set UPG_BRAIN. +""" +import glob +import os +import sys +from pathlib import Path + + +def has_tables(d): + return bool(glob.glob(os.path.join(d, "baseline", "versions", "studio-pro-upgrade-*.md"))) + + +def from_local_env(): + p = Path.home() / ".config" / "upgrade-toolkit" / "local.env" + try: + for line in p.read_text().splitlines(): + line = line.strip() + if line.startswith("UPG_BRAIN="): + return os.path.expanduser(line.split("=", 1)[1].strip().strip("'\"")) + except OSError: + pass + return None + + +def find(): + """(path, how) of the brain, or (None, why not).""" + for how, d in (("UPG_BRAIN", os.environ.get("UPG_BRAIN")), ("local.env", from_local_env())): + if d: + return (d, how) if has_tables(d) else (None, f"{how} points at {d}, which has no baseline/versions tables") + cands = sorted(c for c in glob.glob(str(Path.home() / "Mendix" / "*-brain*")) if os.path.isdir(c) and has_tables(c)) + cands.sort(key=lambda c: c.endswith("-test")) + if cands: + return cands[0], "found under ~/Mendix" + return None, "no folder under ~/Mendix holds brain version tables (set UPG_BRAIN, or UPG_BRAIN= in ~/.config/upgrade-toolkit/local.env)" + + +if __name__ == "__main__": + path, how = find() + if path: + print(path) + sys.exit(0) + print(how, file=sys.stderr) + sys.exit(1) diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index bd7e359..cfa10af 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -146,6 +146,13 @@ set -u : "${UPG_TOOLS:=$RUN/tools}" : "${MXCLI:=${UPG_MXCLI:-}}" B="$TK/bin" +# Apple Silicon: every container call goes to a VM with Rosetta (bin/dockerhost.sh). Cold test WM1: the run used a qemu VM, +# mxcli crashed on all 10 module updates and each was recorded as "kept". +if DH=$(bash "$B/dockerhost.sh"); then + [ -z "$DH" ] || [ "$DH" = "${DOCKER_HOST:-}" ] || { echo "upgrade-pipeline.sh: containers run on $DH (Rosetta)"; export DOCKER_HOST="$DH"; } +elif [ "${DRY:-0}" != 1 ]; then + exit 2 +fi PORT_OFF=${PORT_OFF:-0} APP_PORT=$((8080 + PORT_OFF)); ADMIN_PORT=$((8090 + PORT_OFF)) export TK RUN UPG_TOOLS @@ -243,6 +250,11 @@ decisions_apply() { cockpit_regen() { [ "$DRY" != 1 ] || return 0 python3 "$B/cockpitpage.py" --work "$WORK" --out "$WORK/COCKPIT.html" >> "$WORK/logs/decisions.log" 2>&1 || true + # the page must not contradict the run (cold test WM1: ghost "Not approved yet" rows, a stop bar while running); a problem is + # said in the step log and kept in cockpit-check.txt, never a failed step + if ! python3 "$B/cockpitcheck.py" --work "$WORK" > "$WORK/cockpit-check.txt" 2>&1; then + say "cockpit: the page contradicts the run, see $WORK/cockpit-check.txt"; sed 's/^/ /' "$WORK/cockpit-check.txt" + fi # rung 2 also refreshes the cockpit of rung 1, so the run keeps one page from start to end [ -z "${UPG_COCKPIT_OUT:-}" ] || [ ! -f "$WORK/COCKPIT.html" ] || cp "$WORK/COCKPIT.html" "$UPG_COCKPIT_OUT" 2>/dev/null || true # UPG_COCKPIT_HISTORY=1 keeps every version of the page (a demo video shows the cockpit filling up), next to the one cockpit @@ -601,6 +613,7 @@ Decide, in writing: (a) wrap it in a host app (an app with a home page, a theme note "no module-jumps.txt: release-impact.md holds Studio Pro rows only; module rows NOT generated" fi [ "$rc" = 0 ] || note "release_impact.py exit $rc (2 = a module is UNKNOWN to the brain: fill its row by hand), see $WORK/release-impact.out" + dojo_report # judgment: the plan if [ "$DRY" != 1 ] && [ ! -f "$WORK/plan.txt" ]; then nj_add 13 "Module plan missing: write $WORK/plan.txt" "No script picks module target versions (steps/triage.md item 3: newest version whose min Mendix fits the target, from the brain's modules-index.md and \`mxcli marketplace versions \`). @@ -765,8 +778,9 @@ proposal_stop() { # proposal_stop RC: rc 14 / 15 end the run with NEEDS-JUDGMENT case "$1" in 0) return 0 ;; 14) nj_add 14 "Upgrade proposal awaits sign-off: read $WORK/PROPOSAL.md" "$(head -n 3 "$WORK/proposal.out" 2> /dev/null) -Nothing in the app model has changed. Read PROPOSAL.md, answer the section-3 items in chat (steps/decide.md; the P-n lines go to $WORK/DECISIONS.md), then a person writes one line in section 6: -\`Signed-off: \` to go on, or \`Status: PAUSED \` to stop on purpose. An agent never writes that line. Then run the command at the top of this file." +Nothing in the app model has changed. Answer the questions under Blocked on you in the live cockpit and approve the plan there +(the approval carries your Git name), or answer the section-3 items in chat (steps/decide.md) and approve on the page. +To stop on purpose, write \`Status: PAUSED \` in PROPOSAL.md section 6. Then run the command at the top of this file." nj_file 14 "PROPOSAL.md (head)" "$WORK/PROPOSAL.md" 30 ;; 15) nj_add 15 "Upgrade proposal PAUSED by the owner" "$(head -n 3 "$WORK/proposal.out" 2> /dev/null) The run stopped on purpose; no step after propose has run. Resume: remove the \`Status: PAUSED\` line from $WORK/PROPOSAL.md section 6, write \`Signed-off: \`, then @@ -775,8 +789,20 @@ The run stopped on purpose; no step after propose has run. Resume: remove the \` esac nj_flush } +# Dojo to React (B6-4) on every run, against the last rung: the move is part of each upgrade, not a later project. Cold +# test WM1: the pipeline never ran dojo_report.py, so 144 legacy core widgets in 83 documents showed up nowhere. +dojo_report() { + [ "$DRY" != 1 ] || { say "+ dojo_report.py $WORK/original.mpr --to ${TARGET2:-$TARGET} --out $WORK/dojo-react.md"; return 0; } + [ -f "$WORK/original.mpr" ] || return 0 + local rc=0 + xo "$WORK/dojo-react.out" python3 "$B/dojo_report.py" "$WORK/original.mpr" --to "${TARGET2:-$TARGET}" --out "$WORK/dojo-react.md" || rc=$? + [ "$rc" = 0 ] || note "dojo_report.py exit $rc, see $WORK/dojo-react.out" +} + step_propose() { local rc=0 + # a run whose triage predates the report + [ -s "$WORK/dojo-react.md" ] || dojo_report if [ "$DRY" = 1 ]; then say "+ action_inventory.py $WORK/original.mpr --out $WORK/actions.json; apimock.py inventory $WORK/original.mpr --json $WORK/apimock-inventory.json" say "+ proposal.py --work $WORK --source $SOURCE --target $TARGET (exit 14 unsigned, 15 PAUSED, 0 signed or unattended)" @@ -899,7 +925,8 @@ replay_edits() { # replay_edits [pre]: pre = only modules the post-convert plan [ -f "$f" ] || continue mod=$(basename "$f"); mod=${mod%.*} if [ "${1:-}" = pre ] && grep -qE "^[0-9]+ [^ ]+ $mod\$" "$WORK/plan.txt" 2> /dev/null; then continue; fi - if [ "$DRY" != 1 ] && git_app log --format=%s 2> /dev/null | grep -qxF "replay: $mod"; then note "replay $mod skipped: replayed already"; continue; fi + # one replay per module per rung: the 10.24 rung's replay must not skip the 11 rung's (editroute writes a new replay/ each rung) + if [ "$DRY" != 1 ] && git_app log --format=%s 2> /dev/null | grep -qxF "replay: $mod${UPG_PHASE:+ ($UPG_PHASE)}"; then note "replay $mod skipped: replayed already"; continue; fi if [ "$DRY" != 1 ] && grep -q "^$mod " "$APP/.stepupdate/kept.txt" 2> /dev/null; then note "replay $mod skipped: the module was kept, its edits are in place"; continue; fi rc=0 case "$f" in @@ -928,7 +955,7 @@ replay_edits() { # replay_edits [pre]: pre = only modules the post-convert plan else rc=1; fi ;; esac [ "$rc" = 0 ] || { nj_file 13 "replay of $mod failed (exit $rc): $f" "$WORK/replay-$mod$([ "${f##*.}" = patch ] && echo -patch).out" 30; nj_flush; } - commit_app "replay: $mod" replay "replay-$mod" + commit_app "replay: $mod${UPG_PHASE:+ ($UPG_PHASE)}" replay "replay-$mod" note "replayed the local edits of $mod from $f" done } @@ -950,10 +977,27 @@ assoc_unreplayed() { done } -# update: module updates with the post-convert plan; kept modules are recorded (SKILL stop rules), decided at the gate. +# route_kept (cold test WM1): a module the update refused is not kept at its old version by default. editroute.py routes each one +# (diagram-only, UI and constants replayed, unreadable but unedited, files kept as copies, vendor drops nothing uses, associations +# of other modules replayed with their ids) and a second stepupdate pass updates the routed modules; only HOLD rows stay kept. +route_kept() { + if [ "$DRY" = 1 ]; then x python3 "$B/editroute.py" plan --work "$WORK" --app "$APP" --original "$WORK/original.mpr" --plan "$WORK/plan.txt"; return 0; fi + [ -s "$APP/.stepupdate/kept.txt" ] || return 0 + local rc=0 le=() + [ ! -s "$WORK/localedits.json" ] || le=(--localedits "$WORK/localedits.json") + xo "$WORK/editroute.out" python3 "$B/editroute.py" plan --work "$WORK" --app "$APP" --original "$WORK/original.mpr" --plan "$WORK/plan.txt" "${le[@]}" || rc=$? + cat "$WORK/editroute.out" >> "$LOG" + [ "$rc" = 0 ] || { note "editroute: no kept module could be routed, they stay kept (see $WORK/editroute.out)"; return 0; } + run_stepupdate "$WORK/route-plan.txt" routed + rc=0; xo "$WORK/editroute-collect.out" python3 "$B/editroute.py" collect --work "$WORK" || rc=$? + note "editroute: $(grep -c '^ROUTE' "$WORK/editroute.out") kept module(s) routed, $(grep -c ' HOLD:' "$WORK/editroute.out") held for a person (edit-routes.json)" +} + +# update: module updates with the post-convert plan; kept modules are routed (route_kept), the rest decided at the gate. step_update() { [ -f "$WORK/plan.txt" ] || [ "$DRY" = 1 ] || { nj_add 13 "plan.txt missing" "$WORK/plan.txt does not exist"; nj_flush; } run_stepupdate "$WORK/plan.txt" post-convert + route_kept replay_edits assoc_unreplayed if [ "$DRY" != 1 ]; then diff --git a/upgrade-toolkit/bin/cockpitcheck.py b/upgrade-toolkit/bin/cockpitcheck.py new file mode 100644 index 0000000..9e89d0c --- /dev/null +++ b/upgrade-toolkit/bin/cockpitcheck.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""cockpitcheck.py: the cockpit must never contradict itself or the run. Builds the page model exactly as cockpitpage.py does +and checks what a person reads against the run's own files. + +Why (cold test WM1, 2026-11-16): the owner saw rows that said "Not approved yet" and "Test gap" with nothing to approve, a red +"waits for your answers" bar after the plan was approved and the run was going, and "2 items to decide" next to "0 for you +now". Each was a separate cockpit bug; this check makes the whole class fail a run instead of confusing a person. + +Rules (each line of output names one): + ghost-step an undecided plan-item step (P-n alias) on the page that the current PROPOSAL.md no longer holds + wait-no-ask a row whose plan cell asks for an approval ("Not approved yet", "Needs your OK") while the plan is signed + and Blocked on you does not hold that item + stop-no-ask the red stop bar says the run waits for you, and Blocked on you is empty + stop-and-running the red stop bar and the "Running now" bar at the same time + ask-no-evidence a plan question card that lists no element, file or finding behind it (evidence.py found nothing) + count-mismatch a Blocked on you card that "for you now" does not list while the run is stopped for a person + keep-recommended a module card that recommends keeping the old version without a reason and an end date (an edit is no + reason to stay behind: editroute.py carries edits over; cold test WM1, all seven cards said "keep") + + cockpitcheck.py --work W prints CONSISTENT, or one line per problem; exit 0 / 1 (2 usage) +Run by: upgrade-pipeline.sh after each cockpit render (a problem is a note and a cockpit item, never a silent page), and controls. +""" +import argparse, os, re, sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import cockpitpage # noqa: E402 +import cockpitmatrix # noqa: E402 +import steps as stp # noqa: E402 + +ASK = ("Not approved yet", "Needs your OK") + + +def check(work): + model = cockpitpage.build_model(work) + M = cockpitmatrix.build(work, model) + out = [] + asks = cockpitmatrix.open_questions(model) + ask_names = {re.sub(r"\s*\((module|widget|jar|java)\)$", "", i.get("title", "")) for i in asks} + plan = cockpitpage.rd(os.path.join(work, "PROPOSAL.md")) or "" + in_plan = set(re.findall(r"\[(P-\d+)\]", plan)) + recs, _ = stp.load(work) + for r in recs or []: + decided = (r.get("approval") or {}).get("state") in ("approved", "rejected") # a module the update kept after the sign-off + if plan and r.get("alias", "").startswith("P-") and stp.mine(r) and r["alias"] not in in_plan and not decided: + out.append(f"ghost-step {r['id']} ({r['alias']}, {r['title']}): not in the current plan") + signed = bool(model.get("signed")) + for row in M["rows"]: + c = row["cells"].get("plan") or {} + if signed and c.get("l") in ASK and not any(n and n in row["name"] for n in ask_names): + out.append(f"wait-no-ask {row['id']} ({row['name']}): says \"{c['l']}\" but the plan is signed and nothing asks for it") + for i in asks: # a question with nothing under it cannot be decided (cold test WM1: "local edits refuse the update", no list) + if i["kind"] == "plan-choice" and not (i.get("evidence") or {}).get("what"): + out.append(f"ask-no-evidence {i['id']} ({i['title']}): the card asks a person to decide but shows no element, file or finding") + rec = (i.get("rec_text") or "").lower() + if i["kind"] == "plan-choice" and i.get("title", "").endswith("(module)") and rec.startswith("keep") \ + and not re.search(r"\buntil\b|\bend date\b|\d{4}-\d{2}-\d{2}", rec): + out.append(f"keep-recommended {i['id']} ({i['title']}): recommends the old version without a reason and an end date") + stopped, running = model.get("stopped"), model.get("running") + if stopped and not asks and str(stopped.get("code")) in ("14", "15"): + out.append("stop-no-ask the red bar says the plan waits for your answers, and Blocked on you is empty") + if stopped and running: + out.append(f"stop-and-running the red stop bar (exit {stopped.get('code')}) shows while the run is running") + if stopped: + mine = [o for o in M["band"]["open"]["items"] if o["owner"] != "agent" and o["group"] != "At deploy"] + miss = [i["title"] for i in asks if i["kind"] != "signoff" and not any( + re.sub(r"\s*\((module|widget|jar|java|check)\)$", "", i["title"]) in o["text"] for o in mine)] + if miss: + out.append(f"count-mismatch the run waits for a person: Blocked on you holds {len(asks)}, \"for you now\" lacks {len(miss)} of them ({', '.join(miss[:5])})") + return out + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--work", required=True) + a = ap.parse_args() + if not os.path.isdir(a.work): + print(f"cockpitcheck: no work folder {a.work}", file=sys.stderr) + return 2 + probs = check(os.path.abspath(a.work)) + print("\n".join(probs) if probs else "CONSISTENT") + return 1 if probs else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/bin/cockpitmatrix.py b/upgrade-toolkit/bin/cockpitmatrix.py index 6f52801..9b08864 100644 --- a/upgrade-toolkit/bin/cockpitmatrix.py +++ b/upgrade-toolkit/bin/cockpitmatrix.py @@ -17,7 +17,7 @@ import steps as stp STAGES = ("triage", "plan", "upgrade", "test") -GROUPS = [("app", "App and platform"), ("setup", "Testing setup"), ("golive", "Go-live")] +GROUPS = [("app", "App and platform"), ("setup", "How we prove it still works"), ("golive", "Go-live")] STEP_GROUP = ("steps", "Steps of this run") # added after "app" when work/steps.jsonl exists STATUSES = ("ok", "wait", "bad", "none", "you") LOOP = ("Found", "Proposed", "Trying", "Fixed") @@ -37,8 +37,9 @@ # ---- cells and rows ---- -def cell(s, label, steps=None, tech=None, late=False, deploy=False, dec=None): - return {"s": s, "l": label, "steps": list(steps or []), "tech": list(tech or []), "late": late, "deploy": deploy, "dec": dec} +def cell(s, label, steps=None, tech=None, late=False, deploy=False, dec=None, d=""): + """d: one short line under the label in the matrix (what was found or what the plan does), so a cell says more than its state.""" + return {"s": s, "l": label, "steps": list(steps or []), "tech": list(tech or []), "late": late, "deploy": deploy, "dec": dec, "d": d} def notyet(what="No result file yet, so nothing is claimed."): @@ -49,10 +50,12 @@ def nowork(why="Nothing to do at this stage for this item."): return cell("none", "Not needed", [why]) -def row(rid, name, sub, group, cells, tech=None, fix=None, regression=False): +def row(rid, name, sub, group, cells, tech=None, fix=None, regression=False, story=None): + """story: {found, means, plan} lists of plain sentences, shown at the top of every drawer of the row (cold test WM1: a + cell said "Found" or "Approved" and nobody could tell what was found or what the plan does about it).""" full = {k: cells.get(k) or nowork() for k in STAGES} return {"id": rid, "name": name, "sub": sub, "group": group, "cells": full, "tech": list(tech or []), "fix": fix, - "regression": regression} + "regression": regression, "story": story or {}} def fixd(problem="", proposal="", steps=None, loop=None, result="", help="", learn=""): @@ -221,7 +224,9 @@ def commit_rows(pred): if done["triage"] and src else notyet("Triage has not run yet.")) p = (cell("ok", "Plan written", ["The plan lists the module updates and the choices that need a person.", f"Signed by {signed[0]}." if signed else "Not signed yet."], ["PROPOSAL.md"]) - if done["propose"] and P["present"] else notyet("No plan yet.")) + if done["propose"] and P["present"] else + cell("wait", "Planned, not approved", ["The plan is written. Nothing changes in the app before you approve it."], ["PROPOSAL.md"]) if P["present"] + else notyet("No plan yet.")) if done["convert"]: u = (cell("ok", "Converted, build clean", [f"The model was converted to Mendix {tgt}.", "The build finished with 0 errors."]) if done["build"] and any(r["rung"] == "build" and r["verdict"].startswith("PASS") for r in g["rungs"]) @@ -229,8 +234,14 @@ def commit_rows(pred): u["tech"] = [f"{h} {s}" for h, s in commit_rows(lambda s: s.startswith("convert"))] else: u = notyet("The model has not been converted yet.") + if src: + t["d"] = f"{src} → {tgt}" + p["d"] = p["d"] or "convert, then build" rows.append(row("runtime", "Mendix runtime", f"{src} → {tgt}" if src else "", "app", - {"triage": t, "plan": p, "upgrade": u, "test": rungs_cell(g, ["build", "data"])})) + {"triage": t, "plan": p, "upgrade": u, "test": rungs_cell(g, ["build", "data"])}, + story={"found": [f"The app runs on Mendix {src}; the target is Mendix {tgt}."] if src else [], + "means": ["The whole app model is converted to the new version. Pages, microflows and Java code must still build after that."], + "plan": ["Convert the model with the new Studio Pro, then build it. Every repair after the build is its own Git commit."]})) # Marketplace modules planned = [] @@ -244,7 +255,9 @@ def commit_rows(pred): if P["present"]: t = cell("ok", f"{len(names)} to update" if names else "None to update", [f"{n}: {a} to {b}." for n, a, b in planned] or ["No Marketplace module needs an update."], ["PROPOSAL.md section 2"]) - p = cell("ok", f"{len(planned)} steps planned" if planned else "Nothing planned", ["Each update is its own step, with a data identity check after it."]) if done["propose"] else notyet("No plan yet.") + p = cell("ok", f"{len(planned)} steps planned" if planned else "Nothing planned", ["Each update is its own step, with a data identity check after it."]) if done["propose"] else \ + cell("wait", "Planned, not approved", ["Each update is its own step, with a data identity check after it.", + "Nothing changes in the app before you approve the plan."], ["PROPOSAL.md section 2"]) else: t, p = notyet("Triage has not written a proposal yet."), notyet("No plan yet.") if names and got: @@ -254,8 +267,19 @@ def commit_rows(pred): u = cell("none", "Not started", ["No update step has been committed yet."]) else: u = nowork("No module needed an update.") if P["present"] else notyet() + kept = [i["name"] for i in P["sec3"] if i.get("id") and "keep" in (i["sub"].get("recommendation", "") + i["sub"].get("answer", "")).lower()] + arrow = [f"{n} {a} → {b}" for n, a, b in planned] + if planned: + t["d"] = cp.short(", ".join(n for n, _, _ in planned[:3]) + (f" +{len(planned) - 3}" if len(planned) > 3 else ""), 60) + p["d"] = cp.short(arrow[0] + (f" +{len(arrow) - 1} more" if len(arrow) > 1 else ""), 60) rows.append(row("modules", "Marketplace modules", f"{len(names)} updates" if names else "", "app", - {"triage": t, "plan": p, "upgrade": u, "test": rungs_cell(g, ["const", "logdiff"])})) + {"triage": t, "plan": p, "upgrade": u, "test": rungs_cell(g, ["const", "logdiff"])}, + story={"found": [f"{len(planned)} Marketplace modules have a newer version for the target." if planned else "No Marketplace module needs an update."] + + ([f"Kept at the old version: {', '.join(kept)} (see Our own edits in modules)."] if kept else []), + "means": ["An update replaces the module with the vendor's new version. That brings the fixes for the new Mendix version, " + "and it overwrites anything the app changed inside the module."], + "plan": [f"Update: {x}." for x in arrow[:12]] + ([f"... and {len(arrow) - 12} more."] if len(arrow) > 12 else []) + + ["Why: the old versions are not built for the new Mendix version. Each update is its own step, with a data check after it."]})) # Own edits in Marketplace modules if ledit is None: @@ -268,7 +292,7 @@ def commit_rows(pred): unread.append(m["name"]) if kinds: edited.append((m["name"], kinds)) - sec3 = [i for i in P["sec3"] if i.get("id")] + sec3 = [i for i in P["sec3"] if i.get("id") and i.get("kind") != "check"] # a check that did not run has its own row open_choices = [i for i in sec3 if not (any(l.get("item") == i["id"] for l in ledger) or "CONFIRMED" in ((dm.get(i["id"]) or [""] * 6)[5:6] or [""])[0])] t = (cell("wait", f"{len(edited)} modules with edits", [f"{n}: " + ", ".join(f"{k} {c}" for k, c in ks) + "." for n, ks in edited], @@ -278,30 +302,65 @@ def commit_rows(pred): elif not sec3: p = cell("ok", "Nothing to decide", ["Every edit can be saved and put back by the update."]) else: - p = cell("wait" if open_choices else "ok", f"{len(sec3)} choices" + (f", {len(open_choices)} assumed" if open_choices else ""), + p = cell("wait" if open_choices else "ok", f"{len(sec3)} choice" + ("" if len(sec3) == 1 else "s") + (f", {len(open_choices)} assumed" if open_choices else ""), [f"{i['name']}: {cp.short(i['sub'].get('recommendation', ''), 110)}" for i in sec3], ["PROPOSAL.md section 3", "DECISIONS.md"]) rep = commit_rows(lambda s: s.startswith("replay")) u = (cell("ok", f"{len(rep)} edit sets put back", [cp.short(s.partition(": ")[2], 110) for _, s in rep], [f"{h} {s}" for h, s in rep]) if rep else (cell("none", "Not started", ["No edit has been put back yet."]) if edited else nowork("There were no edits to put back."))) + KW = {"domain": "entities", "ui": "pages", "logic": "microflows", "files": "files"} + said = [f"{n}: " + ", ".join(f"{c} {KW.get(k, k)}" for k, c in ks) for n, ks in edited] + if said: + t["d"] = cp.short(said[0] + (f" +{len(said) - 1}" if len(said) > 1 else ""), 60) + acts = [f"{i['name']}: " + re.sub(r"\s*\((?:DECIDED|CONFIRMED|ASSUMED)[^)]*\)\)?", "", plain(i['sub'].get('answer') or i['sub'].get('recommendation', '')).split(';')[0]) for i in sec3 if "check" not in i.get("kind", "")] + if sec3 and acts: + p["d"] = cp.short(acts[0], 60) rows.append(row("localedits", "Our own edits in modules", f"{len(edited)} modules", "app", - {"triage": t, "plan": p, "upgrade": u, "test": rungs_cell(g, ["idcheck"])})) + {"triage": t, "plan": p, "upgrade": u, "test": rungs_cell(g, ["idcheck"])}, + story={"found": [f"The app changed {len(edited)} Marketplace modules itself." if edited else "No module has edits of its own."] + + [x + "." for x in said[:6]], + "means": ["A Marketplace update overwrites the whole module. Edits the toolkit can save are put back after the update; " + "edits it cannot put back (new entities hold data) need a choice."], + "plan": [x + "." for x in acts] or ["The toolkit saves each edit and puts it back after the update."]})) # Java libraries jc = cp.rd(os.path.join(work, "jarclash.md")) + if jc is None: # before the build only triage's own scan exists (cold test WM1: the cell said "Not yet" after triage scanned) + jc = cp.rd(os.path.join(work, "jarclash-before.md")) + jf = re.search(r"summary: (\d+) Java actions scanned, (\d+) assignments", cp.rd(os.path.join(work, "javafinal.md")) or "") if jc is None: t = notyet("The library check has not run yet.") nclash = None else: nclash = len(re.findall(r"^CLASH ", jc, re.M)) - t = cell("ok", f"{nclash} libraries twice" if nclash else "No clashes", ["Some libraries are present in two versions." if nclash else "No library is present twice."], ["jarclash.md"]) + libs = sorted({m.split(".")[1] if m.count(".") > 1 else m for m in re.findall(r"^CLASH \d+ classes in \d+ packages \(([\w.]+)", jc, re.M)}) + t = cell("ok", f"{nclash} libraries twice" if nclash else "No clashes", + ([f"{nclash} sets of classes are in two jar files ({', '.join(libs[:4])}). The upgrade keeps one copy of each."] if nclash else ["No library is present twice."]) + + ([f"{jf.group(1)} Java actions scanned: {'no' if jf.group(2) == '0' else jf.group(2)} code that breaks on the new Java."] if jf else []), + ["jarclash.md" if os.path.exists(os.path.join(work, "jarclash.md")) else "jarclash-before.md"] + (["javafinal.md"] if jf else [])) fixc = commit_rows(lambda s: re.search(r"jardedupe|java dependencies", s)) u = (cell("ok", "Settled", ["One copy of each library is kept."], [f"{h} {s}" for h, s in fixc]) if fixc else (cell("wait", "Not settled", ["The duplicate libraries are not removed yet."]) if nclash else nowork() if jc is not None else notyet())) + if nclash: + t["d"] = cp.short(", ".join(libs[:3]) + " in two jars", 60) + elif jf: + t["d"] = f"{jf.group(1)} Java actions scanned" + jp = cell("ok", "Keep one copy", ["The build step removes the second copy of each library."], d="remove the duplicate jars") if nclash else None rows.append(row("java", "Java libraries", f"{nclash} duplicates" if nclash else "", "app", - {"triage": t, "upgrade": u, "test": rungs_cell(g, ["build"])})) + {"triage": t, "upgrade": u, "test": rungs_cell(g, ["build"]), **({"plan": jp} if jp else {})}, + story={"found": t["steps"] if jc is not None else [], + "means": (["Two copies of the same classes can make the app load the wrong one after the upgrade, and fail at run time."] if nclash else []) + + (["The app's own Java code uses nothing the new Java version removes."] if jf and jf.group(2) == "0" else []), + "plan": ["Keep one copy of each library (the newest), then build. The build proves the choice."] if nclash else + ["Nothing to change; the build checks it again."] if jc is not None else []})) # Findings from the Mendix 11 catalog - if not finds and not os.path.exists(os.path.join(work, "findings.json")): + if not finds and not os.path.exists(os.path.join(work, "findings.json")) and tgt.startswith("10."): + later = cell("none", "At the 10 → 11 step", ["The Mendix 11 catalog is checked when the run upgrades from Mendix 10 to 11, after this step."], + d="checked after Mendix 10") + rows.append(row("findings", "Mendix 11 findings", "", "app", {"triage": later}, + story={"found": ["Not checked yet: this step goes to Mendix 10."], "means": ["Mendix 11 changes are checked on the Mendix 10 model."], + "plan": ["The second step (10 → 11) scans the app with the Mendix 11 catalog and plans the fixes."]})) + elif not finds and not os.path.exists(os.path.join(work, "findings.json")): rows.append(row("findings", "Mendix 11 findings", "", "app", {"triage": notyet("The scan has not run yet.")})) else: sev = {} @@ -373,12 +432,30 @@ def commit_rows(pred): [f"The app makes {len(eps)} outside calls.", f"{len(mocked)} can be answered by a stand-in.", f"{len(unk)} get their address from data or are not classified." if unk else "All are classified."], ["apimock-inventory.json"]) p = cell("ok" if mocked else "none", f"{len(mocked)} stand-ins" if mocked else "No stand-ins", ["Build a stand-in per call.", "Check the real systems before go-live."]) + t["d"] = f"{len(unk)} of {len(eps)} unclear" if unk else f"{len(eps)} calls" + p["d"] = f"{len(mocked)} answered by stand-ins" if mocked else "" rows.append(row("outside", "Outside systems", f"{len(eps)} calls", "setup", - {"triage": t, "plan": p, "upgrade": nowork("Stand-ins are test side only; the app is unchanged."), "test": apis_cell(g, work)})) + {"triage": t, "plan": p, "upgrade": nowork("Stand-ins are test side only; the app is unchanged."), "test": apis_cell(g, work)}, + story={"found": t["steps"], + "means": ["The test must not call real outside systems. A call with no stand-in is not tested."], + "plan": [f"Answer {len(mocked)} calls with a stand-in during the test.", + f"The other {len(eps) - len(mocked)} are checked on the real test systems by a person before go-live."]})) # Release notes ri = cp.rd(os.path.join(work, "release-impact.md")) - if ri is None: + rq = next((i for i in model.get("items", []) if i.get("title", "").startswith("Release notes") and i["kind"] == "plan-choice"), None) + if ri is None and rq: + why = "the company brain's version tables are not on this machine" if "version tables" in (rq.get("problem") or "") else "see the technical details" + t = cell("bad", "Did not run", [f"The release-note check did not run: {why}.", "So nobody has checked what the new Mendix versions change for this app."], + [rq.get("problem") or "", "release-impact.out"], d="version tables missing") + p = (cell("wait", "You decided, waiting for the pipeline", ["Your answer is recorded; the next step applies it."], dec=rq["id"]) if rq.get("decided") + else cell("you", "Needs your decision", [rq.get("problem") or ""], dec=rq["id"], d="run it first, or go on without it")) + rows.append(row("release", "Release notes", "check did not run", "app", {"triage": t, "plan": p}, + story={"found": t["steps"][:1], + "means": ["A change in Mendix 10 or 11 that hits this app is found only at the build or the test, or after go-live."], + "plan": ["Recommended: give the run the company brain and rerun triage, so the plan covers the release notes.", + "Or a named person reads the release notes for every version before go-live."]})) + elif ri is None: rows.append(row("release", "Release notes", "", "app", {"triage": notyet("The release-note check has not run yet.")})) else: n = len([r for r in ri.splitlines() if re.match(r"^\| \d+ \|", r)]) @@ -388,14 +465,64 @@ def commit_rows(pred): rows.append(row("release", "Release notes", f"{n} rows", "app", {"triage": t, "plan": p, "upgrade": nowork("Release notes are read, not applied."), "test": release_cell(g)})) - # Click paths + # How we prove it still works: one row per kind of app part in test-inventory.md, with the gate rungs that measure it + # (cold test WM1: 13 kinds of test sat in test-inventory.md and the matrix showed one row, "Click paths: Not needed") + inv = {} + for line in (cp.rd(os.path.join(work, "test-inventory.md")) or "").splitlines(): + c = [x.strip() for x in line.strip().strip("|").split("|")] + if len(c) >= 4 and c[0] not in ("app part", "---"): + inv[c[0]] = c + js = cp.rd(os.path.join(work, "journey-scope.md")) or "" + jb = cp.rd(os.path.join(work, "journeys-base.out")) or "" jo = cp.rd(os.path.join(work, "journeys.out")) or "" + scope = re.search(r"journeys in scope: (\d+) reachable", js) + nopath = re.search(r"(\d+) reachable actions have no click path", js) + jbase = re.search(r"(\d+) journeys .*?(\d+) pass on baseline", jb) cov = re.search(r"coverage (\d+)/(\d+) \((\d+)%\)", jo) - ct = rungs_cell(g, ["journeys"]) - if cov and ct["s"] != "none": - ct["l"] = f"{cov.group(3)}% of buttons reached" - ct["steps"].insert(0, f"{cov.group(1)} of {cov.group(2)} buttons have a test that clicks them.") - rows.append(row("clicks", "Click paths", f"{cov.group(1)}/{cov.group(2)} buttons" if cov else "", "setup", {"test": ct})) + shots = len(os.listdir(os.path.join(work, "smoke-before-shots"))) if os.path.isdir(os.path.join(work, "smoke-before-shots")) else 0 + # (inventory key, row name, what we prove, how in plain words, gate rungs) + TESTS = [("Persistent entities", "Data survives", "every table keeps its rows and values", + "Count the rows and checksum every column of every table on a copy of the database, before and after.", ["data"]), + ("Role home pages and menu", "Pages per role", "every role still lands on its home page and opens its menu", + "Log in as each role, open the home page and every menu item, and compare the screens before and after.", ["smoke", "screens"]), + (None, "Buttons and journeys", "the buttons and flows people use still work", + "Click every reachable button and run the recorded journeys, on the old app first and then on the new one.", ["journeys"]), + ("Published REST operations", "Published APIs", "other systems can still call this app", + "Compare the API contract before and after, and call every safe read operation once.", ["apis"]), + ("Scheduled events", "Scheduled jobs", "the background jobs still run on the same schedule", + "Compare each job's schedule and on/off state before and after; trigger it once in the test.", []), + ("Pluggable widgets", "Widgets", "every widget still renders", + "Open each page that uses the widget, check for errors and compare the screen before and after.", ["smoke", "screens"]), + ("Java actions", "Java code", "the app's own Java still compiles and runs", + "The build proves it compiles; the journeys that use it prove it runs.", ["build"])] + for key, name, what, how, rungs in TESTS: + if key is None: + if not (scope or jbase or cov): # nothing measured yet: the row stays, grey (a missing file is never a pass) + rows.append(row("t-buttons-and-journeys", name, "", "setup", {"test": rungs_cell(g, rungs)})) + continue + found = (f"{scope.group(1)} buttons in reach of a user" if scope else "") + (f"; {nopath.group(1)} have no click path yet" if nopath else "") + auto = True + sub = f"{scope.group(1)} buttons" if scope else "" + pl = [how] + ([f"{jbase.group(2)} of {jbase.group(1)} journeys already pass on the old app."] if jbase else []) \ + + ([f"{nopath.group(1)} buttons only show up with the right data; the test reaches them through their flow or a person checks them."] if nopath else []) + else: + c = inv.get(key) + if not c or re.match(r"^0\b", c[1]): + continue + found, sub = c[1], c[1].split(" (")[0] + auto = c[3].startswith("auto") + pl = [how] + ([] if auto else [f"Part of it needs a person: {c[3]}."]) + if key == "Role home pages and menu" and shots: + pl.append(f"{shots} screens of the old app are already taken as the reference.") + t = cell("ok", "Found", [f"Found: {found}."], ["test-inventory.md"], d=found[:60]) + p = cell("ok" if auto else "wait", "Automatic" if auto else "Partly by a person", pl, d=how.split(",")[0][:60]) + ct = rungs_cell(g, rungs) if rungs else notyet("No automatic check measures this yet.") + if name == "Buttons and journeys" and cov and ct["s"] != "none": + ct["l"] = f"{cov.group(3)}% of buttons reached" + ct["steps"].insert(0, f"{cov.group(1)} of {cov.group(2)} buttons have a test that clicks them.") + rows.append(row("t-" + re.sub(r"\W+", "-", name.lower()), name, sub, "setup", + {"triage": t, "plan": p, "upgrade": nowork("Tests do not change the app."), "test": ct}, + story={"found": [f"Found: {found}."], "means": [f"The test has to prove that {what}."], "plan": pl})) # Test repairs (self-healing) hr = heal.get("rows", []) @@ -427,7 +554,7 @@ def commit_rows(pred): rows.append(rw) nopen += 1 # an open ledger question shows as a purple cell on the item it belongs to, never as its own row - home = {"journeys": "clicks", "apis": "outside", "release": "release", "smoke": "findings", "screens": "findings"} + home = {"journeys": "t-buttons-and-journeys", "apis": "outside", "release": "release", "smoke": "findings", "screens": "findings"} for it in pending_j: tgt = next((r for r in rows if r["id"] == home.get(it.get("step"), "findings")), None) or rows[0] c = tgt["cells"]["test"] @@ -632,7 +759,7 @@ def done(x): def per(k): u = [x for x in units if x["kind"] == k] return [sum(1 for x in u if done(x)), len(u)] - measured = bool(any(r["id"] == "heal" or r["regression"] for r in rows)) or any(r["cells"]["test"]["s"] != "none" for r in rows if r["id"] == "clicks") + measured = bool(any(r["id"] == "heal" or r["regression"] for r in rows)) or any(r["cells"]["test"]["s"] != "none" for r in rows if r["id"] == "t-buttons-and-journeys") return {"up": [sum(1 for r in up if r["cells"]["upgrade"]["s"] == "ok"), len(up)], "reg": open_reg if open_reg is not None else (sum(1 for r in rows if r["regression"]) if measured else None), "upg": per("upgrade"), "gap": per("testgap"), "own": per("owner")[1], "gol": per("golive")[1], @@ -779,6 +906,8 @@ def step_cells(rec): lines = [x for x in (pr.get("say"), who if state == "assumed" else None) if x] # why, options and approval are in the action below plan = {"approved": lambda: cell("ok", "Approved", lines), "assumed": lambda: cell("you", "Needs your OK", lines), "rejected": lambda: cell("bad", "Rejected", lines)}.get(state, lambda: cell("wait" if pr.get("say") else "none", "Not approved yet" if pr.get("say") else "Not proposed", lines or [gaps.get("proposed") or "Nothing was proposed for this step."]))() + if (rec.get("topic") or "").startswith("person:"): # cold test WM1: these showed as open approvals after the plan was approved + plan = cell("none", "For a person", [pr.get("say") or "A person does this; the toolkit lists it so it is not forgotten."]) # upgrade: the commits that did it n = len(commits) if n: @@ -789,6 +918,8 @@ def step_cells(rec): upgrade = cell("wait", "Not done yet", ["The step is approved, and no change in the app names it yet."]) elif state == "rejected": upgrade = cell("none", "Not needed", ["The step was rejected, so nothing was changed."]) + elif (rec.get("topic") or "").startswith("person:"): + upgrade = cell("none", "Not needed", ["The toolkit changes nothing in the app for this."]) else: upgrade = cell("none", "Not started", ["Nothing changes in the app before the step is approved."]) # test: one line per rung the step closes by; a rung with no passing result is a gap, never a pass @@ -819,6 +950,8 @@ def step_cells(rec): test = cell("none", "Not tied to a check", lines_t) elif any(x[1] == "fail" for x in rows): test = cell("bad", "Failed", lines_t) + elif gapn and (rec.get("topic") or "").startswith("person:"): + test = cell("none", "Later, by a person", lines_t) elif gapn: test = cell("wait", "Test gap", lines_t) elif rows: @@ -845,11 +978,13 @@ def step_row(rec): r = row(rec["id"], rec["title"] or rec["id"], f"{rec['id']} · {rec['topic']}", STEP_GROUP[0], cells, rec["found"].get("evidence") or []) r["trail"] = [act] r["fromSteps"] = True + r["story"] = {"found": [x for x in [rec["found"].get("say")] if x], "means": [], + "plan": [x for x in [rec["proposed"].get("say"), rec["proposed"].get("why")] if x]} if rec.get("phase"): # a two-version upgrade: the version this step belongs to (steps.py UPG_PHASE) r["phase"] = rec["phase"] ap = rec.get("approval") or {} # slice 8: only a step that is still assumed (or has no approval) gets the buttons; an answered step shows who answered - r["ask"] = ap.get("state") in (None, "assumed") + r["ask"] = ap.get("state") in (None, "assumed") and not (rec.get("topic") or "").startswith("person:") if ap.get("state") in ("approved", "rejected"): r["ans"] = {"state": ap["state"], "by": ap.get("by") or "", "day": _answered(ap), "reason": ap.get("reason") or ""} states = [c["s"] for c in cells.values() if c["s"] not in ("ok", "none")] @@ -1063,7 +1198,8 @@ def band(model, rows, chips, g, tl, kpi=None): if model.get("stopped") and st != "ok": # a judgment stop outranks "in progress": nothing moves until a person decides s = model["stopped"] st, head = "bad", f"Stopped at {s['step']}, waiting for you" - sub = f"{len(s['titles'])} {'item' if len(s['titles']) == 1 else 'items'} to decide, then rerun (see the red bar)." + n = len(open_questions(model)) or len(s["titles"]) # the cards in Blocked on you, not the stop file's older list + sub = f"{n} {'item' if n == 1 else 'items'} to decide, then rerun (see the red bar)." closed =closed_items(rows, g, tl) for it in kpi.get("closed") or []: if isinstance(it, dict) and isinstance(it.get("label"), str) and isinstance(it.get("n"), int): @@ -1076,6 +1212,11 @@ def band(model, rows, chips, g, tl, kpi=None): for it in kpi.get("open") or []: if isinstance(it, dict) and isinstance(it.get("text"), str): opens.append({"group": str(it.get("group") or "Coverage"), "text": it["text"], "owner": str(it.get("owner") or "agent")}) + # every Blocked on you card is also in "for you now" (cold test WM1: 1 here, 7 there): add the ones the matrix did not list + for i in open_questions(model): + nm = re.sub(r"\s*\((module|widget|jar|java|check)\)$", "", i.get("title", "")) + if i["kind"] != "signoff" and not any(nm and nm in o["text"] for o in opens if o["owner"] != "agent"): + opens.append({"group": "Your call", "text": question_of(i), "owner": "you"}) order = {"Regression": 0, "To fix": 1, "Your call": 2, "At deploy": 3, "Coverage": 4} opens.sort(key=lambda x: order.get(x["group"], 9)) # stable: same group keeps its order hide = [str(h).lower() for h in (kpi.get("hide") or []) if isinstance(h, str) and h] @@ -1167,6 +1308,8 @@ def band(model, rows, chips, g, tl, kpi=None): .item small{display:block;font-weight:400;color:var(--muted);font-size:12px;font-family:var(--mono)} .cell{padding:8px 10px;min-width:0} .cell.on{background:var(--accentbg)} +.chip small.cd{display:block;font-size:12px;color:var(--muted);font-weight:400;margin-top:2px} +.story{display:grid;gap:10px}.story section{border-left:3px solid var(--line);padding:2px 0 2px 12px}.story p{margin:0 0 4px} .chip{all:unset;box-sizing:border-box;cursor:pointer;display:flex;gap:8px;align-items:flex-start;width:100%;font-size:13px;padding:6px 8px;border-radius:6px;line-height:1.35;overflow-wrap:anywhere} .chip+.chip{margin-top:6px} .chip:hover{filter:brightness(.97)} @@ -1187,6 +1330,7 @@ def band(model, rows, chips, g, tl, kpi=None): .shade{position:fixed;inset:0;background:var(--shade);z-index:20} .drawer{position:fixed;top:0;right:0;bottom:0;width:min(520px,100%);background:var(--panel);border-left:1px solid var(--line);z-index:21;overflow-y:auto; padding:calc(20px + env(safe-area-inset-top,0px)) max(20px,env(safe-area-inset-right,0px)) calc(28px + env(safe-area-inset-bottom,0px)) 22px;display:grid;gap:16px;align-content:start} +details.kpi summary{cursor:pointer;list-style:none}details.kpi summary b{display:block}details.kpi summary span{font-size:13px;color:var(--muted)}.agentnote{margin:14px 2px 0;font-size:13px;color:var(--muted)}.drawer>*,.drawer *{min-width:0;overflow-wrap:anywhere}.drawer pre,.drawer code{white-space:pre-wrap} .drawer .x{all:unset;cursor:pointer;position:absolute;top:calc(14px + env(safe-area-inset-top,0px));right:max(16px,env(safe-area-inset-right,0px));font-size:22px;color:var(--muted);padding:4px 8px;border-radius:6px} .drawer .x:focus-visible{outline:2px solid var(--accent)} .drawer h3{margin:0;font-size:21px;padding-right:36px;text-wrap:balance} @@ -1286,6 +1430,22 @@ def band(model, rows, chips, g, tl, kpi=None): #feed,#feed *{overflow-wrap:anywhere} .nowrun{display:flex;align-items:center;gap:12px;margin:0 0 14px;padding:10px 14px;border-radius:12px;background:var(--panel);border:1px solid var(--wait);font-size:14px} .nowrun[hidden]{display:none} +.blocked{margin:0 0 14px;padding:12px 14px;border-radius:14px;background:var(--youbg);border:2px solid var(--you)} +.blocked h2{margin:0 0 4px;font-size:17px}.blocked p{margin:0 0 8px;font-size:13px;color:var(--muted)} +.blocked ol{margin:0;padding:0;list-style:none}.blocked li{display:flex;gap:8px;align-items:center;justify-content:space-between;padding:8px 0;border-top:1px solid var(--line);font-size:14px} +.resolve{font:inherit;font-size:13px;font-weight:600;white-space:nowrap;padding:6px 10px;border-radius:9px;border:0;background:var(--you);color:#fff;cursor:pointer} +.qdlg{max-width:640px;width:calc(100% - 32px);border:0;border-radius:16px;padding:0;background:var(--panel);color:var(--fg);box-shadow:0 20px 60px rgba(0,0,0,.35)} +.qdlg::backdrop{background:rgba(0,0,0,.4)}.qcard{padding:18px 20px}.qcard.done{opacity:.6} +.qk{font:12px var(--mono);color:var(--muted)}.qcard h3{margin:4px 0 10px;font-size:19px}.qcard p{margin:6px 0;font-size:14px;line-height:1.45} +.qev{margin:10px 0;padding:10px 12px;border:1px solid var(--line,#ddd);border-radius:8px}.qev ul{margin:6px 0 4px 18px;padding:0}.qev li{margin:2px 0}.qev small{opacity:.7} +.qrec{padding:8px 10px;border-radius:10px;background:var(--accentbg)} +.qopts{display:grid;gap:8px;margin-top:12px} +.qopts button{font:inherit;font-size:14px;text-align:left;padding:10px 12px;border-radius:10px;border:1px solid var(--line);background:var(--bg);color:var(--fg);cursor:pointer} +.qopts button.rec{border:2px solid var(--accent);background:var(--accentbg)}.qopts button em{margin-left:8px;font-size:12px;font-style:normal;color:var(--accent);font-weight:600} +.qopts button small{display:block;margin-top:3px;font-size:13px;color:var(--muted)}.qopts button:disabled{cursor:default;opacity:.7} +.qmsg{margin-top:8px;font-size:13px;font-weight:600}.qhint{font-size:12px!important;color:var(--muted)} +.qclose{font:inherit;font-size:13px;margin-top:6px;padding:6px 12px;border-radius:9px;border:1px solid var(--line);background:var(--bg);color:var(--fg);cursor:pointer} +.legend .gbreak{flex-basis:100%;height:0}.legend .gl b{color:var(--fg);font-weight:600} .stopbar{margin:0 0 14px;padding:10px 14px;border-radius:12px;background:var(--badbg);border:1px solid var(--bad);font-size:14px} .stopbar b{color:var(--bad)}.stopbar ul{margin:6px 0;padding-left:20px}.stopbar small{display:block;color:var(--muted,#6c7a87);overflow-wrap:anywhere} .nowrun .pulse{flex:none;width:10px;height:10px;border-radius:50%;background:var(--wait);animation:nrp 1.4s ease-in-out infinite} @@ -1378,7 +1538,7 @@ def band(model, rows, chips, g, tl, kpi=None): if(pf&&it.phase&&it.phase!==pf)return; // a two-version upgrade: rows of the other version are hidden, rows of the whole run stay if(it.group!==g){g=it.group;h+=`
${esc(GROUPS[g])}
`;} h+=`
${esc(it.name)}${esc(it.sub)}${it.phase&&!pf?` · Mendix ${esc(it.phase)}`:""}
`+ - STAGES.map(s=>{const c=it.cells[s.key];return `
`}).join("")+`
`; + STAGES.map(s=>{const c=it.cells[s.key];return `
`}).join("")+``; }); $("#matrix").innerHTML=h; const pb=$("#phases"); if(pb&&M.phases&&M.phases.length>1){const n=v=>ROWS.filter(r=>r.phase===v).length; @@ -1438,6 +1598,9 @@ def band(model, rows, chips, g, tl, kpi=None): return `
${esc(a.name)}${esc(a.sub)}${esc(t.say||"")}${t.gap?' not linked':""}${TLBL[a.status]||""}
${b}${pf}
`;}).join(""); return `

${TSAY[k]}, per action (${it.trail.length})

Open an action for the reasons and the result. The proof is the record from the run itself.

${acts}
`; } +const STORY=[["found","What we found"],["means","What it means for this app"],["plan","What the plan does, and why"]]; +const storyBox=it=>{const s=it.story||{};const b=STORY.filter(([k])=>(s[k]||[]).length); + return b.length?`
${b.map(([k,h])=>`

${h}

${s[k].map(t=>`

${esc(t)}

`).join("")}
`).join("")}
`:"";}; const planBtn=i=>(M.plan||[]).length?``:""; function topicPlan(it){ const acts=(it.trail||[]).map(a=>{const t=a.stages.plan||{}; @@ -1474,7 +1637,8 @@ def band(model, rows, chips, g, tl, kpi=None): function propBox(it){ const sign=(M.plan||[]).find(s=>/sign-off/i.test(s.h)), st=sign?sign.intro.filter(t=>/^(Status:|Signed-off:)/.test(t)&&!t.includes("<")).pop()||"":""; const signed=/^Signed-off:/.test(st); - const hits=it.id==="runtime"?(M.plan||[]).filter(s=>/summary/i.test(s.h)).flatMap(s=>s.items.map(x=>[s.h,x])):planFor(it); + const hits=it.id==="runtime"?(M.plan||[]).filter(s=>/summary/i.test(s.h)).flatMap(s=>s.items.map(x=>[s.h,x])) + :it.id==="modules"?(M.plan||[]).flatMap(s=>s.items.filter(x=>/^module update /.test(x.text||"")).map(x=>[s.h,x])):planFor(it); const rows=hits.map(([h,x])=>{const g=Object.fromEntries(x.subs.filter(y=>y[0]));const asm=/ASSUMED/.test(g.answer||""); return `
  • ${esc(h)}${x.id?" · "+esc(x.id):""}
    ${esc(x.name||x.text)}${g.recommendation?`Recommended: ${esc(g.recommendation)}`:""}${g.answer?`Answer: ${esc(g.answer.replace(/\s*\(ASSUMED[^)]*\)\)?$/,""))}${asm?' · assumed by the agent':""}`:""}
  • `;}).join(""); const who=signed?`Approved: ${esc(st.replace(/^Signed-off:\s*/,""))}`:`Not approved yet. The plan's sign-off line says "${esc(st||"no status")}".`; @@ -1488,13 +1652,14 @@ def band(model, rows, chips, g, tl, kpi=None): open(`${navBar(esc(it.name)+" › "+esc(s.name))}
    ${tabs}

    ${esc(c.l)}

    + ${storyBox(it)} ${k==="plan"&&!it.fromSteps?propBox(it)+planBtn(i):""} ${k==="plan"&&it.fromSteps&&it.ask?`
    Needs your OK. The agent chose this on its own while it worked. Nothing is final until a person approves it.
    `:""} ${k==="plan"&&it.fromSteps&&it.ans?`
    ${it.ans.state==="approved"?"Approved":"Rejected"} by ${esc(who(it.ans.by))}${it.ans.day?" on "+esc(it.ans.day):""}.${it.ans.reason?" Reason: "+esc(it.ans.reason):""}
    `:""} -

    What happens, step by step

    ${ol(c.steps.length?c.steps:["Nothing recorded for this cell."])}
    +

    This stage, step by step

    ${ol(c.steps.length?c.steps:["Nothing recorded for this cell."])}
    ${trailBox(it,k)} ${decBox(di)} - ${it.trail&&it.trail.length?"":fixBox(it.fix,NONGREEN.includes(c.s)&&!di)} + ${it.trail&&it.trail.length||(it.story&&(it.story.plan||[]).length&&!it.fix)?"":fixBox(it.fix,NONGREEN.includes(c.s)&&!di)} ${c.late?`
    For the toolkit: triage did not flag this item. The new triage checks it from the start.
    `:""}
    ${techBox((it.tech||[]).concat(c.tech||[]))}`); @@ -1561,14 +1726,20 @@ def band(model, rows, chips, g, tl, kpi=None): const el=document.getElementById("sc-"+it.id);if(!el)return; const d=decided[it.id]; if(d&&(d.choice==="approve"||d.choice==="reject")){el.innerHTML=`
    ${d.choice==="reject"?"Rejected":"Approved"} by ${esc(who(d.by))}${d.at?" on "+esc(d.at.slice(0,10)):""}${d.reason?". Reason: "+esc(d.reason):""}. Waiting for the pipeline.
    `;return} - const can=!!(db&&user&&me); + const can=!!(window.UPG_LIVE||(db&&user&&me)); // the live cockpit saves through its own server (cockpitserve.py /decide) el.innerHTML=`
    ${can?"":"Sign in to this page to answer."}
    `; } async function decideStep(btn){ const box=btn.closest(".stepask");const msg=box.querySelector("[data-msg]");box.querySelectorAll("button").forEach(b=>b.disabled=true); try{ + const why=box.querySelector(".reason").value.trim(); + if(window.UPG_LIVE){ // cold test WM1: the step buttons only knew the hosted page's database and stayed grey in the live cockpit + const r=await fetch("decide",{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(Object.assign({item:btn.dataset.sid,choice:btn.dataset.choice},why?{reason:why}:{}))}); + const t=await r.text();if(!r.ok)throw new Error(t||r.status); + msg.textContent=t+" The agent picks it up within 15 seconds."; + setTimeout(()=>location.reload(),1200);return;} // the server rerendered the page before it answered; the item leaves Blocked on you const row={item:btn.dataset.sid,choice:btn.dataset.choice,by:await user.id(),at:new Date().toISOString(),source:"page"}; - const why=box.querySelector(".reason").value.trim();if(why)row.reason=why; + if(why)row.reason=why; await db.collection("decisions").doc(row.item).set(row); msg.textContent="Saved."; }catch(e){box.querySelectorAll("button").forEach(b=>b.disabled=false);msg.textContent="Could not save: "+(e&&e.message?e.message:e)} @@ -1651,13 +1822,27 @@ def h_band(b): bad = b["state"] == "ok" and b["reg"] reg = f'{E(b["sub"])}' chips = "".join(f'{c["n"]}{E(c["label"])}' for c in b["closed"]["chips"]) - rows = "".join(f'
    {E(o["group"])}{E(o["text"])}{E(o["owner"])}
    ' for o in b["open"]["items"]) + # who closes each open topic, and when (cold test WM1: "16 open topics still to validate" and the owner asked "what should a + # person do with this? validate when, by who?"; all 16 were the agent's, closed later in the run) + WHEN = {"Regression": "now", "To fix": "during the upgrade", "Coverage": "at the test stage", "At deploy": "at deploy"} + its = b["open"]["items"] + mine = [o for o in its if o["owner"] != "agent" and o["group"] != "At deploy"] + later = [o for o in its if o["group"] == "At deploy"] + agent = [o for o in its if o not in mine and o not in later] + + def lst(xs, who): + return "".join(f'
    {E(o["group"])}{E(o["text"])}{E(who or WHEN.get(o["group"], ""))}
    ' for o in xs) + you = (f'
    {len(mine)}for you now
    ' if mine + else '
    0for you now: nothing to do, the agent goes on
    ') + agentbox = (f'
    {len(agent)}the agent closes these itself, later in the run (open to see when)' + f'
    ' if agent else "") + laterbox = (f'
    {len(later)}for a person at deploy, not now' + f'
    ' if later else "") closed = (f'
    {b["closed"]["n"]}items closed successfully
    {chips}
    ' if b["closed"]["n"] else "") return (f'
    \n' f'
    {tick}
    {E(b["head"])}{reg}
    \n' - f'
    {closed}
    {b["open"]["n"]}open topics still to validate' - f'
    ') + f'
    {you}{closed}{agentbox}{laterbox}
    ') def h_tiles(M): @@ -1710,9 +1895,16 @@ def fmt(t): return "".join(f'
  • {E(f["text"])}
  • ' for f in M["feed"]) or "
  • Nothing recorded yet.
  • " +GREY = [("Not yet", "the run has not reached this stage for the item"), ("Not started", "planned, waits for the plan approval"), + ("Not needed", "this stage has nothing to do for the item")] + + def h_legend(M): used = {c["s"] for r in M["rows"] for c in r["cells"].values()} | {c["s"] for c in M["ready"]} - return "".join(f'{t}' for k, t in LEGEND if k in used) + labels = {c["l"] for r in M["rows"] for c in r["cells"].values() if c["s"] == "none"} + grey = "".join(f'{l}: {t}' for l, t in GREY if l in labels) + return ("".join(f'{t}' for k, t in LEGEND if k in used) + + (f'Grey cells:{grey}' if grey else "")) def h_running(r): @@ -1730,10 +1922,115 @@ def h_stopped(s): """A judgment stop (NEEDS-JUDGMENT.md): a red bar under the header, so a run that waits for a person says so first.""" if not s: return "" + if str(s["code"]) in ("14", "15"): # the plan waits for sign-off: the Decide now cards are the questions, no paths or commands + return ('') li = "".join(f"
  • {E(t)}
  • " for t in s["titles"]) cmd = f'Then rerun: {E(s["rerun"])}' if s["rerun"] else "" return (f'') + f'{"
      " + li + "
    " if li else ""}Answer under Blocked on you on the right. Details: NEEDS-JUDGMENT.md in the work folder.{cmd}') + + +def open_questions(model): + """The items a person still has to answer, one question card each: plan choices and judgments first, the sign-off last.""" + qs = [i for i in model["items"] if i.get("options") and not i.get("decided") and i["kind"] != "confirmgroup"] + return sorted(qs, key=lambda i: i["kind"] == "signoff") + + +def question_of(i): + if i["title"].endswith("(check)"): # a triage check that did not run (proposal.py, release notes) + return f"The {i['title'][:-8].strip().lower()} check did not run. How do we go on?" + t = re.sub(r"\s*\((module|widget|jar|java)\)$", "", i["title"]) + return {"plan-choice": f"What should the upgrade do with {t}?", "signoff": "Approve the plan and start the upgrade?", + "confirm": f"Accept this change: {t}?"}.get(i["kind"], t) + + +def plain(t): + """An option or recommendation without the file names and rule names behind it (they stay in the technical details).""" + t = re.sub(r"\s*\((kept list|replacements\.tsv row|[a-z0-9_./ ]+\.(?:json|txt|tsv|md))\)", "", t or "") + t = re.sub(r"\beffort ([SML])\b", lambda m: {"S": "small", "M": "medium", "L": "large"}[m.group(1)] + " effort", t) + t = t.replace("a RED edit", "the app's own edit").replace("RED local edit", "the app's own edit") + return "; ".join(x.strip() for x in t.split(";") if x.strip() and "stop rule" not in x).strip() + + +# what an option of a module plan choice does, in words, by the phrase the proposal uses for it +OPTION_SAYS = [("update to the latest", "Take the newest Marketplace version. The pipeline carries the app's edits over: diagram-only changes are dropped, pages, layouts and constants are replayed, and data the vendor drops is checked for use first."), + ("update, move", "Take the newest version after a person moves the app's own logic into a module of its own (_Ext) that works with the Marketplace module."), + ("keep old", "The module stays as it is now. It misses the Marketplace fixes and may not run on the next Mendix version: only with a reason and an end date."), + ("replace", "Take the new Marketplace version. The app's own additions to the module are removed, and their data with them unless it is moved first."), + ("rebuild", "Move the app's own additions into a module of its own, then take the clean Marketplace version. Most work, cleanest result."), + ("hand to", "Park this one. The run waits until the named person answers.")] + + +def context_of(i): + """Two or three plain sentences: what the item is and why it is asked.""" + if i["kind"] == "signoff": + return ["The run has prepared the upgrade and changed nothing in the app yet.", + "Approving starts the upgrade with your answers to the other questions, and with what the toolkit does on its own: the " + "Marketplace module updates, the conversion to the new version and the stand-ins for outside systems (all listed in the matrix).", + "Every change is a Git commit, so you can review or revert each one."] + if i["kind"] == "plan-choice" and (i.get("problem") or "").startswith("the release-note check did not run"): + return ["Before the plan, the toolkit reads what each Mendix version on the route changes (the release notes) and checks this app against it.", + "That check did not run" + (": the company brain's version tables are not on this machine" if "version tables" in i["problem"] else "") + + ". So nobody has checked yet what the new Mendix versions change for this app."] + if i["kind"] == "plan-choice": + ent = [x.split(".")[-1] for x in re.findall(r"ADDED entity ([\w.]+)", i.get("problem") or "")] + more = re.search(r"\(\+(\d+)\)", i.get("problem") or "") + if ent: + n = len(ent) + (int(more.group(1)) if more else 0) + name = re.sub(r"\s*\(.*\)$", "", i["title"]) + return [f"{name} is a Marketplace module, but this app changed it: it added {n} entities of its own ({', '.join(ent)}{', ...' if more else ''}).", + "A Marketplace update overwrites the whole module, so those additions, and the data stored in them, would be lost."] + return [plain(i.get("problem") or "")] if i.get("problem") else [] + + +def h_questions(model, matrix=None): + """The open questions: a "Blocked on you" list in the sidebar, each with a Resolve button that opens one question in a dialog + with its context, one button per option and what each option does. The buttons post to the local server (cockpitserve.py + /decide, live page only); on a static copy they stay off and the answer goes through chat.""" + qs = open_questions(model) + # a choice the agent made on its own (a step with an assumed approval) blocks too; Resolve opens its plan cell, where its buttons are + asks = [(n, r) for n, r in enumerate((matrix or {}).get("rows", [])) if r.get("fromSteps") and r.get("ask") and r["cells"]["plan"]["s"] == "you"] + if not qs and not asks: + return "", "" + rows, dlgs = [], [] + for n, r in asks: + rows.append(f'
  • The agent chose this on its own: {E(r["name"])}. OK?' + f'
  • ') + for n, i in enumerate(qs, 1): + rows.append(f'
  • {E(question_of(i))}
  • ') + btns = [] + for o in i["options"]: + lab = plain(o["label"]) or o["label"] + say = o.get("causes") or next((t for k, t in OPTION_SAYS if k in lab.lower()), "") if i["kind"] == "plan-choice" else "" + if not say and ": " in lab: + lab, say = lab.split(": ", 1) + say = say[:1].upper() + say[1:] + "." + if i["kind"] == "signoff": + lab = "Approve and continue" + rec = o["key"] == i.get("rec") and len(i["options"]) > 1 + btns.append(f'') + ctx = "".join(f"

    {E(t)}

    " for t in context_of(i) if t) + # what exactly is behind it, from the run's own files (cold test WM1: "how can a person ever decide without those details?") + ev = i.get("evidence") or {} + if ev.get("what"): + ctx += ('
    What exactly
      ' + "".join(f"
    • {E(w)}
    • " for w in ev["what"][:15]) + + (f"
    • and {len(ev['what']) - 15} more
    • " if len(ev["what"]) > 15 else "") + "
    " + + f'From: {E(", ".join(ev.get("files") or []))}
    ') + why = plain(i.get("rec_text") or "") + rl = plain(next((o["label"] for o in i["options"] if o["key"] == i.get("rec")), "")) + if rl and why.startswith(rl) and why[len(rl):].startswith(";"): # the option itself is the button; keep effort and risk + why = why[len(rl) + 1:].strip() + why = why[:1].upper() + why[1:] + rec = f'

    Why we recommend it: {E(why)}

    ' if why and len(i["options"]) > 1 else "" + dlgs.append(f'
    Question {n} of {len(qs)} · ' + f'blocks step {E(i.get("step") or "")}

    {E(question_of(i))}

    {ctx}{rec}
    {btns and "".join(btns)}
    ' + '

    The buttons work in the live cockpit. On a copy, answer in chat.

    ' + '
    ') + side = (f'

    Blocked on you: {len(qs) + len(asks)}

    ' + '

    The run waits for these answers. Click Resolve for the full question.

      ' + "".join(rows) + "
    ") + return side, "".join(dlgs) def render(model, matrix): @@ -1741,6 +2038,7 @@ def render(model, matrix): data = data.replace("{e(model['title'])}\n" '\n' '\n' @@ -1762,7 +2060,10 @@ def render(model, matrix): f" {h_health(matrix.get('health'))}\n" '
    \n

    Since you last looked

    \n' f'
      {h_feed(matrix)}
    \n
    \n \n' - f' \n \n\n\n' + f' \n \n\n{qdlgs}\n' '\n' '\n\n' diff --git a/upgrade-toolkit/bin/cockpitpage.py b/upgrade-toolkit/bin/cockpitpage.py index e29bb34..536c20c 100644 --- a/upgrade-toolkit/bin/cockpitpage.py +++ b/upgrade-toolkit/bin/cockpitpage.py @@ -24,6 +24,8 @@ Exit: 0 written, 2 usage. """ import argparse, datetime, glob, html, json, os, re, subprocess, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import evidence # noqa: E402 (what exactly sits behind a plan item) PHASES = [("find", "Find", ("fetch", "triage")), ("plan", "Plan", ("propose",)), ("upgrade", "Upgrade", ("baseline", "convert", "update", "build")), @@ -447,7 +449,9 @@ def build_model(work): nj = rd(os.path.join(work, "NEEDS-JUDGMENT.md")) or "" m = re.match(r"# NEEDS JUDGMENT: step (\w+), exit (\d+)", nj) stopped = (m.group(1), int(m.group(2))) if m else None - done = {s: bool(rd(os.path.join(work, "state", s + ".done"))) for s in STEP_ORDER} + if stopped and running(work): # the run went on after that stop (cold test WM1: red "waits for you" bar after the approval) + stopped = None + done ={s: bool(rd(os.path.join(work, "state", s + ".done"))) for s in STEP_ORDER} per = run_minutes(work) commits = app_commits(work) @@ -467,6 +471,10 @@ def build_model(work): def add(it): it["decided"] = last.get(it["id"]) + # an approval counts only for the plan it was given on (cold test WM1: the plan changed after the approval, the + # pipeline asked again, but the page still hid the button) + if it["kind"] == "signoff" and it["decided"] and it["decided"].get("plan_hash") not in (None, it.get("plan_hash")): + it["decided"] = None items.append(it) for e in rgroups: @@ -505,7 +513,8 @@ def add(it): rerun = (re.search(r"then rerun:\n\n {4}(.+)", nj) or [None, ""])[1] secs = re.findall(r"^## (.+?) \(exit (\d+)\)\n\n(.*?)(?=^## |\Z)", nj, re.M | re.S) stopinfo = {"step": stopped[0], "code": stopped[1], "titles": [s[0] for s in secs], "rerun": rerun} - if stopped and not any(j.get("step") == stopped[0] for j in djo): + # exit 14 / 15 (the plan awaits sign-off, or is paused): the sign-off and the P-n items are the questions, no stop rows + if stopped and stopped[1] not in (14, 15) and not any(j.get("step") == stopped[0] for j in djo): for k, (title, code, body) in enumerate(secs, 1): add({"id": f"stop:{stopped[0]}:{k}", "kind": "judgment", "dot": "bad", "title": title, "step": stopped[0], "meta": f"The run stopped at {stopped[0]} (exit {code}) and waits for you", @@ -553,24 +562,26 @@ def add(it): rec_txt = it["sub"].get("recommendation", "") opts = [{"key": "rec", "label": short(rec_txt.split(";")[0] if rec_txt else "Take the recommendation", 110), "causes": ""}] for n, o in enumerate(opts_txt, 1): - opts.append({"key": f"o{n}", "label": short(o, 110), "causes": ""}) + if short(o, 110) != opts[0]["label"]: # the recommended option is already the first one (WM1: "keep old version" twice) + opts.append({"key": f"o{n}", "label": short(o, 110), "causes": ""}) for o in opts: o["writes"] = [{"decisions_md": {"id": pid, "name": it["name"], "kind": it["kind"], "option": o["label"]}}] is_assumed = bool(row and "ASSUMED" in (row[5] if len(row) > 5 else "")) or not row if is_assumed and not last.get(pid): assumed.append(pid) items.append({"id": pid, "kind": "plan-choice", "hidden": True, "dot": "warn", "title": f"{it['name']} ({it['kind']})", "step": "propose", - "meta": "Plan choice", "problem": short(it["sub"].get("evidence", ""), 260), "rec": "rec", "rec_text": short(rec_txt, 200), - "options": opts, "decided": last.get(pid), "assumed": is_assumed, "effort": effort_of(rec_txt)}) + "meta": "Plan choice", "problem": short(it["sub"].get("evidence", ""), 260), "rec": "rec", "rec_text": short(rec_txt, 320), + "options": opts, "decided": last.get(pid), "assumed": is_assumed, "effort": effort_of(rec_txt), + "evidence": evidence.for_item(work, it["name"])}) plan_signed = bool(signed) if P["present"] and not plan_signed: # Source of the count: every decision the agent recorded in AGENT-DECISIONS.md (len(AD), one per Dn) plus the plan choices # in PROPOSAL.md / DECISIONS.md that nobody answered (len(assumed)). n = len(AD) + len(assumed) if recs is None else sum(1 for r in recs if (r.get("approval") or {}).get("state") == "assumed") add({"id": "signoff", "kind": "signoff", "dot": "warn", "title": "Sign the plan", - "meta": f"{pl(n, 'choice')} {'was' if n == 1 else 'were'} assumed by the agent" if n else "Nothing goes live without your name", - "step": "propose", "problem": "Nothing goes live without your name on the plan. Review it under Plan. To change one choice first, say so in chat (decisions.py pending lists them).", - "rec": "sign", "rec_text": "Read the plan, then sign.", "options": [{"key": "sign", "label": "Sign the plan"}], "plan_hash": P["hash"]}) + "meta": f"{pl(n, 'choice')} {'was' if n == 1 else 'were'} assumed by the agent" if n else "Nothing changes in the app before you approve", + "step": "propose", "problem": "Nothing changes in the app before you approve the plan. Review it under Plan. To change one choice first, answer its question or say so in chat (decisions.py pending lists them).", + "rec": "sign", "rec_text": "", "options": [{"key": "sign", "label": "Approve and continue"}], "plan_hash": P["hash"]}) visible = [i for i in items if not i.get("hidden")] open_visible = [i for i in visible if not (i.get("decided") and i["kind"] != "signoff") and i.get("open", 1)] order = {"bad": 0, "warn": 1} @@ -769,7 +780,7 @@ def phase_time(steps): # --live: poll the server's status (the "Running now" bar, in place) and stamp, reload on change and keep the scroll place; an open drawer holds the reload until it closes -LIVE_JS = """ """ diff --git a/upgrade-toolkit/bin/cockpitserve.py b/upgrade-toolkit/bin/cockpitserve.py index 116a102..b571627 100644 --- a/upgrade-toolkit/bin/cockpitserve.py +++ b/upgrade-toolkit/bin/cockpitserve.py @@ -10,15 +10,58 @@ /stamp the page's change stamp (mtime in ns); the page polls it every few seconds and reloads when it moves. Every /stamp request also touches D/.cockpit-seen: the pipeline reads that heartbeat to know a tab is open, and opens the page again at the next step when no tab has asked for a while (cockpit_live in upgrade-pipeline.sh). + POST /decide one answer from a question dialog: {"item", "choice", "by"?, "reason"?}, or {"item": "signoff"}. Without "by" + the answer carries the machine's Git name (git config user.name): the dev work is tracked in Git anyway. + It goes through decisions.py import (source page; a sign-off carries the current plan hash) and apply, into the + same decisions.json the pipeline reads, then the page is written again and D/.cockpit-decided is touched, so + the session that drives the run sees the answer within seconds. JSON only (a cross-site form cannot send it). Writes D/.cockpit-port and D/.cockpit-pid at start. Exits on its own after --idle-exit seconds without a request (default 6 h). Exit: 0 idle exit, 2 usage. """ -import argparse, http.server, json, os, sys, threading, time +import argparse, http.server, json, os, subprocess, sys, tempfile, threading, time sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import cockpitpage # noqa: E402 VIEW = "COCKPIT-view.html" +BIN = os.path.dirname(os.path.abspath(__file__)) + + +def git_name(): + r = subprocess.run(["git", "config", "user.name"], capture_output=True, text=True) + return r.stdout.strip() if r.returncode == 0 else "" + + +def decide(d, work, body): + """One answer from the page into the run's decision ledger; returns (http status, message).""" + item, by = str(body.get("item") or ""), " ".join(str(body.get("by") or git_name()).split())[:80] + if not item or not by: + return 400, "an answer needs the item and a name (no git config user.name on this machine)" + if item == "signoff": + data = {"signoffs": [{"by": by, "plan": cockpitpage.build_model(work)["planHash"], "source": "page"}]} + else: + row = {"item": item, "choice": str(body.get("choice") or ""), "by": by} + if body.get("reason"): + row["reason"] = str(body["reason"])[:200] + data = {"decisions": [row]} + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f: + json.dump(data, f) + try: + imp = subprocess.run([sys.executable, os.path.join(BIN, "decisions.py"), "import", "--work", work, "--json", f.name], + capture_output=True, text=True) + finally: + os.unlink(f.name) + said = (imp.stdout + imp.stderr).strip() + if imp.returncode != 0 or "imported 0 row" in said: + return 422, said or "not saved" + subprocess.run([sys.executable, os.path.join(BIN, "decisions.py"), "apply", "--work", work], capture_output=True) + new = os.path.join(d, ".COCKPIT-view.new") + if subprocess.run([sys.executable, os.path.join(BIN, "cockpitpage.py"), "--work", work, "--out", new, "--standalone", "--live"], + capture_output=True).returncode == 0 and os.path.exists(new): + os.replace(new, os.path.join(d, VIEW)) + with open(os.path.join(d, ".cockpit-decided"), "a") as f: + f.write(json.dumps({"item": item, "choice": body.get("choice") or "sign", "by": by, "at": time.time()}) + "\n") + return 200, f"Saved: {item} by {by}" def main(): @@ -61,6 +104,24 @@ def do_GET(self): self.end_headers() self.wfile.write(body) + def do_POST(self): + last[0] = time.time() + if self.path.split("?")[0] != "/decide" or "application/json" not in (self.headers.get("Content-Type") or ""): + self.send_error(404) + return + try: + body = json.loads(self.rfile.read(min(int(self.headers.get("Content-Length") or 0), 65536)) or b"{}") + except ValueError: + body = None + work = (cockpitpage.rd(os.path.join(d, ".cockpit-work")) or "").strip() or d + code, msg = (400, "not JSON") if not isinstance(body, dict) else decide(d, work, body) + out = json.dumps({"ok": code == 200, "said": msg}).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(out))) + self.end_headers() + self.wfile.write(out) + def log_message(self, *args): pass diff --git a/upgrade-toolkit/bin/dockerhost.sh b/upgrade-toolkit/bin/dockerhost.sh new file mode 100755 index 0000000..64c4bd9 --- /dev/null +++ b/upgrade-toolkit/bin/dockerhost.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# dockerhost.sh: print the DOCKER_HOST the amd64 tools must use, or explain why there is none (exit 1). +# Apple Silicon only needs it: the Linux mxcli and mxbuild are amd64 and must run through Rosetta. Under qemu (a colima VM +# without Rosetta) the Go mxcli crashes at random and misreads files: cold test WM1 rung 1 had all 10 module updates fail +# with exit 2 ("growslice: len out of range", "zip: checksum error" on a widget that is fine on the host), and stepupdate +# recorded every one as "kept". The doctor knew (doctor.sh, rosetta), but nothing in the run checked it. +# Order: a DOCKER_HOST already set to a Rosetta VM; else the current docker context when it is one; else the first +# running colima profile with rosetta: true. Not Apple Silicon: prints the current DOCKER_HOST (may be empty), exit 0. +[ "$(uname -sm)" = "Darwin arm64" ] || { echo "${DOCKER_HOST:-}"; exit 0; } + +rosetta_sock() { # $1 a unix socket path: is it a colima VM with Rosetta? + case $1 in */.colima/*/docker.sock) grep -q '^rosetta: true' "$(dirname "$1")/colima.yaml" 2> /dev/null ;; *) return 1 ;; esac +} +sock=${DOCKER_HOST#unix://} +[ -n "${DOCKER_HOST:-}" ] || sock=$(docker context inspect --format '{{.Endpoints.docker.Host}}' 2> /dev/null) +sock=${sock#unix://} +[ -L "$sock" ] && sock=$(python3 -c 'import os,sys;print(os.path.realpath(sys.argv[1]))' "$sock") +if rosetta_sock "$sock"; then echo "unix://$sock"; exit 0; fi +# the login home, not $HOME: a run with its own HOME (the cold test) still uses the machine's colima +for home in "$HOME" "$(python3 -c 'import os,pwd;print(pwd.getpwuid(os.getuid()).pw_dir)')"; do + for y in "$home"/.colima/*/colima.yaml; do + s=$(dirname "$y")/docker.sock + [ -S "$s" ] && rosetta_sock "$s" && { echo "unix://$s"; exit 0; } + done +done +echo "dockerhost: no running container VM with Rosetta (current: ${sock:-none}). The amd64 mxcli crashes under qemu." \ + "Start one: colima start -p upgrade --vm-type vz --vz-rosetta" >&2 +exit 1 diff --git a/upgrade-toolkit/bin/editroute.py b/upgrade-toolkit/bin/editroute.py new file mode 100755 index 0000000..837f328 --- /dev/null +++ b/upgrade-toolkit/bin/editroute.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""editroute.py: route each module the update refused to a way of updating it anyway, instead of keeping it at the old version. + +Why (cold test WM1, 2026-11-16): seven Marketplace modules were "kept" because "local edits refuse the update", and the cockpit +recommended "keep old version" for all seven. The owner: an edit does not mean the module cannot be updated; most edits can be +carried over, and an old module misses fixes and may not run on the next Mendix version. Looking closer, Email_Connector's one +"changed" microflow differed only in where an arrow attached; Encryption's edit was its key constant; MendixSSO's update drops an +attribute nothing in the app reads. None of that is a reason to stay behind. + +Routes, per refused module (the module is updated in all of them except HOLD): + COSMETIC every refused microflow/nanoflow/rule equals the installed package once diagram geometry (@position, @anchor, @curve, + @size) is ignored: forced, nothing to replay + REPLAY pages, layouts, snippets, images and constants: saved as MDL by the forced update (--save-edits) and replayed onto the + new version (replay/.mdl); mixed with COSMETIC logic, only the UI elements are replayed + UNREAD mxcli could not read some elements and localedits shows no domain or logic edit: forced, whatever it could save replayed + FILES theme/JavaScript files could not be compared (installed package unreadable): the saved copies under + .stepupdate/saved// stay, the update may commit (accept-unsaved.txt) + VENDOR the new version drops an attribute or entity of its own module and nothing outside the module refers to it + (mxcli refs): accepted in approved.txt with that reason + XASSOC the update drops associations ANOTHER module owns into this one (they hold app data): forced with --save-edits, each + association replayed and its ORIGINAL GUID restored (replay/.guids), the step-only removal in approved-step.txt + HOLD a real logic edit, Java user code, a GUID change, a removal something still uses, a tool or network fault: it stays a + card for a person, with the evidence. The recommendation there is still to update (move the edit to _Ext). + + editroute.py plan --work W --app A --original W/original.mpr --plan W/plan.txt writes the route files, prints ROUTE lines + editroute.py collect --work W after the routed update: builds replay/ +Files: W/edit-routes.json (one row per module), W/route-plan.txt (the plan lines of the routed modules, plan order), and +appends to W/force-ui.txt, W/accept-unsaved.txt, W/approved.txt, W/approved-step.txt. Exit 0 = something routed, 1 = nothing. +""" +import argparse, glob, json, os, re, subprocess, sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import localedits # noqa: E402 (MXCLI, CACHE, fetch_pkg) + +UI_KINDS = {"PAGE", "LAYOUT", "SNIPPET", "IMAGE", "IMAGECOLLECTION", "CONSTANT"} +LOGIC_KINDS = {"MICROFLOW": "microflow", "NANOFLOW": "nanoflow", "RULE": "rule"} +GEOM = re.compile(r"^\s*@(position|anchor|curve|size)\b") +NOISE = re.compile(r"^(✓|Building|Loading|Connected|WARNING)") + + +def rd(p): + try: + with open(p, encoding="utf-8", errors="replace") as f: + return f.read() + except OSError: + return "" + + +def mx(args): + r = subprocess.run([localedits.MXCLI] + args, capture_output=True, text=True) + return r.returncode, "\n".join(ln for ln in (r.stdout or "").splitlines() if not NOISE.match(ln)) + + +def describe(mpr, kind, qn): + rc, out = mx(["describe", "-p", mpr, kind, qn]) + return out if rc == 0 and out.strip() else None + + +def logic_body(text): + return [ln.rstrip() for ln in text.splitlines() if not GEOM.match(ln)] + + +def refusal(log): + """(rows [(KIND, name)], unread count) of the last refusal in an update log.""" + t = rd(log) + rows = [] + for m in re.finditer(r"refusing to update:[^\n]*\n((?:\s{2,}\S[^\n]*\n)+)", t): + rows = [tuple(ln.split(None, 1)) for ln in m.group(1).splitlines() if len(ln.split(None, 1)) == 2 and not ln.strip().startswith("-")] + u = re.findall(r"(\d+) element\(s\) could not be read", t) + return [(k, n.strip()) for k, n in rows], int(u[-1]) if u else 0 + + +def red_rows(idc): + return [m.groups() for m in re.finditer(r"^RED\s+(\w+)\s+(\w+)\s+([\w.]+)", rd(idc), re.M)] + + +def unused(original, kind, qn): + rc, out = mx(["refs", "-p", original, qn]) + return rc == 0 and "no references found" in out + + +def route_module(mod, why, S, ledit, original, cache, mpr): + """-> (route, reason, extra) for one kept module.""" + logs = sorted(glob.glob(os.path.join(S, f"step*-{mod}.log")), key=lambda p: int(re.search(r"step(\d+)-", p).group(1))) + log = logs[-1] if logs else "" + if re.search(r"NETWORK|update failed|Java user code|javasource", why): + return "HOLD", why, {} + if "idcheck REVIEW" in why: + idc = log[:-4] + ".idcheck" if log else "" + rows, own, xassoc, bad = red_rows(idc), [], [], [] + for finding, kind, qn in rows: + if finding == "REMOVED" and kind in ("attribute", "entity") and qn.split(".")[0] == mod: + (own if unused(original, kind, qn) else bad).append(qn) + elif finding == "REMOVED" and kind == "association" and qn.split(".")[0] != mod: + xassoc.append(qn) + else: + bad.append(f"{finding} {kind} {qn}") + if bad or not rows: + return "HOLD", "idcheck finding still used or not a removal: " + ", ".join(bad or ["no RED rows read"]), {} + if xassoc: + return "XASSOC", f"the update drops {len(xassoc)} association(s) other modules own into {mod}; replayed with their original ids", \ + {"assocs": xassoc, "vendor": own} + return "VENDOR", f"the new version drops {', '.join(own)}; nothing outside {mod} refers to it", {"vendor": own} + if "FILES-UNKNOWN" in why: + return "FILES", f"files could not be compared with the installed package; the copies are kept under .stepupdate/saved/{mod}", {} + rows, unread = refusal(log) + if unread and not rows: + e = ledit or {} + if e and not e.get("domain") and not e.get("logic"): + return "UNREAD", f"{unread} elements unreadable by mxcli; localedits shows no domain or logic edit", {} + return "HOLD", f"{unread} elements unreadable and localedits shows domain or logic edits", {} + if not rows: + return "HOLD", why, {} + pkg = None + real, cosmetic, ui = [], [], [] + for kind, name in rows: + if kind in UI_KINDS: + ui.append(f"{kind.lower()} {name}") + elif kind in LOGIC_KINDS: + if pkg is None and ledit and ledit.get("cid") and ledit.get("version"): + pkg, _, _ = localedits.fetch_pkg(ledit["cid"], ledit["version"], cache) + a = describe(mpr, LOGIC_KINDS[kind], f"{mod}.{name}") + b = describe(pkg, LOGIC_KINDS[kind], f"{mod}.{name}") if pkg else None + (cosmetic if a and b and logic_body(a) == logic_body(b) else real).append(f"{kind.lower()} {name}") + else: + real.append(f"{kind.lower()} {name}") + if real: + return "HOLD", "real logic edit: " + ", ".join(real) + f" (move it to {mod}_Ext, then update)", {"cosmetic": cosmetic, "ui": ui} + if ui: + return "REPLAY", f"{len(ui)} UI element(s)/constant(s) saved and replayed onto the new version" + \ + (f"; {len(cosmetic)} diagram-only change(s) dropped" if cosmetic else ""), {"ui": ui, "cosmetic": cosmetic} + return "COSMETIC", "only diagram layout differs (" + ", ".join(cosmetic) + "); the logic equals the Marketplace version", {"cosmetic": cosmetic} + + +def append(path, lines): + have = set(rd(path).splitlines()) + new = [ln for ln in lines if ln not in have] + if new: + with open(path, "a") as f: + f.write("".join(ln + "\n" for ln in new)) + + +def cmd_plan(a): + W, app = os.path.abspath(a.work), os.path.abspath(a.app) + S = os.path.join(app, ".stepupdate") + kept = [ln.split(" ", 1) for ln in rd(os.path.join(S, "kept.txt")).splitlines() if ln.strip()] + try: + led = {x["name"]: x for x in json.load(open(a.localedits or os.path.join(W, "localedits.json"))) if isinstance(x, dict)} + except (OSError, ValueError): + led = {} + cache = os.environ.get("UPG_CACHE") or localedits.CACHE + mpr = os.path.join(app, next((f for f in os.listdir(app) if f.endswith(".mpr")), "")) + rows = [] + for mod, why in kept: + r, reason, extra = route_module(mod, why, S, led.get(mod), a.original, cache, mpr) + rows.append({"module": mod, "route": r, "reason": reason, "kept_because": why, **extra}) + print(f"ROUTE {mod} {r}: {reason}") + json.dump(rows, open(os.path.join(W, "edit-routes.json"), "w"), indent=1) + go = {x["module"] for x in rows if x["route"] != "HOLD"} + plan = [ln for ln in rd(a.plan).splitlines() if len(ln.split()) >= 3 and not ln.startswith("#") and ln.split()[2] in go] + with open(os.path.join(W, "route-plan.txt"), "w") as f: + f.write("# modules the update kept, routed by editroute.py (edit-routes.json)\n" + "".join(ln + "\n" for ln in plan)) + append(os.path.join(W, "force-ui.txt"), [f"{x['module']} # editroute {x['route']}" for x in rows if x["route"] in ("COSMETIC", "REPLAY", "UNREAD", "XASSOC")]) + append(os.path.join(W, "accept-unsaved.txt"), [f"{x['module']} # editroute FILES: saved copies under .stepupdate/saved/{x['module']}" for x in rows if x["route"] == "FILES"]) + append(os.path.join(W, "approved.txt"), [f"{qn} # editroute VENDOR: the new {x['module']} drops it, nothing outside the module refers to it" + for x in rows for qn in x.get("vendor", [])]) + append(os.path.join(W, "approved-step.txt"), [f"{qn} # editroute XASSOC: removed by the {x['module']} update, replayed with its original id" + for x in rows for qn in x.get("assocs", [])]) + return 0 if go else 1 + + +def cmd_collect(a): + W = os.path.abspath(a.work) + try: + rows = json.load(open(os.path.join(W, "edit-routes.json"))) + except (OSError, ValueError): + return 1 + os.makedirs(os.path.join(W, "replay"), exist_ok=True) + n = 0 + for x in rows: + mod, r = x["module"], x["route"] + d = os.path.join(W, "saved-edits", mod) + if r == "REPLAY" or r == "UNREAD": + keep = {re.sub(r"\W", "", e.split(" ", 1)[1]).lower() for e in x.get("ui", [])} + files = [f for f in sorted(glob.glob(os.path.join(d, "*.mdl"))) if not os.path.basename(f).startswith("association-") + and (r == "UNREAD" or re.sub(r"\W", "", os.path.basename(f)[:-4].split("-", 1)[-1]).lower() in keep)] + elif r == "XASSOC": + files = sorted(glob.glob(os.path.join(d, "association-*.mdl"))) + with open(os.path.join(W, "replay", f"{mod}.guids"), "w") as g: + g.write("".join(q + "\n" for q in x.get("assocs", []))) + else: + continue + if not files: + print(f"COLLECT {mod}: nothing saved under {d}") + continue + with open(os.path.join(W, "replay", f"{mod}.mdl"), "w") as out: + out.write(f"mdl 1;\n/* replay of {mod} (editroute {r}), from {d} */\n") + for f in files: + out.write(f"\n/* {os.path.basename(f)} */\n" + re.sub(r"^mdl 1;\s*", "", rd(f).strip(), flags=re.M) + "\n") + print(f"COLLECT {mod}: {len(files)} saved element(s) -> replay/{mod}.mdl") + n += 1 + return 0 if n else 1 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + sp = ap.add_subparsers(dest="cmd", required=True) + p = sp.add_parser("plan") + p.add_argument("--work", required=True); p.add_argument("--app", required=True) + p.add_argument("--original", required=True); p.add_argument("--plan", required=True); p.add_argument("--localedits") + c = sp.add_parser("collect") + c.add_argument("--work", required=True) + a = ap.parse_args() + return cmd_plan(a) if a.cmd == "plan" else cmd_collect(a) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/bin/evidence.py b/upgrade-toolkit/bin/evidence.py new file mode 100644 index 0000000..f1f83c2 --- /dev/null +++ b/upgrade-toolkit/bin/evidence.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""evidence.py: what exactly sits behind one plan item, from the files the run already wrote. A question card shows it, so a person +decides on the elements themselves and not on a label like "local edits refuse the update". + +Why (cold test WM1, 2026-11-16): seven "What should the upgrade do with ?" cards said only "local edits refuse the update"; +the owner asked how anyone can decide that. The run had the answer in its logs: Atlas_Core had 7 layouts and snippets edited, +Encryption only its EncryptionKey constant, Email_Connector one microflow, MendixSSO an attribute the new version drops. + + for_item(work, name) -> {"what": [plain lines], "files": [paths relative to the run folder]} + evidence.py --work W NAME prints the lines and the files (a check by hand) + +Sources, per module (app = state/app-dir, S = app/.stepupdate): + S/step*-.log the elements mxcli refused to overwrite, or why it could not read the module + S/step*-.idcheck data-bearing changes the update makes (RED rows) and a module deleted and imported again + S/step*-.files theme, JavaScript and Java files the update replaced and that could not be proven unedited + W/localedits.md the local edits found in triage (UI, domain model, logic) + W/apimock-inventory.json the outside calls of the module, with their stand-in status + W/edit-routes.json how editroute.py carries the module's edits over (or why it holds them for a person) +""" +import argparse, glob, json, os, re, sys + + +def _rd(p): + try: + with open(p, encoding="utf-8", errors="replace") as f: + return f.read() + except OSError: + return None + + +def _rel(work, p): + run = os.path.dirname(os.path.abspath(work)) + return os.path.relpath(p, run) + + +KIND = {"LAYOUT": "layout", "SNIPPET": "snippet", "PAGE": "page", "MICROFLOW": "microflow", "NANOFLOW": "nanoflow", + "CONSTANT": "constant", "ENTITY": "entity", "ENUMERATION": "enumeration", "JAVA_ACTION": "Java action"} + + +def _refused(text): + m = re.search(r"refusing to update: (\d+) element\(s\) have been changed locally[^\n]*\n((?:\s{2,}\S[^\n]*\n)+)", text) + if not m: + return [] + out = [] + for ln in m.group(2).splitlines(): + f = ln.split(None, 1) + if len(f) == 2 and not f[0].startswith("-"): + out.append(f"{KIND.get(f[0], f[0].lower())} {f[1].strip()}: changed in this app; the update would overwrite it") + return out + + +def for_item(work, name): + work = os.path.abspath(work) + app = (_rd(os.path.join(work, "state", "app-dir")) or "").strip() + S = os.path.join(app, ".stepupdate") if app else "" + what, files = [], [] + for p in sorted(glob.glob(os.path.join(S, f"step*-{name}.log"))) if S else []: + t = _rd(p) or "" + rows = _refused(t) + m = re.search(r"(\d+) element\(s\) could not be read", t) + if m: + rows.append(f"{m.group(1)} elements of the module could not be read by mxcli, so nobody can show yet whether they were edited") + if rows: + what += rows + files.append(_rel(work, p)) + for p in sorted(glob.glob(os.path.join(S, f"step*-{name}.idcheck"))) if S else []: + t = _rd(p) or "" + rows = [] + for ln in t.splitlines(): + r = re.match(r"RED\s+(\w+)\s+(\w+)\s+([\w.]+)", ln) + if r: + rows.append(f"{r.group(2)} {r.group(3)}: {r.group(1).lower()} by the new version, its data is lost") + elif ln.startswith("warn MODULE_RECREATED"): + rows.append("the new version replaces the module as a whole (all ids new), pages that use it are rewritten too") + if rows: + what += rows + files.append(_rel(work, p)) + for p in sorted(glob.glob(os.path.join(S, f"step*-{name}.files"))) if S else []: + t = _rd(p) or "" + rows = [ln.strip() for ln in t.splitlines() if ln.startswith("FILES-") and not ln.startswith("FILES-SAME")] + nf = re.search(r"FILES NOT COMPARED \(([^)]*)\)", t) + if nf: + rows.insert(0, f"its files could not be compared with the Marketplace version ({nf.group(1).split(';')[0]})") + if rows: + what += [re.sub(r"^FILES-\w+ \S+: ", "", r) for r in rows] + files.append(_rel(work, p)) + le = _rd(os.path.join(work, "localedits.md")) or "" + sec = re.search(rf"^## {re.escape(name)} [^\n]*\n((?: [^\n]*\n)*)", le, re.M) + if sec: + rows = [] + for ln in sec.group(1).splitlines(): + r = re.match(r"\s+(\w+)\s+(ADDED|CHANGED|REMOVED)\s+(\w+)\s+([\w.]+)\s+(.*)", ln) + if r: + rows.append(f"{r.group(3)} {r.group(4)}: {r.group(2).lower()} by this app ({r.group(5).split(';')[0]})") + known = {re.split(r"[ :]", w)[1].split(".")[-1] for w in what if len(re.split(r"[ :]", w)) > 1} + rows = [r for r in rows if r.split(" ")[1].rstrip(":").split(".")[-1] not in known] + if rows: + what += rows + files.append(_rel(work, os.path.join(work, "localedits.md"))) + inv = json.loads(_rd(os.path.join(work, "apimock-inventory.json")) or "{}").get("endpoints", []) + rows = [f"outside call {e['doc']} ({(e.get('method') or '').strip()} {e.get('template') or ''}".rstrip() + f"): " + + ("a stand-in answers it in the test" if e.get("status") == "MOCKED" else "no stand-in yet, " + (e.get("why") or "not classified")) + for e in inv if e.get("module") == name] + if rows: + what += rows + files.append(_rel(work, os.path.join(work, "apimock-inventory.json"))) + for x in json.loads(_rd(os.path.join(work, "edit-routes.json")) or "[]"): + if x.get("module") == name: + what.insert(0, ("held for a person: " if x["route"] == "HOLD" else f"the pipeline updates it anyway ({x['route']}): ") + x["reason"]) + files.insert(0, _rel(work, os.path.join(work, "edit-routes.json"))) + return {"what": what, "files": files} + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--work", required=True) + ap.add_argument("name") + a = ap.parse_args() + ev = for_item(a.work, a.name) + print("\n".join("- " + w for w in ev["what"]) or "(no evidence found)") + print("files: " + ", ".join(ev["files"])) + return 0 if ev["what"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/bin/proposal.py b/upgrade-toolkit/bin/proposal.py index 76cd6d8..becc91f 100644 --- a/upgrade-toolkit/bin/proposal.py +++ b/upgrade-toolkit/bin/proposal.py @@ -36,6 +36,10 @@ """ import argparse, datetime, hashlib, json, os, re, sys +# cold test WM1 (2026-11-16): an edited Marketplace module is updated, not kept. Editing a module does not stop its update; an old +# module misses fixes and may not run on the next Mendix version. editroute.py carries the edits over (bin/editroute.py routes). +UPDATE_CARRY = "update to the latest version and carry the edits over (diagram-only dropped, UI and constants replayed, vendor drops checked)" + HERE = os.path.dirname(os.path.abspath(__file__)) TK = os.path.dirname(HERE) sys.path.insert(0, HERE) @@ -128,7 +132,6 @@ def __init__(self, a): self.sizes = {} self.systems = None self.nendpoints = None; self.api_mocked = 0; self.api_unknown = []; self.api_excluded = 0 - self.dojo_adv = [] self.release_open = None self.release_total = None self.actions = None @@ -252,6 +255,14 @@ def localedits(self): def release(self): t = self.need("release-impact.md", "steps/triage.md item 10") if t is None: + # cold test WM1: the release-note check could not run (no brain version tables) and the plan still went to sign-off, + # so nobody had read what Mendix 10 and 11 change. A check that did not run is a question, never a silent pass. + why = (read(self.p("release-impact.out")) or "release-impact.md was not written").strip().splitlines()[0] + self.add3("check", "Release notes", f"the release-note check did not run: {why}", + "run the check first: give the run the company brain (its version tables), then rerun triage | " + "go on without it: a named person reads the Mendix release notes for every version on the route before go-live | " + "pause the upgrade", + "run the check first: give the run the company brain (its version tables), then rerun triage", "S", "a change in the new Mendix versions is only found at the build or the test, or after go-live") return tot = opn = 0 for head, rows in tables(t): @@ -303,6 +314,14 @@ def dojo(self): t = self.need("dojo-react.md", "steps/triage.md item 10: dojo_report.py --out") if t is None: return + # the move to React is part of every upgrade (where a path exists): plan work, not advice. Only a widget without + # a React path is a decision (below) + last = getattr(self.a, "target2", None) or self.a.target or "the target" + for head, rows in tables(t): + if head[:2] == ["widget", "count"]: + for r in rows: + if len(r) > 3 and r[1].isdigit(): + self.items2.append({"key": f"react:{r[0]}", "text": f"Dojo to React on the {last} rung: {r[1]} x {r[0]} -> {r[3].split(';')[0]}"}) for head, rows in tables(t): if head[:3] != ["status", "package", "version"]: continue @@ -312,16 +331,16 @@ def dojo(self): if st in ("can't move", "not researched", "unreadable"): self.add3("widget", (pkg if ver in ("", "?") else f"{pkg} {ver}"), f"dojo-react.md: {st}: {r[3] if len(r) > 3 else ''}", "keep old package, accept the risk | swap to a successor (dojo-successors.tsv) | rebuild as pluggable / native | hand to a named person", - "keep old package, accept the risk (advisory on 11, blocking on 12)", "L" if st == "can't move" else "M", + "rebuild as pluggable / native (no React path found; keep the old package only for this rung, Mendix 12 blocks it)", "L" if st == "can't move" else "M", "does not run on the React client; Mendix 12 removes the Dojo client") elif st in ("can move", "delete"): - self.dojo_adv.append(f"{pkg} ({st})") + self.items2.append({"key": f"react:{pkg}", "text": f"Dojo to React on the {last} rung: widget package {pkg} ({st}: {r[4] if len(r) > 4 else ''})"}) if st == "can move" and re.search(r"htmlsnippet|snippetjs", pkg + " " + (r[3] if len(r) > 3 else ""), re.I): # the successor depends on what each snippet holds (HTML only, a little script, DOM work): a decision per snippet self.add3("widget", (pkg if ver in ("", "?") else f"{pkg} {ver}"), f"dojo-react.md: can move: {r[3] if len(r) > 3 else ''}; the successor depends on the content of each snippet", "keep on the Dojo client for now | HTML Element widget | JavaScript action + nanoflow + Events | pluggable widget | hand to a named person", - "keep on the Dojo client on an 11 target (advisory on 11, blocking on 12); classify every snippet first", "M", + "HTML Element widget for every snippet that holds HTML only; classify the rest (script, DOM work) per snippet", "M", "script in a snippet runs on the Dojo page DOM; Mendix 12 removes the Dojo client") def sizes_and_actions(self): @@ -407,10 +426,11 @@ def build(self): "M", "references and data of the old module are moved by hand" if self.renamed[m]["kind"] == "RENAMED" else "the installed version may not run on the target") continue self.add3(kind, m, why + " (" + ("stranded.md" if stranded else "localedits.json / kept.txt") + ")", - "keep old version (kept list) | replace with the Marketplace equivalent (replacements.tsv row) | rebuild locally, drop the module | hand to a named person" + f"{UPDATE_CARRY} | update, move the edits to {m}_Ext by hand | keep old version for now (a reason and an end date) | " + "replace with the Marketplace equivalent (replacements.tsv row) | hand to a named person" if kind == "module" else "keep old jar, pin | port to the new API | replace with a Marketplace module or a microflow | hand to a named person", "hand to a named person (no version fits; port, replace or remove)" if stranded else - "keep old version (kept list); the stop rule keeps it unattended" if kind == "module" else "keep old jar, pin", + f"{UPDATE_CARRY}; a real logic edit is held until it is moved to {m}_Ext" if kind == "module" else "keep old jar, pin", "L" if stranded else "M", "data loss if a RED edit is replaced" if m in self.red else "behaviour of local edits is lost on update") for m, rows in sorted(self.files.items()): if any(x.get("kind") == "theme" for x in rows): @@ -489,7 +509,9 @@ def write_steps(F, work): groups = [("modules", "Routine module updates", "module", ("module update in the plan that the toolkit does on its own", "module updates in the plan that the toolkit does on its own"), "Update these modules to the versions that fit the new Mendix version."), ("repairs", "Predicted repairs", "repair", ("repair that the toolkit expects to make", "repairs that the toolkit expects to make"), - "Make these repairs during the upgrade, as the repair table describes.")] + "Make these repairs during the upgrade, as the repair table describes."), + ("react", "Dojo to React client", "react", ("move to the React client that the toolkit makes", "moves to the React client that the toolkit makes"), + "Move the app to the React client on the last rung: every legacy widget to its React successor, then test (playbooks/dojo-to-react.md).")] for topic, title, key, what, prop in groups: its = [x for x in F.items2 if x["key"].startswith(key + ":")] if not its: @@ -527,6 +549,7 @@ def write_steps(F, work): "proposed_say": "A person does this. The toolkit lists it so it is not forgotten.", "why": "The toolkit has no way to do this on its own."}) out[sid] = f"4:{it['key']}" + steps.withdraw(work, set(out), "No longer in the plan: a rerun of the proposal dropped this item.") return out @@ -636,8 +659,6 @@ def render(F, hashval, changed_block, signoff_text): L.append(f"- {it['text']}") if it["detail"]: L.append(f" - {it['detail']}") - if F.dojo_adv: - L.append("- React client switch, advisory on 11 (not done in this run): " + ", ".join(F.dojo_adv[:12]) + (" ..." if len(F.dojo_adv) > 12 else "")) # section 5 cnt = {} for it in F.items3: @@ -730,8 +751,17 @@ def main(): F.build() ids_for(F.items3, a.work) sids = write_steps(F, a.work) if os.path.isdir(a.work) else {} - ident = [f"2|{it['key']}|{it['text']}" for it in F.items2] + [f"3|{it['kind']}|{it['name']}|{it['evidence']}" for it in F.items3] \ - + [f"4|{it['key']}|{it['text']}" for it in F.items4] + [f"v|{F.src}|{F.tgt}|{F.tgt2}"] + # the hash is taken without kept.txt: the update step writes it after the sign-off, and a module it kept moves from section 2 to + # section 3 (cold test WM1: 7 kept modules cleared the sign-off at the end of rung 1 and asked the whole plan again). The + # signed path below accepts those items at their recommendation (update, a real logic edit held for _Ext): editroute.py already + # routed what it could, so what is still kept is exactly that held edit. + H = F + if F.kept: + H = Facts(a) + H.keptmods = lambda: None + H.build() + ident = [f"2|{it['key']}|{it['text']}" for it in H.items2] + [f"3|{it['kind']}|{it['name']}|{it['evidence']}" for it in H.items3] \ + + [f"4|{it['key']}|{it['text']}" for it in H.items4] + [f"v|{H.src}|{H.tgt}|{H.tgt2}"] hashval = hashlib.sha256("\n".join(sorted(ident)).encode()).hexdigest() prev_hash = next((m.group(1) for m in (HASH.match(x) for x in (old or "").splitlines()) if m), None) so = parse_signoff(old) diff --git a/upgrade-toolkit/bin/release_impact.py b/upgrade-toolkit/bin/release_impact.py index 10f9365..43e61ba 100755 --- a/upgrade-toolkit/bin/release_impact.py +++ b/upgrade-toolkit/bin/release_impact.py @@ -84,7 +84,7 @@ .facts.json; --facts files given too win over it. One mpr alone gives facts, no module rows: a line "auto cannot use: ..." says each thing that is lost (module rows, kept/stranded, every fact not known). --types default breaking,removal,requirement,deprecation (module rows also migration-step); also security, known-issue, improvement, feature, fix - --brain default $UPG_BRAIN or ~/Mendix/-brain + --brain default: bin/brainpath.py ($UPG_BRAIN, UPG_BRAIN= in local.env, else a ~/Mendix/*-brain with version tables) --out default release-impact.md. A rerun keeps the handled entries of rows it finds again. An entry whose row is gone (the brain reworded it, T2F-4) is listed under "Entries that matched no row", kept until the row returns, and counted in the output. With --auto a rerun also says how many human entries auto would have decided and which @@ -1203,7 +1203,8 @@ def main(): ap.add_argument("--facts", action="append") ap.add_argument("--types") ap.add_argument("--out", default="release-impact.md") - ap.add_argument("--brain", default=os.environ.get("UPG_BRAIN", str(Path("~/Mendix/-brain").expanduser()))) + import brainpath + ap.add_argument("--brain", default=brainpath.find()[0] or os.environ.get("UPG_BRAIN") or str(Path("~/Mendix/-brain").expanduser())) ap.add_argument("--facts-from-mpr", dest="facts_from_mpr", help="count facts from this project (model + javasource), T2F-2") ap.add_argument("--facts-out", dest="facts_out") ap.add_argument("--db-vendor", dest="db_vendor") diff --git a/upgrade-toolkit/bin/steps.py b/upgrade-toolkit/bin/steps.py index 423a3f0..3be9c88 100755 --- a/upgrade-toolkit/bin/steps.py +++ b/upgrade-toolkit/bin/steps.py @@ -199,7 +199,7 @@ def load(work): and (r.get("approval") is None or "state" in r["approval"])) if not ok: raise Damaged(f"step {r['id']} is incomplete") - return recs, None + return [r for r in recs if not r.get("withdrawn")], None except (Damaged, OSError, UnicodeDecodeError, KeyError, TypeError, ValueError) as e: return None, str(e) or "unreadable" @@ -273,6 +273,8 @@ def fold(evs): rec["tested"] = {"results": [fs[k]["value"] for k in sorted(fs, key=lambda k: fs[k]["at"]) if k.startswith("test:")]} rec["gaps"] = {k[4:]: fs[k]["value"] for k in fs if k.startswith("gap:")} rec["status"] = derive_status(rec) + if v("withdrawn"): # the plan no longer holds the item (withdraw): kept in the file, left out of every reader + rec["withdrawn"], rec["status"] = v("withdrawn"), "withdrawn" recs.append(rec) return sorted(recs, key=lambda r: idnum(r["id"])) @@ -302,6 +304,8 @@ def ensure(work, topic, title, fields, by="toolkit"): append(work, sid, "topic", topic, by) append(work, sid, "title", title, by) cur = lastmap(events(work)).get(sid, {}) + if cur.get("withdrawn", {}).get("value"): # back in the plan + append(work, sid, "withdrawn", "", by) for f, v in fields.items(): if v is None or v == [] or (f in cur and cur[f]["value"] == v): continue @@ -309,6 +313,18 @@ def ensure(work, topic, title, fields, by="toolkit"): return sid +def withdraw(work, keep, say, by="toolkit"): + """Mark every plan-item step (one with a P-n alias) of this phase that is not in keep as withdrawn: the plan no longer holds + it. Cold test WM1: ten "keep or update module" steps from a crashed update stayed on the cockpit as "not approved yet" + after the rerun had dropped them from the plan. Returns the ids withdrawn now.""" + out = [] + for r in fold(events(work)): + if r.get("alias", "").startswith("P-") and mine(r) and r["id"] not in keep and not r.get("withdrawn"): + append(work, r["id"], "withdrawn", clean(say), by) + out.append(r["id"]) + return out + + def approval_of(work, sid): """The folded approval of a step ({state, by, at, ...}) or None.""" r = next((r for r in fold(events(work)) if r["id"] == sid), None) diff --git a/upgrade-toolkit/bin/stepupdate.sh b/upgrade-toolkit/bin/stepupdate.sh index 06e3bae..2e7fd7c 100755 --- a/upgrade-toolkit/bin/stepupdate.sh +++ b/upgrade-toolkit/bin/stepupdate.sh @@ -434,6 +434,11 @@ PY fi if [ $rc != 0 ]; then tail -5 "$log" + # a crash of the tool is not a fact about the module: never record it as kept (cold test WM1: mxcli under qemu + # crashed on all 10 updates and every module was "kept") + if grep -qE '^(panic: |fatal error: )|SIGSEGV' "$log"; then + revert; echo "STOP: mxcli crashed while updating $mod (a tool fault, not the module: $(grep -m1 -E '^(panic: |fatal error: )' "$log")); tree reverted. On Apple Silicon run the containers through Rosetta (bin/dockerhost.sh), then rerun"; exit 3 + fi keep "$mod" "update failed, exit $rc" && continue echo "STOP: update of $mod failed (exit $rc, $log); tree left as is"; exit 2 fi diff --git a/upgrade-toolkit/bin/themefiles.py b/upgrade-toolkit/bin/themefiles.py index eccf406..1070ca2 100644 --- a/upgrade-toolkit/bin/themefiles.py +++ b/upgrade-toolkit/bin/themefiles.py @@ -66,8 +66,10 @@ def norm_js_action(text): st = re.sub(r"\{\s*", "{ ", re.sub(r"\s*\}", " }", st)) if st not in JSGEN_IMPORTS: imps.add(st) - blocks = [f"[{m.group(1)}]\n" + "\n".join(ln.strip() for ln in m.group(2).split("\n") if ln.strip()) - for m in JSBLOCK.finditer(text)] + # an empty block is no code: the 3.0.1 NanoflowCommons package ships GetPlatform.js without the EXTRA CODE markers, + # Studio Pro writes them back empty (cold test WM1: 4 such files made the module FILES-EDITED, and it was kept) + blocks = [f"[{m.group(1)}]\n" + body for m in JSBLOCK.finditer(text) + if (body := "\n".join(ln.strip() for ln in m.group(2).split("\n") if ln.strip()))] return "js-action:\n" + "\n".join(sorted(imps)) + "\n" + "\n".join(blocks) diff --git a/upgrade-toolkit/controls/cockpit_control.sh b/upgrade-toolkit/controls/cockpit_control.sh index 4e102d5..3245136 100755 --- a/upgrade-toolkit/controls/cockpit_control.sh +++ b/upgrade-toolkit/controls/cockpit_control.sh @@ -121,6 +121,27 @@ PY echo "cell: $(grep -o '"l": "[0-9] decisions needed"' "$T/out.html" | head -1)" echo "state: $(grep -o '"state": {[^}]*}' "$T/out.html")" rm "$W/NEEDS-JUDGMENT.md"; ck > /dev/null; echo "no file: bars $(cnt -c 'class="stopbar"' "$T/out.html")" ;; + decide) # Blocked on you (cold test WM1): every open question is a sidebar row and a dialog with one button per option, the sign-off last; the local + # server's POST /decide writes the same ledger as the page import, refuses bad answers and form posts, and a card + # disappears once answered + ck > /dev/null + echo "cards: $(grep -o '
    "$d/.cockpit-work" + python3 "$B/cockpitserve.py" --dir "$d" --idle-exit 60 > /dev/null 2>&1 & sp=$! + for _ in 1 2 3 4 5 6 7 8 9 10; do [ -s "$d/.cockpit-port" ] && break; sleep 0.3; done; u=http://127.0.0.1:$(cat "$d/.cockpit-port")/decide + post() { curl -s -X POST -H 'Content-Type: application/json' -d "$1" "$u"; echo; } + echo "rows: $(grep -o '