diff --git a/.github/workflows/controls.yml b/.github/workflows/controls.yml index 93ddbe7..44ad30f 100644 --- a/.github/workflows/controls.yml +++ b/.github/workflows/controls.yml @@ -17,6 +17,14 @@ jobs: with: python-version: '3.12' + - name: Python packages the controls use + # controls/lib/mkmodel.py writes fixture models as BSON (the bson module ships with pymongo) + run: pip install pymongo + + - name: Shell the toolkit scripts use + # bin/*.sh wrappers exec zsh (macOS default); the Ubuntu runner has none + run: sudo apt-get install -y zsh + - name: Toolkit controls (fast) run: bash upgrade-toolkit/controls/run-controls.sh --fast diff --git a/upgrade-toolkit/bin/brainnotes.py b/upgrade-toolkit/bin/brainnotes.py new file mode 100644 index 0000000..546af48 --- /dev/null +++ b/upgrade-toolkit/bin/brainnotes.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""brainnotes.py: the company brain's notes that belong next to a run event (a held module, a route, a failed check). + +Why (cold test WM1, 2026-11-16): the toolkit read only the brain's version tables. The brain also holds the practice and +pitfall notes for exactly the events a run stops on (never edit a Marketplace module, hide Atlas layouts in the company +UI module, PDFBox 2 jars next to PDFBox 3 after a Community Commons update), and no card showed them. The data test hit +the PDFBox case by hand while the brain already had the note. + +Search, no model: every note's title, frontmatter keywords and body are scored against the query words (idf weighted; +title x3, keywords x2, body x1), plain-word stems ("layouts" = "layout"). The brain is found by brainpath.py and only read. +No brain on this machine: prints nothing, exit 0 (the toolkit runs without it). + + brainnotes.py "query text" [-k 3] [--brain DIR] [--json] top notes: score, path under the brain, title + brainnotes.py --eval gold.tsv [-k 3] [--brain DIR] hit@k against a labelled set (id, slugs, why; slugs + comma-separated paths without .md, or NONE); the queries + come from a second file column 'text' or --queries FILE +""" +import argparse, json, math, os, re, sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +STOP = set("a an the and or of to in on for with is are be by it its at as from this that what which how should we our " + "not no do does after before into out than then when would must can there their them one two new old".split()) + + +def stem(w): + for suf in ("ies", "es", "s"): + if len(w) > 4 and w.endswith(suf): + return w[: -len(suf)] + ("y" if suf == "ies" else "") + return w + + +def words(t): + # Atlas_Core is "Atlas Core" in a title; a version digit ("Atlas 4") is a word of its own + return [stem(w) for w in re.findall(r"[a-z0-9]+", (t or "").lower()) if w not in STOP and (len(w) > 1 or w.isdigit())] + + +def load(brain): + notes = [] + for root, _, files in os.walk(os.path.join(brain, "internal")): + for f in files: + if not f.endswith(".md") or f in ("INDEX.md", "INTERNAL-ONLY.md"): + continue + p = os.path.join(root, f) + t = open(p, encoding="utf-8", errors="replace").read() + fm, body = "", t + if t.startswith("---"): + j = t.find("\n---", 3) + fm, body = t[3:j], t[j + 4:] + title = (re.search(r"^title:\s*(.+)$", fm, re.M) or [None, ""])[1].strip("'\" ") + kw = (re.search(r"^keywords:\s*(.+)$", fm, re.M) or [None, ""])[1] + notes.append({"path": os.path.relpath(p, os.path.join(brain, "internal"))[:-3], "title": title, + "f": (set(words(title)), set(words(kw)), words(body))}) + return notes + + +def search(notes, q, k=3): + qs = set(words(q)) + n = len(notes) or 1 + df = {w: sum(1 for x in notes if w in x["f"][0] or w in x["f"][1] or w in x["f"][2]) for w in qs} + out = [] + for x in notes: + t, kw, body = x["f"] + bset = set(body) + s = sum(math.log(1 + n / (1 + df[w])) * (3 * (w in t) + 2 * (w in kw) + (w in bset)) for w in qs) + if s > 0: + out.append((round(s, 1), x["path"], x["title"])) + return sorted(out, key=lambda r: -r[0])[:k] + + +def brain_dir(arg): + if arg: + return arg + try: + import brainpath + return brainpath.find()[0] + except Exception: + return None + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("query", nargs="?") + ap.add_argument("-k", type=int, default=3) + ap.add_argument("--brain"); ap.add_argument("--json", action="store_true") + ap.add_argument("--eval"); ap.add_argument("--queries") + a = ap.parse_args() + b = brain_dir(a.brain) + if not b or not os.path.isdir(os.path.join(b, "internal")): + return 0 + notes = load(b) + if not a.eval: + r = search(notes, a.query or "", a.k) + print(json.dumps([{"score": s, "path": p, "title": t} for s, p, t in r]) if a.json + else "\n".join(f"{s}\t{p}\t{t}" for s, p, t in r)) + return 0 + q = {} + if a.queries: + for ln in open(a.queries): + if "|" in ln: + i, t = ln.split("|", 1) + q[i.strip()] = t.strip() + hit = tot = none_ok = nones = 0 + for ln in open(a.eval): + f = ln.rstrip("\n").split("\t") + if len(f) < 2 or f[0] in ("id", "") or f[0] not in q: + continue + gold = {g.strip() for g in f[1].split(",") if g.strip() and g.strip() != "NONE"} + got = search(notes, q[f[0]], a.k) + paths = [p for _, p, _ in got] + if not gold: + nones += 1 + print(f"{f[0]}\tNONE\t(top: {paths[:1]})") + continue + tot += 1 + ok = bool(gold & set(paths)) + hit += ok + print(f"{f[0]}\t{'HIT ' if ok else 'MISS'}\tgold {sorted(gold)}\tgot {paths}") + print(f"brainnotes hit@{a.k}: {hit}/{tot} situations with a gold note ({nones} with none in the brain)") + return 0 if tot and hit == tot else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/bin/checkwhere.py b/upgrade-toolkit/bin/checkwhere.py new file mode 100755 index 0000000..72e5362 --- /dev/null +++ b/upgrade-toolkit/bin/checkwhere.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""checkwhere.py: add the document of each error to an `mx check` log, from the JSON that `mx check -j` writes. + + checkwhere.py CHECK.json CHECK.log + +The text output of mx check names only the element ("at Action button 'actionButton1'"), and a widget name like that +occurs on many pages (cold CMM: 4 CE0117 on actionButton1/2/4/7, which sat in two Mix_New_WizardStep1 pages, not in the +login templates that also have such buttons). The JSON has the module, document and property. Appends one line per +error location to the log: + + [where] CE0117 MaterialManagement / Page 'Mix_New_WizardStep1': Property 'Dynamic classes expression' of action button 'actionButton1' + +The [error] lines and the summary stay as they are, so checkdiff.py and the error count read the log as before. +Exit: 0 = lines added (or none to add), 2 = the JSON is unreadable (the log is left alone). +""" +import json, sys + + +def where(d): + for e in d.get("errors") or []: + for loc in e.get("locations") or [{}]: + yield (f"[where] {e.get('code', '?')} {loc.get('module-name', '?')} / {loc.get('document-name', '?')}: " + f"{loc.get('element-name', '?')}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit(__doc__) + try: + d = json.load(open(sys.argv[1])) + except (OSError, ValueError) as e: + print(f"checkwhere: {sys.argv[1]}: {e}", file=sys.stderr); sys.exit(2) + lines = list(where(d)) + if lines: + with open(sys.argv[2], "a") as f: + f.write("Where (mx check -j):\n" + "\n".join(lines) + "\n") diff --git a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh index cfa10af..c1cdbcc 100755 --- a/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh +++ b/upgrade-toolkit/bin/cloud/upgrade-pipeline.sh @@ -18,9 +18,10 @@ # heal (steps/heal.md): classifies every failing row, heals what belongs to the test (start noise, selection-gated buttons, a runner # that did not run, outside calls via apimock, known harness log lines) and reruns only the affected journeys, at most 2 rounds. # UPG_HEAL=0 turns it off (journeys then stops on a failure as before). Resume it with --from heal. -# UPG_SMOKE_MICROFLOWS=1: the smoke walks (basewalk, afterwalk) also follow microflow homes and menu items (smokewalk.js -# --microflows). Safe: each walk runs on its own copy of the frozen DB. Needed when a role reaches every page through a -# microflow (WM1: the User home and both menu items), else that role's walk is all SKIPPED and INVALID. +# UPG_SMOKE_MICROFLOWS: the smoke walks (basewalk, afterwalk) follow microflow homes and menu items (smokewalk.js +# --microflows) whenever the plan has them; UPG_SMOKE_MICROFLOWS=0 turns that off. Safe: each walk runs on its own copy of +# the frozen DB. Needed when a role reaches its pages through microflows (WM1: the User home and both menu items; cold +# CMM: 13 of 23 steps, two roles judged on no page at all, and the skip was reported only when the walk was INVALID). # Each finished step writes $WORK/state/.done and is skipped on the next run. Logs: $WORK/logs/.log. # Judgment stops write $WORK/NEEDS-JUDGMENT.md. Exit codes: # 0 gate PASS, nothing left for a person @@ -30,12 +31,16 @@ # 10 mx check / mxbuild / quick start errors after convert+update: repair needed (steps/repair.md) # 11 journeys INVALID or FAIL on the old app or the new one: fix the journey or write the todo (steps/journeys.md) # 12 diff / gate REVIEW, FAIL or INVALID items need a comparison by a person (steps/gate.md). A FAIL is never a pass. -# 13 module plan missing, or a module kept / a data-bearing finding needs a decision (steps/upgrade.md) +# 13 module plan missing, or a module kept / a replay held / a data-bearing finding needs a decision (steps/upgrade.md) # 14 the upgrade proposal awaits sign-off ($WORK/PROPOSAL.md section 6, steps/propose.md); also when a plan input changed # after the sign-off (the sign-off is cleared). Nothing in the app model has changed yet. # 15 the owner paused the run on purpose (`Status: PAUSED ` in PROPOSAL.md); NEEDS-JUDGMENT.md has the resume command # Interview mode unattended (env UPG_INTERVIEW_MODE=unattended, or a line `Interview mode: unattended` in $WORK/interview-mode.txt): # the propose step signs nothing, records ASSUMED (unattended) in DECISIONS.md, goes on, and every report says 'proposal not signed'. +# Interview mode auto (UPG_INTERVIEW_MODE=auto, or `Interview mode: auto`): the same at propose, and a judgment stop that nothing +# after it depends on (journeys, idcheck, smoke walk, diff, gate: nj_park) is parked in $WORK/parked/.md and the run goes on; +# at the end NEEDS-JUDGMENT.md lists every parked item grouped by kind (bin/parked.py) and the run exits with the first code. +# A build or start failure, a missing plan or a replay that cannot be undone still stops at once (nj_flush). # Decisions (bin/decisions.py): at the start of the run and of every step the pipeline applies $WORK/decisions.json (the page buttons # and the chat share it); after every step, and when a step stops, it regenerates $WORK/COCKPIT.html (bin/cockpitpage.py). # Hidden options for rung 2 (set by this script): --db NAME --old-mda PATH --seed-from DB @@ -241,6 +246,31 @@ nj_flush() { # write NEEDS-JUDGMENT.md and exit with the first recorded code say "STOP $STEP: needs judgment (exit $NJ_CODE), see $WORK/NEEDS-JUDGMENT.md" exit "$NJ_CODE" } +# WM1 cold test 2026-11-17: 16 stops, one question each, most answered with the recommendation. Auto mode parks the stops +# nothing later depends on and asks them all at the end (park_finish). +nj_park() { # like nj_flush, but in Interview mode auto the items go to $WORK/parked/$STEP.md and the step goes on + [ "$NJ_CODE" != 0 ] || return 0 + if [ "$DRY" = 1 ] || ! interview_auto; then nj_flush; return 0; fi + mkdir -p "$WORK/parked" + { printf '\n' "$NJ_CODE" "$STEP"; cat "$NJ_TMP"; } >> "$WORK/parked/$STEP.md" + say "PARKED $STEP: exit $NJ_CODE parked (Interview mode: auto), the run goes on; $WORK/parked/$STEP.md" + : > "$NJ_TMP"; NJ_CODE=0 +} +parked_any() { [ -n "$(ls "$WORK/parked" 2> /dev/null)" ] || { [ "$(basename "$WORK")" = rung2 ] && [ -n "$(ls "$WORK/../parked" 2> /dev/null)" ]; }; } +park_finish() { # end of the run: every parked item in one NEEDS-JUDGMENT.md, exit with the first parked code + [ "$DRY" != 1 ] && parked_any || return 0 + local also=() out first code step + [ "$(basename "$WORK")" != rung2 ] || [ ! -d "$WORK/../parked" ] || also=(--also "$(cd "$WORK/.." && pwd)/parked") + out=$(python3 "$B/parked.py" report --work "$WORK" --steps "${STEPS[*]}" ${also[@]+"${also[@]}"}) || return 0 + first=$(printf '%s\n' "$out" | tail -n 1); code=$(echo "$first" | cut -d' ' -f2); step=$(echo "$first" | cut -d' ' -f3) + STEP=$step + { printf '%s\n' "$out" | sed '$d' + printf '\nRerun once, after the answers, from the first parked step (%s; a rung-1 step: the rung-1 command with --from %s):\n\n %s\n' "$step" "$step" "$(rerun_cmd)" + } > "$WORK/NEEDS-JUDGMENT.md" + cockpit_regen + say "STOP auto: $(grep -c '^` and the nj_add text after it. Output: the +grouped markdown on stdout; the last line `first: N S DIR` names the first parked block in step order (rung 1 first): the +exit code of the run and the step to rerun from. Exit 1 when nothing is parked (no output). +""" +import argparse, os, re, sys + +OPEN = re.compile(r"^$") +KINDS = [("13", "Modules and data"), ("11", "Journeys"), ("12", "Screens, comparisons and the gate")] + + +def blocks(d, steps): + out = [] + for s in steps: + p = os.path.join(d, s + ".md") + if not os.path.isfile(p): + continue + cur = None + for ln in open(p, encoding="utf-8", errors="replace").read().splitlines(): + m = OPEN.match(ln) + if m: + cur = {"code": m.group(1), "step": m.group(2), "dir": d, "text": []} + out.append(cur) + elif cur is not None: + cur["text"].append(ln) + return out + + +def assumed(work): + p = os.path.join(work, "DECISIONS.md") + return [ln for ln in open(p, encoding="utf-8", errors="replace").read().splitlines() if "ASSUMED" in ln] if os.path.isfile(p) else [] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + sub = ap.add_subparsers(dest="cmd", required=True) + r = sub.add_parser("report") + r.add_argument("--work", required=True); r.add_argument("--steps", required=True); r.add_argument("--also") + a = ap.parse_args() + steps = a.steps.split() + rung = lambda d: "rung 1" if a.also and d == a.also else "this rung" if a.also else "" + bl = (blocks(a.also, steps) if a.also else []) + blocks(os.path.join(a.work, "parked"), steps) + if not bl: + return 1 + print(f"# NEEDS JUDGMENT: {len(bl)} parked item(s) (Interview mode: auto), answer them in one round\n") + print("The run went on past each of these; nothing below is passed. Decide each in writing (the allow/approved/ack files") + print("its text names, or the cockpit page), then rerun once from the step named in the last line.\n") + for code, title in KINDS + [(c, f"Other (exit {c})") for c in sorted({b["code"] for b in bl} - {k for k, _ in KINDS})]: + group = [b for b in bl if b["code"] == code] + if not group: + continue + print(f"# {title}: {len(group)}\n") + for b in group: + where = " ".join(x for x in (rung(b["dir"]), "step", b["step"]) if x) + print(f"") + print("\n".join(b["text"]).strip("\n").replace("\n## ", "\n### ").replace("## ", "### ", 1) + "\n") + asm = assumed(a.work) + (assumed(os.path.dirname(a.also)) if a.also else []) + if asm: + print(f"# Proposal: {len(asm)} recommendation(s) taken without a sign-off (ASSUMED), confirm them\n") + print("\n".join(f"- {x}" for x in asm) + "\n") + f = bl[0] + print(f"first: {f['code']} {f['step']} {f['dir']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/bin/platform_drops.py b/upgrade-toolkit/bin/platform_drops.py new file mode 100644 index 0000000..2b64384 --- /dev/null +++ b/upgrade-toolkit/bin/platform_drops.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""platform_drops.py: a System-module element the platform removes, approved only when the baseline holds no data in it. + +Why (cold test WM1, 2026-11-17): the 9.24 -> 10.24 gate stopped on 4 RED idcheck rows, all in the System module +(System.DataBrokerEntityMetadata and its attributes, System.WorkflowActivity.IsMigrationActivity). Mendix 10 drops them; +the app cannot keep them. FSM approved the same rows by hand after counting 0 rows, and the 9-to-10 playbook says the +same. The count is the proof, so it is made here, on the baseline snapshot, and written as the reason. + +Rule: only RED REMOVED rows of the System module. Entity: approved when its table has 0 rows. Attribute: approved when +its column holds 0 non-null values (or its table is gone and had 0 rows). Any other count, or a query that fails: +not approved, printed as KEEP with the count, a person decides. App and Marketplace modules are never approved here. + +Vendor drops (WM1 rerun 9): editroute.py approves a Marketplace attribute or entity the new version drops when nothing +outside the module refers to it ('# editroute VENDOR'). "No references" is not "no data": MendixSSO.AuthRequest.Nonce held +5 values, the ID check let it through, and only the data rung saw the loss. So each such approval is counted here too. +0 = it stays. Data, or a count that fails = the line is commented out (HELD), and the gate stops on it for a person. + + platform_drops.py OLD.mda NEW.mda --approved approved.txt --psql "docker exec -i PG psql -U mendix -d DB" [--dry-run] + appends one line per approval: ' # platform drop (platform_drops.py): in ' +""" +import argparse, os, re, shlex, subprocess, sys + +B = os.path.dirname(os.path.abspath(__file__)) +ROW = re.compile(r"^RED\s+REMOVED\s+(entity|attribute)\s+(System\.[\w.]+)", re.M) +TAG = "# platform drop (platform_drops.py)" +VENDOR = re.compile(r"^([A-Za-z]\w*\.\w+(?:\.\w+)?)\s+# editroute VENDOR\b.*$", re.M) + + +def count(psql, sql): + r = subprocess.run(shlex.split(psql) + ["-tAc", sql], capture_output=True, text=True) + out = r.stdout.strip() + return int(out) if r.returncode == 0 and out.isdigit() else None + + +def sql_for(qn): + """(sql, what) counting the data in a removed entity (rows) or attribute (non-null values).""" + parts = qn.split(".") + table = f'"{parts[0].lower()}${parts[1].lower()}"' + if len(parts) == 2: + return f"select count(*) from {table}", "rows" + return f'select count(*) from {table} where "{parts[2].lower()}" is not null', "non-null values" + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("old"); ap.add_argument("new") + ap.add_argument("--approved", required=True); ap.add_argument("--psql", required=True) + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--idcheck-out", help="read the idcheck rows from this file instead of running idcheck.py") + a = ap.parse_args() + rows = open(a.idcheck_out).read() if a.idcheck_out else subprocess.run( + [sys.executable, os.path.join(B, "idcheck.py"), a.old, a.new], capture_output=True, text=True).stdout + have = open(a.approved).read() if os.path.exists(a.approved) else "" + db = a.psql.split()[-1] + new, kept = [], 0 + for kind, qn in ROW.findall(rows): + if re.search(rf"^{re.escape(qn)}\s", have, re.M): + continue + sql, what = sql_for(qn) + n = count(a.psql, sql) + if n == 0: + new.append(f"{qn} {TAG}: Mendix removes it; 0 {what} in {db}") + print(f"APPROVE {qn}: 0 {what} in {db}") + else: + kept += 1 + print(f"KEEP {qn}: {'count failed' if n is None else f'{n} {what}'} in {db}; a person decides") + held = [] + for m in VENDOR.finditer(have): + qn = m.group(1) + sql, what = sql_for(qn) + n = count(a.psql, sql) + if n == 0: + print(f"VENDOR-OK {qn}: 0 {what} in {db}") + continue + got = "count failed" if n is None else f"{n} {what}" + held.append((m.group(0), f"# HELD by platform_drops.py: {got} in {db}; a person decides (was: {m.group(0)})")) + print(f"HELD {qn}: {got} in {db}; the vendor drop loses it, a person decides") + if not a.dry_run and held: + for line, repl in held: + have = have.replace(line, repl, 1) + open(a.approved, "w").write(have) + if new and not a.dry_run: + with open(a.approved, "a") as f: + f.write(("" if not have or have.endswith("\n") else "\n") + "\n".join(new) + "\n") + print(f"platform_drops: {len(new)} approved by count, {kept} kept for a person, {len(held)} vendor drop(s) held for data") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/bin/proposal.py b/upgrade-toolkit/bin/proposal.py index becc91f..d2cb9a9 100644 --- a/upgrade-toolkit/bin/proposal.py +++ b/upgrade-toolkit/bin/proposal.py @@ -499,6 +499,17 @@ def major(v): return int(m.group(1)) if m else None +def edited_modules(work): + """Modules whose comparison with their Marketplace package found the app's own edits, or could not be made (localedits.json).""" + try: + with open(os.path.join(work, "localedits.json"), encoding="utf-8") as f: + le = json.load(f) + except (OSError, ValueError): + return set() + return {m["name"] for m in le if isinstance(m, dict) and m.get("name") and + (m.get("domain_error") or any(m.get(k) for k in ("domain", "ui", "logic", "files")))} + + def write_steps(F, work): """Open (or find) one step per triage item and set found and proposed (STEP-RECORDS-PLAN.md section 3). A rerun with the same input adds no event; the same topic and title is the same step. Returns {step id: key} in item order. @@ -512,8 +523,24 @@ def write_steps(F, work): "Make these repairs during the upgrade, as the repair table describes."), ("react", "Dojo to React client", "react", ("move to the React client that the toolkit makes", "moves to the React client that the toolkit makes"), "Move the app to the React client on the last rung: every legacy widget to its React successor, then test (playbooks/dojo-to-react.md).")] + carry = edited_modules(work) for topic, title, key, what, prop in groups: its = [x for x in F.items2 if x["key"].startswith(key + ":")] + if topic == "modules": + # a module with the app's own edits is no routine swap (the owner, 2026-11-17): it goes ahead and the edits are carried + # over, but the screens that use them are checked on both apps after the build; only a difference asks a person + ed = [x for x in its if x["key"].split(":", 1)[1] in carry] + its = [x for x in its if x not in ed] + if ed: + names = [x["key"].split(":", 1)[1] for x in ed] + sid = steps.ensure(work, "module-edits", "Module updates that carry the app's edits", { + "found_say": f"{len(ed)} module {'update holds' if len(ed) == 1 else 'updates hold'} the app's own edits ({', '.join(names)}).", + "stage": "triage", "evidence": [x["text"] for x in ed], + "proposed_say": "Update these modules and carry every edit over to the new version. After the build, compare the screens " + "that use them on the old and the new app. A difference becomes a decision with its case.", + "why": "Not routine: the update replaces the whole module, so each edit has to come back. It goes ahead with the plan " + "(in auto mode at once) and asks a person only when a screen differs.", "closes_by": ["build", "journeys"]}) + out[sid] = "2:module-edits" if not its: continue sid = steps.ensure(work, topic, title, { @@ -777,6 +804,14 @@ def accept_answers(signer): print(f"proposal: PAUSED by the owner: {so[1]}. Resume: remove the Status line and sign, then rerun the pipeline.") return 15 if so and so[0] == "signed" and prev_hash == hashval: + # same plan, newer wording (cold test WM1: the signed cards kept saying "keep old version" after the toolkit + # recommended the update): re-render the items, the person's sign-off section is copied over as written + i = old.find("\n## 6. Sign-off\n") + cur = render(F, hashval, None, old[i + len("\n## 6. Sign-off\n"):].strip("\n")) if i >= 0 else old + if cur != old and parse_signoff(cur) == so: + with open(out_path + ".tmp", "w", encoding="utf-8") as f: + f.write(cur) + os.replace(out_path + ".tmp", out_path) n = accept_answers(so[1]) approve_steps(a.work, sids, "approved", so[1], "signoff", decisions(a.work)) print(f"proposal: signed by {so[1]} on {so[2]}" + (f"; {n} unanswered section-3 item(s) took their recommendation (DECISIONS.md)" if n else "")) @@ -802,7 +837,11 @@ def accept_answers(signer): rows = [("P-0", "proposal", "proposal", "continue without a sign-off", "Interview mode: unattended signs nothing; the run goes on at the recommendations")] rows += [(it["id"], it["name"], it["kind"], it["rec"], "unattended: recommendation taken, nobody asked") for it in F.items3] write_decisions(F, rows, "-", "ASSUMED (unattended)", "a person signs PROPOSAL.md section 6") - approve_steps(a.work, sids, "assumed", "agent", "unattended") + # routine work with a known result (section 2) goes ahead in auto mode; only a real decision waits for a person (the owner, 2026-11-17) + routine = {s: k for s, k in sids.items() if k.startswith("2:")} + approve_steps(a.work, {s: k for s, k in routine.items() if k != "2:module-edits"}, "approved", "agent", steps.ROUTINE) + approve_steps(a.work, {s: k for s, k in routine.items() if k == "2:module-edits"}, "approved", "agent", steps.CARRY) + approve_steps(a.work, {s: k for s, k in sids.items() if s not in routine}, "assumed", "agent", "unattended") print("proposal: not signed (unattended): ASSUMED (unattended) lines in DECISIONS.md; every report says 'proposal not signed'") return 0 print(f"proposal awaits sign-off: {out_path} ({len(F.items2)} done by the toolkit, {len(F.items3)} to decide, {len(F.items4)} for a person)") diff --git a/upgrade-toolkit/bin/repair/copydoc.py b/upgrade-toolkit/bin/repair/copydoc.py new file mode 100755 index 0000000..4ce0a1f --- /dev/null +++ b/upgrade-toolkit/bin/repair/copydoc.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""copydoc.py: copy a document a module update dropped into an own module, and point its users at the copy (repair row 28). + + copydoc.py --from --doc Module.Doc --to Dest [--roles Dest.R1,Dest.R2] [--retarget] [--apply] + +Why (cold CMM): Administration 2.1.2 -> 4.5.0 dropped the page UserRole_Select and the microflow RetrieveTimeZones; two +own Employee pages used them as the select page of a reference set selector and the source of a reference selector: +3 CE1613. mxcli cannot author those selectors (describe says NOT re-executable), so it cannot repoint them either. + +The baseline is the same app at the same Mendix version BEFORE the update (a git commit of the converted tree, written +out with `git show :.mpr > base.mpr`). The document's BSON is copied with fresh ids into module Dest (at +its top level, same name). Its allowed module roles belong to the old module: --roles names the Dest roles that +replace them (required when it had any). --retarget replaces every string that is exactly Module.Doc with Dest.Doc in +the units of the app's own (non-Marketplace) modules and lists each unit it changed. + +Refusals (exit 2): different Mendix versions; the document missing in the baseline; Dest missing or not an own +module; Dest.Doc already present; a document reference inside the copy that the app does not have (copy that first); +roles needed and not given; an MPR v2 project (mprcontents: not supported yet). Dry run unless --apply; the changes +are one new unit plus one per retargeted user, so commit the app before --apply and run mx check after. +Exit: 0 = done (or the dry run shows it), 2 = refused. +""" +import argparse, base64, copy, hashlib, os, re, sqlite3, sys, uuid + +import bson +from bson.binary import Binary + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from copy_mappings import Mpr # noqa: E402 + +QN = re.compile(r"^[A-Za-z_]\w*\.[A-Za-z_]\w*$") +APPSTORE = ("FromAppStore", "IsThemeModule") + + +def die(msg): + print(f"copydoc: REFUSED: {msg}") + sys.exit(2) + + +def version(m): + try: + return m.con.execute("select _ProductVersion from _MetaData").fetchone()[0] + except sqlite3.DatabaseError: + return None + + +def all_docs(m): + """qualified name -> uid for every named unit inside a module (pages, flows, snippets, enums, ...).""" + out = {} + for uid in m.rows: + d = m.decode(uid) + if d and d.get("Name") and d.get("$Type") not in ("Projects$Folder", "Projects$ModuleImpl", "Projects$Module"): + mod = m.module_of(uid) + if mod: + out[f"{mod}.{d['Name']}"] = uid + return out + + +def module_unit(m, name): + for uid in m.rows: + d = m.decode(uid) + if d and d.get("$Type") in ("Projects$ModuleImpl", "Projects$Module") and d.get("Name") == name: + return uid, d + return None, None + + +def strings(o): + if isinstance(o, dict): + for v in o.values(): + yield from strings(v) + elif isinstance(o, list): + for v in o: + yield from strings(v) + elif isinstance(o, str): + yield o + + +def replace_exact(o, old, new): + n = 0 + if isinstance(o, dict): + for k, v in o.items(): + if v == old: + o[k] = new; n += 1 + else: + n += replace_exact(v, old, new) + elif isinstance(o, list): + for i, v in enumerate(o): + if v == old: + o[i] = new; n += 1 + else: + n += replace_exact(v, old, new) + return n + + +def new_ids(doc): + """Fresh $IDs for the whole copy, and every pointer to one of them follows (a page's ControlBar.DefaultButtonPointer + holds the $ID of its default button: renewing the button but not the pointer makes mx check die on load with + KeyNotFoundException, cold CMM).""" + ids = {} + + def collect(o): + if isinstance(o, dict): + for k, v in o.items(): + if k == "$ID" and isinstance(v, (bytes, Binary)): + ids[bytes(v)] = uuid.uuid4().bytes + else: + collect(v) + elif isinstance(o, list): + for v in o: + collect(v) + + def swap(v): + return (Binary(ids[bytes(v)], v.subtype) if isinstance(v, Binary) else ids[bytes(v)]) \ + if isinstance(v, (bytes, Binary)) and bytes(v) in ids else v + + def remap(o): + if isinstance(o, dict): + for k, v in o.items(): + o[k] = swap(v) if isinstance(v, (bytes, Binary)) else remap(v) + elif isinstance(o, list): + for i, v in enumerate(o): + o[i] = swap(v) if isinstance(v, (bytes, Binary)) else remap(v) + return o + + collect(doc) + return remap(doc), ids + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("app") + ap.add_argument("--from", dest="base", required=True) + ap.add_argument("--doc", required=True) + ap.add_argument("--to", required=True) + ap.add_argument("--roles", default="") + ap.add_argument("--retarget", action="store_true") + ap.add_argument("--apply", action="store_true") + a = ap.parse_args() + try: + app, base = Mpr(a.app), Mpr(a.base) + except ValueError as e: + die(str(e)) + if "Contents" not in app.cols or any(f for _, _, f in app.rows.values()): + die(f"{app.path} is an MPR v2 project (mprcontents): not supported yet") + va, vb = version(app), version(base) + if va != vb: + die(f"Mendix versions differ: app {va}, baseline {vb} (write the baseline out of the converted tree's git history)") + if not QN.match(a.doc): + die(f"--doc {a.doc}: give Module.Doc") + src_mod, name = a.doc.split(".") + bdocs, adocs = all_docs(base), all_docs(app) + if a.doc not in bdocs: + die(f"{a.doc} is not in the baseline {base.path}") + new_qn = f"{a.to}.{name}" + if new_qn in adocs: + die(f"{new_qn} already exists in the app") + muid, mod = module_unit(app, a.to) + if muid is None: + die(f"module {a.to} is not in the app") + if any(mod.get(k) for k in APPSTORE) or mod.get("AppStoreVersion"): + die(f"module {a.to} is a Marketplace module: copy into one of the app's own modules") + doc = copy.deepcopy(base.decode(bdocs[a.doc])) + roles = [r for r in (doc.get("AllowedModuleRoles") or [])[1:] if isinstance(r, str)] + new_roles = [r.strip() for r in a.roles.split(",") if r.strip()] + if roles and not new_roles: + die(f"{a.doc} allows {', '.join(roles)}: name the {a.to} roles that replace them with --roles") + missing = sorted({s for s in strings(doc) if QN.match(s) and s.split(".")[0] not in ("System",) + and s in bdocs and s not in adocs and s != a.doc}) + if missing: + die(f"{a.doc} uses {', '.join(missing)}, which the app does not have: copy those first") + if roles: + doc["AllowedModuleRoles"] = [doc["AllowedModuleRoles"][0]] + new_roles + doc, ids = new_ids(doc) + nid = bytes(doc["$ID"]) + blob = bson.encode(doc) + users = [] + if a.retarget: + for uid in app.rows: + d = app.decode(uid) + if not d or not any(s == a.doc for s in strings(d)): + continue + mname = app.module_of(uid) + _, md = module_unit(app, mname) if mname else (None, None) + if md is None or any(md.get(k) for k in APPSTORE) or md.get("AppStoreVersion"): + continue + d2 = copy.deepcopy(d) + n = replace_exact(d2, a.doc, new_qn) + users.append((uid, f"{mname}.{d.get('Name')}", n, bson.encode(d2))) + print(f"copydoc: {a.doc} ({doc.get('$Type')}) -> {new_qn}" + (f", roles {', '.join(roles)} -> {', '.join(new_roles)}" if roles else "")) + for _, qn, n, _ in users: + print(f" RETARGET {qn}: {n} reference(s) {a.doc} -> {new_qn}") + if a.retarget and not users: + print(f" no own unit uses {a.doc}") + if not a.apply: + print("dry run: nothing written (--apply writes it)") + return 0 + h = lambda b: base64.b64encode(hashlib.sha256(b).digest()).decode() + cols = ["UnitID", "ContainerID", "ContainmentName", "TreeConflict", "ContentsHash", "ContentsConflicts", "Contents"] + have = [c for c in cols if c in app.cols] + vals = {"UnitID": nid, "ContainerID": muid, "ContainmentName": "Documents", "TreeConflict": 0, + "ContentsHash": h(blob), "ContentsConflicts": "", "Contents": blob} + app.con.execute(f"insert into Unit ({', '.join(have)}) values ({', '.join('?' * len(have))})", [vals[c] for c in have]) + app.con.commit() + for uid, qn, n, b in users: + app.write(uid, b) + back = Mpr(app.path) + copied = back.decode(nid) or {} + dangling = [] + + def ptrs(o, p=""): + if isinstance(o, dict): + for k, v in o.items(): + if k.endswith("Pointer") and isinstance(v, (bytes, Binary)) and bytes(v) not in seen: + dangling.append(p + "/" + k) + elif not isinstance(v, (bytes, Binary)): + ptrs(v, p + "/" + k) + elif isinstance(o, list): + for i, v in enumerate(o): + ptrs(v, f"{p}[{i}]") + + seen = set() + + def allids(o): + if isinstance(o, dict): + for k, v in o.items(): + if k == "$ID": + seen.add(bytes(v)) + else: + allids(v) + elif isinstance(o, list): + for v in o: + allids(v) + allids(copied); ptrs(copied) + for x in dangling: + print(f" DANGLING pointer {x}: points at no element of the copy") + ok = not dangling and new_qn in all_docs(back) and all(not any(s == a.doc for s in strings(back.decode(u))) for u, _, _, _ in users) + print(f"copydoc: written, re-read {'OK' if ok else 'MISMATCH'}; run mx check now") + return 0 if ok else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/bin/repair/javacjar.py b/upgrade-toolkit/bin/repair/javacjar.py new file mode 100644 index 0000000..f6a9b63 --- /dev/null +++ b/upgrade-toolkit/bin/repair/javacjar.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""javacjar.py: for each javac "cannot find symbol" in an mxbuild log, name the jars that carry the class and the one that hides it. + + javacjar.py + javacjar.py --strip userlib/.jar [--apply] + +Why (cold CMM, 10.24.28): MendixSSO 10.6.2 calls CookieHeaderNames.SameSite, which netty-codec-http 4.2.17 in +vendorlib has. userlib/server-examples-0.4.1-SNAPSHOT.jar (an OPC UA library, built as one fat jar) carries its +own older Netty without it, and javac took that copy: "cannot find symbol ... location: class CookieHeaderNames". +jarclash.py listed the clash after the updates, as a note; the build stop named only javafinal.py. + +The class of each error is resolved through the imports of the source file, then looked up in every userlib/ and +vendorlib/ jar (and their nested jars). A jar that has the class but not the symbol (no nested class, no name in the +class file) HIDES the jar that has both: + + SYMBOL io.netty.handler.codec.http.cookie.CookieHeaderNames.SameSite (OpenIDUtils.java:237) + HAS vendorlib/netty-codec-http-4.2.17.Final.jar + HIDDEN BY userlib/server-examples-0.4.1-SNAPSHOT.jar (an older copy of the class) + +--strip writes the fat jar again without every class a vendorlib jar also carries (the libraries the modules manage +win), dry run unless --apply; its own classes and the classes only it has stay. The app's git keeps the old jar. +Exit: 0 = nothing hidden (or a strip done), 1 = a class is hidden by an older copy, 2 = input error. +""" +import argparse, io, os, re, shutil, sys, tempfile, zipfile + +ERR = re.compile(r"^(?P\S+\.java):(?P\d+): error: cannot find symbol\n(?:.*\n){0,3}?\s*symbol:\s+(?P\w+)\s+(?P[\w$]+)[^\n]*\n\s*location:\s+(?:\w+\s+)*?(?:of type\s+)?(?P[\w.$]+)\s*$", re.M) + + +def jars(project): + """[(label, ZipFile)] of userlib/ and vendorlib/ jars and the jars nested in them.""" + out = [] + for d in ("userlib", "vendorlib"): + p = os.path.join(project, d) + for f in sorted(os.listdir(p)) if os.path.isdir(p) else []: + if not f.endswith(".jar"): + continue + try: + z = zipfile.ZipFile(os.path.join(p, f)) + except zipfile.BadZipFile: + continue + out.append((f"{d}/{f}", z)) + for n in z.namelist(): + if n.endswith(".jar") and re.search(r"(^|/)(lib|BOOT-INF/lib)/[^/]+\.jar$", n): + try: + out.append((f"{d}/{f}!{n}", zipfile.ZipFile(io.BytesIO(z.read(n))))) + except zipfile.BadZipFile: + pass + return out + + +def fqcn(project, src, simple): + """The full class name of a simple name, from the imports of the source file.""" + if "." in simple: + return simple + path = src if os.path.isabs(src) else os.path.join(project, src) + try: + text = open(path, encoding="utf-8", errors="replace").read() + except OSError: + return None + m = re.search(rf"^import\s+([\w.]+\.{re.escape(simple)})\s*;", text, re.M) + if m: + return m.group(1) + m = re.search(r"^package\s+([\w.]+)\s*;", text, re.M) + return f"{m.group(1)}.{simple}" if m else None + + +def errors(log): + t = open(log, encoding="utf-8", errors="replace").read() + for m in ERR.finditer(t): + yield m.group("file"), m.group("line"), m.group("sym"), m.group("loc") + + +def strip(project, jar, apply): + vend = set() + for label, z in jars(project): + if label.startswith("vendorlib/"): + vend |= {n for n in z.namelist() if n.endswith(".class")} + path = os.path.join(project, jar) + try: + src = zipfile.ZipFile(path) + except (OSError, zipfile.BadZipFile) as e: + print(f"javacjar: {jar}: {e}", file=sys.stderr); return 2 + drop = [i for i in src.infolist() if i.filename in vend] + pk = sorted({i.filename.rsplit("/", 1)[0].replace("/", ".") for i in drop}) + print(f"STRIP {jar}: {len(drop)} class(es) a vendorlib jar also carries, in {len(pk)} package(s)" + + (f" ({', '.join(pk[:4])}{', ...' if len(pk) > 4 else ''})" if pk else "")) + print(f" KEEP {len(src.infolist()) - len(drop)} entries") + if not drop or not apply: + print("dry run (add --apply to write the jar)" if drop else "nothing to strip") + return 0 + gone = {i.filename for i in drop} + fd, tmp = tempfile.mkstemp(suffix=".jar", dir=os.path.dirname(path)); os.close(fd) + with zipfile.ZipFile(tmp, "w") as out: + for i in src.infolist(): + if i.filename not in gone: + out.writestr(i, src.read(i.filename)) + src.close() + shutil.move(tmp, path) + print(f"wrote {jar}; rerun mxbuild, then the quick start (the library now runs on the vendorlib copies)") + return 0 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("project"); ap.add_argument("log", nargs="?") + ap.add_argument("--strip"); ap.add_argument("--apply", action="store_true") + a = ap.parse_args() + if a.strip: + sys.exit(strip(a.project, a.strip, a.apply)) + if not a.log or not os.path.isfile(a.log): + sys.exit(f"javacjar: give the mxbuild log ({a.log}: not a file)") + js = jars(a.project) + seen, hidden, n = set(), 0, 0 + for src, line, sym, loc in errors(a.log): + cls = fqcn(a.project, src, loc.split("<")[0]) + key = (cls, sym) + if key in seen: + continue + seen.add(key); n += 1 + where = f"{os.path.basename(src)}:{line}" + if not cls: + print(f"SYMBOL {loc}.{sym} ({where}): the class name is not in the imports, not looked up"); continue + entry = cls.replace(".", "/") + ".class" + has, lacks = [], [] + for label, z in js: + names = set(z.namelist()) + if entry not in names: + continue + nested = cls.replace(".", "/") + "$" + sym + ".class" + (has if nested in names or sym.encode() in z.read(entry) else lacks).append(label) + print(f"SYMBOL {cls}.{sym} ({where})") + for h in has: + print(f" HAS {h}") + for l in lacks: + print(f" HIDDEN BY {l} (an older copy of the class)" if has else f" LACKS {l}") + if has and lacks: + hidden += 1 + if not has and not lacks: + print(" in no jar of userlib/ or vendorlib/") + print(f"javacjar: {n} missing symbol(s), {hidden} hidden by an older copy of the class in another jar" + + ("; repair: steps/repair.md row 31" if hidden else "")) + sys.exit(1 if hidden else 0) + + +if __name__ == "__main__": + main() diff --git a/upgrade-toolkit/bin/repair/modroles.py b/upgrade-toolkit/bin/repair/modroles.py new file mode 100644 index 0000000..c228890 --- /dev/null +++ b/upgrade-toolkit/bin/repair/modroles.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""modroles.py: module roles a Marketplace update ADDED, mapped to the app's user roles by name. + +Why (cold test WM1, 2026-11-17): Atlas_Core 3 had no module roles; Atlas_Core 4 brings Atlas_Core.Administrator and +Atlas_Core.User, and its layout's language selector calls Atlas_Core.DS_Account_CurrentUser, allowed for those two only. +mxcli's update restored "0 role grant(s)" (there were none), so every page on an Atlas layout failed for every user +role: "Executing action failed for security reasons". The build was clean; only the smoke walk saw it. Company brain: +UI modules' user/administrator module roles must be mapped to the app's user roles. + +Rule: only roles that are NEW in this run (absent from the original model) of a module with a 'step: ' +commit, and held by no user role. Such a role Mod.R goes to every user role that already holds a module role named R in +another module (Administrator holds Administration.Administrator, so it gets Atlas_Core.Administrator). No name match: +UNMATCHED, nothing granted, a person decides. The guest (anonymous) user role is never granted: GUEST, a person decides. A role the app left unmapped on purpose is never new, so never touched. + + modroles.py APP.mpr --original ORIGINAL.mpr [--mdl OUT.mdl] [--mxcli CMD] + prints GRANT / GUEST / UNMATCHED lines and a summary; --mdl writes the ALTER USER ROLE statements (no grants: no file) +""" +import argparse, os, re, shlex, subprocess, sys + + +def mx(cmd, mpr, q): + r = subprocess.run(shlex.split(cmd) + ["-p", mpr, "-c", q], capture_output=True, text=True) + return r.stdout + + +def table_col(out, col=0): + rows = [] + for ln in out.splitlines(): + if not ln.startswith("|") or set(ln) <= set("|-"): + continue + rows.append([c.strip() for c in ln.strip("|").split("|")][col]) + return rows[1:] # drop the header + + +def module_roles(cmd, mpr): + return {r for r in table_col(mx(cmd, mpr, "show module roles")) if "." in r} + + +def user_roles(cmd, mpr): + out = {} + for u in table_col(mx(cmd, mpr, "show user roles")): + d = mx(cmd, mpr, f"describe user role {u}") # "describe userrole X" is a parse error in -c + m = re.search(r"ModuleRoles:\s*\(([^)]*)\)", d) + out[u] = [x.strip() for x in m.group(1).split(",") if x.strip()] if m else [] + return out + + +def updated_modules(app_dir): + r = subprocess.run(["git", "-C", app_dir, "log", "--format=%s"], capture_output=True, text=True) + return {m.group(1) for m in re.finditer(r"^step: (\w+) \S+", r.stdout, re.M)} + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("mpr"); ap.add_argument("--original", required=True) + ap.add_argument("--mdl"); ap.add_argument("--mxcli", default=os.environ.get("UPG_MXCLI_READ", "mxcli")) + a = ap.parse_args() + now, orig = module_roles(a.mxcli, a.mpr), module_roles(a.mxcli, a.original) + users = user_roles(a.mxcli, a.mpr) + held = {r for rs in users.values() for r in rs} + moved = updated_modules(os.path.dirname(os.path.abspath(a.mpr))) + m = re.search(r"Guest User Role:\s*(\w+)", mx(a.mxcli, a.mpr, "describe app security")) + guest = m.group(1) if m else None + grants, unmatched = [], [] + for r in sorted(now - orig - held): + mod, name = r.split(".", 1) + if mod not in moved: + continue + to = [u for u, rs in sorted(users.items()) if any(x.split(".", 1)[1].lower() == name.lower() + and x.split(".", 1)[0] != mod for x in rs)] + gu = guest in to + if gu: # what anonymous visitors may run is a person's call + to.remove(guest) + print(f"GUEST {r}: the guest role {guest} holds a '{name}' role elsewhere; not granted, decide by hand") + if to: + for u in to: + grants.append((u, r)) + print(f"GRANT {r} -> user role {u} (new in the {mod} update; {u} holds a '{name}' role in another module)") + else: + unmatched.append(r) + gu or print(f"UNMATCHED {r}: new in the {mod} update, no user role holds a '{name}' role elsewhere; map it by hand") + if grants and a.mdl: + with open(a.mdl, "w") as f: + f.write("mdl 1;\n" + "".join(f"ALTER USER ROLE {u} ADD MODULE ROLES ({r});\n" for u, r in grants)) + print(f"modroles: {len(grants)} grant(s), {len(unmatched)} unmatched new module role(s)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/upgrade-toolkit/bin/run_journeys.py b/upgrade-toolkit/bin/run_journeys.py index f6422e6..5238383 100644 --- a/upgrade-toolkit/bin/run_journeys.py +++ b/upgrade-toolkit/bin/run_journeys.py @@ -1239,7 +1239,24 @@ def destructive_scope(d): return d.get("_scope") if d.get("_scope") in DESTRUCTIVE_SCOPES else None -def data_waves(jdir, gen_dir, only=None): +def baseline_fail_skips(jdir, cmp_summary): + """--skip-baseline-fail (WM1 2026-11-17: 136 of 160 journeys that failed on the old app were walked on the new app + again, about half the journeys time, to land in "BASELINE DEFECT"): {id: (before verdict, + role folder)} of the journeys the new app need not walk. Only a FAIL on the old app, and only a journey that feeds no later one: one whose path needs data + (second and later passes) or any journey of a set without data passes. A first-pass journey may create the rows a + later one needs, so it still runs. heal.py's old-app recheck runs --only the failing ids, without these, the same way.""" + before = json.load(open(cmp_summary)).get("journeys", {}) + files = sorted(jdir.glob("*/*.journey.json")) if jdir.is_dir() else [] + waves = any(read_journey(f).get("_path_needs") for f in files) + out = {} + for f in files: + j = read_journey(f) + if before.get(j["id"], {}).get("verdict") == "FAIL" and (j.get("_path_needs") or not waves): + out[j["id"]] = ("FAIL", f.parent.name) + return out + + +def data_waves(jdir, gen_dir, only=None, skip=()): """[(key, role, role_dir, files)] in the order they run. First pass: every role (role_dirs order) with its journeys that need no data on the way. Second pass, same role order: the journeys whose path needs data ("_path_needs", gen_journeys.py: a row in a list, a workflow instance, an open user task), so the rows the first pass created @@ -1258,6 +1275,7 @@ def data_waves(jdir, gen_dir, only=None): files = sorted(role_dir.glob("*.journey.json")) if only: files = [f for f in files if read_journey(f)["id"] in only] + files = [f for f in files if read_journey(f)["id"] not in skip] tier = [f for f in files if destructive_scope(read_journey(f))] files = [f for f in files if f not in tier] destr += [(f"{role_dir.name}{DESTR_WAVE}{k}", role_dir.name, role_dir, [f]) for k, f in enumerate(tier, 1)] @@ -1389,6 +1407,9 @@ def main(): ap.add_argument("--work", required=True) ap.add_argument("--positive-control", action="store_true") ap.add_argument("--only", help="comma list of journey ids") + ap.add_argument("--skip-baseline-fail", action="store_true", + help="with --compare: do not walk a journey that FAILed on the old app and feeds no later journey; " + "it is listed as SKIPPED (failed on the old app too), never a FAIL of the new app") ap.add_argument("--resume", action="store_true", help="AD-1: run again only the journeys the last run of this label could not run because the app went " "down (summary.json app_down), and merge them into that summary.json (exit 75 = still some left)") @@ -1504,7 +1525,13 @@ def main(): parked = None dirty = None # PR-P4: the destructive journey that last ran on this database, " ()" sfx = f".resume{summary['resumed']}" if prev else "" - for key, role, role_dir, files in ([] if prev and not only else data_waves(jdir, a.gen_dir, only)): + skip = baseline_fail_skips(jdir, cmp_summary) if a.skip_baseline_fail and cmp_summary and not prev else {} + for jid, (bv, rl) in skip.items(): + summary["journeys"][jid] = {"role": rl, "verdict": "SKIPPED", "compare": f"BASELINE DEFECT (before {bv})", + "why": f"not run on the new app: it {bv}ed on the old app too (--skip-baseline-fail)"} + if skip: + print(f"skip: {len(skip)} journeys failed on the old app and feed no later one: not run on the new app", file=sys.stderr) + for key, role, role_dir, files in ([] if prev and not only else data_waves(jdir, a.gen_dir, only, skip)): prior_dirty = dirty if DESTR_WAVE in key and files: dirty = f"{read_journey(files[0])['id']} ({destructive_scope(read_journey(files[0]))})" @@ -1811,7 +1838,7 @@ def main(): for s in summary["journeys"].values()) # SCREEN / EFFECT INVALID is INVALID, never FAIL (N1R2-8) else: fail = "FAIL" in vs - counts = {v: vs.count(v) for v in ("PASS", "FAIL", "INVALID", "INFO", "NOT CHECKED", "UNREACHABLE", "CONTAMINATED")} + counts = {v: vs.count(v) for v in ("PASS", "FAIL", "INVALID", "INFO", "NOT CHECKED", "UNREACHABLE", "CONTAMINATED", "SKIPPED")} # E2E-11: a baseline defect with checked evidence is PASS* (listed), not INVALID; without evidence it stays INVALID. # B4N1-13: the headline counts it as PASS* (baseline defect), not as FAIL next to "= N PASS*" two lines down; # an unevidenced FAIL stays FAIL @@ -1823,6 +1850,7 @@ def main(): print(f"journeys: {counts['PASS']} PASS, {counts['FAIL']} FAIL, {counts['INVALID']} INVALID" + (f", {len(evid)} PASS* (baseline defect)" if evid else "") + (f", {counts['UNREACHABLE']} PASS* (unreachable in the OLD UI, evidence checked)" if counts["UNREACHABLE"] else "") + + (f", {counts['SKIPPED']} SKIPPED (failed on the old app too, not run on the new one)" if counts["SKIPPED"] else "") + (f", {counts['NOT CHECKED']} NOT CHECKED (excluded: {a.exclude_systems})" if counts["NOT CHECKED"] else "") + (f", {counts['CONTAMINATED']} CONTAMINATED (login failed after a destructive journey on the same database)" if counts["CONTAMINATED"] else "") + (f", {counts['INFO']} INFO (covered, unreachable or out of scope, listed only)" if counts["INFO"] else "")) @@ -1835,7 +1863,7 @@ def main(): if anon: print(anon["line"]) veff = ["PASS*" if j in evid or s["verdict"] == "UNREACHABLE" else s["verdict"] for j, s in summary["journeys"].items()] - ex = (1 if fail or ctl_bad else INVALID_ONLY if any(v not in ("PASS", "INFO", "PASS*", "NOT CHECKED") for v in veff) + ex = (1 if fail or ctl_bad else INVALID_ONLY if any(v not in ("PASS", "INFO", "PASS*", "NOT CHECKED", "SKIPPED") for v in veff) else EXCLUDED_ONLY if counts["NOT CHECKED"] else 0) if cov and a.compare: if a.no_actions: diff --git a/upgrade-toolkit/bin/runapp.py b/upgrade-toolkit/bin/runapp.py index 472f51c..e62d5b2 100755 --- a/upgrade-toolkit/bin/runapp.py +++ b/upgrade-toolkit/bin/runapp.py @@ -378,6 +378,9 @@ def start(name, mda, rtv, db, port_off=0, login_override=None, harness=None): zipfile.ZipFile(mdap).extractall(base) for d in ("data/files", "data/tmp", "data/model-upload", "data/database"): (base / d).mkdir(parents=True, exist_ok=True) + df = os.environ.get("UPG_DATA_FILES") # the package's demo file contents (data-snapshot.zip data/files), back after the wipe + if df and Path(df).is_dir(): + shutil.copytree(df, base / "data/files", dirs_exist_ok=True) note = apply_login_override(base, login_override or os.environ.get("UPG_LOGIN_OVERRIDE", "auto"), name) if note: print("runapp: " + note) diff --git a/upgrade-toolkit/bin/seed_snapshot.py b/upgrade-toolkit/bin/seed_snapshot.py index e3db6a8..065980a 100644 --- a/upgrade-toolkit/bin/seed_snapshot.py +++ b/upgrade-toolkit/bin/seed_snapshot.py @@ -248,6 +248,11 @@ def rows_of(t): return list(tables[t].rows.values()) s_ei = {r[0].v: (int(r[1].v), int(r[2].v)) for r in rows_of("mendixsystem$entityidentifier")} # GUID -> (short, seq) ec = {c: col("mendixsystem$entity", c) for c in ("id", "entity_name", "table_name", "superentity_id")} s_ent = {r[ec["id"]].v: (r[ec["entity_name"]].v, r[ec["table_name"]].v, r[ec["superentity_id"]].v) for r in rows_of("mendixsystem$entity")} + # cold CMM 2026-11-17: an external (OData) entity keeps a $pk table whose "_id" is the REMOTE key, a Long that + # can look like an object id; remapping it made 5 such tables UNMAPPED and nothing loaded. Columns other than id of a + # table whose entity has remote_primary_key are kept as they are. + rk = col("mendixsystem$entity", "remote_primary_key") if "remote_primary_key" in [x[0] for x in tables["mendixsystem$entity"].cols] else None + remote_pk = {r[ec["table_name"]].v for r in rows_of("mendixsystem$entity") if rk is not None and r[rk].k == "t"} skip_re = [(re.compile(p), why) for p, why in SKIP_RULES] waived = set(a.skip) @@ -321,6 +326,7 @@ def pg_table(t): maxseq = collections.defaultdict(int) def remap(t, cname, ctype, v): if v.k != "n" or not ctype.upper().startswith("BIGINT") or f"{t}.{cname}" in a.keep_column: return v + if t in remote_pk and cname != "id": return v x = int(v.v) if x in idmap: return V("n", str(idmap[x])) if x <= (1 << 48): return v diff --git a/upgrade-toolkit/bin/smokewalk.js b/upgrade-toolkit/bin/smokewalk.js index 3c60203..ac6e0d7 100755 --- a/upgrade-toolkit/bin/smokewalk.js +++ b/upgrade-toolkit/bin/smokewalk.js @@ -131,7 +131,8 @@ function landedWhere({ want }) { else { const h = history.state, i = h && h.pageName ? h.pageName.indexOf('.') : -1; if (i > 0 && h.params && h.params.location === 'content') p = h.pageName.slice(0, i) + '/' + h.pageName.slice(i + 1) + '.page.xml'; } if (p === want) return 'content'; - const win = [...document.querySelectorAll('.mx-window')].find(w => w.offsetParent !== null); + // cold CMM: Atlas styles .mx-window position:fixed, whose offsetParent is always null: every popup menu item FAILed + const win = [...document.querySelectorAll('.mx-window')].find(w => w.offsetParent !== null || (w.getClientRects && w.getClientRects().length > 0)); if (win) return 'popup'; return false; } catch (e) { return false; } diff --git a/upgrade-toolkit/bin/snapshot.py b/upgrade-toolkit/bin/snapshot.py index 6522cc2..6ae4b4e 100755 --- a/upgrade-toolkit/bin/snapshot.py +++ b/upgrade-toolkit/bin/snapshot.py @@ -94,6 +94,9 @@ def db_snapshot(psql): return out +ADMIN = re.sub(r"[^\w.@-]", "", os.environ.get("UPG_ADMIN_USER", "MxAdmin")) + + def anonymous_snapshot(psql, ents, attrs, cols): """{"users": n anonymous rows, "total": n, "attrs": {"System.User.X": {...}}, "assoc": {name: {...}}} for the rows of System.User that are not anonymous; {} when System.User or IsAnonymous cannot be found.""" @@ -107,12 +110,20 @@ def anonymous_snapshot(psql, ents, attrs, cols): named = f'coalesce("{acol}", false) = false' n_anon = int(q(psql, f'select count(*) from "{ut}" where not ({named})')[0][0]) n_total = int(q(psql, f'select count(*) from "{ut}"')[0][0]) - res = {"users": n_anon, "total": n_total, "attrs": {}, "assoc": {}} + res = {"users": n_anon, "total": n_total, "attrs": {}, "assoc": {}, "noadmin": {}} + # WM1 2026-11-17: runapp.py sets the local admin's password at every start (create_admin_user), which changes that + # one row's Password and changedDate; the same figures without that row let diff.py tell it from a real change + ncol = next((c for an, c in attrs.get(uid, []) if an == "Name"), None) + ncol = ncol if ncol in cols[ut] else None for an, c in attrs.get(uid, []): if c in cols[ut]: n, h = q(psql, f'select count("{c}"), md5(coalesce(string_agg(coalesce("{c}"::text,\'~\'), \'|\' order by id),\'\')) ' f'from "{ut}" where {named}')[0] res["attrs"][f"System.User.{an}"] = {"nonnull": int(n), "md5": h} + if ncol: + n, h = q(psql, f'select count("{c}"), md5(coalesce(string_agg(coalesce("{c}"::text,\'~\'), \'|\' order by id),\'\')) ' + f'from "{ut}" where {named} and coalesce("{ncol}", \'\') <> \'{ADMIN}\'')[0] + res["noadmin"][f"System.User.{an}"] = {"nonnull": int(n), "md5": h} for name, table, pcol, ccol, peid in q(psql, 'select association_name, table_name, parent_column_name, child_column_name, parent_entity_id ' 'from "mendixsystem$association"'): if peid != uid or table not in cols or ccol not in cols[table]: diff --git a/upgrade-toolkit/bin/steps.py b/upgrade-toolkit/bin/steps.py index 3be9c88..58e9568 100755 --- a/upgrade-toolkit/bin/steps.py +++ b/upgrade-toolkit/bin/steps.py @@ -949,6 +949,7 @@ def topic_title(work, sid): ROUTINE = "routine" +CARRY = "carry-over" # goes ahead like routine work; the screens that use the carried edits are checked on both apps (the owner, 2026-11-17) GENERIC = {"build", "builds", "widget", "widgets", "module", "modules", "error", "errors", "later", "after", "before", "which", "their", "there", "check", "checks", "rows", "journeys", "journey", "pages", "page", "client", "app", "apps", "script", "scripts", "update", "updates", "files", "agent", "notes", "edits", "using", "other", "kept", "cannot", "without", "should"} @@ -1292,7 +1293,7 @@ def routine_ok(sid): "decisions": merge_list(W, host, cur["proposed"].get("decisions") or [], [d]), "evidence": merge_list(W, host, cur["found"].get("evidence") or [], row["rows"])}) keep(host) - if choice and (not cur.get("approval") or cur["approval"].get("source") == ROUTINE): + if choice and (not cur.get("approval") or cur["approval"].get("source") in (ROUTINE, CARRY)): append(W, host, "approval", {"state": "assumed", "source": LEGACY, "needs": "the owner's OK"}, "agent") elif not choice and not cur.get("approval"): routine_ok(host) diff --git a/upgrade-toolkit/bin/stepupdate.sh b/upgrade-toolkit/bin/stepupdate.sh index 2e7fd7c..5a8e9d7 100755 --- a/upgrade-toolkit/bin/stepupdate.sh +++ b/upgrade-toolkit/bin/stepupdate.sh @@ -444,7 +444,19 @@ PY fi case "$forceui" in *",$mod,"*) nsaved=$(find "$saved/$mod" -type f 2>/dev/null | wc -l | tr -d ' ') - if [ $nobase = 1 ] && [ "$nsaved" = 0 ]; then + if [ $nobase = 1 ] && [ "$nsaved" = 0 ] && [ -n "$ledits" ] && python3 - "$ledits" "$mod" <<'PY' +import json, sys +try: + e = next((x for x in json.load(open(sys.argv[1])) if isinstance(x, dict) and x.get("name") == sys.argv[2]), None) +except Exception: + e = None +sys.exit(0 if e and e.get("ui") == [] else 1) +PY + then + # cold CMM 2026-11-17: Administration was forced for the XASSOC route only; localedits listed no UI edit in it, so + # saving 0 files lost nothing. It was KEPT as UNSAVED, MendixSSO moved on without it and mx check had 2 CE0117 + echo " $mod: the --no-baseline route saved 0 files, and localedits ($ledits) lists no UI edits in $mod: nothing to save" + elif [ $nobase = 1 ] && [ "$nsaved" = 0 ]; then # E7-1: the --no-baseline route has no baseline to tell an edit from the package: it saved nothing [ $fstate = edited ] && echo " (its theme/JavaScript FILE edits ARE saved: $st/saved/$mod)" echo " UNSAVED: $mod updated through --no-baseline and saved 0 files; its UI edits (localedits, --force-ui) are NOT saved" @@ -517,7 +529,7 @@ PY echo "STOP: idcheck input error after $mod"; exit 2 fi # NS-7: idcheck warns when the update re-created a module (every unit id new) instead of updating it in place - grep -E '^warn ' "$app/.stepupdate/step$n-$mod.idcheck" | sed 's/^/ /' + grep -E '^(warn|info MODULE_RECREATED) ' "$app/.stepupdate/step$n-$mod.idcheck" | sed 's/^/ /' # NS-6: a module import also replaces shared widgets/*.mpk (notification-service cdd62cc5: the Email Connector import # swapped Datagrid 2.28.3 -> 2.30.6 and 8 more; reverted on day 111). Name every widget the step changed. wout=$(python3 "$bin/widgetdiff.py" "$app" 2>/dev/null) diff --git a/upgrade-toolkit/bin/themefiles.py b/upgrade-toolkit/bin/themefiles.py index 1070ca2..c12f9e2 100644 --- a/upgrade-toolkit/bin/themefiles.py +++ b/upgrade-toolkit/bin/themefiles.py @@ -43,6 +43,9 @@ # nobody edited, and the module read as FILES-EDITED. A file whose first line says it is auto-generated and must not be # modified is the tool's output, not a person's edit: it is not compared. WHOLEGEN = re.compile(r"auto-?generated.*do not (modify|edit)", re.I) +# files the OS writes into any folder it shows (Finder, Explorer): no edit when the app has one or lacks the package's +# (cold CMM: DataWidgets got a plan card "themesource edits" for one .DS_Store that the package held and the app did not) +OSJUNK = re.compile(r"(^|/)(\.DS_Store|\._[^/]*|Thumbs\.db|desktop\.ini)$", re.I) VAR = re.compile(r"^\s*(\$[\w-]+)\s*:\s*(.+?)\s*;\s*(?://.*)?$") @@ -113,7 +116,8 @@ def app_files(app, mod): for base, _, files in os.walk(os.path.join(top, d)): for f in files: p = os.path.join(base, f) - out[os.path.relpath(p, app).replace(os.sep, "/")] = open(p, "rb").read() + if not OSJUNK.search(f): + out[os.path.relpath(p, app).replace(os.sep, "/")] = open(p, "rb").read() return out @@ -122,7 +126,7 @@ def pkg_files(mpk, mod): with zipfile.ZipFile(mpk) as z: for n in z.namelist(): parts = n.split("/") - if len(parts) < 3 or parts[0] not in dict(ROOTS) or parts[1].lower() != mod.lower() or n.endswith("/"): + if len(parts) < 3 or parts[0] not in dict(ROOTS) or parts[1].lower() != mod.lower() or n.endswith("/") or OSJUNK.search(n): continue out[n] = z.read(n) return out diff --git a/upgrade-toolkit/bin/widgetskew.py b/upgrade-toolkit/bin/widgetskew.py new file mode 100644 index 0000000..956348b --- /dev/null +++ b/upgrade-toolkit/bin/widgetskew.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""widgetskew.py: did a module update replace a widget whose styles belong to a module that stayed at its old version? + + widgetskew.py --plan [--cache DIR] + +Why (cold CMM, 9.24.2 -> 10.24.28): Atlas_Web_Content 3.9.0 brought Datagrid 2.30.6 into widgets/ (was 2.2.2), while +DataWidgets, whose package ships that widget and whose themesource/datawidgets holds its styles, was kept (FILES-EDITED). +Every data grid page rendered as unstyled rows; afterwalk passed 23/23 (the data is on the page) and the screens rung +only said REVIEW on each one. + +Reads .stepupdate/widgets.txt (the widgets each module step changed, written by stepupdate) and .stepupdate/kept.txt. +For each kept module that has a plan line (" "), the cached package -.mpk +lists the widgets its target version ships; a widget another step replaced is a skew: + + WIDGET-SKEW Datagrid 2.2.2 -> 2.30.6 (by Atlas_Web_Content 3.9.0) ships with DataWidgets 3.11.8; DataWidgets stays at + the installed version, its theme styles do not match the new widget + +Exit: 0 = no skew, 1 = a skew (update the kept module, or put the old widget back), 2 = input error. +A kept module without a cached package is named on a NOTE line and not counted. +""" +import argparse, os, re, sys, zipfile + + +def steps(app): + """{widget file: (name, old, new, by)} from .stepupdate/widgets.txt (last step wins).""" + out, p = {}, os.path.join(app, ".stepupdate", "widgets.txt") + for line in open(p, encoding="utf-8") if os.path.isfile(p) else []: + by, _, rest = line.strip().partition(": ") + for part in rest.split("; "): + m = re.match(r"(\S+\.mpk) (.+?) (\S+) -> (\S+)$", part.strip()) + if m: + out[os.path.basename(m.group(1))] = (m.group(2), m.group(3), m.group(4), by) + return out + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("app"); ap.add_argument("--plan", required=True) + ap.add_argument("--cache", default=os.environ.get("UPG_CACHE") or os.path.expanduser("~/.cache/upgrade-toolkit/mpk")) + a = ap.parse_args() + if not os.path.isfile(a.plan): + sys.exit(f"widgetskew: {a.plan}: not a file") + kp = os.path.join(a.app, ".stepupdate", "kept.txt") + kept = [l.split()[0] for l in open(kp, encoding="utf-8") if l.strip()] if os.path.isfile(kp) else [] + plan = {} + for l in open(a.plan, encoding="utf-8"): + f = l.split() + if len(f) >= 3 and not l.lstrip().startswith("#"): + plan[f[2]] = (f[0], f[1]) + changed, skew = steps(a.app), 0 + for mod in kept: + if mod not in plan: + continue + cid, ver = plan[mod] + mpk = os.path.join(a.cache, f"{cid}-{ver}.mpk") + try: + ships = {os.path.basename(n) for n in zipfile.ZipFile(mpk).namelist() if re.match(r"widgets/[^/]+\.mpk$", n)} + except (OSError, zipfile.BadZipFile): + print(f"NOTE {mod} {ver}: no cached package ({mpk}), its widgets not checked"); continue + for w in sorted(ships & set(changed)): + name, old, new, by = changed[w] + if by.split()[0] == mod: + continue + skew += 1 + print(f"WIDGET-SKEW {name} {old} -> {new} (by {by}) ships with {mod} {ver}; {mod} stays at the installed version, " + f"its theme styles do not match the new widget") + print(f"widgetskew: {len(kept)} kept module(s), {skew} widget(s) replaced under a kept module" + + ("; repair: steps/repair.md row 32" if skew else "")) + sys.exit(1 if skew else 0) + + +if __name__ == "__main__": + main() diff --git a/upgrade-toolkit/controls/datamove_control.sh b/upgrade-toolkit/controls/datamove_control.sh new file mode 100644 index 0000000..46c64d9 --- /dev/null +++ b/upgrade-toolkit/controls/datamove_control.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# datamove_control.sh CASE: bin/datamove.py check with a fake psql (base and after databases told apart by -d). +# moved every value in the target lost the target has fewer +# fail a count fails bad a malformed plan line +set -euo pipefail +B=$(cd "$(dirname "$0")/../bin" && pwd) +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT +cat > "$T/psql" <<'P' +#!/usr/bin/env bash +db=$2 q=${*: -1} +case "$db:$q" in + base:*'"mendixsso$authrequest" where "nonce"'*) echo 5 ;; + after:*'"mendixsso_ext$authrequestext" where "nonce"'*) echo "${AFTER_N:-5}" ;; + *) echo 'ERROR: relation does not exist' >&2; exit 1 ;; +esac +P +chmod +x "$T/psql" +case "$1" in + bad) echo 'MendixSSO.AuthRequest.Nonce to MendixSSO_Ext.AuthRequestExt.Nonce' > "$T/plan" ;; + fail) echo 'MendixSSO.AuthRequest.Nonce -> Other.Gone.Col' > "$T/plan" ;; + *) printf '# the SSO nonce moves to the extension module\nMendixSSO.AuthRequest.Nonce -> MendixSSO_Ext.AuthRequestExt.Nonce # scenario D\n' > "$T/plan" ;; +esac +[ "$1" != lost ] || export AFTER_N=3 +python3 "$B/datamove.py" check --plan "$T/plan" --before-psql "$T/psql -d base" --after-psql "$T/psql -d after" diff --git a/upgrade-toolkit/controls/editroute_control.sh b/upgrade-toolkit/controls/editroute_control.sh index 63fea43..3bd1661 100644 --- a/upgrade-toolkit/controls/editroute_control.sh +++ b/upgrade-toolkit/controls/editroute_control.sh @@ -2,23 +2,78 @@ # editroute_control.sh CASE: controls of bin/editroute.py (kept modules routed to an update) and the cockpitcheck keep-recommended # rule. A stub mxcli answers describe and refs, so no project or network is needed. # plan one kept module per route; prints the ROUTE lines and the files editroute writes -# collect the replay files built from saved edits after the routed update +# collect the replay files after the routed update: the app's change merged onto the new element, never the saved whole element +# score did each route work: on the plan's version, held, or replay failed; one ledger line per module across runs +# brain each route carries the company brain's notes for it, shown on the card; hit@k measured on a labelled set +# reset a rerun of the update step drops an earlier routing's own lines (a person's stay) and moves its files aside +# reroute a routed update keeps a module for a second reason: routed again, routes add up; the same reason twice is a HOLD # gate cockpitcheck on a module card that recommends the old version, with and without an end date set -u B=$(cd "$(dirname "$0")/../bin" && pwd) T=$(mktemp -d); trap 'rm -rf "$T"' EXIT W=$T/work; A=$T/app; S=$A/.stepupdate; C=$T/cache -mkdir -p "$W" "$S" "$C/7-1.0.0" -touch "$A/App.mpr" "$C/7-1.0.0.mpk" "$C/7-1.0.0/project.mpr" +mkdir -p "$W" "$S" "$C/7-1.0.0" "$T/orig" +touch "$A/App.mpr" "$T/orig/App.mpr" "$C/7-1.0.0.mpk" "$C/7-1.0.0/project.mpr" cat > "$T/mxcli" <<'PY' #!/usr/bin/env python3 import sys +import os a = sys.argv[1:] +nav = os.environ.get("NAVCOPY") # navcopy cases: a Marketplace page on the redirected layout, opened from the menu +if nav and a[0] == "-p": + q = a[3] + if q == "SHOW MODULES": + print("| Module | Source |\n|---|---|\n| Uimod | Marketplace v1.0.0 |\n| Brand | |") + elif q == "SHOW PAGES": + print("| Qualified Name | Module | Name | Folder | Title |\n|---|---|---|---|---|\n| Uimod.Acct | Uimod | Acct | Admin | Accounts |\n" + "| Uimod.Edit | Uimod | Edit | Admin | Edit Account |\n" + f"| Brand.MyEdit | Brand | MyEdit | Pages | {'Something else' if nav == 'nohome' else 'Edit Account'} |") + elif q == "SHOW MODULE ROLES IN Brand": + print("| Qualified Name | Module | Role |\n|---|---|---|\n| Brand.Administrator | Brand | Administrator |") + sys.exit(0) +if nav and a[0] == "refs" and a[3] in ("Uimod.Redir", "Uimod.Acct"): + print("| SourceType | SourceName | RefKind |\n|---|---|---|\n" + ("| PAGE | Uimod.Acct | layout |" if a[3] == "Uimod.Redir" + else "| NAVIGATION | Navigation.Responsive | menu_item |")) + sys.exit(0) +if nav and a[0] == "describe" and a[3] in ("page", "navigation"): + grid = " -- Forms$DataGrid (grid12) -- NOT re-executable: mxcli cannot author this widget\n" if nav in ("grid", "refuse") else "" + print("mdl 1;\ncreate or modify page Uimod.Acct folder 'Admin' (\n Title: 'Accounts',\n Layout: Uimod.Redir\n) {\n" + grid + "};\n" + "grant view on page Uimod.Acct to Uimod.Administrator, Uimod.Auditor;" if a[3] == "page" else + "mdl 1;\ncreate or modify navigation Responsive\n app title 'Uimod App'\n app icon Brand.Icons.Logo\n{\n menu item 'Accounts' ( OnClick: show page Uimod.Acct with (DisabledDuringExecution: false) )\n}") + sys.exit(0) if a[0] == "describe": - mpr, qn = a[2], a[4] + mpr, kind, qn = a[2], a[3], a[4] pkg = mpr.endswith("project.mpr") + side = "base" if pkg else "ours" if "/orig/" in mpr else "theirs" + if kind in ("layout", "snippet", "constant"): # base = installed package, ours = original app, theirs = updated app + el = {"Uimod.L1": {"base": ["class: 'a',", "title: 'x'"], "ours": ["class: 'mine',", "title: 'x'"], "theirs": ["class: 'a',", "title: 'y'"]}, + "Uimod.K": {"base": ["value: '1'"], "ours": ["value: '2'"], "theirs": ["value: '1'"]}, + "Uimod.Added": {"ours": ["class: 'own'"]}, + "Uimod.Bad": {"base": ["class: 'a'"], "ours": ["class: 'BROKEN'"], "theirs": ["class: 'a'"]}, + # layouts: one redirected to the app's design module, one the app added and nothing uses, one edited and used + "Uimod.Redir": {"base": ["class: 'a',", "title: 'x'"], "ours": ["Layout: Brand.Main"], "theirs": ["class: 'a',", "title: 'y'"]}, + "Uimod.Spare": {"ours": ["class: 'copy'"]}, + "Uimod.UsedOwn": {"base": ["class: 'a'"], "ours": ["class: 'own'"], "theirs": ["class: 'a'"]}}.get(qn, {}).get(side) + if el: + print(f"mdl 1;\ncreate or modify {kind} {qn} (\n" + "\n".join(" " + x for x in el) + "\n) {\n};") + sys.exit(0) + if kind == "association": # only the original model still has it + if "/orig/" in mpr and qn == "Other.A_B": + print(f"mdl 1;\ncreate or modify association {qn}\nfrom Other.X to Target.Y;") + sys.exit(0) + if kind == "entity" and qn == "Other.X": + print("create or modify entity Other.X (\n);\ngrant read (A_B, Name), write (A_B) on entity Other.X to Other.User;\ngrant read (Name) on entity Other.X to Other.Guest;") + sys.exit(0) + if kind == "entity" and qn == "Target.Y": + print("create or modify entity Target.Y (\n);\ngrant create, read (Code, A_B), write (Code) on entity Target.Y to Target.User where [Code='x'];\ngrant read (Code) on entity Target.Y to Target.Guest;") + sys.exit(0) body = "logic A" if pkg or "Cosmetic" in qn else "logic B" print(("@position(1, 1)\n" if pkg else "@position(9, 9)\n@anchor(from: right, to: left)\n") + body) +elif a[0] == "check": + t = open(a[1]).read() + bad = "BROKEN" in t or (nav == "refuse" and "ALTER PAGE" in t) # refuse: mxcli check says no to the in-place layout + print(" ✗ class BROKEN is not a class [MDL000]" if bad else "ok") + sys.exit(1 if bad else 0) elif a[0] == "refs": print(f"References to {a[3]}\n| Caller |" if "Used" in a[3] else f"(no references found to attribute {a[3]})") PY @@ -28,12 +83,13 @@ refuse() { printf 'refusing to update: %s element(s) have been changed locally a local m=$2; shift 2; for e in "$@"; do printf ' %s\n' "$e" >> "$S/step$n-$m.log"; done; printf '\n - Save them first: --save-edits \n' >> "$S/step$n-$m.log"; } n=1; refuse 1 Cosmo "MICROFLOW CosmeticFlow" n=2; refuse 2 Realmod "MICROFLOW RealFlow" -n=3; refuse 3 Uimod "LAYOUT L1" "CONSTANT K" +n=3; refuse 3 Uimod "SNIPPET L1" "CONSTANT K" "SNIPPET Added" "SNIPPET Bad" "LAYOUT Redir" "LAYOUT Spare" "LAYOUT UsedOwn" printf '5 element(s) could not be read, so it cannot be shown that the module is unedited.\n' > "$S/step4-Unread.log" touch "$S/step5-Filesmod.log" "$S/step6-Vendmod.log" "$S/step7-Usedmod.log" "$S/step8-Target.log" "$S/step9-Javamod.log" printf 'RED REMOVED attribute Vendmod.E.Gone (column/table dropped; intended?)\n' > "$S/step6-Vendmod.idcheck" printf 'RED REMOVED attribute Usedmod.E.Used (column/table dropped; intended?)\n' > "$S/step7-Usedmod.idcheck" -printf 'RED REMOVED association Other.A_B (column/table dropped; intended?)\n' > "$S/step8-Target.idcheck" +printf 'RED REMOVED association Other.A_B (column/table dropped; intended?)\ninfo REMOVED association Other.Np_B (no data)\n' > "$S/step8-Target.idcheck" +printf 'refusing to update: 1 element(s) have been changed locally\n' > "$S/step12-Target.log" # a later step, no idcheck (WM1) cat > "$S/kept.txt" <<'K' Cosmo local edits refuse the update Realmod local edits refuse the update @@ -42,7 +98,7 @@ Unread elements could not be read, module not shown unedited Filesmod FILES-UNKNOWN: update replaced 1 theme/javascript/java file(s), unproven unedited Vendmod idcheck REVIEW (persistence loss or removal: accept it in approved.txt with a reason, or keep), see x Usedmod idcheck REVIEW (persistence loss or removal: accept it in approved.txt with a reason, or keep), see x -Target idcheck REVIEW (persistence loss or removal: accept it in approved.txt with a reason, or keep), see x +Target idcheck REVIEW (persistence loss or removal: accept it in approved.txt with a reason, or keep), see .stepupdate/step8-Target.idcheck Javamod local Java user code edits (FILES-EDITED javasource) K python3 - "$W/localedits.json" <<'PY' @@ -52,16 +108,46 @@ json.dump([{"name": m, "cid": 7, "version": "1.0.0", "domain": [], "ui": [], "lo PY for m in Cosmo Realmod Uimod Unread Filesmod Vendmod Usedmod Target Javamod; do echo "7 2.0.0 $m"; done > "$W/plan.txt" case "${1:-}" in - plan) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$A/App.mpr" --plan "$W/plan.txt"; echo "rc=$?" + plan) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt"; echo "rc=$?" for f in route-plan.txt force-ui.txt accept-unsaved.txt approved.txt approved-step.txt; do sed "s/^/$f: /" "$W/$f"; done ;; - collect) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$A/App.mpr" --plan "$W/plan.txt" > /dev/null + collect) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null mkdir -p "$W/saved-edits/Uimod" "$W/saved-edits/Target" - printf 'mdl 1;\ncreate or modify layout Uimod.L1;\n' > "$W/saved-edits/Uimod/layout-L1.mdl" - printf 'mdl 1;\ncreate or modify constant Uimod.K;\n' > "$W/saved-edits/Uimod/constant-K.mdl" - printf 'mdl 1;\ncreate or modify page Uimod.NotRefused;\n' > "$W/saved-edits/Uimod/page-NotRefused.mdl" + for e in snippet-L1 constant-K snippet-Added snippet-Bad layout-Redir layout-Spare layout-UsedOwn page-NotRefused entity-Whole; do printf 'mdl 1;\n-- saved whole element %s\n' "$e" > "$W/saved-edits/Uimod/$e.mdl"; done printf 'mdl 1;\ncreate or modify association Other.A_B from Other.X to Target.Y;\n' > "$W/saved-edits/Target/association-Other_A_B.mdl" python3 "$B/editroute.py" collect --work "$W"; echo "rc=$?" sed 's/^/Uimod.mdl: /' "$W/replay/Uimod.mdl"; sed 's/^/Target.mdl: /' "$W/replay/Target.mdl"; sed 's/^/Target.guids: /' "$W/replay/Target.guids" ;; + score) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null + git -C "$A" init -q; git -C "$A" commit -q --allow-empty -m "step: Cosmo 2.0.0 (stepupdate, idcheck PASS, forced)" + sed -i.bak '/^Cosmo /d' "$S/kept.txt" # stepupdate takes a module it committed out of kept.txt (E6-12) + printf '7 2.0.0 Cosmo\n7 2.0.0 Uimod\n' >> "$W/plan.txt" + mkdir -p "$W/replay"; echo x > "$W/replay/Uimod.mdl"; echo "MDL081" > "$W/replay-Uimod.out" + python3 "$B/editroute.py" score --work "$W" --app "$A" --plan "$W/plan.txt" --ledger "$T/ledger.tsv" --label app1; echo "rc=$?" + sed 's/^/ledger: /' "$T/ledger.tsv" | cut -f1-3,5,6,11 | head -3 ;; + brain) Bd="$T/fake-brain"; mkdir -p "$Bd/baseline/versions" "$Bd/internal/architecture" "$Bd/internal/ux-ui" "$Bd/internal/upgrade" + touch "$Bd/baseline/versions/studio-pro-upgrade-10.md" + printf -- '---\ntitle: Never edit Marketplace modules, use an extension module\nkeywords: [extension module, local edits]\n---\nPut changes in a separate module.\n' > "$Bd/internal/architecture/ext.md" + printf -- '---\ntitle: Hide Atlas layouts in the company UI module\nkeywords: [Atlas_Core layouts, design system]\n---\nsettings.json excludes.\n' > "$Bd/internal/ux-ui/atlas.md" + printf -- '---\ntitle: Java dependency sync behind a proxy\nkeywords: [gradle, proxy]\n---\nSet the proxy.\n' > "$Bd/internal/upgrade/proxy.md" + printf 'id\tslugs\twhy\nS1\tarchitecture/ext\tx\nS2\tux-ui/atlas\tx\nS3\tNONE\tx\n' > "$T/gold.tsv" + printf 'S1 | edited Marketplace module, local edits refuse the update\nS2 | Atlas_Core layouts point at the company design system\nS3 | nothing like this\n' > "$T/q.txt" + UPG_BRAIN="$Bd" python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null + python3 -c "import json,sys; [print('brain:', x['module'], [b['path'] for b in x['brain']]) for x in json.load(open(sys.argv[1])) if x['module'] in ('Realmod','Uimod')]" "$W/edit-routes.json" + UPG_BRAIN="$Bd" python3 "$B/evidence.py" --work "$W" Realmod | head -3 + UPG_BRAIN="$Bd" python3 "$B/brainnotes.py" --eval "$T/gold.tsv" --queries "$T/q.txt"; echo "rc=$?" + UPG_BRAIN="$T/none" python3 "$B/brainnotes.py" "anything"; echo "no brain rc=$?" ;; + reset) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null + echo "Handmod # a person forced this" >> "$W/force-ui.txt"; mkdir -p "$W/replay" + python3 "$B/editroute.py" reset --work "$W"; echo "rc=$?" + sed 's/^/force-ui.txt: /' "$W/force-ui.txt"; grep -c 'editroute' "$W/approved-step.txt" | sed 's/^/approved-step editroute lines: /' + ls "$W" | sed 's/^/work: /' ;; + reroute) python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt" > /dev/null + # the routed update keeps Cosmo again, now for associations another module owns into it + printf 'Cosmo idcheck REVIEW (persistence loss or removal: accept it in approved.txt with a reason, or keep), see x\n' > "$S/kept.txt" + printf 'RED REMOVED association Other.C_D (column/table dropped; intended?)\n' > "$S/step10-Cosmo.idcheck"; touch "$S/step10-Cosmo.log" + python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt"; echo "rc=$?" + python3 -c "import json,sys; [print('routes:', x['module'], x['route']) for x in json.load(open(sys.argv[1])) if x['module'] in ('Cosmo','Uimod')]" "$W/edit-routes.json" + sed 's/^/route-plan.txt: /' "$W/route-plan.txt" + python3 "$B/editroute.py" plan --work "$W" --app "$A" --original "$T/orig/App.mpr" --plan "$W/plan.txt"; echo "rc=$?" ;; gate) python3 - "$B" <<'PY' import sys sys.path.insert(0, sys.argv[1]) diff --git a/upgrade-toolkit/controls/fixtures/cockpit/run/work/data.out b/upgrade-toolkit/controls/fixtures/cockpit/run/work/data.out new file mode 100644 index 0000000..b15ec44 --- /dev/null +++ b/upgrade-toolkit/controls/fixtures/cockpit/run/work/data.out @@ -0,0 +1,6 @@ +| verdict | check | detail | +|---|---|---| +| PASS* | db.attr.Shop.Order.Total | values changed (non-null 4 -> 4) (accepted: start noise: also differs between two starts of the SAME old app) | +| PASS* | db.attr.Vendmod.Ent.Field | column gone (noted: the Vendmod update 1.0.0 -> 2.0.0 changed its own attribute, not the app's data) | +| PASS | db.entity.Shop.Order | rows 4 -> 4 | +INVALID 0 | FAIL 0 | REVIEW 0 | PASS* 2 | PASS 40 diff --git a/upgrade-toolkit/controls/fixtures/models/copydoc-base.json b/upgrade-toolkit/controls/fixtures/models/copydoc-base.json new file mode 100644 index 0000000..d2cd6a0 --- /dev/null +++ b/upgrade-toolkit/controls/fixtures/models/copydoc-base.json @@ -0,0 +1,10 @@ +{"modules": [ + {"name": "Src", "appstore": [4444, "1.0.0"], "module_roles": ["User"], + "pages": [{"name": "Pick_Page", "allowed": ["Src.User"]}, {"name": "Pick_Deep", "refs": {"Form": "Src.Pick_Snippet"}}], + "docs": [{"type": "Forms$Snippet", "name": "Pick_Snippet"}, {"type": "Microflows$Microflow", "name": "Get_List"}]}, + {"name": "KeptConn", "appstore": [1111, "3.0.0"], + "docs": [{"type": "Microflows$Microflow", "name": "ACT_Pick", "data": {"Call": {"Page": "Src.Pick_Page"}}}]}, + {"name": "Shop", "module_roles": ["User"], + "pages": [{"name": "Order_Edit", "refs": {"Page": "Src.Pick_Page"}}], + "docs": [{"type": "Microflows$Microflow", "name": "ACT_Open", "data": {"Call": {"Page": "Src.Pick_Page"}}}]} + ]} diff --git a/upgrade-toolkit/controls/fixtures/models/copydoc-new.json b/upgrade-toolkit/controls/fixtures/models/copydoc-new.json new file mode 100644 index 0000000..2df058a --- /dev/null +++ b/upgrade-toolkit/controls/fixtures/models/copydoc-new.json @@ -0,0 +1 @@ +{"from": "copydoc-base.json", "ops": [["set", "modules/0/pages", []], ["set", "modules/0/docs", [{"type": "Microflows$Microflow", "name": "Get_List"}]]]} diff --git a/upgrade-toolkit/controls/fixtures/models/copydoc-v11.json b/upgrade-toolkit/controls/fixtures/models/copydoc-v11.json new file mode 100644 index 0000000..70e6cc3 --- /dev/null +++ b/upgrade-toolkit/controls/fixtures/models/copydoc-v11.json @@ -0,0 +1 @@ +{"from": "copydoc-new.json", "ops": [["set", "version", "11.12.7.1"]]} diff --git a/upgrade-toolkit/controls/fixtures/models/guard6.json b/upgrade-toolkit/controls/fixtures/models/guard6.json new file mode 100644 index 0000000..348d1f0 --- /dev/null +++ b/upgrade-toolkit/controls/fixtures/models/guard6.json @@ -0,0 +1,62 @@ +{ + "version": "9.24.2.0", + "_why": "Cold CMM 2026-11-17: the MaterialHandler home (a data view holding a scroll container, the layout also has a dataView1) got the guard .mx-name-scrollContainer1, which never appeared on the healthy 9.24 page: the login journey FAILed on the OLD app. A scroll container is no guard; the layout grid inside it is.", + "modules": [ + { + "name": "Shop", + "layouts": [ + { + "name": "Layout_Plain", + "named": [ + { + "type": "DataView", + "name": "dataView1" + } + ] + } + ], + "pages": [ + { + "name": "Dash", + "layout": "Shop.Layout_Plain", + "named": [ + { + "type": "DataView", + "name": "dataView1", + "children": [ + { + "type": "ScrollContainer", + "name": "scrollContainer1", + "children": [ + { + "type": "LayoutGrid", + "name": "layoutGrid5", + "children": [ + { + "type": "Label", + "name": "label1", + "text": "Dashboard" + } + ] + } + ] + } + ] + } + ] + }, + { + "name": "Scroll_Only", + "layout": "Shop.Layout_Plain", + "named": [ + { + "type": "ScrollContainer", + "name": "scrollContainer2" + } + ] + } + ], + "docs": [] + } + ] +} diff --git a/upgrade-toolkit/controls/fixtures/models/xcall-own.json b/upgrade-toolkit/controls/fixtures/models/xcall-own.json new file mode 100644 index 0000000..28a4c75 --- /dev/null +++ b/upgrade-toolkit/controls/fixtures/models/xcall-own.json @@ -0,0 +1,3 @@ +{"from": "xcall-page.json", "ops": [["append", "modules", {"name": "Shop", + "docs": [{"type": "Microflows$Microflow", "name": "ACT_Open", "data": {"Call": {"Page": "AuthMod.Login_Page"}}}, + {"type": "Microflows$Microflow", "name": "ACT_Pay", "data": {"Call": {"Action": "AuthMod.ACT_Sign"}}}]}]]} diff --git a/upgrade-toolkit/controls/found_means_control.py b/upgrade-toolkit/controls/found_means_control.py new file mode 100644 index 0000000..1743842 --- /dev/null +++ b/upgrade-toolkit/controls/found_means_control.py @@ -0,0 +1,65 @@ +"""Control for cockpitmatrix.found_detail: what a finding MEANS, per item, built from the run's own files. +Usage: found_means_control.py . Prints "means : ..." and "item : what | tag | tone | meaning" lines.""" +import json, os, sys + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bin")) +import cockpitmatrix as cm + +w = sys.argv[1] +os.makedirs(w, exist_ok=True) +json.dump({"endpoints": [ + {"doc": "Shop.GetRates", "kind": "rest", "status": "MOCKED", "why": "every location is an overridden constant"}, + {"doc": "Mail.GetToken", "kind": "rest", "status": "UNKNOWN", "why": "location from data Mail.Provider.TokenUrl: set it to the printed URL"}, + {"doc": "Sso.Refresh", "kind": "rest", "status": "UNKNOWN", "why": "$Uri is computed by Java action Sso.GetUri at runtime"}]}, + open(os.path.join(w, "apimock-inventory.json"), "w")) +json.dump([{"name": "Theme", "ui": [{"change": "CHANGED", "kind": "layout", "name": "Theme.Default", "detail": "content differs from the package"}]}, + {"name": "Plain", "ui": []}, {"name": "Odd", "domain_error": ["not found"]}, + {"name": "Admin", "domain": [{"change": "ADDED", "kind": "entity", "name": "Admin.Extra", "data": True, "detail": ""}], + "logic": [{"change": "ADDED", "kind": "microflow", "name": "Admin.Save", "detail": "local document, removed by the update"}]}], + open(os.path.join(w, "localedits.json"), "w")) +recs = [{"topic": "standins", "found": {}}, + {"topic": "modules", "found": {"evidence": ["module update Theme 1.0.0 -> 2.0.0 (after the convert)", "module update Plain 1.0.0 -> 1.1.0 (after the convert)", + "module update Odd 3.0.0 -> 3.1.0 (after the convert)"]}}, + {"topic": "module:Admin", "found": {}}, + {"topic": "react", "found": {"evidence": ["Dojo to React on the 11.12.7 rung: 4 x Drop-down (core) -> Combo box"]}}, + {"topic": "nothing:known", "found": {}}] +for r in recs: + d = cm.found_detail(r, w) + if not d: + print(f"empty {r['topic']}") + for m in d.get("means", []): + print(f"means {r['topic']}: {m}") + for i in d.get("items", []): + print(f"item {r['topic']}: " + " | ".join(i)) + +# a case file per decision: loaded by id, screenshots embedded, a broken file ignored +if len(sys.argv) > 2 and sys.argv[2] == "cases": + import base64 + os.makedirs(os.path.join(w, "cases"), exist_ok=True) + open(os.path.join(w, "old.png"), "wb").write(base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==")) + json.dump({"headline": "Seven break", "tiles": [[7, "fail on the new", "bad"]], "shots": [{"old": "old.png", "new": "gone.png", "cap": "look"}], + "why": ["one", "two"], "options": {"fix": {"does": "We fix it.", "catch": "One rerun."}}}, + open(os.path.join(w, "cases", cm.case_id("regression:module-X")), "w")) + open(os.path.join(w, "cases", "broken.json"), "w").write("{not json") + cs = cm.cases(w) + print("case files:", sorted(cs)) + c = cs[cm.case_id("regression:module-X")] + print("old shot:", c["shots"][0]["old"][:22], "| new shot:", repr(c["shots"][0]["new"])) + h = cm.h_case(c) + print("html:", "tiles" if 'class="ct bad"' in h else "-", "why=" + str(h.count("
  • ")), "no-shot" if "no screenshot" in h else "-") + + # a regression nobody wrote a case for: built from heal.json, the walk screenshots and the carried screen edits + for sub in ("e2e-before/shots-A", "e2e-after/shots-A"): + os.makedirs(os.path.join(w, sub), exist_ok=True) + png = base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==") + open(os.path.join(w, "e2e-before/shots-A/act-M.P.b1-step2.png"), "wb").write(png) + open(os.path.join(w, "e2e-after/shots-A/act-M.P.b1-step2-NOT-REACHED.png"), "wb").write(png) + json.dump({"rows": [{"id": "act:act-M.P.b1", "class": "real-regression", + "evidence": "passes on the old app, fails on the new one: FAIL open M.P: button never appeared, screenshot act-M.P.b1-step2-NOT-REACHED.png"}, + {"id": "act:act-M.Q.b2", "class": "real-regression", "evidence": "passes on the old app, fails on the new one: FAIL open M.Q"}]}, + open(os.path.join(w, "heal.json"), "w")) + model = {"items": [{"id": "regression:module-M", "kind": "regression", "tech": ["act:act-M.P.b1", "act:act-M.Q.b2"]}]} + a = cm.cases(w, model)[cm.case_id("regression:module-M")] + print("auto:", a["headline"], "| tiles:", [t[0] for t in a["tiles"]], "| shots:", bool(a["shots"][0]["old"]), bool(a["shots"][0]["new"]), + "| carried:", a["why"][1] if len(a["why"]) > 1 else "-", "| options:", sorted(a["options"])) + print("written case kept:", cm.cases(w, {"items": [{"id": "regression:module-X", "kind": "regression", "tech": ["act:act-M.P.b1"]}]})[cm.case_id("regression:module-X")]["headline"]) diff --git a/upgrade-toolkit/controls/lib/mkheal.py b/upgrade-toolkit/controls/lib/mkheal.py index 780a3fb..fda23de 100644 --- a/upgrade-toolkit/controls/lib/mkheal.py +++ b/upgrade-toolkit/controls/lib/mkheal.py @@ -19,6 +19,7 @@ def put(rel, text): "- REVIEW log.start.NEW_WARNING: WARNING Configuration: Using derived value for SessionKeepAliveUpdatesInterval (SessionTimeout 30000)\n" "- REVIEW log.start.NEW_ERROR: ERROR Shop failed to start the queue\n" "- REVIEW log.start.NEW_WARNING: WARNING Something else changed\n" + "- NEW_ENDPOINT http./rest-doc/shop/openapi.json: status 404 -> 200: endpoint did not exist before\n" "- PASS Shop.Other: same\n") put("twin-old.md", "# twin\n\n## All rows\n- FAIL Shop.Item.changedDate: 3 rows differ\n") J = { diff --git a/upgrade-toolkit/controls/lib/pgtmp.sh b/upgrade-toolkit/controls/lib/pgtmp.sh index e3424fe..8c26dad 100755 --- a/upgrade-toolkit/controls/lib/pgtmp.sh +++ b/upgrade-toolkit/controls/lib/pgtmp.sh @@ -2,6 +2,8 @@ # pgtmp.sh start | stop DIR: a throwaway Postgres (unix socket in a fresh /var/tmp/upg-pg.* folder the postgres user can # reach, no network) for the seed_snapshot controls. start prints the psql command line (it names DIR) on stdout; exit 3 when no postgres binaries are installed (the control SKIPs). bin=$(ls -d /usr/lib/postgresql/*/bin 2>/dev/null | sort -V | tail -1) +# cold CMM 2026-11-17: on a Mac (Homebrew) the binaries are on PATH, not under /usr/lib, and every Postgres control SKIPped +[ -n "$bin" ] || { i=$(command -v initdb 2>/dev/null) && bin=${i%/*}; } [ -n "$bin" ] && [ -x "$bin/initdb" ] || { echo "no postgres binaries"; exit 3; } d=${2:-}; run() { if [ "$(id -u)" = 0 ]; then runuser -u postgres -- "$@"; else "$@"; fi; } case "$1" in diff --git a/upgrade-toolkit/controls/lib/reid_module.py b/upgrade-toolkit/controls/lib/reid_module.py index f40fc0d..429fee7 100644 --- a/upgrade-toolkit/controls/lib/reid_module.py +++ b/upgrade-toolkit/controls/lib/reid_module.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 """reid_module.py: give every unit of one module a new UnitID, as a delete plus re-import of the module does (NS-7). - reid_module.py (in place; MPR v1, run it on a copy) + reid_module.py [--guids] (in place; MPR v1, run it on a copy) + +--guids also gives every element of the module's domain model (entity, attribute, association) a new GUID: a real +delete and re-import, where the data goes too. Without it only the unit ids change, as mxcli's update does (WM1). notification-service cc318c0b: the DataWidgets 2.27.6 import re-created the module, so its ModuleImpl, folder, enumeration, security, domain model, settings and JavaScript action units all got new ids. The new ids here are @@ -13,6 +16,23 @@ import bson path, mod = sys.argv[1], sys.argv[2] +guids = "--guids" in sys.argv[3:] + + +def regen(x): + """New GUIDs (uuid5 of the old, same BSON type) everywhere under x; the number replaced.""" + n = 0 + if isinstance(x, dict): + for k, v in x.items(): + if k == "GUID" and isinstance(v, (bytes, bytearray)): + b = uuid.uuid5(uuid.NAMESPACE_OID, "reguid:" + bytes(v).hex()).bytes_le + x[k] = type(v)(b, v.subtype) if hasattr(v, "subtype") else b + n += 1 + else: + n += regen(v) + elif isinstance(x, list): + n = sum(regen(i) for i in x) + return n con = sqlite3.connect(path) rows = [(bytes(u), bytes(c) if c else None, b) for u, c, b in con.execute("select UnitID, ContainerID, Contents from Unit")] parent = {u: c for u, c, _ in rows} @@ -27,9 +47,16 @@ inside.add(u); break seen.add(x); x = parent.get(x) new = {u: uuid.uuid5(uuid.NAMESPACE_OID, "reid:" + u.hex()).bytes_le for u in inside} +ng = 0 +if guids: + for u, _, b in rows: + if u in inside and b and bson.decode(b).get("$Type") == "DomainModels$DomainModel": + d = bson.decode(b) + ng += regen(d) + con.execute("update Unit set Contents = ? where UnitID = ?", (bson.encode(d), u)) for u, c, _ in rows: nu, nc = new.get(u, u), new.get(c, c) if c else c if (nu, nc) != (u, c): con.execute("update Unit set UnitID = ?, ContainerID = ? where UnitID = ?", (nu, nc, u)) con.commit() -print(f"reid_module: {mod}: {len(inside)} units got new ids") +print(f"reid_module: {mod}: {len(inside)} units got new ids" + (f", {ng} element GUIDs new" if guids else "")) diff --git a/upgrade-toolkit/controls/modroles_control.sh b/upgrade-toolkit/controls/modroles_control.sh new file mode 100644 index 0000000..3e9df8e --- /dev/null +++ b/upgrade-toolkit/controls/modroles_control.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# modroles_control.sh: bin/repair/modroles.py on a fake model (WM1 2026-11-17: Atlas_Core 4's new module roles unmapped, +# every page on an Atlas layout failed "for security reasons"). Prints the script's output; run-controls.sh greps it. +set -euo pipefail +B=$(cd "$(dirname "$0")/../bin" && pwd) +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT +mkdir -p "$T/app"; : > "$T/app/App.mpr"; : > "$T/original.mpr" +git -C "$T/app" init -q; git -C "$T/app" -c user.name=c -c user.email=c@c commit -q --allow-empty -m "step: Atlas_Core 4.1.0 (stepupdate)" +git -C "$T/app" -c user.name=c -c user.email=c@c commit -q --allow-empty -m "step: Widgets 2.0.0 (stepupdate)" +cat > "$T/mxcli" <<'EOF' +#!/usr/bin/env bash +# fake: -p MPR -c QUERY +mpr=$2 q=$4 +roles() { printf '| Qualified Name | Module | Role |\n|---|---|---|\n'; for r in "$@"; do printf '| %s | x | x |\n' "$r"; done; } +case "$q" in + "show module roles") + if [[ $mpr == *original.mpr ]]; then roles Administration.Administrator Administration.User Legacy.Viewer + else roles Administration.Administrator Administration.User Legacy.Viewer Atlas_Core.Administrator Atlas_Core.User \ + Atlas_Core.Designer Widgets.Reader Other.User; fi ;; + "show user roles") printf '| Name | Module Roles |\n|---|---|\n| Administrator | 2 |\n| User | 1 |\n| Anonymous | 1 |\n' ;; + "describe user role Administrator") echo " ModuleRoles: (Administration.Administrator, Administration.User)," ;; + "describe user role User") echo " ModuleRoles: (Administration.User)," ;; + "describe user role Anonymous") echo " ModuleRoles: (Administration.User)," ;; + "describe app security") echo "Guest User Role: Anonymous" ;; +esac +EOF +chmod +x "$T/mxcli" +python3 "$B/repair/modroles.py" "$T/app/App.mpr" --original "$T/original.mpr" --mdl "$T/out.mdl" --mxcli "$T/mxcli" +[ ! -f "$T/out.mdl" ] || sed 's/^/mdl: /' "$T/out.mdl" diff --git a/upgrade-toolkit/controls/platform_drops_control.sh b/upgrade-toolkit/controls/platform_drops_control.sh new file mode 100644 index 0000000..fd7dd5c --- /dev/null +++ b/upgrade-toolkit/controls/platform_drops_control.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# platform_drops_control.sh: bin/platform_drops.py with fake idcheck rows and a fake psql (WM1 2026-11-17: System-module +# drops of Mendix 10 approved only on a count of 0; +# rerun 9: an editroute VENDOR approval re-counted, MendixSSO.AuthRequest.Nonce held 5 values). Prints the script's output and the approved file. +set -euo pipefail +B=$(cd "$(dirname "$0")/../bin" && pwd) +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT +cat > "$T/rows" <<'R' +RED REMOVED entity System.Gone (column/table dropped; intended?) +RED REMOVED attribute System.Kept.HasData (column/table dropped; intended?) +RED REMOVED attribute System.Broken.Col (column/table dropped; intended?) +RED REMOVED attribute Shop.Order.Note (column/table dropped; intended?) +RED REMOVED entity System.Already (column/table dropped; intended?) +info REMOVED attribute System.Info.Col (no data) +R +printf '%s\n' 'System.Already # a person: fine' \ + 'MendixSSO.AuthRequest.Nonce # editroute VENDOR: the new MendixSSO drops it, nothing outside the module refers to it' \ + 'Vend.Old # editroute VENDOR: the new Vend drops it, nothing outside the module refers to it' \ + 'Vend.Odd.Col # editroute VENDOR: the new Vend drops it, nothing outside the module refers to it' > "$T/approved.txt" +cat > "$T/psql" <<'P' +#!/usr/bin/env bash +q=${*: -1} +case "$q" in + *'"system$gone"'*) echo 0 ;; + *'"system$kept"'*) echo 3 ;; + *'"system$broken"'*|*'"vend$odd"'*) echo 'ERROR: relation does not exist' >&2; exit 1 ;; + *'"mendixsso$authrequest" where "nonce"'*) echo 5 ;; + *) echo 0 ;; +esac +P +chmod +x "$T/psql" +python3 "$B/platform_drops.py" old new --idcheck-out "$T/rows" --approved "$T/approved.txt" --psql "$T/psql -d basedb" +sed 's/^/approved: /' "$T/approved.txt" diff --git a/upgrade-toolkit/controls/replay_hold_control.sh b/upgrade-toolkit/controls/replay_hold_control.sh new file mode 100644 index 0000000..e4c12c9 --- /dev/null +++ b/upgrade-toolkit/controls/replay_hold_control.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# replay_hold_control.sh CASE: replay_edits and step_gate from the pipeline on a real git app with a fake mxcli. +# A failed replay is held (its changes undone, the run goes on) and the gate stops 13 on replay-held.txt. +# fail the replay writes half the model and a new unit, then fails +# dirty the same, on a tree that was not clean before the replay (cannot be undone: the old stop) +# ok the replay succeeds after an earlier run held it (the held line goes) +# gate step_gate with a held replay: stop 13 +# gate-ack the same with replay-held-ack.txt: the gate goes through +set -euo pipefail +P=$(cd "$(dirname "$0")/../bin/cloud" && pwd)/upgrade-pipeline.sh +T=$(mktemp -d); trap 'rm -rf "$T"' EXIT +APP=$T/app WORK=$T/work DRY=0 MPR=$T/app/App.mpr B=$T/b +mkdir -p "$APP" "$WORK/replay" "$WORK/state" "$B" +echo orig > "$MPR" +git -C "$APP" init -q; git -C "$APP" add -A; git -C "$APP" -c user.name=c -c user.email=c@c commit -qm original +echo 'create page Mod.P;' > "$WORK/replay/Mod.mdl" +git_app() { git -C "$APP" -c user.name=c -c user.email=c@c "$@"; } +xo() { local o=$1; shift; "$@" > "$o" 2>&1; } +note() { echo "note: $*"; } +Q="" +nj_file() { echo "STOP $1: $2"; Q=$1; } +nj_add() { echo "STOP $1: $2"; Q=$1; } +nj_flush() { [ -z "$Q" ] || { echo "EXIT $Q"; exit "$Q"; }; } +nj_park() { nj_flush; } # Interview mode is not auto here: a parkable stop stops +parked_any() { false; } +commit_app() { git_app add -A; git_app commit -qm "$1"; echo "commit: $1"; } +write_summary() { :; } +record_runlog() { :; } +case "$1" in + fail|dirty) mxcli_run() { echo half > "$MPR"; echo unit > "$APP/new.unit"; echo "MDL081 refused" >&2; return 3; } ;; + *) mxcli_run() { echo replayed > "$MPR"; } ;; +esac +eval "$(sed -n '/^replay_edits() {/,/^}/p' "$P")" +eval "$(sed -n '/^step_gate() {/,/^}/p' "$P")" +case "$1" in + dirty) echo stray > "$APP/stray.txt"; replay_edits ;; + ok) echo "Mod exit 3: old (log: x)" > "$WORK/replay-held.txt"; echo "Other exit 1: y (log: z)" >> "$WORK/replay-held.txt"; replay_edits ;; + gate|gate-ack) echo 'Mod exit 3: r (log: l)' > "$WORK/replay-held.txt"; : > "$WORK/release-impact.md"; printf '#!/bin/sh\n' > "$B/release_impact.py" + [ "$1" = gate ] || echo 'Mod exit 3: accepted, the page edit is redone by hand' > "$WORK/replay-held-ack.txt" + step_gate; echo "gate went through"; exit 0 ;; + *) replay_edits ;; +esac +echo "mpr: $(cat "$MPR")" +echo "untracked: $(git_app ls-files --others --exclude-standard | tr '\n' ' ')" +[ ! -f "$WORK/replay-held.txt" ] || sed 's/^/held: /' "$WORK/replay-held.txt" +echo "commits: $(git_app log --format=%s | tr '\n' '|')" diff --git a/upgrade-toolkit/controls/run-controls.sh b/upgrade-toolkit/controls/run-controls.sh index 56573be..e363325 100755 --- a/upgrade-toolkit/controls/run-controls.sh +++ b/upgrade-toolkit/controls/run-controls.sh @@ -101,7 +101,11 @@ fast=0; only="" while [ $# -gt 0 ]; do case "$1" in --fast) fast=1; shift;; --only) only=$2; shift 2;; *) echo "usage: run-controls.sh [--fast] [--only REGEX]"; exit 2;; esac done -T=$(mktemp -d "${TMPDIR:-/tmp}/upg-controls.XXXXXX") +# One spelling of the temp folder: TMPDIR ends in '/' on macOS ('/T//' in paths) and /var is /private/var; tools that +# resolve their paths then never matched the expected ones (gate-config-relative-*, mxrun-*). Fixtures are stamped in UTC: +# on a +08 machine the archive and timing controls read local time as UTC. Controls never write bytecode under controls/. +T=$(cd "$(mktemp -d "${TMPDIR:-/tmp}"/upg-controls.XXXXXX)" && pwd -P) +export TZ=UTC PYTHONDONTWRITEBYTECODE=1 trap 'rm -rf "$T"' EXIT mkdir -p "$T/out" "$T/m" "$T/field" export UPG_LOCK=$T/test.lock UPG_LOCK_POLL=1 # controls never wait on the real heavy lock (except the start control) @@ -109,6 +113,9 @@ export UPG_LOCK=$T/test.lock UPG_LOCK_POLL=1 # controls never wait on the real # container exports JAVA_TOOL_OPTIONS: the container-wrapper controls then saw the run's own docker arguments. Controls set # their own environment; these never come from the caller. unset UPG_EXTRA UPG_ENV_FILE UPG_CA UPG_DRYRUN UPG_NAME +# Controls prepend stubs with an unquoted `env PATH=$X:$PATH`; a PATH entry with a space (VS Code's "Application Support/ +# .../copilot-chat/debugCommand") split it, and 11 controls failed exit 127 on a healthy toolkit (2026-11-17) +PATH=$(printf '%s' "$PATH" | tr ':' '\n' | grep -v ' ' | paste -sd: -); export PATH export UPG_COCKPIT_OPEN=0 # a control run never opens browser tabs (the live-cockpit controls turn it on with a stub opener) npass=0; nfail=0; nskip=0; failed=""; skipped="" @@ -219,6 +226,109 @@ ctl crosscalls-page-listed-first-catch 1 'e\.g\. page KeptConn\.ACT_Show -> Auth ctl crosscalls-no-kept-invalid 2 'give --kept' -- python3 $B/crosscalls.py $M/xcall --planned AuthMod ctl crosscalls-unknown-module-invalid 2 'not a module of the model: Typo' -- python3 $B/crosscalls.py $M/xcall --kept KeptConn --planned Typo ctl crosscalls-empty-kept-pass 0 'kept 0' '!CAP ' -- python3 $B/crosscalls.py $M/xcall --kept '' --planned AuthMod +# cold CMM: Administration 2.1.2 -> 4.5.0 dropped a page and a microflow two OWN employee pages used: 3 CE1613 at the build, +# nothing said so before. --own judges the app's own modules against the plan version package, any major (own code is +# changed, not held back); the target still in the package passes, no package in the cache is a NOTE +model xcall-own > /dev/null +mkdir -p $T/cache-xc-own-gone $T/cache-xc-own-here $T/cache-xc-own-none +cp $T/cache-xc-gone/2222-3.9.0.mpk $T/cache-xc-own-gone/2222-4.1.0.mpk; cp $T/cache-xc-here/2222-3.9.0.mpk $T/cache-xc-own-here/2222-4.1.0.mpk +ctl crosscalls-own-gone-catch 1 '^OWN AuthMod 3\.2\.0 -> 4\.1\.0: 1 target\(s\) own modules use are gone from the 4\.1\.0 package' '^ GONE page AuthMod\.Login_Page \(used by own Shop\.ACT_Open\)' '!GONE .*ACT_Sign' '!GONE .*KeptConn' 'repair\.md row 28' -- python3 $B/crosscalls.py $M/xcall-own --kept '' --own --plan-file $T/xc-plan-major.txt --cache $T/cache-xc-own-gone +ctl crosscalls-own-present-pass 0 '^crosscalls: 0 target\(s\) used by own modules are gone' '!GONE' -- python3 $B/crosscalls.py $M/xcall-own --kept '' --own --plan-file $T/xc-plan-major.txt --cache $T/cache-xc-own-here +ctl crosscalls-own-unchecked-pass 0 '^NOTE AuthMod 3\.2\.0 -> 4\.1\.0: used by own Shop; targets NOT checked' '!GONE' -- python3 $B/crosscalls.py $M/xcall-own --kept '' --own --plan-file $T/xc-plan-major.txt --cache $T/cache-xc-own-none +ctl crosscalls-own-off-pass 0 '!OWN|GONE' -- python3 $B/crosscalls.py $M/xcall-own --kept '' --plan-file $T/xc-plan-major.txt --cache $T/cache-xc-own-gone +# repair row 28 as a tool: copydoc.py copies the dropped document from the pre-update model into an own module, with +# fresh ids (pointers follow) and its roles mapped, and repoints the OWN users only (a Marketplace caller is left alone) +model copydoc-base > /dev/null; model copydoc-new > /dev/null; model copydoc-v11 > /dev/null +rm -rf $T/cd; mkdir -p $T/cd; cp $M/copydoc-new/App.mpr $T/cd/App.mpr +python3 - $M/copydoc-base/App.mpr <<'PY' +import sqlite3, sys, bson +con = sqlite3.connect(sys.argv[1]) +for uid, blob in con.execute("select UnitID, Contents from Unit").fetchall(): + d = bson.decode(blob) + if d.get("Name") == "Pick_Page": # a default button and the pointer to it, as a real popup page has + d["Widgets"].append({"$ID": b"\x01" * 16, "$Type": "Forms$ActionButton", "Name": "okButton"}) + d["ControlBar"] = {"$ID": b"\x02" * 16, "$Type": "Forms$ControlBar", "DefaultButtonPointer": b"\x01" * 16} + con.execute("update Unit set Contents=? where UnitID=?", (bson.encode(d), uid)) +con.commit() +PY +cat > $T/cd/look.py <<'PY' +import sqlite3, sys, bson +con = sqlite3.connect(sys.argv[1]) +units = {bytes(u): (bytes(c) if c else None, bson.decode(bytes(b))) for u, c, b in con.execute("select UnitID, ContainerID, Contents from Unit") if b} +mod = {u: d["Name"] for u, (c, d) in units.items() if d["$Type"] == "Projects$ModuleImpl"} +for u, (c, d) in units.items(): + if d.get("Name") == "Pick_Page" and c in mod: + ids = set() + def walk(o): + if isinstance(o, dict): + ids.add(bytes(o.get("$ID", b""))); [walk(v) for v in o.values()] + elif isinstance(o, list): + [walk(v) for v in o] + walk(d) + p = d["ControlBar"]["DefaultButtonPointer"] + print(f"page {mod[c]}.Pick_Page roles {d.get('AllowedModuleRoles')[1:]} pointer {'inside' if bytes(p) in ids else 'DANGLING'} " + f"fresh {bytes(p) != b'\x01' * 16}") +for u, (c, d) in units.items(): + s = bson.encode(d) + for q in (b"Src.Pick_Page", b"Shop.Pick_Page"): + if q in s and d.get("Name") != "Pick_Page": + print(f"ref {d['Name']} -> {q.decode()}") +PY +ctl copydoc-dry-catch 0 '^copydoc: Src\.Pick_Page \(Forms\$Page\) -> Shop\.Pick_Page, roles Src\.User -> Shop\.User$' '^ RETARGET Shop\.Order_Edit: 1 reference' '^ RETARGET Shop\.ACT_Open: 1 reference' '!RETARGET KeptConn' '^dry run' -- python3 $B/repair/copydoc.py $T/cd --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to Shop --roles Shop.User --retarget +ctl copydoc-apply-catch 0 '^page Shop\.Pick_Page roles \[.Shop\.User.\] pointer inside fresh True$' '^ref Order_Edit -> Shop\.Pick_Page$' '^ref ACT_Open -> Shop\.Pick_Page$' '^ref ACT_Pick -> Src\.Pick_Page$' '!ref (Order_Edit|ACT_Open) -> Src' -- bash -c "python3 $B/repair/copydoc.py $T/cd --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to Shop --roles Shop.User --retarget --apply | grep -q 're-read OK' && python3 $T/cd/look.py $T/cd/App.mpr" +ctl copydoc-again-invalid 2 'Shop\.Pick_Page already exists' -- python3 $B/repair/copydoc.py $T/cd --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to Shop --roles Shop.User +ctl copydoc-roles-needed-invalid 2 'allows Src\.User: name the Shop roles' -- python3 $B/repair/copydoc.py $M/copydoc-new --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to Shop +ctl copydoc-missing-dep-invalid 2 'uses Src\.Pick_Snippet, which the app does not have' -- python3 $B/repair/copydoc.py $M/copydoc-new --from $M/copydoc-base/App.mpr --doc Src.Pick_Deep --to Shop +ctl copydoc-into-marketplace-invalid 2 'KeptConn is a Marketplace module' -- python3 $B/repair/copydoc.py $M/copydoc-new --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to KeptConn --roles KeptConn.User +ctl copydoc-version-invalid 2 'Mendix versions differ: app 11\.12\.7\.1' -- python3 $B/repair/copydoc.py $M/copydoc-v11 --from $M/copydoc-base/App.mpr --doc Src.Pick_Page --to Shop --roles Shop.User +# javacjar.py (cold CMM 10.24.28): MendixSSO 10.6.2 needs CookieHeaderNames.SameSite (vendorlib netty 4.2); a userlib fat +# jar's older Netty hid it from javac. The error names the jar that hides the class, --strip drops the vendorlib copies +JC=$T/jc; mkdir -p $JC/app/userlib $JC/app/vendorlib $JC/app/javasource/sso $JC/solo/vendorlib +python3 - "$JC" <<'PY' +import sys, zipfile +d = sys.argv[1]; c = "io/netty/handler/codec/http/cookie/CookieHeaderNames" +with zipfile.ZipFile(f"{d}/app/userlib/fat-0.4.1.jar", "w") as z: + z.writestr(c + ".class", b"\xca\xfe\xba\xbe old PATH DOMAIN"); z.writestr("org/opc/Server.class", b"\xca\xfe\xba\xbe") +for j in (f"{d}/app/vendorlib/netty-codec-http-4.2.17.Final.jar", f"{d}/solo/vendorlib/netty-codec-http-4.2.17.Final.jar"): + with zipfile.ZipFile(j, "w") as z: + z.writestr(c + ".class", b"\xca\xfe\xba\xbe new"); z.writestr(c + "$SameSite.class", b"\xca\xfe\xba\xbe") +PY +printf 'package sso;\nimport io.netty.handler.codec.http.cookie.CookieHeaderNames;\nclass U {}\n' > $JC/app/javasource/sso/U.java +cp -R $JC/app/javasource $JC/solo/ +printf '%s\n' 'ERROR: An error occurred while compiling Java actions' "$JC/app/javasource/sso/U.java:7: error: cannot find symbol" ' c.setSameSite(CookieHeaderNames.SameSite.Lax);' ' ^' ' symbol: variable SameSite' ' location: class CookieHeaderNames' > $JC/build.log +ctl javacjar-hidden-catch 1 '^SYMBOL io\.netty\.handler\.codec\.http\.cookie\.CookieHeaderNames\.SameSite \(U\.java:7\)$' '^ HAS vendorlib/netty-codec-http-4\.2\.17\.Final\.jar$' '^ HIDDEN BY userlib/fat-0\.4\.1\.jar' 'row 31' -- python3 $B/repair/javacjar.py $JC/app $JC/build.log +ctl javacjar-not-hidden-pass 0 '^ HAS vendorlib/' '!HIDDEN BY' '0 hidden' -- python3 $B/repair/javacjar.py $JC/solo $JC/build.log +ctl javacjar-strip-dry-pass 0 '^STRIP userlib/fat-0\.4\.1\.jar: 1 class\(es\) a vendorlib jar also carries' '^dry run' '^entries 2$' -- bash -c "python3 $B/repair/javacjar.py $JC/app --strip userlib/fat-0.4.1.jar && python3 -c 'import zipfile,sys; print(\"entries\", len(zipfile.ZipFile(sys.argv[1]).namelist()))' $JC/app/userlib/fat-0.4.1.jar" +ctl javacjar-strip-apply-catch 0 '^wrote userlib/fat-0\.4\.1\.jar' "^left \['org/opc/Server\.class'\]$" -- bash -c "cp -R $JC/app $JC/app2 && python3 $B/repair/javacjar.py $JC/app2 --strip userlib/fat-0.4.1.jar --apply && python3 -c 'import zipfile,sys; print(\"left\", zipfile.ZipFile(sys.argv[1]).namelist())' $JC/app2/userlib/fat-0.4.1.jar" +# the mxbuild stop runs it on a javac "cannot find symbol" and quotes it when a class is hidden +awk '/if grep -q .error: cannot find symbol/{p=1} p{print} p&&/^ fi$/{exit}' $B/cloud/upgrade-pipeline.sh > $JC/nj.sh +printf '%s\n' 'xo() { o=$1; shift; "$@" > "$o"; }' 'nj_file() { echo "NJ $1 $2"; grep -E "$5" "$3"; }' > $JC/stub.sh +cp $JC/build.log $JC/build-10.24.1.log +ctl pipeline-build-javacjar-catch 0 '^NJ 10 An older copy of a class inside another jar hides what the code needs \(steps/repair\.md row 31\)' '^ HIDDEN BY userlib/fat' -- bash -c ". $JC/stub.sh; WORK=$JC APP=$JC/app TARGET=10.24.1 B=$B; . $JC/nj.sh; echo done" +ctl pipeline-build-javacjar-not-hidden-pass 0 '!^NJ' '^done$' -- bash -c ". $JC/stub.sh; WORK=$JC APP=$JC/solo TARGET=10.24.1 B=$B; . $JC/nj.sh; echo done" +# widgetskew.py (cold CMM 10.24.28): Atlas_Web_Content 3.9.0 replaced Datagrid 2.2.2 with 2.30.6 while DataWidgets (its styles) +# was kept; every grid page lost its layout and afterwalk passed. A widget replaced under a kept module's target package stops +WS=$T/ws; mkdir -p $WS/app/.stepupdate $WS/cache $WS/solo/.stepupdate +printf '%s\n' 'Atlas_Web_Content 3.9.0: Badge.mpk Badge 3.1.0 -> 3.2.2; com.mendix.widget.web.Datagrid.mpk Datagrid 2.2.2 -> 2.30.6' 'DataWidgets 3.11.8: com.mendix.widget.web.Gallery.mpk Gallery 1.0.0 -> 1.4.0' > $WS/app/.stepupdate/widgets.txt +printf '%s\n' 'DataWidgets local theme/javascript file edits refuse the update (FILES-EDITED)' 'NoCache diagram-only edits (UNKNOWN)' > $WS/app/.stepupdate/kept.txt +cp $WS/app/.stepupdate/widgets.txt $WS/solo/.stepupdate/ +printf '%s\n' '# plan' '117183 3.9.0 Atlas_Web_Content' '116540 3.11.8 DataWidgets' '999 1.0.0 NoCache' > $WS/plan.txt +python3 -c 'import zipfile,sys; z=zipfile.ZipFile(sys.argv[1],"w"); [z.writestr(n,b"x") for n in ("widgets/com.mendix.widget.web.Datagrid.mpk","widgets/com.mendix.widget.web.Gallery.mpk","themesource/datawidgets/web/datagrid.scss")]' $WS/cache/116540-3.11.8.mpk +ctl widgetskew-catch 1 '^WIDGET-SKEW Datagrid 2\.2\.2 -> 2\.30\.6 \(by Atlas_Web_Content 3\.9\.0\) ships with DataWidgets 3\.11\.8' '!Badge' '!Gallery' '^NOTE NoCache 1\.0\.0: no cached package' '1 widget\(s\) replaced under a kept module; repair: steps/repair\.md row 32' -- python3 $B/widgetskew.py $WS/app --plan $WS/plan.txt --cache $WS/cache +ctl widgetskew-nothing-kept-pass 0 '!WIDGET-SKEW' '0 kept module\(s\), 0 widget' -- python3 $B/widgetskew.py $WS/solo --plan $WS/plan.txt --cache $WS/cache +# step_update parks on it, unless widgetskew-ack.txt accepts it +awk '/xo "\$WORK\/widgetskew.out"/{p=1} p{print} p&&/^ fi$/{exit}' $B/cloud/upgrade-pipeline.sh > $WS/nj.sh +printf '%s\n' 'xo() { o=$1; shift; "$@" > "$o"; }' 'nj_file() { echo "NJ $1 $2"; grep -E "$5" "$3"; }' 'nj_add() { echo "NJ $1 $2: $3"; }' 'nj_park() { echo PARK; }' 'export UPG_CACHE='"$WS"'/cache; rc=0' > $WS/stub.sh +mkdir -p $WS/w1 $WS/w2; cp $WS/plan.txt $WS/w1/; cp $WS/plan.txt $WS/w2/; : > $WS/w2/widgetskew-ack.txt +ctl pipeline-update-widgetskew-catch 0 '^NJ 13 A module update replaced a widget whose styles stay with a kept module \(steps/repair\.md row 32\)$' '^WIDGET-SKEW Datagrid' '^NJ 13 Decision: Update the kept module' '^PARK$' -- bash -c ". $WS/stub.sh; WORK=$WS/w1 APP=$WS/app B=$B; . $WS/nj.sh; echo done" +ctl pipeline-update-widgetskew-ack-pass 0 '!^NJ' '!PARK' '^done$' -- bash -c ". $WS/stub.sh; WORK=$WS/w2 APP=$WS/app B=$B; . $WS/nj.sh; echo done" +# the build stop quotes owngone.out when mx check has CE1613 (and only then) +mkdir -p $T/og; printf '%s\n' 'crosscalls: own modules 1' 'OWN AuthMod 3.2.0 -> 4.1.0: 1 target(s) own modules use are gone from the 4.1.0 package' ' GONE page AuthMod.Login_Page (used by own Shop.ACT_Open)' > $T/og/owngone.out +printf '%s\n' '[error] [CE1613] "The selected page no longer exists." at Microflow ACT_Open' > $T/og/check-1613.log; printf '%s\n' '[error] [CE0117] "x"' > $T/og/check-0117.log +awk '/if has_code CE1613 && grep/{p=1} p{print} p&&/^ fi$/{exit}' $B/cloud/upgrade-pipeline.sh > $T/og/nj.sh +printf '%s\n' 'has_code() { grep -q "\[$1\]" "$CHECK_LOG"; }' 'nj_file() { echo "NJ $1 $2"; grep -E "$5" "$3"; }' > $T/og/stub.sh +ctl pipeline-build-owngone-catch 0 '^NJ 10 Own documents use targets the module update dropped \(CE1613; steps/repair\.md row 28\)' '^ GONE page AuthMod\.Login_Page \(used by own Shop\.ACT_Open\)' '!crosscalls: own' -- bash -c ". $T/og/stub.sh; WORK=$T/og CHECK_LOG=$T/og/check-1613.log; . $T/og/nj.sh; echo done" +ctl pipeline-build-owngone-other-code-pass 0 '^done$' '!NJ' -- bash -c ". $T/og/stub.sh; WORK=$T/og CHECK_LOG=$T/og/check-0117.log; . $T/og/nj.sh; echo done" # T6R-7: the other direction. An UPDATED module that needs what a KEPT module lacks (a name, a parameter of a flow, a Java # class) is FAIL with the pair named; all names present is REVIEW (signatures unproven), never PASS; a kept module calling @@ -258,6 +368,11 @@ ctl constcheck-restore-config-file-pass 0 "^ALTER SETTINGS CONSTANT 'MFAmodule\. # --- gate: start log beyond ERROR lines (E1 18, E1 24) ------------------------------------------------------------- ctl logdiff-mfa-off-catch 1 'REVIEW +CHANGED.*MFA module login NOT active' 'was: .*MFA module login active' -- python3 $B/logdiff.py $FX/logs/old.log $FX/logs/new-mfa-off.log ctl logdiff-same-pass 0 'FAIL 0 \| REVIEW 0' -- python3 $B/logdiff.py $FX/logs/old.log $FX/logs/new-same.log +# WM1 2026-11-17: 9.24 logs the trial-license line as Core, 10.x as LicenseService: same line, renamed logger, noted +sed 's/WARNING - Core: Cache size/WARNING - CacheService: Cache size/' $FX/logs/old.log > $T/new-logger.log +sed 's/WARNING - Core: Cache size for entity/WARNING - CacheService: Cache limit for entity/' $FX/logs/old.log > $T/new-logger-text.log +ctl logdiff-renamed-logger-noted-pass 0 'PASS\* +NEW_WARNING +WARNING CacheService: Cache size' 'noted: the old app logs the same line as Core; only the logger name changed' 'FAIL 0 \| REVIEW 0' -- python3 $B/logdiff.py $FX/logs/old.log $T/new-logger.log +ctl logdiff-renamed-logger-other-text-catch 1 'REVIEW +NEW_WARNING +WARNING CacheService: Cache limit' -- python3 $B/logdiff.py $FX/logs/old.log $T/new-logger-text.log ctl logdiff-start-failed-catch 1 'FAIL +START_FAILED' 'deprecated' -- python3 $B/logdiff.py $FX/logs/old.log $FX/logs/new-start-failed.log # an ERROR logged after the start window before and inside it now is MOVED (REVIEW), not NEW_ERROR (E4 24) ctl logdiff-moved-pass 1 'REVIEW +MOVED +ERROR SAML_SSO' '!NEW_ERROR' -- python3 $B/logdiff.py $FX/logs/old-lazy-error.log $FX/logs/new-early-error.log @@ -313,16 +428,43 @@ EOF ctl diff-anonymous-users-grouped-pass 0 'PASS\* \| db\.known\.anonymous_users \| 18 anonymous System\.User rows gone \(18 -> 0\)' '!FAIL \| db\.(attr|assoc|entity)\.System\.User' 'FAIL 0 \| REVIEW 0' -- python3 $B/diff.py $T/anon-ok-b.json $T/anon-ok-a.json --allow $FX/allow/narrow.txt ctl diff-anonymous-named-lost-catch 1 'FAIL \| db\.attr\.System\.User\.Name \| non-null 3 -> 2 \(named users\)' 'FAIL \| db\.entity\.System\.User \| named user rows 3 -> 2' -- python3 $B/diff.py $T/anon-named-lost-b.json $T/anon-named-lost-a.json --allow $FX/allow/narrow.txt ctl diff-anonymous-old-snapshot-catch 1 'FAIL \| db\.entity\.System\.User \| rows 21 -> 3 \(if these are Anonymous_\* users' '!known\.anonymous' -- python3 $B/diff.py $T/anon-ok-old-b.json $T/anon-ok-old-a.json --allow $FX/allow/narrow.txt +# WM1 2026-11-17: the gate card said "Data compare: Failed" on 4 rows that were no loss: a vendor module's own attribute +# changed by its update (MendixSSO, MxModelReflection) and the local admin's row (runapp.py sets its password at start). +python3 - "$T" <<'EOF' +import json, sys +t = sys.argv[1] +for nm in ("jump", "jump-rows", "admin", "admin-other"): + b = json.load(open(f"{t}/sb.json")); a = json.load(open(f"{t}/sa.json")) + for d, h in ((b, "v1"), (a, "v2")): + d["db"]["entities"]["Vend.Thing"] = 5 if d is b or nm != "jump-rows" else 3 + d["db"]["attributes"]["Vend.Thing.Field"] = {"nonnull": 5, "md5": h if nm.startswith("jump") else "same"} + d["db"]["entities"]["System.User"] = 3 + d["db"]["attributes"]["System.User.changedDate"] = {"nonnull": 3, "md5": "c" + h if nm.startswith("admin") else "same"} + rest = "same" if nm == "admin" or d is b else "other" + d["db"]["anonymous"] = {"users": 0, "total": 3, "attrs": {"System.User.changedDate": d["db"]["attributes"]["System.User.changedDate"]}, + "assoc": {}, "noadmin": {"System.User.changedDate": {"nonnull": 2, "md5": rest}}} + json.dump(b, open(f"{t}/{nm}-b.json", "w")); json.dump(a, open(f"{t}/{nm}-a.json", "w")) +open(f"{t}/jumps.txt", "w").write("Vend 1.0.0 -> 2.0.0\n") +EOF +ctl diff-module-jump-attr-noted-pass 0 'PASS\* \| db\.attr\.Vend\.Thing\.Field \| values changed .*the Vend update 1\.0\.0 -> 2\.0\.0' 'FAIL 0 \| REVIEW 0' -- python3 $B/diff.py $T/jump-b.json $T/jump-a.json --allow $FX/allow/narrow.txt --module-jumps $T/jumps.txt +ctl diff-module-jump-attr-unlisted-catch 1 'FAIL \| db\.attr\.Vend\.Thing\.Field \| values changed' -- python3 $B/diff.py $T/jump-b.json $T/jump-a.json --allow $FX/allow/narrow.txt +ctl diff-module-jump-rows-lost-catch 1 'FAIL \| db\.entity\.Vend\.Thing \| rows 5 -> 3' -- python3 $B/diff.py $T/jump-rows-b.json $T/jump-rows-a.json --allow $FX/allow/narrow.txt --module-jumps $T/jumps.txt +ctl diff-admin-row-noted-pass 0 'PASS\* \| db\.attr\.System\.User\.changedDate \| .*local admin row alone' 'FAIL 0 \| REVIEW 0' -- python3 $B/diff.py $T/admin-b.json $T/admin-a.json --allow $FX/allow/narrow.txt +ctl diff-admin-other-user-catch 1 'FAIL \| db\.attr\.System\.User\.changedDate \| values changed' '!local admin' -- python3 $B/diff.py $T/admin-other-b.json $T/admin-other-a.json --allow $FX/allow/narrow.txt ctl diff-owner-catch 1 'FAIL \| db\.identity\.assoc:System\.owner' -- python3 $B/diff.py $T/sb.json $T/s-owner-lost.json --allow $FX/allow/narrow.txt ctl diff-owner-pass 0 '!System\.owner' 'FAIL 0 \| REVIEW 0' -- python3 $B/diff.py $T/sb.json $T/s-owner-order.json --allow $FX/allow/narrow.txt -ctl diff-new-endpoint-catch 2 'NEW_ENDPOINT \| http\./rest-doc/notify/openapi\.json' -- python3 $B/diff.py $T/sb.json $T/sa.json --allow $FX/allow/email-only.txt +# WM1 2026-11-17: openapi.json of a service whose swagger.json the old app served is the same docs in the 10.x format: noted. +# A service the old app had no docs for at all stays NEW_ENDPOINT +ctl diff-openapi-next-to-swagger-noted-pass 0 'PASS\* \| http\./rest-doc/notify/openapi\.json \| .*noted: the same service.s docs in the OpenAPI 3 format' -- python3 $B/diff.py $T/sb.json $T/sa.json --allow $FX/allow/email-only.txt +python3 -c "import json,sys; d=json.load(open(sys.argv[1])); assert d['http'].pop('/rest-doc/notify/swagger.json'); json.dump(d, open(sys.argv[2], 'w'))" $T/sb.json $T/sb-nosw.json +ctl diff-new-endpoint-catch 2 'NEW_ENDPOINT \| http\./rest-doc/notify/openapi\.json' -- python3 $B/diff.py $T/sb-nosw.json $T/sa.json --allow $FX/allow/email-only.txt ctl diff-allowlist-pass 0 'PASS\* \| http\./rest-doc/notify/openapi\.json' 'PASS\* \| db\.attr\.Administration\.Account\.Email' -- python3 $B/diff.py $T/sb.json $T/sa.json --allow $FX/allow/narrow.txt ctl diff-allow-detail-catch 1 'FAIL \| db\.attr\.Administration\.Account\.Email \| values changed' -- python3 $B/diff.py $T/sb.json $T/sa.json --allow $FX/allow/wrong-detail.txt # PV-7: the restart control. diff --emit-allow turns the FAIL rows between two starts of the SAME old app into allow lines # (start noise); fed back as --allow they leave 0 FAIL, and a clean pair writes an empty file. ctl diff-emit-allow-catch 1 '^--emit-allow: [1-9][0-9]* check\(s\) written' -- python3 $B/diff.py $T/sb.json $T/sa.json --emit-allow $T/noise.txt ctl diff-emit-allow-lines-pass 0 'db\.attr\.Administration\.Account\.Email values changed[^#]* # start noise: differs between two starts of the same old app \(' -- cat $T/noise.txt -ctl diff-emit-allow-roundtrip-pass 2 'FAIL 0 \| REVIEW 0' -- python3 $B/diff.py $T/sb.json $T/sa.json --allow $T/noise.txt +ctl diff-emit-allow-roundtrip-pass 0 'FAIL 0 \| REVIEW 0 \| NEW_ENDPOINT 0' -- python3 $B/diff.py $T/sb.json $T/sa.json --allow $T/noise.txt ctl diff-emit-allow-clean-pass 0 '^--emit-allow: 0 check\(s\)' -- python3 $B/diff.py $T/sb.json $T/sb.json --db-only --emit-allow $T/noise-clean.txt # NMR-1: the emitted line carries the noise row's own detail, so one noise error accepts that error only, never every new error { cat $FX/logs/old.log; echo "2026-09-14 10:16:02.296 ERROR - Cache: NoiseError: cache warmup"; } > $T/old-noise.log @@ -481,6 +623,45 @@ for f in bcprov-jdk18on-1.85.jar commons-io-2.21.0.jar xercesImpl-2.12.2.jar; do ctl jardedupe-catch 1 'bcprov-jdk18on-1\.78' 'commons-io-2\.11\.0' 'KEEP.*xercesImpl' -- python3 $B/repair/jardedupe.py $J ctl jardedupe-apply-stale-refused 2 'REFUSED: vendorlib is stale' -- python3 $B/repair/jardedupe.py $J --apply ctl jardedupe-apply 0 'vendorlib current: stated with --vendorlib-synced' -- python3 $B/repair/jardedupe.py $J --apply --vendorlib-synced +# WM1 2026-11-17: the 9->10 gate stopped on System-module drops; approved only when the baseline count is 0 +PD="bash $B/../controls/platform_drops_control.sh" +ctl platform-drops-zero-approved-pass 0 '^approved: System.Gone # platform drop \(platform_drops.py\): Mendix removes it; 0 rows in basedb$' '^platform_drops: 1 approved by count, 2 kept' -- $PD +ctl platform-drops-data-kept-catch 0 '^KEEP System.Kept.HasData: 3 non-null values' '!approved: System.Kept' -- $PD +ctl platform-drops-query-fail-kept-catch 0 '^KEEP System.Broken.Col: count failed' '!approved: System.Broken' -- $PD +ctl platform-drops-app-and-info-untouched-pass 0 '!Shop.Order' '!System.Info' '!APPROVE System.Already' -- $PD +# rerun 9: editroute approved MendixSSO.AuthRequest.Nonce as a VENDOR drop ("nothing refers to it"); it held 5 values +ctl platform-drops-vendor-data-held-catch 0 '^HELD MendixSSO.AuthRequest.Nonce: 5 non-null values' '^approved: # HELD by platform_drops.py: 5 non-null values' '!^approved: MendixSSO' -- $PD +ctl platform-drops-vendor-empty-stays-pass 0 '^VENDOR-OK Vend.Old: 0 rows' '^approved: Vend.Old # editroute VENDOR' -- $PD +ctl platform-drops-vendor-count-fail-held-catch 0 '^HELD Vend.Odd.Col: count failed' '!^approved: Vend.Odd' '2 vendor drop\(s\) held for data$' -- $PD +# a failed replay is held: its changes undone, the run goes on, the gate stops 13 until a person fixes or acknowledges it +RH="bash $B/../controls/replay_hold_control.sh" +ctl replay-hold-undone-catch 0 '^note: replay Mod HELD \(exit 3\)' '^mpr: orig$' '^untracked: $' '^held: Mod exit 3: ' '^commits: original\|$' '!^STOP' -- $RH fail +ctl replay-hold-dirty-tree-stops-catch 13 '^STOP 13: replay of Mod failed \(exit 3\) on a tree that was not clean' '!HELD' -- $RH dirty +ctl replay-hold-cleared-on-success-pass 0 '^commit: replay: Mod$' '^held: Other exit 1' '!^held: Mod' '^mpr: replayed$' -- $RH ok +ctl replay-hold-gate-stops-catch 13 '^STOP 13: upgrade complete except held replays' '!gate went through' -- $RH gate +ctl replay-hold-gate-ack-pass 0 '^gate went through$' '!^STOP' -- $RH gate-ack +# scenario D: a run-once data move is proven by count (target after >= source on the baseline) +DM="bash $B/../controls/datamove_control.sh" +ctl datamove-moved-pass 0 '^MOVED MendixSSO.AuthRequest.Nonce -> MendixSSO_Ext.AuthRequestExt.Nonce: 5 before, 5 after$' '^datamove: 1 of 1 moves proven$' -- $DM moved +ctl datamove-lost-catch 1 '^LOST .*: 5 before, 3 after$' -- $DM lost +ctl datamove-count-fail-catch 1 '^UNCOUNTED .*count failed after$' -- $DM fail +ctl datamove-bad-plan-catch 2 "not 'Src.Entity.Attr -> Dst.Entity.Attr'" -- $DM bad +# WM1 rerun 7: baseline ran microflow steps, the after walk skipped them: 4 false REGRESSIONs +ctl pipeline-smoke-mf-follows-baseline-pass 0 '^--microflows$' -- bash -c "WORK=\$(mktemp -d); echo '{\"landed\": \"microflow ran\"}' > \$WORK/smoke-before.json; eval \"\$(awk '/^smoke_mf\(\)/{f=1} f{print} f&&/^}|; }\$/{exit}' \${PIPE:-$B/cloud/upgrade-pipeline.sh})\"; smoke_mf" +ctl pipeline-smoke-mf-off-without-baseline-pass 0 '!microflows' -- bash -c "WORK=\$(mktemp -d); eval \"\$(awk '/^smoke_mf\(\)/{f=1} f{print} f&&/^}|; }\$/{exit}' \${PIPE:-$B/cloud/upgrade-pipeline.sh})\"; smoke_mf; echo done" +# cold WM1 #12 and cold CMM: without ~/.config/upgrade-toolkit/local.env the scanner lookup's sed failed inside $(...) and the ERR +# trap printed "unexpected failure (exit 1) at line 624 in step 'triage'" on every machine but the author's; the run went on +ctl pipeline-scanner-lookup-no-local-env-pass 0 '^EA=\[\]$' '!unexpected failure' -- bash -c "set -Eeuo pipefail; trap 'echo unexpected failure \$? >&2; exit 4' ERR; HOME=\$(mktemp -d); unset EA UPG_SCANNER; eval \"f() { \$(grep 'upgrade-toolkit/local.env\" 2>' \${PIPE:-$B/cloud/upgrade-pipeline.sh} | grep -v '^ *#'); echo \\\"EA=[\\\$EA]\\\"; }\"; f" +ctl pipeline-scanner-lookup-local-env-pass 0 '^EA=\[/opt/scan\]$' -- bash -c "set -Eeuo pipefail; trap 'echo unexpected failure \$? >&2; exit 4' ERR; HOME=\$(mktemp -d); mkdir -p \$HOME/.config/upgrade-toolkit; echo UPG_SCANNER=/opt/scan > \$HOME/.config/upgrade-toolkit/local.env; unset EA UPG_SCANNER; eval \"f() { \$(grep 'upgrade-toolkit/local.env\" 2>' \${PIPE:-$B/cloud/upgrade-pipeline.sh} | grep -v '^ *#'); echo \\\"EA=[\\\$EA]\\\"; }\"; f" +# WM1 2026-11-17: Atlas_Core 4's new module roles unmapped; every Atlas page failed for security reasons (smoke walk only) +MR="bash $B/../controls/modroles_control.sh" +ctl modroles-grant-by-name-pass 0 '^mdl: ALTER USER ROLE Administrator ADD MODULE ROLES \(Atlas_Core.Administrator\);' '^mdl: ALTER USER ROLE User ADD MODULE ROLES \(Atlas_Core.User\);' '^modroles: 3 grant' -- $MR +ctl modroles-guest-held-catch 0 '^GUEST Atlas_Core.User' '!ALTER USER ROLE Anonymous' -- $MR +ctl modroles-unmatched-catch 0 '^UNMATCHED Atlas_Core.Designer' '^UNMATCHED Widgets.Reader' -- $MR +ctl modroles-keeps-old-and-unmoved-pass 0 '!Legacy.Viewer' '!Other.User' -- $MR +ctl pipeline-runs-modroles-pass 0 '^1$' -- bash -c "grep -c 'repair/modroles.py\" \"\\\$MPR\" --original' $B/cloud/upgrade-pipeline.sh" +# WM1 rerun 6: the build repair ran jardedupe without --updated-from-git; pdfbox 2 stayed next to pdfbox 3, javac failed +ctl pipeline-jardedupe-updated-from-git-pass 0 '^2$' -- bash -c "grep -c 'jardedupe.py\" \"\\\$APP\" --updated-from-git' $B/cloud/upgrade-pipeline.sh" ctl jardedupe-pass 0 '!bcprov-jdk18on-1\.78' -- python3 $B/repair/jardedupe.py $J ctl jardedupe-keeps-own-and-newer 0 -- bash -c "[ -f $J/userlib/own-helper-1.0.jar ] && [ -f $J/userlib/xercesImpl-2.12.2-sp1.jar ] && [ ! -f $J/userlib/bcprov-jdk18on-1.78.jar.SAML20.RequiredLib ]" # U4: -sources and -javadoc jars are their own artifact: never compared with the class jar, never kept over it @@ -576,9 +757,9 @@ cp -r $T/jarid $T/jarid-b; cp -r $T/jarid $T/jarid-c; cp -r $T/jarid $T/jarid-d ctl jarid-apply-stale-refused 2 'REFUSED: vendorlib is stale' -- python3 $B/repair/jardedupe.py $T/jarid-b --apply ctl jarid-apply-stale-removes-nothing 0 -- bash -c "cd $T/jarid-b/userlib; [ -f org.apache.commons.io-2.3.0.jar ] && [ -f commons-io-2.11.0.jar.CommunityCommons.RequiredLib ] && [ -f org.apache.commons.lang3.jar ]" # an SBOM OLDER than the last module change is no proof (the sync ran for the previous module set) -echo '{}' > $T/jarid-c/vendorlib-sbom.json; touch -d '2001-01-01' $T/jarid-c/vendorlib-sbom.json +echo '{}' > $T/jarid-c/vendorlib-sbom.json; touch -t 200101010000 $T/jarid-c/vendorlib-sbom.json ctl jarid-apply-old-sbom-refused 2 'vendorlib-sbom\.json is older than the last module change' -- python3 $B/repair/jardedupe.py $T/jarid-c --apply -echo '{}' > $T/jarid-d/vendorlib-sbom.json; touch -d '+1 hour' $T/jarid-d/vendorlib-sbom.json +echo '{}' > $T/jarid-d/vendorlib-sbom.json; python3 -c "import os, sys, time; t = time.time() + 3600; os.utime(sys.argv[1], (t, t))" $T/jarid-d/vendorlib-sbom.json ctl jarid-apply-current-sbom-pass 0 'vendorlib current: vendorlib-sbom\.json is newer than the last module change' 'removed 4 jars and 3 markers' -- python3 $B/repair/jardedupe.py $T/jarid-d --apply ctl jarid-apply-result 0 -- bash -c "cd $T/jarid-d/userlib; [ ! -f org.apache.commons.io-2.3.0.jar ] && [ ! -f commons-io-2.11.0.jar ] && [ ! -f org.apache.commons.lang3.jar ] && [ ! -f org.apache.commons.io-2.3.0.jar.TcConnector.RequiredLib ] && [ -f commons-codec-1.15.jar ] && [ -f own-helper-1.0.jar ] && [ ! -f commons-codec.commons-codec-1.15.jar ]" ctl jarid-clean-after-apply 0 '!REMOVE' -- python3 $B/repair/jardedupe.py $T/jarid-d @@ -714,7 +895,7 @@ ctl scsscase-input-catch 2 -- python3 $B/repair/scsscase.py $T/empty-nothing # B4T1-3, B4T2-2, B4T4-1, B4T5-2, B4T7-2, B4T8-1: a fixture that .gitignore swallows (*.log) is never committed, so its # controls FAIL in every fresh clone. No file under controls/ may be ignored (needs git; SKIP outside a work tree). if git -C "$here" rev-parse --git-dir >/dev/null 2>&1; then - IGN='n=$(git -C "$1" ls-files -o -i --exclude-standard . | tee /dev/stderr | wc -l); echo "ignored under controls: $n"' + IGN='n=$(git -C "$1" ls-files -o -i --exclude-standard . | tee /dev/stderr | wc -l | tr -d " "); echo "ignored under controls: $n"' ctl controls-fixtures-not-ignored-pass 0 '^ignored under controls: 0$' -- bash -c "$IGN" _ "$here" # catch: a file git still ignores (.DS_Store) placed under fixtures is found (the probe is removed again) mkdir -p "$here/fixtures/zz-ignore-probe" && : > "$here/fixtures/zz-ignore-probe/.DS_Store" @@ -882,6 +1063,10 @@ printf '%s\n' '/*C1*/INSERT INTO BLOCKS VALUES(0,2147483647,0)' > $T/snapblocks/ ctl seed-snapshot-blocks-bookkeeping-catch 0 '!UNMAPPED' '!statement not understood' 'to load 18' -- $SS $T/snapblocks --dry-run printf '%s\n' '/*C1*/INSERT INTO BLOCKS VALUES(0,2147483647,0)' '/*C1*/INSERT INTO BLOCKSX VALUES(1)' > $T/snapblocks/d/default.log ctl seed-snapshot-blocks-not-blanket-pass 1 'UNMAPPED \(1\)' 'statement not understood: INSERT INTO BLOCKSX' -- $SS $T/snapblocks --dry-run +# the throwaway cluster also starts from binaries on PATH (Homebrew), not only /usr/lib/postgresql (cold CMM: every Postgres control SKIPped on a Mac) +if command -v initdb > /dev/null && ! ls -d /usr/lib/postgresql/*/bin > /dev/null 2>&1; then + ctl pgtmp-path-binaries-catch 0 '^psql -h /var/tmp/upg-pg\.' '^stopped$' -- bash -c "c=\$(bash $here/lib/pgtmp.sh start) && echo \"\$c\" && bash $here/lib/pgtmp.sh stop \$(echo \"\$c\" | awk '{print \$3}') && echo stopped" +else skip pgtmp-path-binaries "no initdb on PATH, or the /usr/lib/postgresql layout is there"; fi if PGC=$(bash $here/lib/pgtmp.sh start 2>/dev/null); then PGD=$(echo "$PGC" | awk '{print $3}'); pgx() { psql -h "$PGD" -U mendix -qtA "$@"; } pgx -d fx -f $SNAP/pg-schema.sql > /dev/null && for n in ok colgone noguid skipt; do pgx -d fx -c "create database t_$n template fx" > /dev/null; done @@ -898,6 +1083,21 @@ if PGC=$(bash $here/lib/pgtmp.sh start 2>/dev/null); then ctl seed-snapshot-column-missing-loaded-nothing-pass 0 '^0$' -- pgx -d t_colgone -c 'select count(*) from "demo$product"' ctl seed-snapshot-skip-waiver-pass 1 'waived \(--skip\)' 'column demo\$product.price' '!has no counterpart in the database \(' -- $SS $SNAP --psql "$(mypsql colgone)" --dry-run --skip demo\$order_product ctl seed-snapshot-entity-guid-catch 1 'GUID b2222222-0000-4000-8000-000000000001\) is not in the database' 'carry entity short id 3, which' -- $SS $SNAP --psql "$(mypsql noguid)" --dry-run + # cold CMM 2026-11-17: an external entity's $pk table holds the REMOTE key in "_id" (remote_primary_key in + # mendixsystem$entity); remapping it as an object id made 5 such tables UNMAPPED and nothing loaded. id is remapped, _id kept + rm -rf $T/snapremote; cp -R $SNAP $T/snapremote; RS=$T/snapremote/data/database/hsqldb/default.script + sed -i.bak -e 's/"superentity_id" VARCHAR(36))$/"superentity_id" VARCHAR(36),"remote" BOOLEAN,"remote_primary_key" BOOLEAN)/' \ + -e '/^INSERT INTO "mendixsystem\$entity" VALUES/s/)$/,FALSE,FALSE)/' $RS && rm -f $RS.bak + printf '%s\n' "INSERT INTO \"mendixsystem\$entityidentifier\" VALUES('c3333333-0000-4000-8000-000000000001',6,2)" \ + "INSERT INTO \"mendixsystem\$entity\" VALUES('c3333333-0000-4000-8000-000000000001','Demo.Ext','demo\$ext\$pk',NULL,TRUE,TRUE)" \ + 'CREATE MEMORY TABLE PUBLIC."demo$ext$pk"("id" BIGINT NOT NULL PRIMARY KEY,"_id" BIGINT)' \ + 'INSERT INTO "demo$ext$pk" VALUES(1688849860263937,27866022694354949)' >> $RS + pgx -d fx -c "create database t_remote template fx" > /dev/null + pgx -d t_remote -c 'create table "demo$ext$pk"(id bigint primary key, _id bigint)' \ + -c "insert into \"mendixsystem\$entityidentifier\" values ('c3333333-0000-4000-8000-000000000001',16,0)" \ + -c "insert into \"mendixsystem\$entity\" values ('c3333333-0000-4000-8000-000000000001','Demo.Ext','demo\$ext\$pk',NULL)" > /dev/null + ctl seed-snapshot-remote-key-catch 0 'applied: 15 rows' '!UNMAPPED' -- $SS $T/snapremote --psql "$(mypsql remote)" + ctl seed-snapshot-remote-key-kept-pass 0 '^16\|27866022694354949$' -- pgx -d t_remote -F'|' -c 'select id>>48, _id from "demo$ext$pk"' ctl seed-snapshot-psql-error-catch 2 'psql failed' -- $SS $SNAP --psql "psql -h $PGD -U mendix -d nope" --dry-run bash $here/lib/pgtmp.sh stop "$PGD" else skip seed-snapshot-postgres "no postgres binaries or the throwaway cluster did not start"; fi @@ -907,6 +1107,8 @@ if command -v node > /dev/null; then ctl smokewalk-predicates 0 '^ok +loggedIn w # R1: the React client has no mx.session.getUserName / mx.ui.getContentForm; an anonymous session is not logged in, a content page maps to its Dojo path ctl smokewalk-react-catch 0 '^ok +react loggedIn when anonymous' '^ok +react contentPath without page' '^ok +react landedWhere on a popup state' -- node $here/smokewalk_predicates.js ctl smokewalk-react-pass 0 '^ok +react loggedIn when ready' '^ok +react homeReady when ready' '^ok +react contentPath maps to the page path' '^ok +react landedWhere on the wanted page' '^ok +react landedWhere on another page' -- node $here/smokewalk_predicates.js +# cold CMM 2026-11-17: Atlas styles .mx-window position:fixed (offsetParent null): every "Switch user" popup menu item FAILed +ctl smokewalk-popup-fixed-catch 0 '^ok +landedWhere fixed popup window' '^ok +landedWhere hidden popup window' -- node $here/smokewalk_predicates.js # T3-6: a step is judged by what the browser shows: the login form and the "An error occurred" dialog FAIL, a clean page passes ctl smokewalk-landedok-pass 0 '^ok +landedOk login form page .*FAIL wanted' '^ok +landedOk error dialog .*FAIL wanted' '^ok +landedOk clean page .*PASS wanted' \ '^ok +landedOk hidden login form .*PASS wanted' '^ok +landedOk when inspection throws is not a pass' '!^FAIL' -- node $here/smokewalk_predicates.js @@ -1429,11 +1631,15 @@ ctl stepupdate-widgets-unchanged-pass 0 'committed' '!WIDGETS' -- bash $B/stepup ctl widgetdiff-bad-rev-catch 2 'not a commit' -- python3 $B/widgetdiff.py $S/wg --rev no-such-rev # NS-7: an update that deletes and re-imports a module (cc318c0b, DataWidgets 2.27.6): every unit id of it is new mkdir -p $S/recreate; cp $M/base/App.mpr $S/recreate/App.mpr; python3 $here/lib/reid_module.py $S/recreate/App.mpr Administration > /dev/null -ctl idcheck-module-recreated-catch 0 '^warn MODULE_RECREATED Administration: all 5 unit ids new \(5 units now\)' '^verdict: PASS' -- python3 $B/idcheck.py $M/base/App.mpr $S/recreate/App.mpr +# WM1: mxcli's update gives new unit ids but keeps every element identity: info, the data is safe +ctl idcheck-module-recreated-ids-kept-info-pass 0 '^info MODULE_RECREATED Administration: all 5 unit ids new \(5 units now\), but all 7 element identities kept' '!^warn MODULE_RECREATED' '^verdict: PASS' -- python3 $B/idcheck.py $M/base/App.mpr $S/recreate/App.mpr +# a real delete and re-import: the element GUIDs are new too: warn, and the GUID changes are RED +mkdir -p $S/recreate-g; cp $M/base/App.mpr $S/recreate-g/App.mpr; python3 $here/lib/reid_module.py $S/recreate-g/App.mpr Administration --guids > /dev/null +ctl idcheck-module-recreated-catch 1 '^warn MODULE_RECREATED Administration: all 5 unit ids new \(5 units now\), 0 of 7 element identities kept' '!^info MODULE_RECREATED' 'GUID_CHANGED' -- python3 $B/idcheck.py $M/base/App.mpr $S/recreate-g/App.mpr ctl idcheck-module-updated-pass 0 '!MODULE_RECREATED' -- python3 $B/idcheck.py $M/base/App.mpr $M/step1-adds/App.mpr ctl idcheck-module-recreated-json-pass 0 '"module": "Administration"' -- sh -c "python3 $B/idcheck.py $M/base/App.mpr $S/recreate/App.mpr --json $T/recreate.json > /dev/null && cat $T/recreate.json" newapp rc '1 1.0 Recreate\n' -ctl stepupdate-module-recreated-catch 0 '^ +warn MODULE_RECREATED Administration' 'committed' -- bash $B/stepupdate.sh $S/rc $S/rc.plan +ctl stepupdate-module-recreated-catch 0 '^ +info MODULE_RECREATED Administration: .*element identities kept' 'committed' -- bash $B/stepupdate.sh $S/rc $S/rc.plan # NS-10/NS-13: notification-service 3fbf1566 is the 11.12.4 convert (model units, Badge.mpk and 22 more without their # appstore.id entry, widgets-appstore-metadata.json, regenerated JSDoc) squashed together with toolkit files for k in cn cnok cnw; do rm -rf $S/$k; mkdir -p $S/$k/mprcontents/0a $S/$k/javascriptsource/nanoflowcommons/actions @@ -1531,6 +1737,56 @@ mkdir -p $V/papp; touch $V/papp/App.mpr # F2: an --app-dir outside the run folder is refused up front (exit 3), not left for mxbuild to fail on at baseline mkdir -p $V/outside/oapp; touch $V/outside/oapp/App.mpr ctl pipeline-appdir-outside-catch 3 'is outside the run folder' '!step ' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/outside/oapp --source 10.24.26.1 --target 11.15.0.1 --work $V/pw2/work +# cold CMM 2026-11-17: two run folders both named "run" shared the database names upg_run_*; the second app's baseline said +# "reusing existing frozen database upg_run_frozen" and started on the first app's data. Names now differ per run folder, +# and a frozen database whose owner stamp names another run (or that has no stamp and no ledger line here) is refused +mkdir -p $V/dbn1/run/app $V/dbn2/run/app; touch $V/dbn1/run/app/App.mpr $V/dbn2/run/app/App.mpr +ctl pipeline-db-name-per-run-catch 0 '^2$' -- bash -c "for d in dbn1 dbn2; do bash \${PIPE:-$B/cloud/upgrade-pipeline.sh} --app-dir $V/\$d/run/app --source 10.24.26.1 --target 11.15.0.1 --work $V/\$d/run/work --only baseline --dry-run 2>&1 | sed -n 's/.*createdb -U mendix -T \([a-z0-9_]*\) .*_frozen\$/\1/p'; done | sort -u | wc -l | tr -d ' '" +ctl pipeline-db-name-kept-pass 0 '^upg_keep$' -- bash -c "mkdir -p $V/dbn3/run/app $V/dbn3/run/work/state; touch $V/dbn3/run/app/App.mpr; echo upg_keep > $V/dbn3/run/work/state/db-prefix; bash \${PIPE:-$B/cloud/upgrade-pipeline.sh} --app-dir $V/dbn3/run/app --source 10.24.26.1 --target 11.15.0.1 --work $V/dbn3/run/work --only baseline --dry-run 2>&1 | sed -n 's/.*createdb -U mendix -T \([a-z0-9_]*\) .*_frozen\$/\1/p'" +# an explicit --db is kept for the next resume: the prefix block is evaluated with an older name already kept +awk '/^RUNP=/{f=1} /^frozen_stamp\(\)/{exit} f' "${PIPE:-$B/cloud/upgrade-pipeline.sh}" > $V/dbblock.sh +ctl pipeline-db-arg-persisted-catch 0 '^upg_new$' -- bash -c "mkdir -p $V/dba/run/work/state; echo upg_old > $V/dba/run/work/state/db-prefix; RUN=$V/dba/run WORK=$V/dba/run/work DB_ARG=upg_new DRY=0; usage_die() { echo \"\$*\"; exit 2; }; . $V/dbblock.sh; cat $V/dba/run/work/state/db-prefix" +ctl pipeline-db-kept-no-arg-pass 0 '^upg_old$' -- bash -c "mkdir -p $V/dbb/run/work/state; echo upg_old > $V/dbb/run/work/state/db-prefix; RUN=$V/dbb/run WORK=$V/dbb/run/work DB_ARG= DRY=0; usage_die() { echo \"\$*\"; exit 2; }; . $V/dbblock.sh; cat $V/dbb/run/work/state/db-prefix" +mkdir -p $V/fdk; cat > $V/fdk/docker <<'FAKE' +#!/bin/bash +case "$*" in *shobj_description*) cat "$FAKE_OWNER" 2>/dev/null;; *COMMENT*) echo "stamped: $*" >> "$FAKE_LOG";; esac +FAKE +chmod +x $V/fdk/docker +cat > $V/fdk/fo.sh <<'FO' +# fo.sh WORK: frozen_owner from the pipeline (or $PIPE), run folder /r/cmm, database upg_run_frozen +set -u; PGC=mig-pg; X=upg_run; X_FROZEN=upg_run_frozen; RUNP=/r/cmm; WORK=$1 +must() { "$@"; }; die4() { echo "DIE: $*"; exit 4; } +eval "$(sed -n '/^frozen_stamp()/,/^}/p' "$PIPE")" +frozen_owner && echo owner-ok +FO +FO="env PIPE=${PIPE:-$B/cloud/upgrade-pipeline.sh} bash $V/fdk/fo.sh" +echo 'upgrade-toolkit run /r/wm1' > $V/fdk/other; echo 'upgrade-toolkit run /r/cmm' > $V/fdk/own; : > $V/fdk/none +mkdir -p $V/fdk/w-legacy $V/fdk/w-new; echo 'upg_run_base snapshot before 2026-11-15T23:33:59' > $V/fdk/w-legacy/db-ledger.txt +ctl pipeline-frozen-other-run-catch 4 '^DIE: frozen database upg_run_frozen exists but this run did not make it \(upgrade-toolkit run /r/wm1' '!owner-ok' -- env PATH=$V/fdk:"$PATH" FAKE_OWNER=$V/fdk/other FAKE_LOG=$V/fdk/log $FO $V/fdk/w-new +ctl pipeline-frozen-no-stamp-no-ledger-catch 4 'no owner stamp' -- env PATH=$V/fdk:"$PATH" FAKE_OWNER=$V/fdk/none FAKE_LOG=$V/fdk/log $FO $V/fdk/w-new +ctl pipeline-frozen-own-pass 0 '^owner-ok$' -- env PATH=$V/fdk:"$PATH" FAKE_OWNER=$V/fdk/own FAKE_LOG=$V/fdk/log $FO $V/fdk/w-new +ctl pipeline-frozen-legacy-adopted-pass 0 '^owner-ok$' -- bash -c "rm -f $V/fdk/log2; env PATH=$V/fdk:\"\$PATH\" FAKE_OWNER=$V/fdk/none FAKE_LOG=$V/fdk/log2 $FO $V/fdk/w-legacy && grep -q 'COMMENT ON DATABASE upg_run_frozen IS .upgrade-toolkit run /r/cmm' $V/fdk/log2" +# cold CMM 2026-11-17: the package carried 9,235 demo rows (data-snapshot.zip) and seed_snapshot.py existed, yet baseline +# seeded 5 synthetic rows per entity ("hsql data present, not replayed (no tool)"): walks then judge near-empty pages. +mkdir -p $V/demo/run/app $V/nodemo/run/app; touch $V/demo/run/app/App.mpr $V/nodemo/run/app/App.mpr +( cd $V/demo && mkdir -p data/database/hsqldb/default && touch data/database/hsqldb/default/default.script && zip -qr run/app/data-snapshot.zip data && rm -rf data ) +ctl pipeline-demo-data-seeded-catch 0 'seed_snapshot\.py .*data-snapshot\.zip --psql' '!seed_pg\.py' -- bash -c "bash \${PIPE:-$B/cloud/upgrade-pipeline.sh} --app-dir $V/demo/run/app --source 10.24.26.1 --target 11.15.0.1 --work $V/demo/run/work --only baseline --dry-run 2>&1" +ctl pipeline-demo-data-none-seed-pg-pass 0 'seed_pg\.py' '!seed_snapshot' -- bash -c "bash \${PIPE:-$B/cloud/upgrade-pipeline.sh} --app-dir $V/nodemo/run/app --source 10.24.26.1 --target 11.15.0.1 --work $V/nodemo/run/work --only baseline --dry-run 2>&1" +# seed_demo: UNMAPPED with nothing loaded -> 1 (the caller seeds synthetic rows) and a note; a clean load -> 0 and the files +mkdir -p $V/sdm/bin $V/sdm/w/logs $V/sdm/w/state +printf '#!/bin/sh\necho "$SD_OUT"; exit $SD_RC\n' > $V/sdm/bin/seed_snapshot.py +cat > $V/sdm/sd.sh <<'SDEOF' +set -Eeuo pipefail +eval "$(sed -n '/^demo_snapshot()/p; /^seed_demo()/,/^}/p; /^x()/,/^}/p; /^xo()/,/^}/p; /^note()/p; /^psql_str()/p' "$PIPE")" +quoted() { printf ' %q' "$@"; } +python3() { sh "$@"; } +B=$SDB; WORK=$SDW; APP=$SDA; X=upg_t; PGC=pg; DRY=0; STEP=baseline; LOG=$SDW/logs/b.log +rc=0; seed_demo || rc=$?; echo "rc=$rc"; cat "$WORK/state/notes.txt" 2> /dev/null; ls "$WORK/demo-files" 2> /dev/null || true +SDEOF +( cd $V/sdm && mkdir -p data/files/00 && echo img > data/files/00/u1 && zip -qr app.zip data && rm -rf data && mkdir app && mv app.zip app/data-snapshot.zip ) +SD="env PIPE=${PIPE:-$B/cloud/upgrade-pipeline.sh} SDB=$V/sdm/bin SDW=$V/sdm/w SDA=$V/sdm/app" +ctl pipeline-seed-demo-unmapped-falls-back-catch 0 '^rc=1$' 'demo data NOT loaded' -- $SD SD_RC=1 SD_OUT='nothing was loaded' bash $V/sdm/sd.sh +ctl pipeline-seed-demo-loaded-pass 0 '^rc=0$' 'demo data loaded from data-snapshot.zip \(applied: 9 rows in 2 tables\)' '^00$' -- bash -c "rm -f $V/sdm/w/state/notes.txt; $SD SD_RC=0 SD_OUT='applied: 9 rows in 2 tables; every table' bash $V/sdm/sd.sh" # F1: force-ui.txt reaches stepupdate as --force-ui (else --unattended keeps every edited module), replay/.mdl runs after the updates mkdir -p $V/pw3/replay; printf '# update anyway, replay\nMendixSSO\nAtlas_Core\n' > $V/pw3/force-ui.txt; echo 'alter entity X.Y drop validation rule;' > $V/pw3/replay/MendixSSO.mdl; : > $V/pw3/replay/Atlas_Core.patch ctl pipeline-dry-forceui-replay-pass 0 'stepupdate\.sh .* --unattended --force-ui MendixSSO.?,Atlas_Core --saved-edits .*/pw3/saved-edits' 'exec .*/pw3/replay/MendixSSO\.mdl -p ' 'git -C .*/papp apply --check .*/pw3/replay/Atlas_Core\.patch' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pw3 --only update --dry-run @@ -1538,12 +1794,21 @@ ctl pipeline-dry-forceui-replay-pass 0 'stepupdate\.sh .* --unattended --force-u mkdir -p $V/ltools/mxbuild-10.24.9.1 $V/ltools/mxbuild-10.24.27.1 ctl pipeline-ladder-9-goes-through-10-24-pass 0 'version ladder 9\.24\.2\.1 -> 10\.24\.27\.1 -> 11\.12\.7$' -- env UPG_TOOLS=$V/ltools bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 9.24.2.1 --to 11.12.7 --work $V/lw --dry-run ctl pipeline-ladder-10-24-goes-straight-pass 0 'version ladder 10\.24\.3\.1 -> 11\.12\.7$' -- env UPG_TOOLS=$V/ltools bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.3.1 --to 11.12.7 --work $V/lw --dry-run +# cold CMM 2026-11-17: 13 of 23 walk steps were microflows, SKIPPED without a word (the notice fired only on an INVALID walk) +ctl pipeline-smoke-mf-plan-microflow-catch 0 '^--microflows$' -- bash -c "WORK=\$(mktemp -d); echo '{\"kind\": \"microflow\"}' > \$WORK/plan.json; eval \"\$(awk '/^smoke_mf\(\)/{f=1} f{print} f&&/^}|; }\$/{exit}' \${PIPE:-$B/cloud/upgrade-pipeline.sh})\"; smoke_mf" +ctl pipeline-smoke-mf-opt-out-pass 0 '!microflows' '^done$' -- bash -c "WORK=\$(mktemp -d); echo '{\"kind\": \"microflow\"}' > \$WORK/plan.json; echo '{\"landed\": \"microflow ran\"}' > \$WORK/smoke-before.json; UPG_SMOKE_MICROFLOWS=0; eval \"\$(awk '/^smoke_mf\(\)/{f=1} f{print} f&&/^}|; }\$/{exit}' \${PIPE:-$B/cloud/upgrade-pipeline.sh})\"; smoke_mf; echo done" +# cold CMM 2026-11-17: no git identity on the machine: stepupdate's plain `git commit` stopped the run (exit 4) +GI="eval \"\$(sed -n '/^git_identity()/,/^}/p' \${PIPE:-$B/cloud/upgrade-pipeline.sh})\"; git_identity 2> /dev/null || true" +ctl pipeline-git-identity-none-catch 0 '^committed$' -- bash -c "export HOME=\$(mktemp -d) GIT_CONFIG_NOSYSTEM=1; unset GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL EMAIL; $GI; r=\$(mktemp -d); git -C \$r init -q && touch \$r/f && git -C \$r add f && git -C \$r commit -qm t && echo committed" +ctl pipeline-git-identity-kept-pass 0 '^Person $' -- bash -c "export HOME=\$(mktemp -d) GIT_CONFIG_NOSYSTEM=1; unset GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL EMAIL; git config --global user.name Person; git config --global user.email p@x; $GI; r=\$(mktemp -d); git -C \$r init -q && touch \$r/f && git -C \$r add f && git -C \$r commit -qm t && git -C \$r log -1 --format='%an <%ae>'" ctl pipeline-ladder-no-10-24-toolset-catch 2 'goes to 11\.12\.7 through 10\.24: no mxbuild-10\.24\.\* toolset found, give --via' '!unexpected failure' -- env UPG_TOOLS=$V/none bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 9.24.2.1 --to 11.12.7 --work $V/lw --dry-run ctl pipeline-ladder-refuses-to-with-target-catch 2 'give one or the other' -- env UPG_TOOLS=$V/ltools bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 9.24.2.1 --to 11.12.7 --target 10.24.1 --work $V/lw --dry-run -# UPG_SMOKE_MICROFLOWS=1 passes --microflows to the smoke walk (WM1: a role whose home and menu are all microflows); off by default +# UPG_SMOKE_MICROFLOWS=1 passes --microflows to the smoke walk (WM1: a role whose home and menu are all microflows); off when the plan has no microflow step ctl pipeline-dry-smoke-microflows-pass 0 'smokewalk\.js .* --microflows --login-path' -- env UPG_SMOKE_MICROFLOWS=1 bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pmf1 --only basewalk --dry-run ctl pipeline-dry-smoke-microflows-off-catch 0 'smokewalk\.js .* --login-path' '!--microflows' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pmf0 --only basewalk --dry-run ctl pipeline-dry-noforceui-catch 0 '!--force-ui' '!replay/' 'stepupdate\.sh .* --unattended' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pw --only update --dry-run +# the update step runs crosscalls --own on the original model BEFORE the module updates (cold CMM: 3 CE1613 found only at the build) +ctl pipeline-dry-update-owngone-catch 0 '^own-before-update$' -- bash -c "bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pw --only update --dry-run | awk '/crosscalls\.py .*\/pw\/original\.mpr --kept .*--own --plan-file .*\/pw\/plan\.txt/{if(!c)c=NR} /stepupdate\.sh/{if(!s)s=NR} END{print (c && c.guids restores the original GUIDs after the MDL replay; approved-step.txt reaches the step idcheck only mkdir -p $V/pw4/replay; printf 'MendixSSO\n' > $V/pw4/force-ui.txt; echo 'create or modify association A.R_X from A.R to B.U;' > $V/pw4/replay/MendixSSO.mdl printf '# original GUIDs back\nA.R_X\nA.R_Y\n' > $V/pw4/replay/MendixSSO.guids; printf 'A.R_X # restored by replay/MendixSSO.guids\n' > $V/pw4/approved-step.txt; printf '# accepted\n' > $V/pw4/approved.txt @@ -1578,6 +1843,17 @@ mkdir -p $NJ/w13; printf '1 1.0 Administration\n' > $NJ/w13/plan.txt ctl pipeline-no-plan-pre-catch 0 'needs judgment \(exit 13\)' 'plan-pre.txt missing on a 10.x to 11 run' '!mx convert exit' -- bash -c "MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-dir $NJ/app --source 10.24.26.1 --target 11.15.0.1 --work $NJ/w13 --from convert --only convert; cat $NJ/w13/NEEDS-JUDGMENT.md" mkdir -p $NJ/w13b; printf '1 1.0 Administration\n' > $NJ/w13b/plan.txt; printf '# nothing before the convert\n' > $NJ/w13b/plan-pre.txt ctl pipeline-plan-pre-stated-pass 0 '!plan-pre.txt missing' 'convert' -- bash -c "MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-dir $NJ/app --source 10.24.26.1 --target 11.15.0.1 --work $NJ/w13b --from convert --only convert; cat $NJ/w13b/NEEDS-JUDGMENT.md 2>/dev/null; true" +# WM1 cold test 2026-11-17: 16 judgment stops, one question each. Interview mode auto parks a stop nothing later depends on +# (here the gate's release-impact and kept-modules stops), goes on, and asks every parked item in one grouped list at the end +for d in ga gm; do mkdir -p $NJ/$d; printf 'Admin 1.0\n' > $NJ/$d/kept.txt; printf '# release impact\n' > $NJ/$d/release-impact.md; done +printf 'Interview mode: auto\n' > $NJ/ga/interview-mode.txt +ctl pipeline-auto-parks-all-pass 12 '^PARKED gate: exit 12 parked' '^PARKED gate: exit 13 parked' '2 parked item\(s\) to answer in one round \(exit 12\)' '^# Modules and data: 1$' '^# Screens, comparisons and the gate: 1$' '--from gate$' '!PASS gate' -- bash -c "MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-dir $NJ/app --source 10.24.26.1 --target 11.15.0.1 --work $NJ/ga --only gate; rc=\$?; cat $NJ/ga/NEEDS-JUDGMENT.md; exit \$rc" +ctl pipeline-auto-off-stops-first-catch 12 'needs judgment \(exit 12\)' '!PARKED' '!kept modules' -- bash -c "MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-dir $NJ/app --source 10.24.26.1 --target 11.15.0.1 --work $NJ/gm --only gate; rc=\$?; cat $NJ/gm/NEEDS-JUDGMENT.md; exit \$rc" +# a stop that later steps depend on is never parked: the plan-pre stop of F13 still ends the run in auto mode +mkdir -p $NJ/w13a; printf '1 1.0 Administration\n' > $NJ/w13a/plan.txt +ctl pipeline-auto-hard-stop-still-stops-catch 0 'needs judgment \(exit 13\)' '!PARKED' '!mx convert exit' -- bash -c "UPG_INTERVIEW_MODE=auto MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-dir $NJ/app --source 10.24.26.1 --target 11.15.0.1 --work $NJ/w13a --from convert --only convert; true" +# a step that runs again parks afresh: the gate rerun after the acks leaves nothing parked and reaches the PASS summary +ctl pipeline-auto-rerun-clears-pass 0 'PASS gate' '!PARKED' '!parked item' -- bash -c "cp $NJ/ga/kept.txt $NJ/ga/kept-ack.txt; printf '## Studio Pro\n\n| 1 | 11.15 | behaviour | x | not relevant: control fixture |\n' > $NJ/ga/release-impact.md; MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-dir $NJ/app --source 10.24.26.1 --target 11.15.0.1 --work $NJ/ga --only gate" newapp o11b '1 1.0 Old11Base\n' ctl stepupdate-pre1021-baseline-pass 0 'installed Old11Base cannot serve as the local-edit baseline \(range from 10.21.0 to 10.24.99\)' 'committed' '!SKIP' -- bash $B/stepupdate.sh $S/o11b $S/o11b.plan --unattended newapp o11 '1 1.0 Old11Pkg\n' @@ -1589,6 +1865,41 @@ newapp nbu '1 1.0 NoBaseUi\n' ctl stepupdate-unsaved-unattended-catch 1 'KEPT: NoBaseUi .*UNSAVED' 'kept: NoBaseUi UNSAVED' -- bash $B/stepupdate.sh $S/nbu $S/nbu.plan --force-ui NoBaseUi --unattended newapp nbu '1 1.0 NoBaseUi\n' ctl stepupdate-unsaved-accept-pass 0 'accepted \(--accept-unsaved NoBaseUi\)' 'committed' 'UNSAVED \(replay by hand\): NoBaseUi 1.0' -- bash $B/stepupdate.sh $S/nbu $S/nbu.plan --force-ui NoBaseUi --accept-unsaved NoBaseUi +# cold CMM 2026-11-17: a module forced for another route (XASSOC) whose localedits list no UI edit loses nothing when the +# --no-baseline route saves 0 files: committed, not KEPT as UNSAVED; one whose localedits list UI edits is still kept +printf '[{"name": "NoBaseUi", "domain": [], "ui": [], "logic": [], "files": []}]' > $V/le-noui.json +printf '[{"name": "NoBaseUi", "domain": [], "ui": [{"change": "CHANGED", "kind": "page", "name": "NoBaseUi.P"}], "logic": [], "files": []}]' > $V/le-ui.json +newapp nbu '1 1.0 NoBaseUi\n' +ctl stepupdate-unsaved-no-ui-edits-catch 0 'lists no UI edits in NoBaseUi: nothing to save' 'committed' '!KEPT' '!UNSAVED' -- bash $B/stepupdate.sh $S/nbu $S/nbu.plan --force-ui NoBaseUi --unattended --localedits $V/le-noui.json +newapp nbu '1 1.0 NoBaseUi\n' +ctl stepupdate-unsaved-ui-edits-listed-pass 1 'KEPT: NoBaseUi .*UNSAVED' '!nothing to save' -- bash $B/stepupdate.sh $S/nbu $S/nbu.plan --force-ui NoBaseUi --unattended --localedits $V/le-ui.json +mkdir -p $V/pwle; echo '[]' > $V/pwle/localedits.json +ctl pipeline-dry-stepupdate-localedits-catch 0 'stepupdate\.sh .* --unattended.* --localedits .*/pwle/localedits\.json' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pwle --only update --dry-run +# cold CMM 2026-11-17: the mx check text names only the element ("at Action button 'actionButton1'", on many pages); the +# build check passes -j (mx 10 and later) and checkwhere.py appends the module and document of each error as [where] lines +mkdir -p $V/cw +printf '%s\n' '{"serialization_version": 1, "errors": [{"code": "CE0117", "message": "Error(s) in expression.", "locations": [{"module-name": "Shop", "document-name": "Page '"'"'Order_Step1'"'"'", "element-name": "Property '"'"'Dynamic classes expression'"'"' of action button '"'"'actionButton1'"'"'"}]}]}' > $V/cw/check.json +printf '%s\n' '[error] [CE0117] "Error(s) in expression." at Action button '"'"'actionButton1'"'"'' 'The app contains: 1 errors.' > $V/cw/check0.log +cp $V/cw/check0.log $V/cw/check.log +ctl checkwhere-document-catch 0 '^\[where\] CE0117 Shop / Page .Order_Step1.: Property .Dynamic classes expression. of action button .actionButton1.$' '^\[error\] \[CE0117\]' -- bash -c "python3 $B/checkwhere.py $V/cw/check.json $V/cw/check.log && cat $V/cw/check.log" +ctl checkwhere-checkdiff-count-pass 0 '^OK: errors 1 -> 1' -- python3 $B/checkdiff.py $V/cw/check0.log $V/cw/check.log +awk '/^CHECK_N=0/{p=1} /^has_code\(\)/{p=0} p' $B/cloud/upgrade-pipeline.sh > $V/cw/mxcheck.sh +cat > $V/cw/stub.sh < $V/cw/openerr.sh +printf '%s\n' 'Loading the mpr file.' "ERROR: System.Collections.Generic.KeyNotFoundException: The given key 'cb8a' was not present in the dictionary." ' at Mendix.Modeler.Storage.Operations.UnitContentsLoader.ConvertFromStorage' > $V/cw/crash.log +printf '%s\n' 'Checking app for errors...' 'The app contains: 0 errors.' > $V/cw/clean.log +ctl pipeline-mxcheck-crash-catch 0 '^\[error\] \[MXCHECK\] mx check ended without a result .*KeyNotFoundException' '^errors 1$' -- bash -c "B=$B WORK=$V/cw/crash TARGET=11.12.7 MPR=x.mpr DRY=0; . $V/cw/stub.sh; heavy() { cat $V/cw/crash.log; return 1; }; . $V/cw/mxcheck.sh; . $V/cw/openerr.sh; mxcheck; cat \$CHECK_LOG; echo errors \$(open_errors \$CHECK_LOG)" +ctl pipeline-mxcheck-clean-pass 0 '^errors 0$' '!MXCHECK' -- bash -c "B=$B WORK=$V/cw/clean TARGET=11.12.7 MPR=x.mpr DRY=0; . $V/cw/stub.sh; heavy() { cat $V/cw/clean.log; }; . $V/cw/mxcheck.sh; . $V/cw/openerr.sh; mxcheck; cat \$CHECK_LOG; echo errors \$(open_errors \$CHECK_LOG)" +ctl pipeline-dry-check-json-catch 0 'mx10\.sh 11\.15\.0\.1 mx check .*/papp/App\.mpr -j .*/pwle/check-1\.json > ' -- bash $B/cloud/upgrade-pipeline.sh --app-dir $V/papp --source 10.24.26.1 --target 11.15.0.1 --work $V/pwle --only build --dry-run # v3.1 E6-7: a removal (idcheck verdict REVIEW) is named REVIEW in the KEPT line, a GUID change stays a plain finding mkdir -p $S/base; cp $M/base/App.mpr $S/base/ newapp rv '1 1.0 Removes\n'; cp $M/step1-adds/App.mpr $S/rv/App.mpr; git -C $S/rv commit -qam 'start at step1-adds' @@ -1736,6 +2047,15 @@ cp $TH/gen-pkg.scss $TH/fg-genlike/themesource/administration/web/_generated-exc printf '// generated from the brand sheet\n$brand-x: #222;\n' > $TH/fg-genlike/themesource/administration/web/_brand.scss ctl localedits-autogenerated-file-pass 0 'files unchanged' '!FILES-EDITED' -- env UPG_CACHE=$TH/cache-gen python3 $B/localedits.py $TH/fg-gen/App.mpr --module Administration --files-only ctl localedits-generated-comment-edit-catch 1 'FILES-EDITED themesource/administration/web/_brand\.scss' '!_generated-exclusion' -- env UPG_CACHE=$TH/cache-gen python3 $B/localedits.py $TH/fg-genlike/App.mpr --module Administration --files-only +# cold CMM: DataWidgets got a plan card "themesource edits" for one package .DS_Store the app did not have; files the OS +# writes into a folder (.DS_Store, ._x, Thumbs.db, desktop.ini) are no edit either way, a deleted real file still is +mkdir -p $TH/cache-junk; cp $TH/cache/23513-4.1.0.mpk $TH/cache-junk/; : > $TH/junk.bin +python3 -c "import zipfile; z=zipfile.ZipFile('$TH/cache-junk/23513-4.1.0.mpk','a'); z.write('$TH/junk.bin','themesource/administration/.DS_Store'); z.write('$TH/junk.bin','javascriptsource/administration/actions/._Decode.js'); z.close()" +for k in junk real; do rm -rf $TH/fj-$k; cp -R $TH/same $TH/fj-$k; done +: > $TH/fj-junk/themesource/administration/web/Thumbs.db; : > $TH/fj-junk/themesource/administration/desktop.ini +rm $TH/fj-real/themesource/administration/web/_variables.scss +ctl localedits-os-junk-files-pass 0 'files unchanged' '!FILES-EDITED' '!DS_Store' -- env UPG_CACHE=$TH/cache-junk python3 $B/localedits.py $TH/fj-junk/App.mpr --module Administration --files-only +ctl localedits-os-junk-real-removed-catch 1 'FILES-EDITED themesource/administration/web/_variables\.scss' '!DS_Store' -- env UPG_CACHE=$TH/cache-junk python3 $B/localedits.py $TH/fj-real/App.mpr --module Administration --files-only # after the update (themefiles report): a regenerated action is not FILES-OVERWRITTEN, a replaced USER CODE line is rm -rf $TH/js-pkgform; cp -R $TH/js-regen $TH/js-pkgform; cp $TH/jsact-pkg.js $TH/js-pkgform/javascriptsource/administration/actions/Decode.js python3 -c "import sys; sys.path.insert(0,'$B'); import themefiles as t; t.snapshot('$TH/js-regen','Administration','$TH/snap-regen'); t.snapshot('$TH/js-user','Administration','$TH/snap-user')" @@ -1778,9 +2098,9 @@ hgapp() { # hgapp NAME: App.mpr 4.0.0 + files, then the install commit (4.1.0, hgit $HG/$1 add -A .; hgit $HG/$1 commit -qm "step: Administration 4.1.0 (stepupdate, content 23513)" } hgapp hclean; echo notes > $HG/hclean/README.md; hgit $HG/hclean add README.md; hgit $HG/hclean commit -qm "docs" -sed -i 's/\r$//' $HG/hclean/themesource/administration/web/_variables.scss; hgit $HG/hclean commit -qam "line ends only" +sedi 's/\r$//' $HG/hclean/themesource/administration/web/_variables.scss; hgit $HG/hclean commit -qam "line ends only" ctl themefiles-history-clean-pass 0 '^FILES-UNEDITED-BY-HISTORY Administration since [0-9a-f]{7} [0-9-]+ "step: Administration 4\.1\.0 \(stepupdate' '[0-9]+ files, provenance: toolkit step' '!FILES-EDITED' '!FILES-UNKNOWN' -- python3 $B/themefiles.py history $HG/hclean Administration --mpr App.mpr -hgapp hedit; sed -i 's/f26522/00ff00/; s/^\$brand-primary: *[^;]*/$brand-primary: #00ff00/' $HG/hedit/themesource/administration/web/_variables.scss; echo '$brand-extra: #123;' >> $HG/hedit/themesource/administration/web/_variables.scss; hgit $HG/hedit commit -qam "brand colour" +hgapp hedit; sedi 's/f26522/00ff00/; s/^\$brand-primary: *[^;]*/$brand-primary: #00ff00/' $HG/hedit/themesource/administration/web/_variables.scss; echo '$brand-extra: #123;' >> $HG/hedit/themesource/administration/web/_variables.scss; hgit $HG/hedit commit -qam "brand colour" ctl themefiles-history-edited-catch 1 '^FILES-EDITED themesource/administration/web/_variables\.scss \(by history, since [0-9a-f]{7}\)' '!FILES-UNEDITED' -- python3 $B/themefiles.py history $HG/hedit Administration --mpr App.mpr --json $HG/hedit.json ctl themefiles-history-edited-vars-pass 0 'brand-extra: #123;' -- cat $HG/hedit.json git clone -q --depth 1 file://$HG/hclean $HG/hshallow @@ -1919,7 +2239,7 @@ ctl stepupdate-java-action-identical-usercode-forced-pass 0 'IDENTICAL: user cod ctl stepupdate-java-action-commit-pass 0 'step: JavaAct 2.0 \(stepupdate, idcheck PASS, convert-made Java action metadata forced, user code identical\)' -- git -C $S/ja log -1 --format=%s javaapp jc '\t\tLog.info("local tweak"); return Login.start(this.ctx);' ctl stepupdate-java-action-changed-usercode-keep-catch 1 'DIFFERENT: Java code of JavaAct 1\.0' 'changed user code: javasource/javaact/actions/StartLogin\.java' 'local LOGIC edits \(JAVA_ACTION' 'KEPT: JavaAct' '!committed [0-9a-f]{7}' '0 committed, 1 kept' -- env UPG_CACHE=$UC/cache bash $B/stepupdate.sh $S/jc $S/jc.plan --localedits $UC/le.json --unattended -ctl usercode-no-package-catch 2 'CANNOT COMPARE: installed package JavaAct 1\.1 not cached' -- env UPG_CACHE=$UC/cache UPG_MXCLI=/bin/false python3 $B/usercode.py $S/ja JavaAct --cid 999 --version 1.1 +ctl usercode-no-package-catch 2 'CANNOT COMPARE: installed package JavaAct 1\.1 not cached' -- env UPG_CACHE=$UC/cache UPG_MXCLI=$(command -v false) python3 $B/usercode.py $S/ja JavaAct --cid 999 --version 1.1 if [ -d "$LAB/f1/runs/e4/work" ] && [ -d "$LAB/e5/run2/work" ]; then # field: replay the real refusal output (E4 ExcelImporter local edits, E5b NanoflowCommons pre-9.18 baseline) newapp fe4 '1 1.0 ExcelImporter\n' @@ -2115,6 +2435,15 @@ cp -R $SH/old/shots-Administrator $J3/old/before/; cp -R $SH/empty/shots-Adminis for d in old/before new/after; do echo '{"journeys": {"login-Administrator": {"role": "Administrator", "verdict": "PASS"}}, "roles": {}}' > $J3/$d/summary.json; done ctl journeys-compare-screens-orphan-catch 1 '^screens: 1 PASS, 0 PASS\*, 1 REVIEW, 0 INVALID' -- python3 $B/run_journeys.py x --label after --work $J3/new --compare $J3/old --rejudge ctl journeys-screen-accept-pass 0 '!SCREEN REVIEW' -- python3 $B/run_journeys.py x --label after --work $J/new --compare $J/old/before/summary.json --rejudge --shot-accept $T/shot-accept.txt "${NOACT[@]}" +# --skip-baseline-fail (WM1 2026-11-17): the new app does not walk a journey that failed on the old app and feeds no later one +ctl journeys-skip-baseline-fail-pick-pass 0 '^waves skip: use-User=FAIL/User$' '^waves walked: make-User ok-User$' '^no-waves skip: make-User$' -- python3 $here/skip_baseline_control.py $(mkdir -p $T/skc && echo $T/skc) +# a SKIPPED journey is counted and named in the headline, is no REGRESSION, and its old-app failure stays the baseline rule's +# (exit 3, which gate.sh journeys_noted turns into a note) +J4=$T/rj4s; mkdir -p $J4; cp -R $J2/old $J2/new $J4/ +python3 -I -c "import json,sys +for f,v in ((sys.argv[1],{'role':'Administrator','verdict':'FAIL'}),(sys.argv[2],{'role':'Administrator','verdict':'SKIPPED','compare':'BASELINE DEFECT (before FAIL)','why':'not run on the new app'})): + d=json.load(open(f)); d['journeys']['report-Administrator']=v; json.dump(d,open(f,'w'))" $J4/old/before/summary.json $J4/new/after/summary.json +ctl journeys-skipped-counted-pass 3 '^journeys: 2 PASS, 0 FAIL, 0 INVALID, 1 SKIPPED \(failed on the old app too, not run on the new one\)' ' 0 regression ' '!^ *REGRESSION' -- python3 $B/run_journeys.py x --label after --work $J4/new --compare $J4/old --rejudge "${NOACT[@]}" ctl journeys-compare-path-catch 2 'no summary.json there.*nothing was run' -- python3 $B/run_journeys.py x --label after --work $J/new --compare $J/nothere --rejudge # N1R2-8 / T2R3-11: INVALID only (SCREEN INVALID, a BASELINE DEFECT) exits 3 with its own verdict line, never 1 (FAIL) J4=$T/rj4; mkdir -p $J4/old/before $J4/new/after; cp -R $SH/old/shots-Administrator $J4/old/before/ @@ -2567,7 +2896,8 @@ for d in $(echo "$PATH" | tr ':' ' '); do for f in "$d"/*; do n=${f##*/}; case $ ctl toolcheck-host-no-timeout-catch 0 'WARNING, no GNU timeout or gtimeout' 'host OK' -- env PATH=$NT UPG_PYTHON=$T/py-ok zsh $B/toolcheck.sh --host ctl toolcheck-host-timeout-pass 0 'host OK' '!no GNU timeout' -- env UPG_PYTHON=$T/py-ok zsh $B/toolcheck.sh --host NS=$T/nogsed; mkdir -p $NS $T/bsdsed; printf '#!/bin/sh\nexit 1\n' > $T/bsdsed/sed; chmod +x $T/bsdsed/sed -ctl toolcheck-host-bsd-sed-catch 0 'WARNING, sed is not GNU sed' -- env PATH=$T/bsdsed:$NT UPG_PYTHON=$T/py-ok zsh $B/toolcheck.sh --host +for f in "$NT"/*; do [ "${f##*/}" = gsed ] || ln -s "$(readlink "$f")" "$NS/${f##*/}" 2>/dev/null; done # a host with Homebrew gsed never warned +ctl toolcheck-host-bsd-sed-catch 0 'WARNING, sed is not GNU sed' -- env PATH=$T/bsdsed:$NS UPG_PYTHON=$T/py-ok zsh $B/toolcheck.sh --host # B4T2-1, B4N1-3: setup-container.sh pre-places the ANTLR jar (Maven Central 429 -> mirror; a proxy page is no jar), never # takes a credential on a command line, and runs toolcheck AFTER the pip/apt step: REFUSED or no rsync = exit 1 (it ran # before the pip step with `|| true`: "REFUSED" for modules installed a step later, exit 0 with modules or rsync missing) @@ -2879,6 +3209,14 @@ ctl gen-journeys-guard-dataview-sources-catch 0 '^Home_Nf label4$' '^Home_Path l ctl gen-journeys-guard-dataview-page-param-pass 0 '^Order_Edit dataView7$' -- python3 -c "$G3PY; print('Order_Edit', w('Order_Edit'))" ctl gen-journeys-guard-dataview-around-last-pass 0 '^Grid_Only layoutGrid3$' -- python3 -c "$G3PY; print('Grid_Only', w('Grid_Only'))" ctl gen-journeys-guard-dataview-only-catch 0 '^Only_Dependent None$' -- python3 -c "$G3PY; print('Only_Dependent', w('Only_Dependent'))" +# cold CMM 2026-11-17: a scroll container has no .mx-name- in the DOM (the MaterialHandler home's scrollContainer1 +# never appeared on the healthy 9.24 page): no guard, not even as the last resort; the layout grid inside it is taken, and +# a kept impact.json landing on one is re-picked (guard_names leaves it out) +model guard6 > /dev/null +G6PY="import sys; sys.path.insert(0,'$B'); import impact_map as I, mprmodel; m=mprmodel.Model.load('$M/guard6'); w=lambda p: (I.landing_widget(m, 'Shop.' + p) or {}).get('widget')" +ctl gen-journeys-guard-scrollcontainer-catch 0 '^Dash layoutGrid5$' '!scrollContainer' -- python3 -c "$G6PY; print('Dash', w('Dash'))" +ctl gen-journeys-guard-scrollcontainer-only-catch 0 '^Scroll_Only None$' -- python3 -c "$G6PY; print('Scroll_Only', w('Scroll_Only'))" +ctl gen-journeys-guard-scrollcontainer-names-catch 0 '^kept False$' -- python3 -c "$G6PY; print('kept', 'scrollContainer1' in I.guard_names(m, 'Shop.Dash'))" cp $FX/journeys/impact-guard3.json $T/impact-guard3.json G3=$T/gen-guard3; (cd / && python3 $B/gen_journeys.py $T/impact-guard3.json --out-dir $G3 --roles-with-users Inspector,Planner > /dev/null 2>&1) ctl gen-journeys-guard-dataview-home-closes-pass 0 "^ready .mx-name-actionButton2 closes \['home:Shop.NAV_Inspector'\]$" -- python3 -c "import json; j=json.load(open('$G3/journeys/Inspector/login-Inspector.journey.json')); print('ready', j['steps'][0]['ui']['ready'], 'closes', j.get('closes'))" @@ -3175,7 +3513,7 @@ ctl release-impact-auto-one-mpr-pass 0 'facts: 1[0-9] counted from .*facts-clean ctl release-impact-auto-one-mpr-flags-pass 0 'facts: 1[0-9] counted from' '!auto cannot use: runtime_os' '!auto cannot use: db_vendor' 'dotnet-ide-host' 'other-db-version: row is about postgresql 13' -- sh -c "$RIN --auto --from-mpr $M/facts-clean/App.mpr --db-vendor postgresql --db-version 16 --runtime-os linux --out $RI/one2.md; cat $RI/one2.md" ctl release-impact-one-mpr-no-auto-catch 1 'go together' -- $RIN --from-mpr $M/facts-clean/App.mpr --out $RI/one3.md # T2F-4: an entry whose row the brain reworded is listed, kept across reruns, and returns with its row -cp -r $RN $RI/brain-n19b; sed -i 's/Intel-based Macs are no longer supported/Intel Macs are dropped/' $RI/brain-n19b/baseline/versions/studio-pro-upgrade-11.md +cp -r $RN $RI/brain-n19b; sedi 's/Intel-based Macs are no longer supported/Intel Macs are dropped/' $RI/brain-n19b/baseline/versions/studio-pro-upgrade-11.md [ -f $RI/n19.md ] && sed -E 's/^(\| 4 \|.*\|) +\|$/\1 handled: evidence\/mac.md (arm64 checked) |/' $RI/n19.md > $RI/n19h.md ctl release-impact-orphan-catch 0 'orphan entries: 1 handled entries matched no row' '^## Entries that matched no row \(1\)' '\| o1 \| Studio Pro \| 11\.6\.8 \| Studio Pro on macOS now runs natively on Apple Silicon \(arm64\)\. Intel-based Macs are no longer supported\. \| handled: evidence/mac\.md' -- sh -c "python3 $B/release_impact.py --brain $RI/brain-n19b --from 11.6.3 --to 11.12.3 --auto --facts $RF/facts-n19.json --out $RI/n19h.md; cat $RI/n19h.md" ctl release-impact-orphan-check-pass 1 '^auto: 10, human: 0, open: 2$' '!OPEN .*o1' -- python3 $B/release_impact.py --check $RI/n19h.md @@ -3629,7 +3967,13 @@ ctl gate-journey-coverage-rc3-catch 2 '\| 9 \| journeys \| INVALID \| 3 \| INVAL # a decision, not a fix; one untagged FAIL line keeps it a regression JOUTB=$'journeys: 4 PASS, 1 FAIL, 0 INVALID\n FAIL j3-User x [BASELINE DEFECT (before FAIL): no evidence]' JOUTU=$'journeys: 3 PASS, 2 FAIL, 0 INVALID\n FAIL j3-User x [BASELINE DEFECT (before FAIL): no evidence]\n FAIL j4-User y' -ctl gate-journey-baseline-only-decision-pass 1 'journeys: needs a decision' '!journeys: REGRESSION to fix' -- gcase jbd GATE_STUB_RC_run_journeys=1 GATE_STUB_OUT_run_journeys="$JOUTB" +# 2026-11-17 (owner: "if it fails on the new one and the old one, it's not a fail; mention it"): FSM rung 2 read FAIL with +# 277 PASS, 0 FAIL, 243 failing on the old app too. Failing on both is PASS* with a note; it still needs >= 20% passing on both +ctl gate-journey-baseline-only-noted-pass 0 '\| journeys \| PASS\* \| 1 \| noted, not a failure: nothing that passed on the old app fails' '!journeys: REGRESSION to fix' '!journeys: needs a decision' -- gcase jbd GATE_STUB_RC_run_journeys=1 GATE_STUB_OUT_run_journeys="$JOUTB" +JOUTF=$'journeys: 277 PASS, 0 FAIL, 2 INVALID, 243 PASS* (baseline defect), 1 CONTAMINATED (x)\n coverage INVALID: instrument not measuring: 233 of 457 actions fail on the baseline (50% > 30%)\n coverage INVALID: 3 baseline defects without evidence (FIXTURE / OLD-APP / FLAKY, --baseline-evidence): a, b, c\nverdict: INVALID only (exit 3)' +ctl gate-journey-fsm-shape-noted-pass 0 '\| journeys \| PASS\* \| 3 \| noted, not a failure' -- gcase jfsm GATE_STUB_RC_run_journeys=3 GATE_STUB_OUT_run_journeys="$JOUTF" +JOUTN=$'journeys: 5 PASS, 0 FAIL, 0 INVALID, 95 PASS* (baseline defect)\n coverage INVALID: instrument not measuring: 95 of 100 actions fail on the baseline (95% > 30%)\nverdict: INVALID only (exit 3)' +ctl gate-journey-not-measuring-stays-invalid-catch 2 '\| journeys \| INVALID \| 3 \|' '!noted, not a failure' -- gcase jnm GATE_STUB_RC_run_journeys=3 GATE_STUB_OUT_run_journeys="$JOUTN" ctl gate-journey-untagged-fail-regression-catch 1 'journeys: REGRESSION to fix' -- gcase jbu GATE_STUB_RC_run_journeys=1 GATE_STUB_OUT_run_journeys="$JOUTU" # the config file: relative paths resolve against the config's own folder, so a worker's cwd cannot break them printf 'WORK=w\nLABEL=cf\nBEFORE_JSON=before.json\nAFTER_JSON=after.json\n' > "$GI/gate.conf" @@ -5612,8 +5956,15 @@ ctl proposal-steps-rerun-no-duplicates-catch 0 '^same events: yes$' '^same steps printf '88 2.0.0 Newmod\n' >> $PR/rerun/plan.txt; bash $PR/prop.sh $PR/rerun > /dev/null [ \"\$b\" = \"\$(bash $PR/ssum.sh $PR/rerun | cut -d' ' -f1,2 | tr '\\n' ' ')\" ] && echo 'plan change keeps ids: yes'" ctl proposal-steps-plan-change-updates-found-pass 0 'plan holds 2 module updates' -- sh -c "grep -Eo 'plan holds [0-9]+ module updates?' $PR/rerun/steps.jsonl | tail -1" -# unattended: every step has an approval, assumed, by agent, source unattended; none is approved -ctl proposal-steps-unattended-assumed-catch 0 '^S-[0-9]+ modules .* approval=assumed by=agent source=unattended ' '^S-[0-9]+ module:DataWidgets .* approval=assumed by=agent source=unattended ' '!approval=none' '!approval=approved' -- bash $PR/ssum.sh $PR/unatt +# unattended (auto mode): routine work goes ahead (approved, source routine); a real decision stays assumed until a person answers (the owner, 2026-11-17) +ctl proposal-steps-unattended-assumed-catch 0 '^S-[0-9]+ modules .* approval=approved by=agent source=routine ' '^S-[0-9]+ module:DataWidgets .* approval=assumed by=agent source=unattended ' '!approval=none' '!module:DataWidgets .*approval=approved' -- bash $PR/ssum.sh $PR/unatt +# a routine update of a module with the app's own edits is not routine (the owner, 2026-11-17): its own step, carried over in auto mode +# (source carry-over), and the plain routine step no longer holds it +mkw carry kept; python3 -c "import json,sys; p=sys.argv[1]; d=json.load(open(p)); d.append({'name': 'Plainmod', 'domain': [], 'ui': [{'kind': 'snippet', 'change': 'CHANGED', 'name': 'Plainmod.Card'}]}); json.dump(d, open(p, 'w'))" $PR/carry/localedits.json +bash $PR/prop.sh $PR/carry --unattended > $PR/carry.out 2>&1 || cat $PR/carry.out >&2 +ctl proposal-steps-module-edits-carried-pass 0 '^S-[0-9]+ module-edits .* approval=approved by=agent source=carry-over ' '!^S-[0-9]+ module:Plainmod ' '^evidence module-edits: module update Plainmod ' '!^evidence modules: module update Plainmod ' -- sh -c "bash $PR/ssum.sh $PR/carry; python3 $B/steps.py show --work $PR/carry --json | python3 -c 'import json,sys +for r in json.load(sys.stdin): + [print(\"evidence\", r[\"topic\"] + \":\", e) for e in r[\"found\"].get(\"evidence\") or [] if r[\"topic\"] in (\"modules\", \"module-edits\")]'" # signed: every step is approved with the signer; the item Jan answered by hand (P-1) is approved by Jan, not by the signer ctl proposal-steps-signed-approved-pass 0 '^S-[0-9]+ modules .* approval=approved by=Jan Jansen source=signoff ' '^S-[0-9]+ person:go-live .* approval=approved by=Jan Jansen source=signoff ' \ '^S-[0-9]+ module:Administration alias=P-1 approval=approved by=Jan source=decisions ' '!approval=none' '!approval=assumed' '!by=agent' -- bash $PR/ssum.sh $PR/signed @@ -5755,7 +6106,7 @@ ctl remedy-apply-fail-branch-kept-pass 0 '^main$' 'remedy-fx-card-a' 'model v1' ctl remedy-apply-tried-again-catch 2 'already tried: fx-card A FAIL' -- rapply fx-card A ctl remedy-apply-check-fail-catch 1 'mx check rc 0, error lines 1: FAIL' 'fx-card B3 FAIL .*remedy-check-fx-card-b3\.log' -- rapply fx-card B3 'CHECK_CMD=echo "[error] CE0001 broken"' ctl remedy-apply-release-pause-catch 5 'rung release INVALID' 'PAUSED for a person \(release\): .*remedy-todo-fx-card-b2\.md' '!recorded in' -- rapply GATE_STUB_RC_release_impact=1 fx-card B2 -ctl remedy-apply-release-todo-pass 0 '^## Release rows' '\| 10 \| release \| INVALID \|' 'finish fx-card B2 PASS' -- cat $AW/remedy-todo-fx-card-b2.md +ctl remedy-apply-release-todo-pass 0 '^## Release rows' '\| [0-9]+ \| release \| INVALID \|' 'finish fx-card B2 PASS' -- cat $AW/remedy-todo-fx-card-b2.md ctl remedy-apply-finish-fail-catch 1 'fx-card B2 FAIL .* finished by a person: rows stay open' 'app back on main' -- rapply --finish fx-card B2 FAIL --note 'rows stay open' ctl remedy-apply-eye-pause-catch 5 'rung idcheck PASS' 'PAUSED for a person \(eye\)' -- rapply fx-card B ctl remedy-apply-eye-todo-pass 0 '^## By eye' 'eye ' -- cat $AW/remedy-todo-fx-card-b.md @@ -5861,7 +6212,7 @@ ctl runlog-code-name-accepted-pass 0 'rc=0' '!customer name' 'rows=1' \ -- sh -c "python3 -I $B/runlog.py --work $RL/w1 --log $RL/log6.md --app CODE-A --type 'template app' 2>&1 >/dev/null; echo rc=\$?; echo rows=\$(grep -c '^| 2' $RL/log6.md)" # backfill: a root with two run folders (one copy of the fixture, one more); dry run prints and writes nothing mkdir -p $RL/root/a/work $RL/root/b/work; cp -R $RLF/. $RL/root/a/work/; cp -R $RLF/. $RL/root/b/work/ -sed -i 's/ 20261013041248\./ 20261013999999./' $RL/root/b/work/gate-summary.md +sedi 's/ 20261013041248\./ 20261013999999./' $RL/root/b/work/gate-summary.md ctl runlog-backfill-dry-run-catch 0 '^ADD +a/work' '^ADD +b/work' '2 run folders under .*: 2 to add' 'Dry run: nothing written' 'log=absent' \ -- sh -c "python3 -I $B/runlog.py --backfill $RL/root --log $RL/log7.md --app CODE-A; [ -e $RL/log7.md ] && echo log=present || echo log=absent" ctl runlog-backfill-write-pass 0 '^ADDED +a/work' '2 added' 'rows=2' \ @@ -5881,7 +6232,7 @@ mkdir -p $RL/xroot/x1 $RL/xroot/x2; cp -R $RLF/. $RL/xroot/x1/; printf '# journa ctl runlog-explain-skipped-reason-catch 0 '^x1: WOULD-ADD' '^x2: SKIPPED no gate-summary\.md and no SCORECARD verdict' 'log=absent' \ -- sh -c "python3 -I $B/runlog.py --explain $RL/xroot --log $RL/log12.md --app CODE-A; [ -e $RL/log12.md ] && echo log=present || echo log=absent" mkdir -p $RL/sroot/a $RL/sroot/b; cp -R $RLF/. $RL/sroot/a/; cp -R $RLF/. $RL/sroot/b/ -sed -i 's/ 20261013041248\./ 20261015999999./; s/2026-10-13T/2026-10-15T/' $RL/sroot/b/gate-summary.md +sedi 's/ 20261013041248\./ 20261015999999./; s/2026-10-13T/2026-10-15T/' $RL/sroot/b/gate-summary.md ctl runlog-summary-latest-per-app-idempotent-pass 0 '^- \*\*CODE-A\*\* \(template app\): latest run 2026-10-15; highest version reached 11\.12\.5; .*; 2 runs$' 'same=yes' 'apps=1' \ -- sh -c "L=$RL/log13.md; python3 -I $B/runlog.py --backfill $RL/sroot --log \$L --app CODE-A --type 'template app' --write >/dev/null; python3 -I $B/runlog.py --summary --log \$L >/dev/null; a=\$(md5sum < \$L); python3 -I $B/runlog.py --summary --log \$L >/dev/null; [ \"\$a\" = \"\$(md5sum < \$L)\" ] && echo same=yes; grep '^- \*\*' \$L; echo apps=\$(grep -c '^- \*\*' \$L)" # the pipeline hook: record_runlog and cleanup, extracted and run under the pipeline's own set -Eeu / ERR trap @@ -5911,7 +6262,8 @@ set -Eeu trap 'exit 4' ERR WORK=$1; TIMES=$2; shift 2 . "$FN" -date() { local n; n=$(cat "$WORK/.tick" 2> /dev/null || echo 0); n=$((n + 1)); echo "$n" > "$WORK/.tick"; sed -n "${n}p" "$TIMES"; } +# the cockpit's running marker asks `date +%s` (b4ddb02); it must not use up a tick of the listed clock +date() { [ "${1:-}" != +%s ] || { echo 0; return; }; local n; n=$(cat "$WORK/.tick" 2> /dev/null || echo 0); n=$((n + 1)); echo "$n" > "$WORK/.tick"; sed -n "${n}p" "$TIMES"; } mkdir -p "$WORK/state" "$WORK/logs" ONLY=""; FROM=""; DRY=${DRY:-0}; NJ_TMP="$WORK/state/nj.tmp"; NJ_CODE=0; RUN=$WORK; SOURCE=10.24.0.1; TARGET=11.12.5; ARGS_ORIG=(); SELF=pipeline; STEP=""; LOG=$WORK/logs/x.log; APPS_UP=() mxcli_pin() { :; }; proposal_recheck() { :; }; cockpit_regen() { :; } @@ -6165,8 +6517,8 @@ echo big > $BT9Z/src/shop/deployment/model/model.mdp; echo j > $BT9Z/src/shop/ja (cd $BT9Z/src && zip -qr ../shop.mpk shop) mkdir -p $BT9Z/run1 $BT9Z/run2; cp -r $BTR/tools $BT9Z/run1/tools; cp -r $BTR/tools $BT9Z/run2/tools bt9fetch() { env PATH=$BTR/bin:$PATH MXCLI= bash $B/cloud/upgrade-pipeline.sh --app-zip $BT9Z/shop.mpk --source 10.24.26.1 --target 11.15.0.1 --work $1/work --only fetch; } -ctl bt9-fetch-skips-deployment-catch 2 '^theme/a.css$' '^javasource/deployment/Keep.java$' '!deployment/model' -- sh -c "$(declare -f bt9fetch); BTR=$BTR B=$B BT9Z=$BT9Z; bt9fetch $BT9Z/run1 >/dev/null 2>&1; cd $BT9Z/run1/app/shop && find . -type f ! -path './.git/*' | sed 's#^\./##' | sort; cd $BT9Z/run1/app/shop; bash -c 'exit 2'" -ctl bt9-fetch-note-pass 0 'fetch: the package ships a deployment/ folder' -- cat $BT9Z/run1/work/state/notes.txt +ctl bt9-fetch-skips-deployment-catch 2 '^theme/a.css$' '^javasource/deployment/Keep.java$' '!deployment/model' -- sh -c "$(declare -f bt9fetch); BTR=$BTR B=$B BT9Z=$BT9Z; bt9fetch $BT9Z/run1 > $BT9Z/run1.log 2>&1; cd $BT9Z/run1/app/shop && find . -type f ! -path './.git/*' | sed 's#^\./##' | sort; cd $BT9Z/run1/app/shop; bash -c 'exit 2'" +ctl bt9-fetch-note-pass 0 'fetch: the package ships a deployment/ folder' -- sh -c "cat $BT9Z/run1/work/state/notes.txt || tail -25 $BT9Z/run1.log" # BT-11: a module in both plan-pre.txt and plan.txt: the post-convert line starts from the pre-convert target. Golden: the real # DOCGEN plan-pre.txt and plan.txt (Atlas_Core 3.21.1 then 4.5.1, DataWidgets once); installed versions from its PROPOSAL.md. @@ -6271,7 +6623,13 @@ else: json.dump({"results": [{"rung": "ui", "name": "login", "verdict": v, "detail": d}], "walks": []}, open("artifacts/journey-findings.json", "w")) EOF2 chmod +x $BT7/bin/node -bt7() { local w=$1 refusals=$2; shift 2; rm -f $BT7/count.*; env PATH=$BT7/bin:$PATH BT7_COUNT=$BT7/count BT7_REFUSALS=$refusals python3 $B/run_journeys.py $BT7/gen --label before --port 1 --admin-port 2 --admin-pass x --users $BT7/users.json --psql "$SHOP" --work $BT7/$w "$@"; } +# AD-1 (app_alive): run_journeys.py first asks whether the app answers at all; on --port 1 nothing did, so every role was +# INVALID "app down" before the stub runner was reached. A plain HTTP server stands in for the app. +bt7() { local w=$1 refusals=$2 port srv rc=0; shift 2; rm -f $BT7/count.* + port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])') + python3 -m http.server "$port" --bind 127.0.0.1 --directory $BT7 > /dev/null 2>&1 & srv=$! + for _ in 1 2 3 4 5 6 7 8 9 10; do curl -s -o /dev/null "http://127.0.0.1:$port/" && break; sleep 0.3; done + env PATH=$BT7/bin:$PATH BT7_COUNT=$BT7/count BT7_REFUSALS=$refusals python3 $B/run_journeys.py $BT7/gen --label before --port "$port" --admin-port 2 --admin-pass x --users $BT7/users.json --psql "$SHOP" --work $BT7/$w "$@" || rc=$?; kill $srv 2> /dev/null; wait $srv 2> /dev/null; return $rc; } bt7sum() { python3 -c "import json; r=json.load(open('$BT7/$1/before/summary.json'))['roles']; print({k: (v['login'], v.get('session_retries', 0)) for k, v in sorted(r.items())})"; } ctl bt7-retry-after-refusal-catch 0 'session cap: RoleB refused by the runtime.s session limit; waiting 1s for sessions to expire, retry 1 of 2' -- bash -c "$(declare -f bt7); BT7=$BT7 B=$B SHOP=\"$SHOP\"; bt7 w1 1 --session-wait 1 >/dev/null 2>$BT7/w1.err; cat $BT7/w1.err | grep 'session cap'" ctl bt7-retry-recovers-login-catch 0 "^{'RoleA': \('PASS', 0\), 'RoleB': \('PASS', 1\)}\$" -- bt7sum w1 @@ -6433,15 +6791,21 @@ hcls() { python3 -I -c " import json,sys for r in json.load(open('$HL/h1/heal.json'))['rows']: if r['id']==sys.argv[1]: print(r['id'], '=>', r['class'], '/', r['heal_kind'], '/', r['evidence'])" "$1"; } -ctl heal-classes-line-pass 0 'classes: startup-noise 1, no-test-data 1, no-click-path 1, outside-call 1, test-setup 4, real-regression 3, likely-real 2' 'CHANGED 6' -- cat $HL/plan1.out +ctl heal-classes-line-pass 0 'classes: startup-noise 1, no-test-data 1, no-click-path 1, outside-call 1, test-setup 4, real-regression 3, likely-real 3' 'CHANGED 6' -- cat $HL/plan1.out ctl heal-startup-noise-evidence-catch 0 'data:Shop\.Item\.changedDate => startup-noise / allow / start noise: .* also differs between two starts of the SAME old app' -- hcls data:Shop.Item.changedDate ctl heal-allow-line-reason-pass 0 '^Shop\.Item\.changedDate 3 rows differ # start noise: ' '!Shop\.Customer' '!Shop\.Order' -- cat $HL/h1/heal-allow.txt ctl heal-no-twin-no-noise-catch 0 'data:Shop\.Customer\.name => likely-real' '!startup-noise' -- hcls data:Shop.Customer.name +# WM1 2026-11-17: 'status 404 -> 200' was read as counts 404 -> 200 and called lost data (real-regression) +ctl heal-new-endpoint-not-lost-data-catch 0 'data:http\./rest-doc/shop/openapi\.json => likely-real / None / .*a new surface the old app did not have' '!real-regression' '!missing after the upgrade' -- hcls data:http./rest-doc/shop/openapi.json ctl heal-lost-rows-real-regression-catch 0 'data:Shop\.Order\.total => real-regression / None' 'missing after the upgrade' -- hcls data:Shop.Order.total ctl heal-harness-log-accepted-catch 0 'test-setup / logallow / known harness line: .*SessionTimeout 30000' -- hcls 'log:log.start.NEW_WARNING:WARNING_Configuration_Using_derived_value_for_Se' ctl heal-log-allow-line-pass 0 '^WARNING Configuration: Using derived value for SessionKeepAliveUpdatesInterval\* # test harness: ' '!Something else' -- cat $HL/h1/heal-log-allow.txt ctl heal-new-error-never-accepted-catch 0 'log:log\.start\.NEW_ERROR:.* => real-regression / None' -- hcls 'log:log.start.NEW_ERROR:ERROR_Shop_failed_to_start_the_queue' ctl heal-unknown-log-line-stays-open-catch 0 'Something_else_changed => likely-real' 'not a known harness line' -- hcls 'log:log.start.NEW_WARNING:WARNING_Something_else_changed' +# WM1 2026-11-17: 10.24 also derives ClusterManagerActionInterval from the harness's short SessionTimeout (a REVIEW every run) +python3 $here/lib/mkheal.py $HL/h3 && sed -i.bak 's/^- PASS Shop.Other: same$/- REVIEW log.start.NEW_WARNING: WARNING Configuration: Using derived value for ClusterManagerActionInterval (# ms) based on SessionTimeout\n&/' $HL/h3/diff.md \ + && python3 $B/heal.py plan --work $HL/h3 --round 1 > $HL/plan3.out 2>&1 +ctl heal-cluster-interval-harness-line-pass 0 '^WARNING Configuration: Using derived value for ClusterManagerActionInterval\* # test harness: .*SessionTimeout' -- cat $HL/h3/heal-log-allow.txt ctl heal-effective-allow-has-both-pass 0 '^# written by heal.py' 'Shop\.Item\.changedDate' -- cat $HL/h1/allow-effective.txt ctl heal-no-test-data-seed-catch 0 'journey:j-empty => no-test-data / seed / the page Shop\.Empty shows an empty list' -- hcls journey:j-empty ctl heal-seed-entity-in-plan-pass 0 '"Shop\.Thing"' '!Shop\.Item"' -- sh -c "sed -n '/\"seed\"/,/\]/p' $HL/h1/heal-plan.json" @@ -6484,13 +6848,15 @@ ctl heal-round2-changes-nothing-catch 0 'CHANGED 0' -- cat $HL/plan2.out ctl heal-finalize-pass-result-pass 0 'passes on the old and the new app' -- sh -c "python3 -I -c \"import json;print([r['result'] for r in json.load(open('$HL/h1/heal.json'))['rows'] if r['id']=='journey:j-flake'])\"" ctl heal-open-rows-nj-text-catch 0 'real-regression: 3 row\(s\)' 'Options: fix the app' 'Recommendation: ' 'likely-real: ' -- python3 $B/heal.py nj --work $HL/h1 ctl heal-summary-line-pass 0 '^heal: [0-9]+ failing row\(s\) classified, [0-9]+ accepted with a written reason' 'stay open \(3 real-regression' -- python3 $B/heal.py summary --work $HL/h1 -# a rerun that still fails turns the row into likely-real (the heal did not work), never into a pass +# a rerun that still fails on both apps is a note for the owner and a toolkit gap (2026-11-17), never a pass and never likely-real python3 $here/lib/mkheal.py $HL/h3 && python3 $B/heal.py plan --work $HL/h3 --round 1 > /dev/null 2>&1 mkdir -p $HL/ro3/before-picked $HL/rn3/after echo '{"journeys": {"j-flake": {"verdict": "FAIL"}}}' > $HL/ro3/before-picked/summary.json echo '{"journeys": {"j-flake": {"verdict": "FAIL", "not_pass": ["x"]}}}' > $HL/rn3/after/summary.json python3 $B/heal.py merge --work $HL/h3 --old $HL/ro3 --new $HL/rn3 > /dev/null 2>&1; python3 $B/heal.py finalize --work $HL/h3 > /dev/null 2>&1 -ctl heal-rerun-still-fails-likely-real-catch 0 "\['still FAIL on the new app, old app FAIL'\]" 'likely-real' -- sh -c "python3 -I -c \"import json;r=[r for r in json.load(open('$HL/h3/heal.json'))['rows'] if r['id']=='journey:j-flake'][0];print([r['result']], r['class'])\"" +ctl heal-rerun-fails-both-noted-pass 0 "\['fails on both apps after the heal \(new FAIL, old FAIL\): listed for the owner, not an issue'\] test-setup" -- sh -c "python3 -I -c \"import json;r=[r for r in json.load(open('$HL/h3/heal.json'))['rows'] if r['id']=='journey:j-flake'][0];print([r['result']], r['class'])\"" +# the learning loop's input (steps/heal.md): rows no rule explains, and tests that run on neither app; a real regression is not a gap +ctl heal-toolkit-gaps-pass 0 '^# Toolkit gaps: ' '`data:Shop\.Customer\.name` \(unexplained difference\).*rule goes in: bin/diff\.py' '`journey:j-flake` \(test does not run on either app\)' '!Shop\.Order\.total' -- cat $HL/h3/toolkit-gaps.md # a rerun that now FAILs on the new app while the old app passes is a regression echo '{"journeys": {"j-flake": {"verdict": "PASS"}}}' > $HL/ro3/before-picked/summary.json python3 $B/heal.py merge --work $HL/h3 --old $HL/ro3 --new $HL/rn3 > /dev/null 2>&1; python3 $B/heal.py finalize --work $HL/h3 > /dev/null 2>&1 @@ -6614,25 +6980,78 @@ ctl cockpit-render-needs-you-from-run-catch 0 '"id": "regression:overlay"' '"id" ctl cockpit-render-effort-from-proposal-catch 0 '"Gridmod", "kept at its version|"Gridmod", "local edits refuse the update[^"]*", "M"' '"Themepack styles", "[^"]*", "S"' -- $CK render ctl cockpit-render-upgrade-from-git-log-catch 0 '"Widgetpack 2\.0\.0", "Automatic module update' '"Repair D2"' '"Rebuilds"' -- $CK render ctl cockpit-render-gate-and-heal-catch 0 'Click through pages|"Smoke"' '[0-9] real problem' '"1 check not run"' '1 of 2 page checks fails\. Cause: Gridmod\.' 'Replace Gridmod' 'Overlay blocks 2 clicks' -- $CK render +# WM1 2026-11-17: "All pass, 1 with accepted differences" said nothing about what was accepted; each reason is on the card +ctl cockpit-render-accepted-reasons-pass 0 'Accepted, no loss: start noise; the Vendmod update 1\.0\.0 -> 2\.0\.0 changed its own attribute' -- $CK render +# the Test stage shows what ran on the old and the new app; a new-app file older than the old one is not this run's +ctl cockpit-test-runs-both-pass 0 '^row: Journeys \| 2 journeys: 2 passed \| 2 journeys: 1 passed, 1 failed \| 1 pass on both, 1 passed on the old app and not on the new one$' '^list Journeys: b-User \| passed \| failed$' '^row: Page walk \| 1 page: 1 passed' -- python3 $here/test_runs_control.py $T/trb both +ctl cockpit-test-runs-stale-catch 0 '^row: Journeys \| 2 journeys: 2 passed \| not run yet in this run \| needs both apps$' '!1 passed, 1 failed' -- python3 $here/test_runs_control.py $T/trs stale +# what a finding MEANS, per item, from the run's own files (the owner, 2026-11-17: "not just what, but what it means") +ctl cockpit-found-means-pass 0 '^means standins: 1 of 3 outside calls are covered in the tests\.$' '^item standins: Mail\.GetToken \| address from data \| wait \| .*Mail\.Provider\.TokenUrl' '^item modules: Theme 1\.0\.0 → 2\.0\.0 \| 1 local edit \| wait \| .*restyled this shared layout' '^item modules: Plain .* \| no local edits \| ok \|' '^means modules: 1 could not be compared' '^item module:Admin: Admin\.Extra \| added \| bad \| .*data stored in it' '^means module:Admin: 1 of them holds the app' '^means react: 4 widgets in 1 kind move' '^empty nothing:known$' -- python3 $here/found_means_control.py $T/fm +# a decision's case file (work/cases/.json): found by id, screenshots embedded, a missing shot shown as such, a broken file ignored +ctl cockpit-case-file-pass 0 "^case files: \['regression_module-X\.json'\]$" '^old shot: data:image/png;base64, \| new shot: .{2}$' '^html: tiles why=2 no-shot$' -- python3 $here/found_means_control.py $T/fmc cases +# a regression without a case file gets one from the run's own files (no manual case writing): count, old and new shot of the failing +# step, the carried screen edits as the first place to look; a written case file wins +ctl cockpit-auto-case-pass 0 '^auto: 2 actions that work on the old app fail on the new one \(M\.P, M\.Q\)\. \| tiles: \[2, 2, 1\] \| shots: True True \| carried: .*screen edits in Theme\. .*options: \[.accept., .fix., .old-wrong.\]$' '^written case kept: Seven break$' -- python3 $here/found_means_control.py $T/fmc2 cases ctl cockpit-render-time-from-timing-pass 0 'Shop|App upgrade' '1 h 40|1 h 30' -- $CK render ctl cockpit-page-content-only-pass 0 'no page skeleton tags' '^' -- $CK nodoc ctl cockpit-no-hardcoded-run-pass 0 'no hard-coded run names' -- $CK nohard ctl cockpit-check-ghost-catch 0 '^ghost-step S-4 \(P-99, Keep or update module Oldmod\)' 'rc=1' -- $CK check-ghost ctl cockpit-check-withdraw-pass 0 "withdrawn \['S-4'\]" '!ghost-step' 'rows Oldmod: 0' -- $CK check-withdraw ctl cockpit-check-running-no-stopbar-pass 0 'stopbar: 0' '^0$' -- $CK check-running +# cold CMM: the "Release notes (check)" and "<M> themesource edits (theme)" cards were ask-no-evidence on every run, as only +# module names were looked up; each kind has its own source now, and a name that has none still shows nothing +mkdir -p $T/evw/work; printf 'release_impact: no brain version tables under /x/baseline/versions\n' > $T/evw/work/release-impact.out +printf '[{"name": "Shopmod", "files": [{"change": "CHANGED", "kind": "theme", "path": "themesource/shopmod/web/main.scss", "detail": "1 variable(s) changed: $brand"}]}]' > $T/evw/work/localedits.json +ctl evidence-check-release-notes-catch 0 '^- release_impact: no brain version tables' '^files: work/release-impact\.out$' -- python3 $B/evidence.py --work $T/evw/work --kind check "Release notes" +ctl evidence-theme-files-catch 0 '^- themesource/shopmod/web/main\.scss: changed \(1 variable' '^files: work/localedits\.json$' -- python3 $B/evidence.py --work $T/evw/work --kind theme "Shopmod themesource edits" +ctl evidence-theme-other-module-pass 1 '\(no evidence found\)' '!main\.scss' -- python3 $B/evidence.py --work $T/evw/work --kind theme "Othermod themesource edits" +ctl cockpitpage-evidence-passes-kind-pass 0 'evidence\.for_item\(work, it\["name"\], it\["kind"\]\)' -- grep -F 'evidence.for_item(work, it["name"], it["kind"])' $B/cockpitpage.py # editroute.py (cold test WM1): a module the update refused is routed to an update; only a real logic edit, Java code, a removal # something still uses or a tool fault stays kept. A stub mxcli answers describe and refs. ER="bash $here/editroute_control.sh" ctl editroute-cosmetic-pass 0 '^ROUTE Cosmo COSMETIC: only diagram layout differs' '^force-ui.txt: Cosmo ' -- $ER plan ctl editroute-real-logic-held-catch 0 '^ROUTE Realmod HOLD: real logic edit: microflow RealFlow \(move it to Realmod_Ext' '!route-plan.txt: .* Realmod$' -- $ER plan -ctl editroute-ui-constant-replay-pass 0 '^ROUTE Uimod REPLAY: 2 UI' '^force-ui.txt: Uimod ' -- $ER plan +ctl editroute-ui-constant-replay-pass 0 '^ROUTE Uimod REPLAY: 7 UI' '^force-ui.txt: Uimod ' -- $ER plan ctl editroute-unread-unedited-pass 0 '^ROUTE Unread UNREAD' -- $ER plan ctl editroute-files-accept-pass 0 '^ROUTE Filesmod FILES' '^accept-unsaved.txt: Filesmod ' -- $ER plan ctl editroute-vendor-drop-unused-pass 0 '^ROUTE Vendmod VENDOR' '^approved.txt: Vendmod.E.Gone # editroute VENDOR' -- $ER plan ctl editroute-vendor-drop-used-catch 0 '^ROUTE Usedmod HOLD' '!approved.txt: Usedmod' -- $ER plan ctl editroute-xassoc-pass 0 '^ROUTE Target XASSOC' '^approved-step.txt: Other.A_B ' -- $ER plan ctl editroute-java-held-catch 0 '^ROUTE Javamod HOLD' '!route-plan.txt: .* Javamod$' -- $ER plan -ctl editroute-collect-pass 0 'Uimod.mdl: create or modify layout Uimod.L1' 'Uimod.mdl: create or modify constant Uimod.K' '!NotRefused' 'Target.mdl: create or modify association Other.A_B' '^Target.guids: Other.A_B$' -- $ER collect +ctl editroute-collect-pass 0 'Uimod.mdl: create or modify snippet Uimod.L1' 'Uimod.mdl: create or modify constant Uimod.K' '!NotRefused' 'Target.mdl: create or modify association Other.A_B' '^Target.guids: Other.A_B$' -- $ER collect +# cold test WM1 Atlas_Core: a saved whole element lost the master-layout link and carried the old widget's properties; the change +# is merged onto the new element instead, and a result mxcli check refuses is held, never executed +ctl editroute-collect-prop-merge-pass 0 "^Uimod.mdl: class: 'mine',$" "^Uimod.mdl: title: 'y'$" '!saved whole element' -- $ER collect +ctl editroute-collect-app-added-pass 0 "snippet Added: added by the app" "^Uimod.mdl: class: 'own'$" "constant K: the vendor did not change it" -- $ER collect +# data test 2026-11-16: save-edits wrote no association file; a re-created association lost its member access; a saved +# whole entity put widened vendor attributes back +ctl editroute-collect-assoc-from-original-pass 0 'association Other.A_B: from the original model' '^Target.mdl: grant read \(A_B, Name\), write \(A_B\) on entity Other.X to Other.User;$' '!Other.Guest' -- $ER collect +# cold CMM run 8: the re-created Employee_Account (Employee -> Administration.Account) had member access only on the owner +# side; Account's rules said None, 5 CE2729 on the Employee pages. The other entity gets just the association member back +ctl editroute-collect-assoc-other-side-catch 0 "^Target.mdl: grant read \\(A_B\\) on entity Target.Y to Target.User where \\[Code='x'\\];$" '!Target.mdl: .*Target.Guest' '!Target.mdl: grant create' -- $ER collect +ctl editroute-collect-no-whole-entity-catch 0 '!entity-Whole' '!saved whole element entity' -- $ER collect +# WM1 rerun: mxcli never writes a layout into a Marketplace module; the app's redirect to its design module becomes a page repoint +ctl editroute-collect-layout-repoint-pass 0 '^Uimod.mdl: ALTER PAGES SET LAYOUT = Brand.Main WHERE LAYOUT = Uimod.Redir;$' '^SKIP Uimod layout Spare: used by nothing' '^HELD Uimod layout UsedOwn: mxcli never writes a layout' '!create or modify layout' -- $ER collect +# WM1 gate 2026-11-17: ALTER PAGES skips Marketplace pages, so the Accounts menu opened Administration.Account_Overview on plain +# Atlas 4 (screens REVIEW); the page is copied to the app's module for copies, on the company layout, the menu pointed at it +ctl editroute-collect-nav-page-copied-pass 0 "^Uimod.mdl: create or modify page Brand.Acct folder 'Pages' \\($" '^Uimod.mdl: Layout: Brand.Main$' '^Uimod.mdl: grant view on page Brand.Acct to Brand.Administrator;$' '^Uimod.mdl: menu item .Accounts. \( OnClick: show page Brand.Acct ' 'no Brand role for Uimod.Auditor: not granted' '!^Uimod.mdl: app (title|icon)' -- env NAVCOPY=home $ER collect +ctl editroute-collect-nav-page-no-home-inplace-pass 0 '^Uimod.mdl: ALTER PAGE Uimod.Acct \{ SET Layout = Brand.Main; \};$' 'no app module holds copies of Uimod pages: its layout set to Brand.Main in place, the menu keeps it' '!Brand.Acct' '!^HELD Uimod page' -- env NAVCOPY=nohome $ER collect +# WM1 2026-11-17: the copy of a page with a widget mxcli cannot write came out without it (Account_Overview, no user list) +# WM1 2026-11-17: the page's own layout is set in place (ALTER PAGE): every widget stays, the menu keeps the page +ctl editroute-collect-nav-page-unwritable-widget-inplace-pass 0 '^Uimod.mdl: ALTER PAGE Uimod.Acct \{ SET Layout = Brand.Main; \};$' 'a copy would drop Forms\$DataGrid \(grid12\) \(mxcli cannot write it\): its layout set to Brand.Main in place' '!create or modify page Brand.Acct' '!show page Brand.Acct' '!^HELD Uimod page' -- env NAVCOPY=grid $ER collect +ctl editroute-collect-nav-page-inplace-refused-held-catch 0 '^HELD Uimod page Uimod.Acct: opened from the navigation, a copy would drop .*: mxcli check refuses the result' '!ALTER PAGE Uimod.Acct' -- env NAVCOPY=refuse $ER collect +# WM1 rerun 5: a dropped association with no data (non-persistent owner) is an info row, and the app still used it (CE1613) +ctl editroute-xassoc-no-data-assoc-pass 0 '^ROUTE Target XASSOC: the update drops 2 association' '^approved-step.txt: Other.Np_B ' -- $ER plan +ctl editroute-collect-check-held-catch 0 '^HELD Uimod snippet Bad: mxcli check refuses' '!BROKEN.*Uimod.mdl|Uimod.mdl:.*BROKEN' -- $ER collect +# cold test WM1 Email_Connector: diagram-only, then the routed update kept it for 3 associations; routed again, routes add up +# does routing work across real runs: a route that ends off the plan's version is counted, never hidden +ctl editroute-score-pass 0 '^SCORE Cosmo COSMETIC -> ON TARGET' '^SCORE Uimod REPLAY -> REPLAY FAILED' '^SCORE Realmod HOLD -> HOLD' 'ended on the plan.s version' '^rc=1$' '^ledger: .*app1' -- $ER score +# the brain's practice notes reach the card that needs them (WM1: the PDFBox note existed, nobody saw it); measured, not assumed +ctl editroute-brain-notes-pass 0 "^brain: Realmod \['architecture/ext'" "^brain: Uimod \['ux-ui/atlas'" '^- company brain \(internal\): Never edit Marketplace modules' 'brainnotes hit@3: 2/2' '^rc=0$' '^no brain rc=0$' -- $ER brain +# WM1 rerun: the earlier run's 'Atlas_Core # editroute REPLAY' forced Atlas_Core before routing; its edits were never merged back +ctl editroute-reset-pass 0 '^editroute-reset: [1-9][0-9]* line' '^force-ui.txt: Handmod # a person forced this$' '!^force-ui.txt: .*# editroute' '^approved-step editroute lines: 0$' '!^work: edit-routes.json$' '^rc=0$' -- $ER reset +ctl editroute-reroute-adds-route-pass 0 '^ROUTE Cosmo XASSOC' '^routes: Cosmo COSMETIC\+XASSOC$' '^route-plan.txt: 7 2.0.0 Cosmo$' -- $ER reroute +ctl editroute-reroute-same-reason-held-catch 0 '^ROUTE Cosmo HOLD: the routed update kept it again for the same reason' '^rc=1$' -- $ER reroute ctl cockpit-check-keep-recommended-catch 0 "'keep old version \(kept list\); ': CAUGHT" "'keep old version until 2027-03': ok" "'update to the latest version a': ok" -- $ER gate ctl cockpit-db-pattern-pass 0 'has claude\.use\("db"\)' 'has claude\.use\("user"\)' 'decisions"\)\.doc\(it\.id\)\.set' 'signoffs"\)\.doc\(await user\.id\(\)\)\.set' 'decisions"\)\.onSnapshot' 'never reads window\.claude\.db' -- $CK pattern ctl cockpit-live-script-only-with-flag-pass 0 'live: polls stamp 1' 'standalone without --live: polls stamp 0' 'content page: polls stamp 0' -- $CK live-script diff --git a/upgrade-toolkit/controls/skip_baseline_control.py b/upgrade-toolkit/controls/skip_baseline_control.py new file mode 100644 index 0000000..f7309b9 --- /dev/null +++ b/upgrade-toolkit/controls/skip_baseline_control.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""skip_baseline_control.py DIR: run_journeys --skip-baseline-fail picks, on a fake journey set. Prints the skipped and the +walked ids. A set with data passes: the failing consumer (a _path_needs journey) is skipped, the failing creator still runs, +a passing one runs. A set without data passes: every journey that failed on the old app is skipped.""" +import json +import os +import sys +from pathlib import Path + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bin")) +import run_journeys as rj # noqa: E402 + +t = Path(sys.argv[1]) + + +def mk(d, jid, needs): + p = t / d / "journeys" / "User" + p.mkdir(parents=True, exist_ok=True) + j = {"id": jid, "role": "User", "steps": []} + if needs: + j["_path_needs"] = ["a row in a list"] + (p / f"{jid}.journey.json").write_text(json.dumps(j)) + + +(t / "b.json").write_text(json.dumps({"journeys": {"make-User": {"verdict": "FAIL"}, "use-User": {"verdict": "FAIL"}, + "ok-User": {"verdict": "PASS"}}})) +mk("w", "make-User", False); mk("w", "use-User", True); mk("w", "ok-User", True) +mk("n", "make-User", False); mk("n", "ok-User", False) +sw = rj.baseline_fail_skips(t / "w" / "journeys", t / "b.json") +sn = rj.baseline_fail_skips(t / "n" / "journeys", t / "b.json") +walked = [rj.read_journey(f)["id"] for _, _, _, fs in rj.data_waves(t / "w" / "journeys", str(t / "w"), None, sw) for f in fs] +print("waves skip:", " ".join(f"{k}={v[0]}/{v[1]}" for k, v in sorted(sw.items()))) +print("waves walked:", " ".join(sorted(walked))) +print("no-waves skip:", " ".join(sorted(sn))) diff --git a/upgrade-toolkit/controls/smokewalk_predicates.js b/upgrade-toolkit/controls/smokewalk_predicates.js index 293d3af..2a91da8 100644 --- a/upgrade-toolkit/controls/smokewalk_predicates.js +++ b/upgrade-toolkit/controls/smokewalk_predicates.js @@ -23,6 +23,12 @@ global.mx = { session: { getUserName: () => 'smoke_user' }, ui: { getContentForm check('loggedIn when ready', loggedIn(), true); check('homeReady when ready', homeReady(), true); check('landedWhere on the wanted page', landedWhere({ want: 'P/Home.page.xml' }), 'content'); +// cold CMM: a popup opened from the menu (Atlas: .mx-window position:fixed, offsetParent null, but it has client rects) +global.document = { querySelectorAll: s => (s === '.mx-window' ? [{ offsetParent: null, getClientRects: () => [1] }] : []) }; +check('landedWhere fixed popup window', landedWhere({ want: 'P/Other.page.xml' }), 'popup'); +global.document = { querySelectorAll: s => (s === '.mx-window' ? [{ offsetParent: null, getClientRects: () => [] }] : []) }; +check('landedWhere hidden popup window', landedWhere({ want: 'P/Other.page.xml' }), false); +global.document = { querySelectorAll: () => [] }; // R1: React client (no getUserName / getContentForm). catch case: anonymous session on the login page global.mx = { session: { sessionData: { user: null } }, ui: {} }; global.history = { state: null }; diff --git a/upgrade-toolkit/controls/test_runs_control.py b/upgrade-toolkit/controls/test_runs_control.py new file mode 100644 index 0000000..96721bb --- /dev/null +++ b/upgrade-toolkit/controls/test_runs_control.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""test_runs_control.py DIR CASE: cockpitmatrix.test_runs on a fake work folder, the Test stage's "what ran, on the old and +the new app" table. Prints one line per row and per listed test. Cases: + both both apps walked: a journey that passed on the old app only is counted and listed + stale the new app's journeys file is older than the old app's (a rerun that has not walked the new app yet)""" +import json +import os +import sys +import time + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bin")) +import cockpitmatrix as cm # noqa: E402 + +w, case = sys.argv[1], sys.argv[2] + + +def put(rel, obj): + p = os.path.join(w, rel) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "w") as f: + json.dump(obj, f) + return p + + +for sd, v in (("before", "PASS"), ("after", "PASS")): + put(f"smoke-{sd}.json", {"roles": [{"role": "User", "steps": [{"caption": "Orders", "verdict": v}]}]}) +old = put("e2e-before-picked/before-picked/summary.json", {"journeys": {"a-User": {"verdict": "PASS"}, "b-User": {"verdict": "PASS"}}}) +new = put("e2e-after/after/summary.json", {"journeys": {"a-User": {"verdict": "PASS"}, "b-User": {"verdict": "FAIL"}}}) +t0 = os.path.getmtime(old) +os.utime(new, (t0 - 3600, t0 - 3600) if case == "stale" else (t0 + 60, t0 + 60)) +with open(os.path.join(w, "data.out"), "w") as f: + f.write("INVALID 0 | FAIL 0 | REVIEW 0 | PASS* 1 | PASS 9 | INFO 2\n") +r = cm.test_runs(w) +for row in r["rows"]: + print("row: " + " | ".join(row)) +for k, l in r["lists"].items(): + for t in l: + print(f"list {k}: " + " | ".join(t)) diff --git a/upgrade-toolkit/docs/feedback-ledger.md b/upgrade-toolkit/docs/feedback-ledger.md new file mode 100644 index 0000000..84fed9b --- /dev/null +++ b/upgrade-toolkit/docs/feedback-ledger.md @@ -0,0 +1,38 @@ +# Feedback ledger + +Every piece of the owner's feedback on the toolkit, the conclusion we reached, and where it landed. One row per point. +Status: **done** (merged to main), **branch** (committed on `fix/signed-cards`, no PR yet), **PR** (open pull request), +**open** (agreed, not built), **owner** (waits for the owner's call). "this PR" rows close with the PR that adds them. Newest first. Update this file in the same commit +that closes a row. + +| date | feedback (owner's words, short) | conclusion | status | where | +|---|---|---|---|---| +| 11-17 | CI on GitHub was red since #10 (setup: no bson, no zsh) | CI installs pymongo and zsh: 2624 passed, 10 failed on Linux (smokewalk, stepupdate x6, remedy, smoke-users, decide-cards; all pass on macOS); make those 10 run on Linux | open | #11 | +| 11-17 | "Make the sidebar 50% of screen ... show what was found per action, and what it MEANS" | drawer is 50% wide; per action a "What it means" box and an item-by-item table, worst first | branch | ba76a0e | +| 11-17 | "If we could write such a thing per issue which needs a decision ... embed this" | a decision opens its case in the cockpit: headline, tiles, old vs new screenshot, why, options with the catch | branch | 775ddae | +| 11-17 | the step writes its case on its own, not by hand | a regression without a case file gets one built from the run's own files | PR | this PR | +| 11-17 | "All routine decisions just have one sign off ... in auto mode they just upgrade" | routine work: one plan sign-off, or approved by the agent in auto mode; only a deviation asks a person | branch | ba76a0e | +| 11-17 | modules with screen edits counted as routine (5 of 10 carry local edits) | a module with the app's edits gets its own step: carried over, screens compared, a difference is a decision with its case | PR | this PR | +| 11-17 | "No manual copying" (Accounts page) | a Marketplace page the menu opens: copied onto the app layout, or its layout set in place when a copy is not safe | branch | 132323a, 183d044, 8aa073d | +| 11-17 | 7 Accounts actions work on the old app, fail on the new | in-place layout keeps the user grid; rerun WM1 from build to prove the 7 pass | open | | +| 11-17 | "we still don't understand what fails means" | a FAIL is only "worked on old, broken on new"; fails on both = a note; known platform differences PASS* with reason | branch | 44d9111, ba76a0e | +| 11-17 | "see which tests actually ran? On old & new" | test step shows the runs on old and new; repeated context hideable | branch | ba76a0e | +| 11-17 | "build the SQL standin skill into toolkit" | 9.24 data reads go through the toolkit's own Postgres route | branch | e1b677f, 186c0f1 | +| 11-17 | CLI testing does not work on the old app: skill + bug report for mxcli | playbook entry in the toolkit; mxcli issue filed | branch + issue | e1b677f, mendixlabs/mxcli#1879 | +| 11-17 | "we don't spend 50 hours of testing when it's not needed" | e2e strategy: journeys once at the end, skip what failed on the old app | open (partly) | ba76a0e | +| 11-17 | proper e2e journeys incl. unhappy paths | rendered widget check | PR | #8 | +| 11-16 | "Is our advice at all correct to keep it on the old version?" | update instead of keep: refused modules routed to an update that carries the edits | done | #10 | +| 11-16 | "we give people 000 information on what the local edits are" | every plan card shows the evidence behind it; cockpitcheck fails a card without it | done | #10 | +| 11-16 | "Is our brain and script so far routing as it should? Are we measuring that?" | route score per run and a ledger; brain notes hit@k | branch | 6d73740, 081799b | +| 11-16 | rows "not approved yet" with nothing to approve; "create a gate" | cockpitcheck gate; grey when unmeasured | done | #10 | +| 11-16 | "Dojo React will always be a topic" | Dojo to React report on every upgrade | done | #10 | +| 11-16 | "Don't we want more than click paths?" | one test row per kind of app part | done | #10 | +| 11-16 | "Cockpit not up to date? Talk to your agent" | sidebar note | done | #10 | +| 11-16 | "how do I now let this continue? ... a mode for upgrades" | judgment stop as a red bar with the rerun command; interview mode auto | done + branch | #9, acdabb2 | +| 11-16 | "I signed off ... but it still said blocked" | a sign-off counts for the plan it was given on; signed item leaves Blocked | done | #10 | +| 11-16 | decisions hard to find; buttons like ask-user-question, read every 15 s | Decide now cards, one click per question, saved by the local server | done | #10 | +| 11-16 | end-to-end testing was poor | test resilience | done | #6 | +| 11-15 | cockpit: show when stuff is running | live cockpit shows the running step | done | #4 | +| 11-15 | open the cockpit as localhost after triage | live cockpit opens after triage | done | #4 | +| 11-15 | harvest learnings and submit them back | harvest and submit learnings | PR | #3 | +| 11-15 | version chip label | says what is open | PR | #2 | diff --git a/upgrade-toolkit/playbooks/9-to-10.md b/upgrade-toolkit/playbooks/9-to-10.md index 0cb6609..e8f83aa 100644 --- a/upgrade-toolkit/playbooks/9-to-10.md +++ b/upgrade-toolkit/playbooks/9-to-10.md @@ -19,6 +19,17 @@ Route: 9.x must be on 9.24 first, then convert to 10.24, then update modules ONC SKILL.md, NR1-4), with only convert/step/build/jar commits since the "original" commit, is CONVERT-MADE and forced (T1-15, T3-10, T4-3, T8-12). Commit the app's first commit as "original" and the convert as "convert ...". +- No data reads on the OLD app through `mxcli oql` (WM1 CLI test, 2026-11-17; mendixlabs/mxcli#1879): the 9.24 runtime + has no `preview_execute_oql` admin action, so every call answers `{"result":-5,"message":"Action not found."}` and + mxcli's hint about the live-preview dev flags cannot help. Seeds, row counts, outcome checks and fixture-manifest.sh + on the old app are an instrument FAULT, so the run has no "before" data. One call tells: post `preview_execute_oql` + with `SELECT COUNT(*) AS n FROM System.Session` to the admin port; 10.24 answers `{"n":"5"}`-style data. + The pipeline needs nothing for this: run_journeys.py sets `UPG_PSQL` and the engine asks the app's Postgres + (bin/pgdata.js + bin/pgmap.py, names read from the runtime's `mendixsystem$` tables, the same on 9.24; WM1 rerun: + 97 entities, 111 associations on the 9.24 database). Driving an e2e engine by hand outside the pipeline, give it + the same route (`UPG_PSQL`, or `pgmap.py query --psql CMD "<OQL>"`); never hand-map OQL to SQL. pgmap refuses + what it cannot translate (exit 1: the check is INVALID, never PASS). A hand mapping was proved once (WM1 CLI test: + 10/10 queries equal to real mxcli on 10.24), but it covers only the queries someone typed in. - Coverage gap of localedits on 9.24 (E5b-7): pages and layouts are compared only when both models can be read by the same mxcli; a 9.24 app against a 10.x package often cannot. Run localedits again right after the convert, on 10.24, before the post-convert updates: that run sees the page, layout and logic layers. diff --git a/upgrade-toolkit/steps/baseline.md b/upgrade-toolkit/steps/baseline.md index 09d1e9d..abb4835 100644 --- a/upgrade-toolkit/steps/baseline.md +++ b/upgrade-toolkit/steps/baseline.md @@ -37,7 +37,9 @@ copy's converted model. the script and log in order, remaps ids by entity GUID, matches System.User / UserRole by name, and lists every table or column it cannot map under UNMAPPED with exit 1 (`--skip TABLE` waives one, reported). Start the app again, then freeze. File contents (`data/files/*` in the zip) are copied into the app's `data/files` after EVERY start (runapp.py - wipes `run/<name>/`, T5-5). Open: autonumber counters are not advanced. + wipes `run/<name>/`, T5-5). Open: autonumber counters are not advanced. `upgrade-pipeline.sh` baseline does all of + this itself (seed_demo, log in `logs/seed-snapshot.txt`; the file contents go to `$W/demo-files`, which runapp.py + copies in at every start through UPG_DATA_FILES); only when nothing loads does it fall back to seed_pg.py, with a note. 2. Start the OLD app on X: `WORKDIR=$W UPG_TOOLS=<run>/tools python3 $B/runapp.py start old <old.mda> <full ver> X` (Java image by major: 9 to temurin 11, 10 and every 11.x to temurin 21, also for an 11.x to 11.y run, N1-14; `CONST_OVERRIDES` = the environment's values, the SAME for old and new: JSON text or the path of a JSON file, B4T2-12). Slow start (E5b-13): a start above a few minutes is usually a module calling home at @@ -91,7 +93,8 @@ copy's converted model. the baseline commit; in a parallel run `app/` is owned by triage and is converted later, NR1-6): `python3 $B/smokewalk_gen.py <old model folder> -o $W/plan.json` `node $B/smokewalk.js --plan $W/plan.json --url http://localhost:<port> --user <Role>=smoke_<role>:<pw> ... --out $W/smoke-before.json` - The runner skips microflow steps unless it gets `--microflows`. When smokewalk_gen prints `WARN <role>: ...` (the + The runner skips microflow steps unless it gets `--microflows`; upgrade-pipeline.sh passes it whenever the plan has + microflow steps (UPG_SMOKE_MICROFLOWS=0 turns it off, with a note). When smokewalk_gen prints `WARN <role>: ...` (the home is a microflow, or over half the menu items are; PV-6: every page of Product View 360 was one), add `--microflows` to the baseline AND the gate walk, on the X_smoke copy only: they run for real. Add `--shots $W/smoke-before-shots` (and `--shots $W/smoke-after-shots` after the upgrade) so the gate can compare diff --git a/upgrade-toolkit/steps/heal.md b/upgrade-toolkit/steps/heal.md index 5e9c21e..6dfe7d6 100644 --- a/upgrade-toolkit/steps/heal.md +++ b/upgrade-toolkit/steps/heal.md @@ -45,6 +45,70 @@ listed as "needs a decision": check production configurations with an empty or w - The merge keeps the stored summaries as `summary.pre-heal.json` and lists the healed ids under `healed`. - `heal-pre-diff.md` is the data diff the classifier reads (the heal step runs `diff.py` itself, before the gate). +## The learning loop (toolkit-gaps.md) + +A FAIL is only "worked on the old app, broken on the new one". Everything else that turns a run red is a rule the toolkit +lacks, and the rule is closed in the toolkit, so the next app gets it for free. `heal.py finalize` writes +`work/toolkit-gaps.md`: every likely-real row (a difference no rule explains) and every test that runs on neither app, +each with the script its rule belongs in. The `heal:` summary line counts them. Work the list, one row at a time: + +1. **Prove it on both apps.** The old run, `twin-old.md` (two starts of the old app), the screenshots, the log lines. + A guess is not evidence. +2. **A real defect of the app stays open.** It is in NEEDS-JUDGMENT; nothing in the loop accepts it. +3. **Otherwise write the rule in the owner script, with its evidence condition.** Never a bare allow line for one app: + the rule names what makes the row safe, so the same row without that evidence is still caught. +4. **Two controls:** one that fails on the old toolkit and passes on the new one, and a guard that still catches the real + case. Run the group, then check the first control against the old code. +5. **Rejudge the stored run** (`--from heal`, or the script's `--rejudge`); no new walk. +6. **Write it down.** The rule and its why in this step doc or the playbook; a Mendix fact (not a toolkit fact) also as a + note in the company brain (`internal/upgrade/`). + +The rules closed this way on a 9.24 -> 10.24 -> 11.12 app (2026-11-17), each printed as PASS* with its reason: + +| row | rule | where | +|---|---|---| +| data: a Marketplace module's own attributes changed (MendixSSO, MxModelReflection) | that module was updated in this run (`--module-jumps`) and no rows of other entities are lost | `diff.py` | +| data: the admin's password and `changedDate` | the toolkit signs in as `UPG_ADMIN_USER`; only that one row may differ (`noadmin` figures) | `snapshot.py`, `diff.py` | +| http: `.../openapi.json` 404 -> 200 | the same service's `swagger.json` answered 2xx on the old app (10.x serves both formats) | `diff.py` | +| log: a new WARNING whose text the old app logs under another logger | same text, only the logger name changed (trial license: `Core` -> `LicenseService`) | `logdiff.py` | +| log: `Using derived value for ClusterManagerActionInterval` | `runapp.py` starts every app with a short `SessionTimeout` | `HARNESS_LOG` in `heal.py` | +| journeys: failures on the old app too | nothing that passed on the old app fails on the new one, and at least 20% pass on both (otherwise the run is not measuring) | `gate.sh journeys_noted` | +| journeys: a heal that leaves a journey failing on both apps | listed for the owner, `test-setup`, and a toolkit gap; not likely-real | `heal.py finalize` | + +## A decision for a person comes with its case (`work/cases/<id>.json`) + +Ask a person only when there is a deviation: an edit of the app, a check that worked on the old app and fails on the new +one, or no clear path. Routine work goes ahead (auto mode) or under the one plan sign-off. When the loop raises a decision, +the agent writes its case next to it, so the person decides on one screen (the cockpit's Resolve opens it): + +```json +{"headline": "what is wrong, in one sentence", + "tiles": [[30, "pass on both apps", "ok"], [7, "pass on the old app, fail on the new", "bad"], [1, "cause", "wait"]], + "shots": [{"old": "path/in/work/old.png", "new": "path/in/work/new.png", "cap": "what to look at"}], + "why": ["step 1 of the cause", "step 2", "step 3"], + "options": {"<option key>": {"does": "what happens when you pick it", "catch": "what it costs or breaks"}}, + "details": ["the evidence: tests, files, what was checked with which tool"]} +``` + +The file name is the decision id with every character outside `[A-Za-z0-9._-]` as `_` (`regression:module-Administration` +-> `regression_module-Administration.json`). Every sentence comes from evidence on both apps; the screenshots are embedded, +so the cockpit stays one file. A regression without a case file gets one built by the cockpit from the run's own files +(`cockpitmatrix.auto_case`): how many actions pass on the old app and fail on the new one, the old and new screenshot of +the first failing step, the modules whose own screen edits the update carried over (the first place to look), and the +heal evidence lines. A written case file wins over the built one. + +## A module update that carries the app's own edits is not routine + +A Marketplace module the app edited (a changed layout, snippet, page, JavaScript action or entity in `localedits.json`) +is never counted as a routine update: the update replaces the whole module, and the toolkit carries each edit back. +`proposal.py` puts these modules in their own step, `module-edits` ("Module updates that carry the app's edits"), +apart from the routine `modules` step. Three cases: + +1. No edits: routine, goes ahead under the plan sign-off (auto mode: approved with source `routine`). +2. Edits carried over and the screens that use them look and work the same on both apps: goes ahead. In auto mode the + step is approved with source `carry-over`, so the cockpit says which edits were carried, with the found detail. +3. An edit could not be carried, or a screen differs on the new app: a decision for a person, with its case (above). + ## What the gate reads from it `gate.sh` keys `HEAL_JSON` (prints the one-line `heal:` summary), `GATE_NOTES` (L56: every non-empty line that is not a diff --git a/upgrade-toolkit/steps/journeys.md b/upgrade-toolkit/steps/journeys.md index 718d2ba..679a85d 100644 --- a/upgrade-toolkit/steps/journeys.md +++ b/upgrade-toolkit/steps/journeys.md @@ -224,6 +224,16 @@ with the `defects:` line; a defect without evidence stays in FAIL, B4N1-13): arguments, or a journey file that is empty or not a journey: the file is named, nothing runs, B4T2-18). The gate shows 3 and 2 as INVALID. +### Journeys that failed on the old app are not walked again (`--skip-baseline-fail`) + +A failure is only "worked on the old app, broken on the new one". On a 9.24 app 160 of 421 journeys failed on the old app; +walking them on the new app took about half the journeys time and could only end in BASELINE DEFECT. The pipeline passes +`--skip-baseline-fail` to the new-app run: a journey that FAILed on the old app and feeds no later one (a second-pass +journey with `_path_needs`, or any journey of a set without data passes) is listed as `SKIPPED` ("failed on the old app +too, not run on the new one"), counted in the headline, never a FAIL. A first-pass journey still runs, because it may +create the rows a later one needs. `UPG_JOURNEYS_ALL=1` walks every journey. The cockpit's Test tab lists every journey +with its old and its new verdict. + ## Action inventory (E2E-8): the denominator, from the old model ``` diff --git a/upgrade-toolkit/steps/propose.md b/upgrade-toolkit/steps/propose.md index 75b1774..681194f 100644 --- a/upgrade-toolkit/steps/propose.md +++ b/upgrade-toolkit/steps/propose.md @@ -31,6 +31,12 @@ Signing an item nobody answered accepts its recommendation (recorded as a P-n li records `ASSUMED (unattended)` P-lines, goes on, and the gate summary and SUMMARY.md say "proposal not signed". A person who later signs and answers the P-n lines closes them; the ASSUMED lines block the owner sign-off until then. +`Interview mode: auto` does the same at propose and also parks the later judgment stops that nothing after them depends on +(journeys, idcheck, smoke walk, diff, gate) in `$W/parked/<step>.md`; the run goes on. At the end NEEDS-JUDGMENT.md lists every +parked item and every ASSUMED line, grouped (modules and data, journeys, screens and gate), for one round of answers and one +rerun from the first parked step (bin/parked.py). A build or start failure, a missing plan or a replay that cannot be undone +still stops at once. A run with a parked item never ends in PASS. + Section-3 items that fit a pattern in playbooks/rearchitecture.md carry its options, effort and recommendation and cite `[RA-n, status]` (bin/proposal.py); the answer updates that entry (steps/decide.md). ## Step records diff --git a/upgrade-toolkit/steps/repair.md b/upgrade-toolkit/steps/repair.md index 7b49ef4..1afe51d 100644 --- a/upgrade-toolkit/steps/repair.md +++ b/upgrade-toolkit/steps/repair.md @@ -25,7 +25,7 @@ keep the fix and continue with the new class. Without `--fixed` a rise is always | 12 | CE1571 "No argument has been selected" after a 9 to 10 convert, at a call to a flow in a protected add-on module (`modules/*.mxmodule`, not in app.mpr): convert could not read the flow's parameters, so the mapping stayed empty (Opcenter ClientConfiguration_NewEdit.dataView1, T2-4, T2R3-7) | `python3 $B/repair/copy_mappings.py app/ --scan` lists the empty calls to flows outside the model; per page `python3 $B/repair/copy_mappings.py app/ --page <Module.Page> --ref <publisher's newer app dir or .mpr>` (dry run: COPY / SAME / DIFFERS / NOREF), then `--apply` | do NOT fix it with `mxcli alter page ... set DataSource = microflow M(P = $x)`: for an add-on flow it prints "Altered page" and saves nothing (twice in Opcenter). Run any `alter page` through `python3 $B/repair/alter_guard.py app/ -- mxcli ...`: REFUSED (exit 2, not run) for a page in an add-on, WARN for an add-on flow called with arguments, NO-OP or MAPPING NOT SAVED (exit 1) when mxcli exited 0 and the unit did not change. A call to an own flow can be set in MDL (`$currentObject`, T2R3 HANDWORK 10). NOREF (no reference maps it): call the add-on flow from an own wrapper microflow that sets the arguments (T3). Then mx check (CE1571 gone) and `convertcheck.py --mappings` (the copied mapping must map the data view's object; `$currentObject` counts as that object, T2R3-8; a REVIEW you checked and keep is answered with `--answer FILE`, `<Page>: <site> @maps:<hash> # what you checked`, B4T2-8) | | 13 | CE1613 "<Module>.<Layout> no longer exists" on own pages: an updated module dropped a layout (Logging 2.3.0 dropped Logging.PopupLayout: 6 errors on 3 pages, T8R-5; 7 on own pages in T3) | When only the layout differs (also the replay of pages a no-baseline update moved to another layout): `python3 $B/repair/alter_guard.py app/ -- mxcli exec -p app/<app>.mpr $W/replay.mdl` with one line per page, `ALTER PAGE <Module.Page> { SET Layout = <Module.Layout>; };` (MAP for other placeholders); alter_guard says OK or REFUSED (NR1-15: 4 replays, one line each). When more than the layout differs: `mxcli -p app/<app>.mpr -c "describe page <Module.Page>" > $W/<page>.mdl`, set `Layout:` to the sibling layout of the same type (a popup: `Atlas_Core.PopupLayout`), then `python3 $B/repair/alter_guard.py app/ -- mxcli exec -p app/<app>.mpr $W/<page>.mdl` | `mxcli -c "exec file"` is a parse error, use the `exec` subcommand (T8R2 R-3). One commit per module. CE1613 on menu items (an icon or a page that is gone) after an add-on swap: `describe menu`, drop the dead targets and run it back as `create or modify menu ...` (T2R3-9) | | 14 | CE6911 "index.html missing the cachebust template tag(s)" after a 9 to 10 convert (9.x hard-coded `?637643641483639052`, 34 places in FSM) | `python3 $B/repair/cachebust.py app/`, then `--apply` | also rewrites the dojoConfig `cacheBust: "<number>"` that CE6911 does not flag and that otherwise stays until the gate (T8R-5, T8R2 G-5). Commit as "theme: cachebust tags" | -| 15 | CE0161 "Error(s) in XPath constraint" at a widget (`Data grid 'grid2'`) with no document named | `mx check app/<app>.mpr -j $W/check-<n>.json` (`-j` AFTER the mpr; before it, mx prints usage, T8R2 R-6) names the document and unit id | the 9 to 10 cause seen twice (FSM batch 1 and T8R2): `[%CurrentObject%]` in a classic data grid inside a data view now means the data view's object. mxcli cannot author the classic DataGrid, so the constraint was rewritten by a BSON patch of that unit (T8R2 R-5). A behaviour change: list it in the report and have a human look at the page. No tool yet | +| 15 | CE0161 "Error(s) in XPath constraint" at a widget (`Data grid 'grid2'`) with no document named | `mx check app/<app>.mpr -j $W/check-<n>.json` (`-j` AFTER the mpr; before it, mx prints usage, T8R2 R-6) names the document and unit id; the build step now does this itself and lists `[where]` lines in check-N.log and in the stop report (checkwhere.py, cold CMM) | the 9 to 10 cause seen twice (FSM batch 1 and T8R2): `[%CurrentObject%]` in a classic data grid inside a data view now means the data view's object. mxcli cannot author the classic DataGrid, so the constraint was rewritten by a BSON patch of that unit (T8R2 R-5). A behaviour change: list it in the report and have a human look at the page. No tool yet | | 16 | CE1707 "layout ... not found" (and CE1613 on menu items) after replacing an add-on by the publisher's newer build (playbooks/10-to-11.md row 9; Opcenter `EXFN_CrossNavigation_InCanvas`, T2R3-9) | `mxcli describe layout` returns an empty body, so the layout cannot be re-created in MDL: copy the unit by name from the publisher's MPR with fresh `$ID`s (T2R3 used a BSON script, runs/t2r3-opcenter-ef evidence/rung2-upgrade/portunits.py; no toolkit tool yet) | then the menu as in row 13, mx check, and `idcheck.py` old against new before the convert | | 17 | CE0114 at an own microflow that does not apply entity access and calls one that does: an updated module turned it on (DeepLink 10.0.10 `DeepLink.DeepLinkHome`, called first by the role home microflows; T6R and batch 1 T6-12: Masterdata `NAV_ApplicationSettings_Admin`, `NAV_Dashboard_DepartmentManager`, `NAV_Dashboard_Inspector`) | mx check does not name the calling microflow (T6-12), so list the callers: `mxcli -p app/<app>.mpr -c "show callers of DeepLink.DeepLinkHome"`. Per own caller: `mxcli -p app/<app>.mpr -c "describe microflow <Module.Flow>" > $W/rep-<Flow>.mdl`, put `@applyentityaccess` on the line above `create or modify microflow`, `mxcli exec -p app/<app>.mpr $W/rep-<Flow>.mdl` (the T6R files: runs/t6r2-checklist evidence/rep-NAV_*.mdl) | a decision, not a mechanical fix (no tool on purpose): applying entity access in the caller changes what that role may read in the rest of the flow. The alternative the publisher took (Checklist 2.0.0) is an own copy of the callee without entity access (`Mobile_Inspector.DeepLinkHome_custom`). Say which in the report; then mx check (T6R: 3 CE0114 to 0) and the role's login journey | | 18 | CE0117 in a KEPT Administration (2.1.x) after MendixSSO 4.2.0 or newer: `Administration.MendixSSO_CreateUser` / `MendixSSO_UpdateUser` still read `$UserProfile/EmailAddress`, which MendixSSO removed; its `UserMapper` now passes an `EmailAddress` parameter (T6R and batch 1 T6-13: 2 errors). triage's `crosscalls.py` predicts it (T5F-4) | `describe microflow` each one, add the parameter `$EmailAddress: String` and use it where `$UserProfile/EmailAddress` was (`Email = $EmailAddress`), then run the file. `create or modify microflow` refuses this change when the free annotations differ ("cannot be spliced"): write `drop microflow <Module.Flow>;` above the `create` in the SAME file so the flow is never missing between two commands (T6R: evidence/rep-Administration.MendixSSO_*.mdl) | update Administration with MendixSSO instead when the stop rules allow it (the publisher moved to Administration 4.1.0). A drop + create gives the flow a new id: name both flows in the report. Then mx check (T6R: 2 CE0117 to 0) and the SSO login journey | @@ -37,6 +37,12 @@ keep the fix and continue with the new class. Without `--fixed` a rise is always | 24 | CE0463 after a hand-set value on a pluggable widget property; or `mx update-widgets` puts a property value back (FM-5) | read the widget's `getProperties()` in its editorConfig (inside the .mpk): a property that the config hides unless another property is on (for SprintrFeedback `targetContainerSelector` needs `showAdvancedSettings` = true) is hidden in the model, and a hidden property with a non-default value is a CE0463 and is reset by update-widgets | set the controlling property first (`showAdvancedSettings` true), then the value, in one commit; run `mx update-widgets` AFTER and check the value survived (diff the unit), then mx check. Seen: SprintrFeedback 12.0.4 (bundled with Atlas_Core 4.5.1) portals its button into `targetContainerSelector` and renders nothing when the default `.mx-scrollcontainer-center` is absent (an app layout without a centre region): the Feedback tab vanished from every page and shotdiff said "lost feedback". The remedy card `sprintr-feedback-target` (playbooks/remedies.tsv) carries it | | 25 | `mx update-widgets` moves a widget across a technology jump (Slider 1.3.2 Dojo to 3.0.4 pluggable) and then CE7006 on every attribute of it, or the widget lost its min/max/step attributes and its on-change microflow (QI-6) | a Dojo-to-pluggable version jump is a widget SWAP, not an update: before it, list the widget's properties in the old model (BSON of the page or snippet unit) and after it again, and write every dropped binding down; `bin/widgetdiff.py` names the package versions only | re-bind the dropped properties by BSON (not through mxcli describe, which keeps only the value attribute); a pluggable widget gets no entity context from a snippet parameter, so wrap it in a data view on the snippet parameter (the CE7006 goes); a microflow action on a pluggable widget takes empty ParameterMappings. Seen: Slider on a Quality Inspection snippet: only valueAttribute survived, 4 CE7006 until the data view wrap, 0 after. Then mx check and a by-eye look at the page | | 26 | screens rung REVIEW on looks that changed by design: the Charts 6 palette, the Data grid 2 column selector (eye) icon size, the MxAdmin changedDate stamped at runtime start on the 10.24 rung (data rung accept) (QI-10) | look at the pair first; these three are known and each has a fix or an accept | Charts 6 (6.x) comes without the old series palette: re-apply it in the theme (a `.widget-charts` series colour rule) when the dashboard look is wanted; the Data grid 2 column selector icon grows: add a theme rule for its size; the 10.24 runtime writes MxAdmin changedDate at start: accept it in the data rung accept file with that reason. Never accept an unseen pair | +| 27 | CE0117 at the "Dynamic classes expression" of a button after the 9 to 10 convert, where the expression follows an association from `$currentObject` (`if $currentObject/Module.A_B = empty then 'disabled' else ''`) | the `[where]` lines of check-N.log name the page and property | Cause: the button sits in a data view nested in another one, and in Mendix 10 `$currentObject` there is the INNERMOST data view's object, which has no such association; the expression meant the outer object. Cold CMM (10.24.28): 4 buttons in the inner Recipes data view of Mix_New_WizardStep1 (desktop and phone) tested `$currentObject/MaterialManagement.RecipeHelper_Recipes`, where RecipeHelper is the outer data view. The 9.24 build accepted it. Remedy (verified, cold CMM run 8: the 4 CE0117 gone): name the outer data view instead, `$dataView1/Module.A_B` (the outer data view's widget name), in each expression; check the page renders the same. They can be HIDDEN: in cold CMM run 5 the 4 vanished while 3 CE1613 (row 28) were open, and came back unchanged once those were repaired (run 7), so mx check reports no CE0117 while CE1613 errors stand; a check that lost its CE0117 next to new CE1613 has not fixed them. Fixing them can reveal the next phase in turn (run 8: 5 CE2729, row 30). No tool yet | +| 28 | CE1613 "the selected page/microflow no longer exists" in an OWN module after a Marketplace module update: an own reference selector's select page, a microflow data source or a Call microflow that points into the updated module | `owngone.out` (crosscalls.py --own, written by the update step before the update) lists each `GONE <kind> <target> (used by own <document>)`; the build stop quotes it | cold CMM: Administration 2.1.2 -> 4.5.0 dropped `Administration.UserRole_Select` (the select page of the user role selector on two own Employee pages) and `Administration.RetrieveTimeZones` (the time zone data source on Employee_New): 3 CE1613. Remedy (verified, cold CMM run 7: the 3 CE1613 gone): `bin/repair/copydoc.py <app> --from base.mpr --doc Administration.UserRole_Select --to EmployeeManagement --roles EmployeeManagement.Administrator --retarget` (dry run; then `--apply`, commit, mx check). base.mpr is the app at the SAME Mendix version before the update: `git -C app show <commit before the update>:'<App>.mpr' > base.mpr`. It copies with fresh ids (pointers follow), maps the roles, repoints only own users; old widget versions in the copy are CE0463 that the build loop's update-widgets clears (cold CMM: 11). mxcli cannot author reference selectors (not re-executable), so it cannot do this. Alternatives: switch the selector's source to a database/XPath source in Studio Pro, or keep the module (a held update). The copy is old code in an own module: a judgment item; check the selector on the page as a real user afterwards | +| 29 | `[MXCHECK] mx check ended without a result` in check-N.log: mx check died (an `ERROR: System.…Exception` and a stack, no "The app contains" line) | the ERROR line it quotes; `git -C app log -1` names the last change | mx check cannot load the model, so it reports nothing: the pipeline counts it as an error and stops (before, it counted 0 and ran mxbuild). The last commit broke the MPR: a raw unit edit (copydoc.py, copy_mappings.py, restoreunit.py) or a tool crash mid-write. KeyNotFoundException with a GUID = a pointer to an element id that is not in the unit (cold CMM: a page copied with fresh ids kept its old `DefaultButtonPointer`; copydoc.py now remaps them). Tag, `git reset --hard` to the commit before, fix the tool, redo. EntryPointNotFound or NullReferenceException were seen too (an earlier run, twice), cause not established: find the last change first | +| 30 | CE2729 "No read access to association 'A_B' in entity 'Other.E' for user role ..." after an update that re-created an association to another module's entity (editroute XASSOC replay) | `work/replay/<Module>.mdl` has `member access on <owner>` but no `the other side` part; `mxcli describe entity Other.E` shows the member gone from its grants, the original model has it | The re-created association starts with no member access on BOTH entities. The replay restored only the owner side; since fix 15 it also restores the other side, as only that member merged into each rule (mxcli's grant adds to a role's rule and keeps the rest). Cold CMM run 8: EmployeeManagement.Employee_Account to Administration.Account, ReadOnly for Administrator and User became None, 5 CE2729 on the Employee text boxes. Remedy for a run replayed before the fix (verified, cold CMM run 9): per grant line of the entity in the ORIGINAL model, `grant read (A_B) on entity Other.E to <role> [where ...];` (editroute.member_grant prints these), mxcli exec, commit | +| 31 | javac `cannot find symbol` in an updated module's Java (MendixSSO, CommunityCommons) at mxbuild, where the class exists but the member does not | `javacjar.out` (the build stop runs `bin/repair/javacjar.py <app> <build log>` and quotes it): `SYMBOL <class>.<member>`, `HAS <jar>`, `HIDDEN BY <jar>` | An older copy of the class sits in another jar, often a fat (shaded, jar-with-dependencies) jar in userlib that bundles its libraries, and javac takes that copy. Cold CMM (10.24.28): MendixSSO 10.6.2 calls `CookieHeaderNames.SameSite` (vendorlib netty-codec-http 4.2.17); userlib/server-examples-0.4.1-SNAPSHOT.jar (OPC UA, 12,588 entries) carries its own older Netty, Guava and BouncyCastle. jardedupe keeps such a jar (only copy of some classes) and jarclash lists it as a note. Remedy (verified, cold CMM run 10: mxbuild passes): `python3 $B/repair/javacjar.py app/ --strip userlib/<fat>.jar` (dry run: how many classes a vendorlib jar also carries), then `--apply`, commit `jar:`; the library then runs on the vendorlib versions (newer majors: Netty 4.2, Guava 33): a judgment item, so test what uses the library at the gate. Alternatives: a thin jar plus its separate dependencies, or a newer version of the library | +| 32 | After the module updates, data grid (or other widget) pages show their rows without layout: no columns, no header styling; afterwalk passes (the data is on the page), the screens rung says REVIEW | `widgetskew.out` (step_update runs `bin/widgetskew.py <app> --plan plan.txt` and parks): `WIDGET-SKEW <widget> <old> -> <new> (by <module>) ships with <kept module> <version>`. One module's package replaced a shared widget in widgets/ (`.stepupdate/widgets.txt`), while the module that carries its styles (themesource) was kept at its old version | Update the kept module too: a one-line plan (`<content id> <version> <module>`) through `bin/stepupdate.sh` (its local edits are often OS junk, `.DS_Store`); or put the old widget .mpk back from git. Accepting the skew: `widgetskew-ack.txt` in the work dir. Cold CMM: Atlas_Web_Content 3.9.0 brought Datagrid 2.30.6, DataWidgets stayed 3.x (FILES-EDITED on `.DS_Store`); 23 grid pages unstyled | mxcli v0.28.0 (#1408, #1474), what it changes for the rows above: - `mxcli exec` is all-or-nothing. A script whose statement N fails now changes NOTHING and exits non-zero; before, statements 1 to N-1 diff --git a/upgrade-toolkit/steps/upgrade.md b/upgrade-toolkit/steps/upgrade.md index 38ef1ed..47bcd6f 100644 --- a/upgrade-toolkit/steps/upgrade.md +++ b/upgrade-toolkit/steps/upgrade.md @@ -16,6 +16,8 @@ Goal: reach the target version with every step proven not to change a runtime id re-imported instead of updated in place; NS-7, notification-service cc318c0b: DataWidgets 2.27.6 rewrote 12 own pages and was reverted the next day). After a MODULE_RECREATED line, diff the own pages the step touched (`git show --stat HEAD`) before going on, or revert the step and update the module with Studio Pro. + `info MODULE_RECREATED` is the same with every element identity kept (WM1: mxcli's update copies the documents + with new unit ids; 28 of 28 such lines had no GUID change): the data is safe, only the page diff remains. - approved.txt: one fnmatch pattern per line on the element or "FINDING kind name", then `# reason`. Only for findings accepted in writing. - Local edits refusal ("Save them first: --save-edits"). stepupdate names the edited element kinds. UI only