Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions upgrade-toolkit/bin/cockpitmatrix.py
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,11 @@ def commit_rows(pred):
for it in pending_j:
tgt = next((r for r in rows if r["id"] == home.get(it.get("step"), "findings")), None) or rows[0]
c = tgt["cells"]["test"]
if c.get("s") == "you" and c.get("dec", "").startswith("stop:") and it["id"].startswith("stop:"):
# several sections of one judgment stop on one row: list them all (the first one kept only its own title)
n = c.get("nstop", 1) + 1
c.update(nstop=n, l=f"{n} decisions needed", steps=c["steps"] + [it["title"]])
continue
was = c["l"]
c.update(s="you", l="Needs your decision", dec=it["id"], steps=[it["title"], it["problem"], f"Before: {was}."][:3] if it["problem"] else [it["title"]])
return rows, ready_chips(work, P, g, heal, inv, ledit, done), P, g, done, per, ledger
Expand Down Expand Up @@ -1055,7 +1060,11 @@ def band(model, rows, chips, g, tl, kpi=None):
st, head, sub = "bad", "The build does not pass yet", "The agent is fixing it."
else:
st, head, sub = "neutral", (f"Upgrade to Mendix {tgt} in progress" if tgt else "Upgrade in progress"), "The build has not run yet."
closed = closed_items(rows, g, tl)
if model.get("stopped") and st != "ok": # a judgment stop outranks "in progress": nothing moves until a person decides
s = model["stopped"]
st, head = "bad", f"Stopped at {s['step']}, waiting for you"
sub = f"{len(s['titles'])} {'item' if len(s['titles']) == 1 else 'items'} to decide, then rerun (see the red bar)."
closed =closed_items(rows, g, tl)
for it in kpi.get("closed") or []:
if isinstance(it, dict) and isinstance(it.get("label"), str) and isinstance(it.get("n"), int):
hit = next((c for c in closed if c["label"].lower() == it["label"].lower()), None)
Expand Down Expand Up @@ -1277,6 +1286,8 @@ def band(model, rows, chips, g, tl, kpi=None):
#feed,#feed *{overflow-wrap:anywhere}
.nowrun{display:flex;align-items:center;gap:12px;margin:0 0 14px;padding:10px 14px;border-radius:12px;background:var(--panel);border:1px solid var(--wait);font-size:14px}
.nowrun[hidden]{display:none}
.stopbar{margin:0 0 14px;padding:10px 14px;border-radius:12px;background:var(--badbg);border:1px solid var(--bad);font-size:14px}
.stopbar b{color:var(--bad)}.stopbar ul{margin:6px 0;padding-left:20px}.stopbar small{display:block;color:var(--muted,#6c7a87);overflow-wrap:anywhere}
.nowrun .pulse{flex:none;width:10px;height:10px;border-radius:50%;background:var(--wait);animation:nrp 1.4s ease-in-out infinite}
@keyframes nrp{0%,100%{opacity:1}50%{opacity:.25}}
@media (prefers-reduced-motion:reduce){.nowrun .pulse{animation:none}}
Expand Down Expand Up @@ -1715,6 +1726,16 @@ def h_running(r):
f'<small>{E(what)}</small></div></div>')


def h_stopped(s):
"""A judgment stop (NEEDS-JUDGMENT.md): a red bar under the header, so a run that waits for a person says so first."""
if not s:
return ""
li = "".join(f"<li>{E(t)}</li>" for t in s["titles"])
cmd = f'<small>Then rerun: <code>{E(s["rerun"])}</code></small>' if s["rerun"] else ""
return (f'<div class="stopbar" role="alert"><b>The run stopped at {E(s["step"])} (exit {s["code"]}) and waits for you.</b>'
f'{"<ul>" + li + "</ul>" if li else ""}<small>Details: NEEDS-JUDGMENT.md in the work folder.</small>{cmd}</div>')


def render(model, matrix):
data = json.dumps(dict({k: model[k] for k in ("title", "sub", "state", "planHash", "signed", "phases", "items")}, matrix=matrix), ensure_ascii=False)
data = data.replace("</", "<\\/").replace("<!--", "<\\!--")
Expand All @@ -1728,7 +1749,7 @@ def render(model, matrix):
f" <h1>{e(model['title'])}</h1>\n <div class=\"ver\">{e(model['sub'])}</div>\n"
" </div>\n"
f" {h_ladder(matrix.get('ladder'))}\n </header>\n"
f" {h_running(model.get('running'))}\n"
f" {h_stopped(model.get('stopped'))}{h_running(model.get('running'))}\n"
' <div class="layout">\n <div class="main">\n'
' <details class="card more"><summary>Counts by kind (tap one to mark its rows)</summary>\n'
f' <div class="headline" id="headline">{head_}</div>\n <div class="tiles" id="tiles">{tiles_}</div>\n </details>\n'
Expand Down
19 changes: 18 additions & 1 deletion upgrade-toolkit/bin/cockpitpage.py
Original file line number Diff line number Diff line change
Expand Up @@ -498,6 +498,20 @@ def add(it):
"title": j.get("question", j.get("id")), "step": j.get("step", ""), "meta": f"Open since step {j.get('step', '?')}, effort {opts[0].get('effort', '') if opts else ''}".replace(", effort ", ""),
"problem": short(j.get("what", ""), 260) + (" " + short(j.get("why_matters", ""), 160) if j.get("why_matters") else ""),
"rec": rec, "rec_text": j.get("recommended_why", ""), "options": opts})
# a judgment stop: each NEEDS-JUDGMENT.md section is a row, unless decisions-open.json already holds that step's items
# (cold test WM1: basewalk stopped with exit 11 and three sections, the page said "0 your tasks" and the run sat overnight)
stopinfo = None
if stopped:
rerun = (re.search(r"then rerun:\n\n {4}(.+)", nj) or [None, ""])[1]
secs = re.findall(r"^## (.+?) \(exit (\d+)\)\n\n(.*?)(?=^## |\Z)", nj, re.M | re.S)
stopinfo = {"step": stopped[0], "code": stopped[1], "titles": [s[0] for s in secs], "rerun": rerun}
if stopped and not any(j.get("step") == stopped[0] for j in djo):
for k, (title, code, body) in enumerate(secs, 1):
add({"id": f"stop:{stopped[0]}:{k}", "kind": "judgment", "dot": "bad", "title": title, "step": stopped[0],
"meta": f"The run stopped at {stopped[0]} (exit {code}) and waits for you",
"problem": short(re.sub(r"```.*?```", "", body, flags=re.S).strip() or body.strip(), 400),
"rec": "rerun", "rec_text": "Decide in writing as the text says, then rerun from this step.",
"options": [{"key": "rerun", "label": "Decided, rerun", "causes": rerun or "Rerun the pipeline from this step."}]})
conf = {} # change id -> {"label","tech","commit"}; from AGENT-DECISIONS rows marked confirm and gate-notes lines that ask for a look
for bid, d in AD.items():
if d["confirm"]:
Expand Down Expand Up @@ -731,6 +745,8 @@ def phase_time(steps):
nn = len(open_visible)
if nb:
state = ("bad", f"Not ready yet. Testing found {pl(nb, 'real problem')}, {pl(nn, 'thing')} {'needs' if nn == 1 else 'need'} you.")
elif stopped:
state = ("bad", f"Stopped at {stopped[0]}. {pl(nn, 'thing')} {'needs' if nn == 1 else 'need'} you before the run goes on.")
elif nn:
state = ("warn", f"Waiting for you. {pl(nn, 'thing')} {'needs' if nn == 1 else 'need'} you.")
elif gate["overall"] == "PASS":
Expand All @@ -748,7 +764,8 @@ def phase_time(steps):
if stopped:
sub += f" · stopped at {stopped[0]}"
return {"title": f"{app_name(work)} upgrade", "sub": sub, "state": {"dot": state[0], "text": state[1]}, "planHash": P["hash"],
"signed": list(signed) if signed else None, "phases": phases, "items": items, "running": run}
"signed": list(signed) if signed else None, "phases": phases, "items": items, "running": run,
"stopped": stopinfo}


# --live: poll the server's status (the "Running now" bar, in place) and stamp, reload on change and keep the scroll place; an open drawer holds the reload until it closes
Expand Down
7 changes: 5 additions & 2 deletions upgrade-toolkit/bin/gen_journeys.py
Original file line number Diff line number Diff line change
Expand Up @@ -458,7 +458,10 @@ def create_journey(spec, data_rung=False):
acts = [{"do": "nav", "group": spec["nav"].get("group"), "item": spec["nav"]["item"]}]
acts += [{"do": "click", "widget": c["widget"], "label": c.get("label") or c["widget"]} for c in spec.get("path", [])]
acts += [{"do": "click", "widget": spec["newButton"], "label": "New"}]
acts += [{"do": "fill", "widget": f["widget"], "value": f["value"]} for f in spec.get("fields", [])]
# a field with "do": "select" picks from a reference selector ({"match": caption} or {"first": true}); the rest fill
# (cold test WM1: an Office needs its Time zone, a reference selector, before Save is accepted)
acts += [{"do": "select", "widget": f["widget"], **({"match": f["match"]} if f.get("match") else {"first": True})}
if f.get("do") == "select" else {"do": "fill", "widget": f["widget"], "value": f["value"]} for f in spec.get("fields", [])]
acts += [{"do": "click", "widget": spec["saveButton"], "label": "Save"}]
ui = {"ready": spec.get("ready") or ".mx-page"}
for k in ("textPresent", "textAbsent"):
Expand All @@ -470,7 +473,7 @@ def create_journey(spec, data_rung=False):
# E2E-9: +1 says some row arrived; the own-row check says it is THIS run's row with the typed value
own = [{"label": f"the run's own row: {f['attr']} = {f['value']}", "expect": 1,
"sql": f"SELECT COUNT(*) AS n FROM {spec['entity']} WHERE {f['attr']} = '{f['value']}'"}
for f in spec.get("fields", []) if f.get("attr")]
for f in spec.get("fields", []) if f.get("attr") and f.get("do") != "select"]
if own:
step["data"]["oql"] = own
j = journey("create-save", f"Core create-and-save: {spec['entity']}", spec["role"],
Expand Down
4 changes: 3 additions & 1 deletion upgrade-toolkit/bin/run_journeys.py
Original file line number Diff line number Diff line change
Expand Up @@ -271,14 +271,16 @@
# of the role on the login form: Purchase Request pipe-19, 1 Requester journey ended it and the next 21 FAILed at
# "role home". Signing in again here is the reset between independent journeys, not a mid-walk recovery; it is
# said in the log, and a role whose sign-in fails again stops the walk at the next journey's first step as before.
# The anonymous role never signs in here: its home is the sign-in form, and signing in with no credentials put
# "username or password incorrect" on the page its journey checks (cold test WM1, anon-signin).
("reset: sign in again when the previous journey ended the session (PR-F26)",
"async function resetToStart(page) {\n await page.goto(cfg.baseUrl, { waitUntil: 'domcontentloaded' });\n"
" await page.waitForTimeout(2500);\n}",
"async function resetToStart(page) {\n await page.goto(cfg.baseUrl, { waitUntil: 'domcontentloaded' });\n"
" await page.waitForTimeout(2500);\n"
" const out = await page.locator('#loginButton, .mx-name-loginButton, input[type=password]').first()\n"
" .isVisible().catch(() => false);\n"
" if (out) {\n"
" if (out && !process.env.UPG_ANONYMOUS) { // anonymous role: the sign-in form IS its page (cold test WM1)\n"
" const li = await H.login(page);\n"
" console.log(` reset: the previous journey ended the session; signed in again: ${li.ok ? 'ok' : 'FAILED ' + li.reason}`);\n"
" }\n}"),
Expand Down
12 changes: 11 additions & 1 deletion upgrade-toolkit/controls/cockpit_control.sh
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,16 @@ PY
printf 'basewalk\t%s\t999999\n' "$(date +%s)" > "$W/state/running"; ck > /dev/null
echo "dead pid: hidden $(grep -c '<div class="nowrun" id="nowrun" hidden' "$T/out.html")"
rm "$W/state/running"; ck > /dev/null; echo "no file: hidden $(grep -c '<div class="nowrun" id="nowrun" hidden' "$T/out.html")" ;;
stopped) # a judgment stop (cold test WM1): a red bar with each NEEDS-JUDGMENT.md section and the rerun command, one
# needs-you row per section, and the matrix cell lists every section; no NEEDS-JUDGMENT.md = no bar
printf '# NEEDS JUDGMENT: step basewalk, exit 11\n\nwork: %s\nRule: the items below are pending, not passed: ..., then rerun:\n\n upgrade-pipeline.sh --work W --from basewalk\n\n## The smoke walk SKIPPED microflow steps (exit 11)\n\nRerun with UPG_SMOKE_MICROFLOWS=1.\n\n## Role Anonymous has no walk (exit 11)\n\nfile: x\n\n```\nAnonymous: none\n```\n' "$W" > "$W/NEEDS-JUDGMENT.md"; ck > /dev/null
echo "bar: $(grep -o '<div class="stopbar" role="alert"><b>[^<]*' "$T/out.html" | sed 's/.*<b>//')"
echo "sections: $(grep -o '<div class="stopbar".*</ul>' "$T/out.html" | grep -o '<li>[^<]*' | sed 's/<li>//' | paste -sd '|' -)"
echo "rerun: $(grep -o 'Then rerun: <code>[^<]*' "$T/out.html" | sed 's/.*<code>//')"
echo "rows: $(grep -o '"id": "stop:basewalk:[0-9]"' "$T/out.html" | sort -u | paste -sd ' ' -)"
echo "cell: $(grep -o '"l": "[0-9] decisions needed"' "$T/out.html" | head -1)"
echo "state: $(grep -o '"state": {[^}]*}' "$T/out.html")"
rm "$W/NEEDS-JUDGMENT.md"; ck > /dev/null; echo "no file: bars $(cnt -c 'class="stopbar"' "$T/out.html")" ;;
live-open) # cockpit_live as the pipeline runs it, with a stub opener that counts the opens
{ sed -n '/^say() /p' "$B/cloud/upgrade-pipeline.sh"; sed -n '/^cockpit_live() {/,/^}/p' "$B/cloud/upgrade-pipeline.sh"; } > "$T/fn.sh"
. "$T/fn.sh"; DRY=0; WORK=$W; mkdir -p "$W/state" "$W/logs"; rm -f "$W/state/triage.done"
Expand Down Expand Up @@ -193,7 +203,7 @@ ns=re.sub(r"<script>.*?</script>","",t,flags=re.S) # what a viewer without
chips={c["label"]:c["n"] for c in B["closed"]["chips"]}
if case=="static":
print("band without JS:", 'class="hero' in ns and "items closed successfully" in ns and "open topics still to validate" in ns)
print("headline without JS:", bool(re.search(r"<b>The app (runs|builds) on Mendix [\d.]+</b>|Upgrade to Mendix [\d.]+ in progress", ns)))
print("headline without JS:", bool(re.search(r"<b>The app (runs|builds) on Mendix [\d.]+</b>|Upgrade to Mendix [\d.]+ in progress|Stopped at \w+, waiting for you", ns)))
print("matrix cells without JS:", len(re.findall(r'data-i="\d+" data-s="', ns)), "of", 4*len(D["matrix"]["rows"]))
print("feed and readiness without JS:", ns.count("<li>")>0, ns.count('data-ready=')+ns.count('class="chip ok"')>0)
print("viewport meta:", 'name="viewport" content="width=device-width,initial-scale=1"' in ns)
Expand Down
6 changes: 6 additions & 0 deletions upgrade-toolkit/controls/run-controls.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2767,6 +2767,10 @@ ctl gen-create-data-rung-pass 0 '"creates": "Shop.Item"' -- sh -c "$GJ --out-dir
python3 -c "import json; d=json.load(open('$FX/journeys/create-spec.json')); d['fields'][0]['attr']='Name'; json.dump(d, open('$T/create-spec-attr.json', 'w'))"
ctl gen-create-own-row-pass 0 "SELECT COUNT\\(\\*\\) AS n FROM Shop.Item WHERE Name = 'Item \\$\\{RUN_ID\\}'" '"expect": 1' -- sh -c "$GJ --out-dir $T/gen-create3 --create $T/create-spec-attr.json --data-rung > /dev/null 2>&1; cat $T/gen-create3/journeys/Admin/create-save.journey.json"
ctl gen-create-own-row-missing-catch 0 'no field names its attribute.*own .*row' -- $GJ --out-dir $T/gen-create4 --create $FX/journeys/create-spec.json --data-rung
# cold test WM1: a "do": "select" field picks from a reference selector (by caption, else its first option) and is no own-row attribute
python3 -c "import json; d=json.load(open('$FX/journeys/create-spec.json')); d['fields'] += [{'widget':'referenceSelector2','do':'select','match':'Europe/Amsterdam'},{'widget':'referenceSelector3','do':'select'}]; json.dump(d, open('$T/create-spec-select.json', 'w'))"
ctl gen-create-select-pass 0 '"do": "select", "widget": "referenceSelector2", "match": "Europe/Amsterdam"' '"do": "select", "widget": "referenceSelector3", "first": true' '!referenceSelector2.*"value"' \
-- sh -c "$GJ --out-dir $T/gen-create-sel --create $T/create-spec-select.json > /dev/null 2>&1; python3 -c 'import json,sys; [print(json.dumps(a)) for st in json.load(open(sys.argv[1]))[\"steps\"] for a in st[\"actions\"]]' $T/gen-create-sel/journeys/Admin/create-save.journey.json"
# R1: a "path" in the spec puts its clicks between the menu item and the New button, in order (Vendors overview link)
python3 -c "import json; d=json.load(open('$FX/journeys/create-spec.json')); d['path']=[{'widget':'linkVendors','label':'Vendors overview'}]; json.dump(d, open('$T/create-spec-path.json', 'w'))"
ctl gen-create-path-pass 0 '"do": "nav"[^}]*}, \{"do": "click", "widget": "linkVendors", "label": "Vendors overview"}, \{"do": "click", "widget": "[A-Za-z0-9]+", "label": "New"}' -- sh -c "$GJ --out-dir $T/gen-create5 --create $T/create-spec-path.json > /dev/null 2>&1; python3 -c \"import json,sys; d=json.load(open('$T/gen-create5/journeys/Admin/create-save.journey.json')); print(json.dumps(d['steps'][0]['actions']))\""
Expand Down Expand Up @@ -6610,6 +6614,8 @@ ctl cockpit-db-pattern-pass 0 'has claude\.use\("db"\)' 'has claude\.use\("user"
ctl cockpit-live-script-only-with-flag-pass 0 'live: polls stamp 1' 'standalone without --live: polls stamp 0' 'content page: polls stamp 0' -- $CK live-script
ctl cockpit-serve-stamp-heartbeat-pass 0 'page served: True' 'heartbeat written: True' 'stamp moved after a change: True' -- $CK serve
ctl cockpit-running-bar-pass 0 'bar: Running now: Clicking through every page of the old app \(basewalk\) \| 5 min · now in runapp\.py · starting the old app on db_x' '"dot": "bad"' 'status: \{"step": "basewalk".*"tool": "runapp\.py"' 'dead pid: hidden 1' 'no file: hidden 1' -- $CK running
ctl cockpit-stopped-bar-pass 0 'bar: The run stopped at basewalk \(exit 11\) and waits for you\.' 'sections: The smoke walk SKIPPED microflow steps\|Role Anonymous has no walk$' \
'rerun: upgrade-pipeline\.sh --work W --from basewalk' 'rows: "id": "stop:basewalk:1" "id": "stop:basewalk:2"' 'cell: "l": "2 decisions needed"' 'no file: bars 0' -- $CK stopped
ctl cockpit-live-opens-after-triage-catch 0 'before triage: opened 0' 'stopped in triage: opened 1' 'after triage: opened 1' 'tab polling: opened 1' 'tab gone: opened 2' 'off: opened 2' -- $CK live-open
# doctor.sh: a dead docker socket is a FAIL with its fix, never a pass; a missing toolset names the version
ctl doctor-docker-down-catch 1 'FAIL docker' 'fix: ' 'doctor: [1-9][0-9]* FAIL' -- env DOCKER_HOST=unix://$T/no.sock bash $B/doctor.sh
Expand Down
3 changes: 2 additions & 1 deletion upgrade-toolkit/steps/journeys.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,8 @@ the instrument cannot fail: stop. The run exits 0 when every ui-landing row is P
counted as INFO UNPROVEN: they stay INVALID in the gate, but they are not control failures (E7-12).

create-spec.json, once per app: `{"role", "nav": {"group", "item"}, "newButton", "fields": [{"widget", "value", "attr"}],
"saveButton", "entity", "ready", "textPresent": [...], "textAbsent": [...]}`. Without it create-and-save stays a todo,
"saveButton", "entity", "ready", "textPresent": [...], "textAbsent": [...]}`. A reference selector is a field
`{"widget", "do": "select", "match": "<option caption>"}` (without `match`: its first option). Without it create-and-save stays a todo,
which is INVALID, never covered. `textPresent` is the saved value as the grid shows it, `textAbsent` the validation
messages: a ready selector alone was satisfied by the grid BEHIND a popup whose save was refused (E4-14, E5b-12), and
gen_journeys warns when the spec has no textPresent. Put `${RUN_ID}` in the field value and in textPresent: it is
Expand Down
Loading