-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_required_workflow_queue_contract.py
More file actions
1817 lines (1591 loc) · 75.8 KB
/
Copy pathtest_required_workflow_queue_contract.py
File metadata and controls
1817 lines (1591 loc) · 75.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""Verify central required-workflow queue, security, and dispatch contracts."""
import json
import os
import re
import shutil
import subprocess
import sys
import textwrap
import time
from pathlib import Path
import pytest
REPO_ROOT = Path(__file__).resolve().parents[1]
def workflow_text(name: str) -> str:
"""Read one central workflow for contract assertions."""
return (REPO_ROOT / ".github" / "workflows" / name).read_text(encoding="utf-8")
def workflow_level_concurrency_group(workflow: str) -> str:
"""Return only the workflow-level ``concurrency.group`` value, comments removed.
Asserting that an expression "appears in the concurrency block" is satisfied by
a comment that merely documents the key while the key itself says something
else, because the block's raw text carries its comments. That is not
hypothetical: the block above this workflow's group explains the key in prose,
so a maintainer quoting the expressions there while another change collapsed
the group to the repository alone would leave every pull request in one group,
cancelling each other, with the contract still green. Slice to the group's own
value so the assertion tests the key rather than the documentation beside it.
"""
header = workflow.split("permissions:", 1)[0]
block = header.split("concurrency:", 1)[1]
value: list[str] = []
collecting = False
for line in block.splitlines():
if line.strip().startswith("#"):
continue
if not collecting:
if re.match(r"^\s*group:", line):
collecting = True
value.append(line.split("group:", 1)[1])
continue
if re.match(r"^\s*[A-Za-z][\w-]*:", line):
break
value.append(line)
if not collecting:
raise AssertionError("workflow-level concurrency block declares no group")
return "\n".join(value)
def workflow_step(workflow: str, name: str) -> str:
"""Extract one named workflow step without parsing YAML dynamically."""
step = f" - name: {name}\n"
start = workflow.index(step)
try:
end = workflow.index("\n - name:", start + len(step))
except ValueError:
end = len(workflow)
return workflow[start:end]
def test_merge_scheduler_dispatches_one_review_by_default() -> None:
"""Keep the default scheduler dispatch bounded to one review."""
workflow = workflow_text("pr-review-merge-scheduler.yml")
assert workflow.count('default: "1"') >= 2
assert "vars.REVIEW_DISPATCH_LIMIT || '1'" in workflow
assert "SCHEDULER_ALLOW_CROSS_REPO_REPOSITORY_DISPATCH" in workflow
assert (
"secrets.PR_REVIEW_MERGE_TOKEN != '' || secrets.OPENCODE_APPROVE_TOKEN != ''"
in workflow
)
def test_scheduler_uses_bounded_run_state_without_cache_lock_claims() -> None:
"""Keep each run bounded without treating immutable cache snapshots as locks."""
workflow = workflow_text("pr-review-merge-scheduler.yml")
assert workflow.count(
'--admission-state-path "${RUNNER_TEMP}/review-admission/state.json"'
) == 1
assert workflow.count("--admission-dispatch-budget") == 1
assert workflow.count("--admission-sequence \"$GITHUB_RUN_ID\"") == 1
assert "actions/cache/restore" not in workflow
assert "actions/cache/save" not in workflow
assert "actions/upload-artifact" not in workflow
def test_organization_readiness_does_not_echo_untrusted_http_method(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Keep arbitrary HTTP method text out of organization-loop diagnostics."""
from types import SimpleNamespace
from scripts.ci.organization_commercial_readiness_loop import (
GitHubClient,
GitHubError,
)
token = "ghp_abcdefghijklmnopqrstuvwxyz0123456789AB"
monkeypatch.setattr(
"subprocess.run",
lambda *_args, **_kwargs: SimpleNamespace(
returncode=1,
stdout="",
stderr="request rejected",
),
)
with pytest.raises(GitHubError) as raised:
GitHubClient("client-token").request("/repos/example", method=token)
message = str(raised.value)
assert token.upper() not in message
assert "[REDACTED_METHOD]" in message
def test_merge_scheduler_rejects_untrusted_stale_timeout_values() -> None:
"""Dispatch payloads must not smuggle shell syntax into scheduler arguments."""
workflow = workflow_text("pr-review-merge-scheduler.yml")
assert workflow.count("STALE_OPENCODE_MINUTES must contain only decimal digits") == 1
assert workflow.count("STALE_OPENCODE_MINUTES must be between 1 and 1440") == 2
assert workflow.count("stale_opencode_minutes=$((10#$STALE_OPENCODE_MINUTES))") == 1
assert workflow.count('STALE_OPENCODE_MINUTES="$stale_opencode_minutes"') == 1
def test_merge_scheduler_uses_native_auto_merge_after_required_checks() -> None:
"""Do not enqueue a scheduler run after every required workflow completion."""
workflow = workflow_text("pr-review-merge-scheduler.yml")
concurrency_contract = workflow.split("concurrency:", 1)[1].split(
"permissions:", 1
)[0]
assert "org-sweep" not in concurrency_contract
assert "format('repo-dispatch-{0}', github.repository)" in concurrency_contract
assert "workflow_run:" not in workflow.split("workflow_call:", 1)[0]
assert "github.event.workflow_run" not in concurrency_contract
assert "github.event_name == 'repository_dispatch' && github.run_id" not in (
concurrency_contract
)
assert "cancel-in-progress: ${{" in concurrency_contract
assert "github.event_name == 'repository_dispatch'" in concurrency_contract
def test_merge_scheduler_provides_same_repository_dispatch_credential() -> None:
"""Guard the runner-token dispatch credential for central review workflows.
The scheduler runs inside the same repository as the central required
workflows, so its repository-scoped token is the single dispatch credential.
"""
workflow = workflow_text("pr-review-merge-scheduler.yml")
assert workflow.count("SCHEDULER_DISPATCH_TOKEN: ${{ github.token }}") == 1
def test_targeted_scheduler_dispatch_is_allowlisted_and_exact_pr_scoped() -> None:
"""Central single-PR dispatch accepts a bounded fork head without trusting it."""
workflow = workflow_text("pr-review-merge-scheduler.yml")
validation = workflow_step(workflow, "Validate targeted repository dispatch")
inspect = workflow_step(workflow, "Inspect PR review and merge queue")
assert "TARGET_REPOSITORY_INPUT:" in validation
assert "TARGET_PR_NUMBER:" in validation
assert "TARGET_BASE_BRANCH_INPUT:" in validation
assert (
"ALLOWED_TARGET_REPOSITORIES: ${{ "
"vars.OPENCODE_REPOSITORY_DISPATCH_TARGETS }}"
) in validation
assert 'GITHUB_REPOSITORY" != "ContextualWisdomLab/.github"' in validation
assert "target_allowed=0" in validation
assert '"repos/${TARGET_REPOSITORY_INPUT}/pulls/${TARGET_PR_NUMBER}"' in validation
assert '[ "$live_state" != "open" ]' in validation
assert '[ "$live_base_repository" != "$TARGET_REPOSITORY_INPUT" ]' in validation
assert 'target_default_branch="$(gh api "repos/${TARGET_REPOSITORY_INPUT}" --jq' in validation
assert 'printf \'base_branch=%s\\n\' "$target_default_branch"' in validation
assert "PR base %s; scheduler default branch %s" in validation
assert (
'! [[ "$live_head_repository" =~ '
'^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]'
) in validation
assert '[ "$live_head_repository" != "$TARGET_REPOSITORY_INPUT" ]' not in validation
assert "Targeted scheduler dispatch base branch does not match the live PR" in validation
assert "TARGET_REPOSITORY: ${{ steps.targeted_dispatch.outputs.repository }}" in inspect
assert (
"TARGET_DEFAULT_BRANCH: ${{ steps.targeted_dispatch.outputs.base_branch }}"
in inspect
)
assert '--repo "$TARGET_REPOSITORY"' in inspect
assert '--base-branch "$TARGET_DEFAULT_BRANCH"' in inspect
assert 'args+=(--pr-number "$PULL_REQUEST_NUMBER")' in inspect
assert (
"github.event_name == 'repository_dispatch' && "
"github.event.client_payload.target_repository != '' && "
"github.event.client_payload.target_repository != github.repository && "
"(secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || "
"steps.scheduler_app_token.outputs.token) || github.token"
) in inspect
assert (
"format('target-{0}-pr-{1}', "
"github.event.client_payload.target_repository, "
"github.event.client_payload.pr_number)"
) in workflow
def test_privileged_review_retries_use_default_branch_repository_dispatch() -> None:
"""Privileged retries must never load workflow code from a selected ref."""
expected_types = {
"opencode-review-dispatch.yml": "opencode-review",
"noema-review.yml": "noema-review",
"strix.yml": "strix-scan",
"pr-review-merge-scheduler.yml": "merge-scheduler",
}
for filename, event_type in expected_types.items():
workflow = workflow_text(filename)
trigger_contract = workflow.split("concurrency:", 1)[0]
assert "repository_dispatch:" in trigger_contract
assert f"types: [{event_type}]" in trigger_contract
assert "workflow_dispatch:" not in trigger_contract
assert "github.event.inputs" not in workflow
assert "github.event.client_payload" in workflow
scheduler = (
REPO_ROOT / "scripts" / "ci" / "pr_review_merge_scheduler.py"
).read_text(encoding="utf-8")
assert 'f"repos/{dispatch_repo}/dispatches"' in scheduler
assert '"event_type": "opencode-review"' in scheduler
assert '"event_type": "strix-scan"' in scheduler
autofix_workflow = workflow_text("pr-review-autofix.yml")
assert "repository_dispatch:" in autofix_workflow
assert "types: [pr-review-autofix]" in autofix_workflow
assert "workflow_dispatch:" not in autofix_workflow
assert "github.event.client_payload" in autofix_workflow
autofix_scheduler = (
REPO_ROOT / "scripts" / "ci" / "pr_review_fix_scheduler.py"
).read_text(encoding="utf-8")
assert 'f"repos/{dispatch_repo}/dispatches"' in autofix_scheduler
assert 'AUTOFIX_REPOSITORY_DISPATCH_TYPE = "pr-review-autofix"' in autofix_scheduler
assert '"gh",\n "workflow",\n "run"' not in autofix_scheduler
def test_privileged_review_dispatch_coalesces_superseded_runs_before_admission() -> None:
"""A superseded dispatch must be cancelled while queued, not after it takes a runner.
``opencode-review-dispatch.yml`` carried its concurrency group only on the
long ``opencode-review-target`` job. A job-level group is not evaluated
while the whole run waits behind the organization job ceiling, so two
dispatches for one pull request each waited hours and each was allocated a
runner before the older one could be discarded. Measured on 2026-09-06:
four of the five dispatch runs that passed ``validate-pr-metadata`` were
then rejected by the privileged metadata check because the head had moved
while they queued, every one of them after ``coverage-source-tree`` and
``coverage-evidence`` had already run.
The workflow-level group is keyed by the dispatched pull request, matching
``codeql-scan-dispatch.yml``'s workflow-level group and the job-level group
this workflow keeps for the review job itself.
"""
workflow = workflow_text("opencode-review-dispatch.yml")
header = workflow.split("permissions:", 1)[0]
concurrency_contract = header.split("concurrency:", 1)[1]
group_value = workflow_level_concurrency_group(workflow)
assert re.search(r"(?m)^concurrency:", header)
assert "opencode-review-dispatch-" in group_value
assert (
"github.event.client_payload.target_repository || github.repository"
in group_value
)
assert "github.event.client_payload.pr_number || github.run_id" in group_value
assert "cancel-in-progress: true" in concurrency_contract
assert "github.event.client_payload.pr_head_sha" not in concurrency_contract
assert re.search(r"(?m)^ concurrency:", workflow)
@pytest.mark.parametrize(
("workflow_name", "group_prefix"),
(
("agent-mention-opencode-dispatch.yml", "agent-mention-opencode-"),
("agent-mention-noema-dispatch.yml", "agent-mention-noema-"),
),
)
def test_agent_mention_dispatch_coalesces_while_queued(
workflow_name: str, group_prefix: str
) -> None:
"""A superseded agent mention must be discarded before it holds a queue slot.
Both mention dispatchers carried the same defect
``opencode-review-dispatch.yml`` carried before #1958: the group sat on the
single ``validate-and-forward`` job, and a job-level group is not evaluated
while the run waits behind the organization job ceiling. Measured on the
review dispatcher over the 39.7 hours ending 2026-09-06T12:41Z, 23 pairs of
runs for one pull request overlapped -- the older run was still open when its
successor arrived -- and none was coalesced; the five that ended
``cancelled`` were cancelled between 0.7 and 2.9 hours after the newer run
was created, which is a sweep, not concurrency.
The group moves to workflow level and is not duplicated on the job. Every
workflow here that keys a group at both levels (``strix.yml``,
``opencode-review-dispatch.yml``) gives the two levels different names,
because a job that requests the group its own run already holds waits on
itself.
"""
workflow = workflow_text(workflow_name)
header = workflow.split("permissions:", 1)[0]
group = workflow_level_concurrency_group(workflow)
assert re.search(r"(?m)^concurrency:", header)
# Read the group's value, not the block: the comment above these keys quotes
# the very expressions asserted here, so a raw-block assertion would survive
# the key being collapsed. That is the hole #1970 closed.
assert group.strip().startswith(group_prefix)
assert "github.event.client_payload.target_repository" in group
assert "github.event.client_payload.pr_number || github.run_id" in group
# ``cancel-in-progress`` is a sibling key, so it is outside the group value.
# Anchor it to its own line at the block's indent; a comment starts with
# ``#`` and cannot satisfy this.
assert re.search(r"(?m)^ cancel-in-progress: true$", header)
# ``\s`` also matches the newline before a column-0 key, so anchor the
# job-level search on horizontal whitespace only.
assert not re.search(r"(?m)^[ \t]+concurrency:", workflow)
def test_agent_mention_router_keeps_its_two_distinct_job_groups() -> None:
"""The router must not be hoisted: its two jobs need different groups.
``agent-mention-router.yml`` runs a per-issue local route that supersedes
itself and an organization-wide sweep that must never be cancelled midway.
A workflow carries at most one workflow-level group, so hoisting either one
would silently give the sweep the route's ``cancel-in-progress: true`` and
let a later comment kill a sweep that is part way through the organization.
"""
workflow = workflow_text("agent-mention-router.yml")
assert not re.search(r"(?m)^concurrency:", workflow)
assert (
"group: review-agent-mention-router-local-${{ github.repository }}"
in workflow
)
assert "group: review-agent-mention-router-sweep-${{ github.repository }}" in workflow
sweep = workflow.split("sweep-organization-agent-mentions:", 1)[1]
assert "cancel-in-progress: false" in sweep.split("steps:", 1)[0]
def test_concurrency_group_slice_ignores_the_comment_that_documents_it() -> None:
"""A comment quoting the key must not satisfy an assertion about the key.
This is the negative control for ``workflow_level_concurrency_group``. The
synthetic workflow below is exactly the shape that defeated the previous
contract: the real group is collapsed to the repository alone, so every pull
request in that repository shares one group and they cancel each other, while
a comment directly above still quotes both expressions the contract looks for.
Reading the raw block finds them; reading the group's value does not.
"""
defeated = textwrap.dedent(
"""\
name: Example
on:
repository_dispatch:
concurrency:
# Key: github.event.client_payload.target_repository || github.repository
# with github.event.client_payload.pr_number || github.run_id
group: opencode-review-dispatch-${{ github.repository }}
cancel-in-progress: true
permissions:
contents: read
"""
)
raw_block = defeated.split("permissions:", 1)[0].split("concurrency:", 1)[1]
group_value = workflow_level_concurrency_group(defeated)
assert "github.event.client_payload.pr_number || github.run_id" in raw_block
assert "github.event.client_payload.pr_number || github.run_id" not in group_value
assert "github.event.client_payload.target_repository" not in group_value
assert "opencode-review-dispatch-${{ github.repository }}" in group_value
def test_concurrency_group_slice_reads_a_folded_multi_line_key() -> None:
"""The real key is a folded block, so the slice must join its continuation lines."""
folded = textwrap.dedent(
"""\
concurrency:
group: >-
opencode-review-dispatch-${{
github.event.client_payload.target_repository || github.repository }}-${{
github.event.client_payload.pr_number || github.run_id }}
cancel-in-progress: true
permissions:
contents: read
"""
)
group_value = workflow_level_concurrency_group(folded)
assert (
"github.event.client_payload.target_repository || github.repository"
in group_value
)
assert "github.event.client_payload.pr_number || github.run_id" in group_value
assert "cancel-in-progress" not in group_value
def test_required_opencode_dispatch_does_not_wait_on_merge_scheduler() -> None:
"""Dispatch review execution directly so polling cannot starve its producer."""
workflow = workflow_text("opencode-review.yml")
dispatch = workflow_step(workflow, "Request current-head OpenCode review execution")
assert 'event_type:"opencode-review"' in dispatch
assert 'event_type:"merge-scheduler"' not in dispatch
assert 'required_run_id:$required_run_id' in dispatch
for field in (
"target_repository",
"pr_number",
"pr_base_ref",
"pr_base_sha",
"pr_head_ref",
"pr_head_sha",
):
assert f"{field}:${field}" in dispatch
def test_no_central_workflow_exposes_branch_selected_manual_dispatch() -> None:
"""Every central manual entrypoint must load code from the default branch."""
workflow_files = sorted((REPO_ROOT / ".github" / "workflows").glob("*.yml"))
offenders = [
path.name
for path in workflow_files
if "workflow_dispatch:" in path.read_text(encoding="utf-8")
]
assert offenders == []
def test_required_pull_request_workflows_cancel_superseded_runs() -> None:
"""Ensure required pull-request workflows cancel obsolete executions."""
for filename in (
"codeql-pr.yml",
"noema-review.yml",
"opencode-review.yml",
"security-scan.yml",
):
workflow = workflow_text(filename)
concurrency_contract = workflow.split("concurrency:", 1)[1].split(
"permissions:", 1
)[0]
group_value = workflow_level_concurrency_group(workflow)
assert "concurrency:" in workflow
assert "github.event.pull_request.base.repo.full_name" in group_value
assert "github.repository" in group_value
assert "github.event.pull_request.number" in workflow
assert re.search(r"(?m)^concurrency:", workflow)
assert "cancel-in-progress: true" in concurrency_contract
if filename == "security-scan.yml":
assert (
"github.event_name == 'pull_request_target'" in group_value
or ("github.event_name == 'pull_request'" in group_value)
)
elif filename == "opencode-review.yml":
assert "required-opencode-review-${{" in group_value
assert "outputs.admitted == 'true'" in workflow
elif filename == "noema-review.yml":
assert not re.search(r"(?m)^ concurrency:", workflow)
assert "github.event.workflow_run" not in concurrency_contract
assert "required-noema-review-${{" in group_value
assert "outputs.admitted == 'true'" in workflow
else:
if filename == "codeql-pr.yml":
assert "github.event_name == 'pull_request'" in group_value
else:
assert "github.event_name == 'pull_request_target'" in group_value
assert "github.event.pull_request.head.sha" not in concurrency_contract
assert "format('pr-{0}-{1}'" not in concurrency_contract
def test_pr_quality_workflows_isolate_concurrency_by_repository_and_pr() -> None:
"""Quality runs from different repositories must never share a PR queue."""
groups = {
"agent-mention-router-quality-ci.yml": "agent-mention-router-quality",
"cloudflare-dns.yml": "cloudflare-dns",
"javascript-coverage-quality-ci.yml": "javascript-coverage-quality",
"trusted-uv-materializer-quality-ci.yml": (
"trusted-uv-materializer-quality"
),
}
for filename, group_name in groups.items():
workflow = workflow_text(filename)
concurrency = workflow.split("concurrency:", 1)[1].split("jobs:", 1)[0]
assert (
f"group: {group_name}-${{{{ github.repository }}}}-"
"${{ github.event.pull_request.number || github.ref }}"
) in concurrency
if filename == "cloudflare-dns.yml":
assert (
"cancel-in-progress: ${{ github.event_name == 'pull_request' }}"
in concurrency
)
else:
assert "cancel-in-progress: true" in concurrency
def test_central_semgrep_logs_every_finding_and_distinguishes_engine_failure() -> None:
"""Keep Semgrep finding output distinct from scanner-engine failures."""
workflow = workflow_text("sast-semgrep.yml")
assert "Report every Semgrep finding in the job log" in workflow
assert "--exclude='docs/research/**/standards'" in workflow
assert "SEMGREP_FINDING_COUNT=" in workflow
assert "SEMGREP_FINDING rule=" in workflow
assert 'level=\\(.level // $levels[.ruleId] // "unknown")' in workflow
assert 'path=\\($location.artifactLocation.uri // "unknown")' in workflow
assert "line=\\($location.region.startLine // 0)" in workflow
assert "message=" in workflow
assert "SEMGREP_ENGINE_FAILURE rc=" in workflow
assert "semgrep_sarif.outputs.finding_count != '0'" in workflow
assert 'if [ "${SEMGREP_FINDING_COUNT:-missing}" != "0" ]' in workflow
assert "Every rule, path, line, and message is listed" in workflow
assert "Semgrep engine/configuration failed with rc=${SEMGREP_RC}" in workflow
def test_central_semgrep_binds_pr_scans_and_sarif_to_the_exact_head() -> None:
"""Reject GitHub's synthetic merge as SAST source or SARIF identity."""
workflow = workflow_text("sast-semgrep.yml")
checkout = workflow_step(workflow, "Checkout exact submitted revision")
verify = workflow_step(workflow, "Verify exact submitted revision")
upload = workflow_step(workflow, "Upload Semgrep SARIF to code scanning")
assert (
"repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}"
in checkout
)
assert (
"ref: ${{ github.event.pull_request.head.sha || github.sha }}" in checkout
)
assert "persist-credentials: false" in checkout
assert (
"EXPECTED_CHECKOUT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}"
in verify
)
assert 'actual_sha="$(git rev-parse HEAD)"' in verify
assert 'if [ "$actual_sha" != "$EXPECTED_CHECKOUT_SHA" ]; then' in verify
assert "exit 1" in verify
assert (
"ref: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/head', github.event.pull_request.number) || github.ref }}"
in upload
)
assert (
"sha: ${{ github.event.pull_request.head.sha || github.sha }}" in upload
)
def test_strix_serializes_provider_evidence_per_repository_and_pr() -> None:
"""Scope Strix workflow admission per repository AND PR.
History: from 2026-08-24 through 2026-09-03 the concurrency group was
deliberately repository-wide (not PR-scoped) because PR-scoping is what
caused a real litellm.RateLimitError storm against the shared NVIDIA NIM
key on 2026-08-23/24 -- sibling PRs scanned concurrently, each retrying the
shared key three times, producing fail-closed gate failures on every open
PR. That repository-wide scoping fixed the storm but starved cross-PR
Strix evidence within the same repository instead (a different PR's scan
always queued behind whichever scan was already running there).
Restored to PR-scoped on explicit owner authorization (2026-09-03) after
confirming NVIDIA_NIM_API_KEY and NVIDIA_NIM_API_KEY_SUB have independent
rate limits rather than a shared pool. The workflow-level group now retires
superseded runs before runner admission, including runs still blocked by
the organization-wide job ceiling. Native and dispatched evidence share
one group; non-PR events use a unique run id.
"""
workflow = workflow_text("strix.yml")
concurrency_contract = workflow.split("concurrency:", 1)[1].split(
"permissions:", 1
)[0]
strix_job = workflow.split("\n strix:\n", 1)[1]
group_value = workflow_level_concurrency_group(workflow)
assert re.search(r"(?m)^concurrency:", workflow)
assert "needs: [changed-scope, admit-current-head]" in strix_job
assert "needs.admit-current-head.outputs.admitted == 'true'" in strix_job
assert "strix-security-scan-${{" in group_value
assert "github.event.pull_request.base.repo.full_name" in group_value
assert "github.event.client_payload.target_repository" in group_value
assert "github.event.pull_request.number" in group_value
assert "github.event.client_payload.pr_number" in group_value
assert "github.run_id" in group_value
assert "github.event.pull_request.head.sha" not in concurrency_contract
assert "github.event.client_payload.pr_head_sha" not in concurrency_contract
assert "cancel-in-progress: true" in concurrency_contract
assert " concurrency:" not in strix_job.split(" permissions:", 1)[0]
assert "queue: max" not in workflow
assert workflow.index("admit-current-head:") < workflow.index("\n strix:\n")
cleanup_job = workflow.split(" cancel-superseded-pr-runs:", 1)[1].split(
" strix:", 1
)[0]
assert "github.event.action == 'synchronize'" in cleanup_job
assert 'endswith("@" + $head_sha)' in cleanup_job
assert "/force-cancel" in cleanup_job
assert 'gh api "repos/${TARGET_REPOSITORY}/pulls/${TARGET_PR_NUMBER}"' in cleanup_job
assert "could not verify the live pull request" in cleanup_job
assert "target changed before run selection" in cleanup_job
assert "target changed before cancellation" in cleanup_job
assert cleanup_job.index("if ! live_target_matches") < cleanup_job.index(
'runs_url="repos/${TARGET_REPOSITORY}/actions/runs?status=${status}&per_page=100"'
)
assert cleanup_job.rindex("if ! live_target_matches") < cleanup_job.index(
'gh api --method POST "repos/${TARGET_REPOSITORY}/actions/runs/${run_id}/cancel"'
)
assert "actions: write" in cleanup_job
assert "pull-requests: read" in cleanup_job
assert "actions/checkout" not in cleanup_job
assert (
"refs/pull/<n>/head has already advanced before this queued run starts"
in workflow
)
def test_strix_install_normalizes_executable_permissions_before_hashing() -> None:
"""Normalize the Strix executable before its trusted hash is computed."""
workflow = workflow_text("strix.yml")
install_step = workflow_step(workflow, "Install Strix")
assert install_step.index("umask 022") < install_step.index(
"python3 -m pip install"
)
permission_normalization = 'chmod go-w -- "$strix_scripts_root" "$strix_executable"'
assert install_step.index('strix_scripts_root="') < install_step.index(
permission_normalization
)
assert install_step.index(permission_normalization) < install_step.index(
'strix_executable_sha256="'
)
def test_strix_cleanup_uses_pr_metadata_when_custom_title_is_absent() -> None:
"""Required-workflow runs retain exact PR/head cleanup without run-name rendering."""
jq = shutil.which("jq")
if jq is None:
pytest.skip("jq is required to execute the production cleanup selector")
workflow = workflow_text("strix.yml")
marker = '--arg action "$PR_ACTION" --arg repo "$TARGET_REPOSITORY" --arg current "$CURRENT_RUN_ID" \'\n'
start = workflow.index(marker) + len(marker)
end = workflow.index('\n \' <<<"$runs_json"', start)
runs = {
"workflow_runs": [
{"id": 1, "name": "Strix Security Scan", "event": "pull_request_target", "pull_requests": [{"number": 7, "head": {"sha": "old"}}]},
{"id": 2, "name": "Strix Security Scan", "event": "pull_request_target", "pull_requests": [{"number": 7, "head": {"sha": "current"}}]},
{"id": 3, "name": "Strix Security Scan", "event": "pull_request_target", "pull_requests": [{"number": 7}]},
{"id": 4, "name": "Strix Security Scan", "event": "pull_request_target", "display_title": "Strix Security Scan owner/repo#7@old", "pull_requests": [{"number": 7, "head": {"sha": "current"}}]},
{"id": 5, "name": "Strix Security Scan", "event": "pull_request_target", "pull_requests": [{"number": 8, "head": {"sha": "old"}}]},
]
}
result = subprocess.run(
[jq, "-r", "--arg", "pr", "7", "--arg", "head_sha", "current", "--arg", "action", "synchronize", "--arg", "repo", "owner/repo", "--arg", "current", "99", workflow[start:end]],
input=json.dumps(runs),
text=True,
capture_output=True,
check=True,
)
assert result.stdout.splitlines() == ["1"]
def _run_strix_cleanup(
tmp_path: Path, pull_states: list[dict[str, object]], *, action: str = "synchronize"
) -> str:
"""Execute the production cleanup step against a stateful fake ``gh``."""
jq = shutil.which("jq")
if jq is None:
pytest.skip("jq is required to execute the production cleanup")
step = workflow_step(
workflow_text("strix.yml"),
"Cancel queued and running scans for superseded or inactive pull requests",
)
run_block = step.split(" run: |\n", 1)[1].split("\n strix:", 1)[0]
script = textwrap.dedent(run_block)
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
calls = tmp_path / "calls"
pulls = tmp_path / "pulls"
pulls.write_text(
"\n".join(json.dumps(state) for state in pull_states) + "\n",
encoding="utf-8",
)
fake_gh = fake_bin / "gh"
fake_gh.write_text(
"""#!/usr/bin/env bash
set -euo pipefail
printf '%s\n' "$*" >>"$FAKE_CALLS"
if [[ "$*" == *"/pulls/7"* ]]; then
count_file="${FAKE_PULLS}.count"
count=0
[[ ! -f "$count_file" ]] || count="$(cat "$count_file")"
count=$((count + 1))
printf '%s' "$count" >"$count_file"
sed -n "${count}p" "$FAKE_PULLS"
exit 0
fi
if [[ "$*" == *"actions/runs?status=queued"* ]]; then
printf '%s\n' '{"workflow_runs":[{"id":100,"name":"Strix Security Scan","event":"pull_request_target","pull_requests":[{"number":7,"head":{"sha":"old"}}]}]}'
exit 0
fi
if [[ "$*" == *"actions/runs?status="* ]]; then
printf '%s\n' '{"workflow_runs":[]}'
exit 0
fi
exit 0
""",
encoding="utf-8",
)
fake_gh.chmod(0o755)
env = {
**os.environ,
"PATH": f"{fake_bin}{os.pathsep}{os.environ['PATH']}",
"FAKE_CALLS": str(calls),
"FAKE_PULLS": str(pulls),
"TARGET_REPOSITORY": "owner/repo",
"TARGET_PR_NUMBER": "7",
"TARGET_PR_HEAD_SHA": "current",
"PR_ACTION": action,
"CURRENT_RUN_ID": "999",
}
subprocess.run(["bash", "-c", script], env=env, check=True, capture_output=True, text=True)
return calls.read_text(encoding="utf-8")
def test_old_strix_cleanup_never_lists_or_cancels_after_live_head_advanced(
tmp_path: Path,
) -> None:
"""A late old synchronize job must stop before selecting current runs."""
calls = _run_strix_cleanup(
tmp_path, [{"state": "open", "head": {"sha": "newer"}}] * 5
)
assert "actions/runs?status=" not in calls
assert "/cancel" not in calls
assert "/force-cancel" not in calls
def test_strix_cleanup_revalidates_after_selection_before_cancellation(
tmp_path: Path,
) -> None:
"""A head advance after selection must prevent the pending mutation."""
calls = _run_strix_cleanup(
tmp_path,
[
{"state": "open", "draft": False, "head": {"sha": "current"}},
{"state": "open", "draft": False, "head": {"sha": "newer"}},
]
+ [{"state": "open", "draft": False, "head": {"sha": "newer"}}] * 4,
)
assert "actions/runs?status=queued" in calls
assert "/actions/runs/100/cancel" not in calls
assert "/actions/runs/100/force-cancel" not in calls
def test_strix_draft_transition_cancels_current_scan(tmp_path: Path) -> None:
"""A verified Draft transition retires the current expensive Strix run."""
calls = _run_strix_cleanup(
tmp_path,
[{"state": "open", "draft": True, "head": {"sha": "current"}}] * 6,
action="converted_to_draft",
)
assert "/actions/runs/100/cancel" in calls
def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() -> None:
"""Close events should cancel old runs without starting expensive jobs."""
workflows = (
"codeql-pr.yml",
"noema-review.yml",
"pr-review-merge-scheduler.yml",
"python-security.yml",
"sast-semgrep.yml",
"security-scan.yml",
"strix.yml",
)
for filename in workflows:
workflow = workflow_text(filename)
assert "closed" in workflow
if filename == "strix.yml":
assert "cancel-superseded-pr-runs:" in workflow
assert "Cancel queued and running scans for superseded or inactive pull requests" in workflow
assert (
"secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN "
"|| github.token"
) in workflow
assert "DISPATCH_REPOSITORY" not in workflow
assert "TARGET_PR_HEAD_SHA" in workflow
assert 'select(.event == "pull_request_target")' in workflow
assert 'select(.event == "repository_dispatch")' not in workflow
assert "(.pull_requests // [])" in workflow
assert ".head.sha // \"\"" in workflow
assert "leaving runs unchanged" in workflow
assert (
"for active_status in queued in_progress requested waiting pending"
in workflow
)
cleanup_job = workflow.split(" cancel-superseded-pr-runs:", 1)[1].split(
" strix:", 1
)[0]
elif filename == "noema-review.yml":
assert "cancel-closed-pr-runs:" in workflow
assert "Cancel queued and running Noema reviews for the inactive pull request" in workflow
assert "leaving runs unchanged" in workflow
cleanup_job = workflow.split(" cancel-closed-pr-runs:", 1)[1].split(
" noema-review:", 1
)[0]
assert "actions: write" in cleanup_job
assert "actions/checkout" not in cleanup_job
assert "cleanup skipped" not in cleanup_job
elif filename in {
"codeql-pr.yml",
"pr-review-merge-scheduler.yml",
"python-security.yml",
"sast-semgrep.yml",
"security-scan.yml",
}:
assert "cancel-closed-pr-runs:" not in workflow
concurrency_contract = workflow.split("concurrency:", 1)[1].split(
"permissions:", 1
)[0]
assert "github.event.pull_request.number" in concurrency_contract
assert "github.event.pull_request.head.sha" not in concurrency_contract
assert "cancel-in-progress:" in concurrency_contract
else:
raise AssertionError(f"unclassified close-event workflow: {filename}")
assert "github.event.action != 'closed'" in workflow
if filename in {"noema-review.yml", "strix.yml"}:
assert "github.event.action != 'converted_to_draft'" in workflow
opencode_bootstrap = workflow_text("opencode-review.yml")
assert "types: [opened, synchronize, reopened, ready_for_review, converted_to_draft, closed]" in (
opencode_bootstrap
)
assert "actions/checkout" not in opencode_bootstrap
assert "${{ secrets." not in opencode_bootstrap
strix_workflow = workflow_text("strix.yml")
# Strix admits the live head before same-PR cancellation while cleanup stays
# outside that queue so synchronize and close events can retire old work.
assert "admit-current-head:" in strix_workflow
assert "skipping stale evidence" in strix_workflow
assert "cancel-in-progress: true" in strix_workflow
def test_merge_scheduler_owns_empty_pr_cleanup_without_checkout() -> None:
"""Keep empty-PR cleanup in the existing metadata-only scheduler job."""
workflow = workflow_text("pr-review-merge-scheduler.yml")
scheduler = workflow_step(workflow, "Inspect PR review and merge queue")
assert not (REPO_ROOT / ".github/workflows/close-empty-pr.yml").exists()
assert "pr_review_merge_scheduler.py" in scheduler
assert "actions/checkout" not in workflow
def test_review_workflow_completions_do_not_spawn_scheduler_runs() -> None:
"""Required checks rely on GitHub auto-merge instead of a follow-up workflow."""
workflow = workflow_text("pr-review-merge-scheduler.yml")
assert "github.event.workflow_run" not in workflow
def test_required_workflow_trusted_source_refs_are_not_input_controlled() -> None:
"""Ensure privileged workflows resolve trusted source code independently of inputs."""
for filename in (
"opencode-review-dispatch.yml",
"noema-review.yml",
"pr-review-merge-scheduler.yml",
):
workflow = workflow_text(filename)
assert "canonical_ref:" not in workflow
assert "INPUT_CANONICAL_REF" not in workflow
assert "github.event.client_payload.canonical_ref" not in workflow
assert "inputs.canonical_ref" not in workflow
assert "workflow_sha" in workflow
if filename == "opencode-review-dispatch.yml":
assert "ref: ${{ steps.trusted_source.outputs.ref }}" in workflow
assert "ref: ${{ github.workflow_sha }}" not in workflow
else:
assert (
"ref: ${{ github.workflow_sha }}" in workflow
or "TRUSTED_SOURCE_REF: ${{ steps.trusted_source.outputs.ref }}"
in workflow
)
assert "JOB_CONTEXT_JSON: ${{ toJSON(job) }}" in workflow
assert "GITHUB_CONTEXT_JSON: ${{ toJSON(github) }}" in workflow
def test_noema_triggers_preserve_standalone_pull_request_review() -> None:
"""Noema reviews PRs independently of the other review workflows."""
workflow = workflow_text("noema-review.yml")
noema_job = workflow.split("\n noema-review:\n", 1)[1]
concurrency_contract = workflow.split("\nconcurrency:\n", 1)[1].split(
"\npermissions:\n", 1
)[0]
assert "workflow_run:" not in concurrency_contract
assert "github.event.workflow_run" not in workflow
assert "github.event.pull_request.number" in concurrency_contract
assert "github.event.client_payload.pr_number" in concurrency_contract
assert "required-noema-review-${{" in concurrency_contract
assert "github.event_name" not in concurrency_contract.split(
"cancel-in-progress:", 1
)[0]
assert "cancel-in-progress: true" in concurrency_contract
assert re.search(r"(?m)^concurrency:", workflow)
assert not re.search(r"(?m)^ concurrency:", workflow)
assert "needs.admit-current-head.outputs.admitted == 'true'" in noema_job
assert '[ "${live_head_sha,,}" != "${EXPECTED_HEAD_SHA,,}" ]' in workflow
def test_noema_review_credentials_and_orchestrator_configuration_fail_closed() -> None:
"""Require explicit reviewer credentials and the trusted orchestrator sidecar."""
workflow = workflow_text("noema-review.yml")
assert "fail_unavailable()" in workflow
assert 'echo "::error::$message"' in workflow
assert "vars.NOEMA_TOKEN_EXCHANGE_URL || vars.NOEMA_EXCHANGE_URL || ''" in workflow
assert (
"Noema reviewer credential is unconfigured: set NOEMA_GITHUB_APP_CLIENT_ID with "
"NOEMA_GITHUB_APP_PRIVATE_KEY, NOEMA_REVIEW_TOKEN, or NOEMA_TOKEN_EXCHANGE_URL. "
"Review cannot be skipped."
) in workflow
assert (
"Noema app token exchange unavailable: OIDC request environment is missing."
in workflow
)
assert (
"Noema app token exchange unavailable: OIDC token request did not complete."
in workflow
)
assert (
"Noema app token exchange unavailable: OIDC token response was empty."
in workflow
)
assert (
"Noema app token exchange unavailable: app token request did not complete."
in workflow
)
assert (
"Noema app token exchange unavailable: app token response was empty."
in workflow
)
assert (
"Noema reviewer credential selection succeeded but no token was minted"
in workflow
)
assert "Resolve Noema target repository visibility" in workflow
assert "target_visibility.outputs.require_zdr" in workflow
assert "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" in workflow
assert "https://integrate.api.nvidia.com/v1/chat/completions" not in workflow
assert "nvidia/nemotron-3-ultra-550b-a55b" not in workflow
assert "contextual_orchestrator_review_sidecar.sh" in workflow
assert 'export NOEMA_LLM_MODEL="orchestrator/free"' in workflow
assert (
"contextual-orchestrator review sidecar must be provisioned before Noema LLM review."
in workflow
)
assert "BYTEZ_API_KEY: ${{ secrets.BYTEZ_API_KEY }}" in workflow
assert "NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}" in workflow
assert "NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }}" in workflow
assert "OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}" in workflow
assert "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}" in workflow
assert "COPILOT_GITHUB_TOKEN" not in workflow
assert "secrets: inherit" not in workflow
assert "mark_unconfigured()" not in workflow
assert "review skipped until Noema is deployed" not in workflow
assert "Noema app token is unavailable; review skipped." not in workflow
def test_strix_gateway_default_and_noema_sidecar_fail_closed(
tmp_path: Path,
) -> None:
"""Keep Strix on the gateway and fail Noema closed without its sidecar."""
bash_executable = shutil.which("bash") or "/bin/bash"
strix_output = tmp_path / "strix-output"
strix = subprocess.run( # noqa: S603, S607
[
bash_executable,
"-c",
textwrap.dedent(
workflow_step(
workflow_text("strix.yml"),
"Gate Strix secrets",
)
.split(" run: |\n", 1)[1]
),
],
env={