From 0ffbb2c3f48d40a76b95f4455ca0c24693d4bed9 Mon Sep 17 00:00:00 2001 From: Neil Getty Date: Tue, 27 Oct 2026 16:42:15 +0000 Subject: [PATCH 1/2] Widen the site-path gate and scrub the leaks it surfaced Gate (tools/ci/check_site_paths.py): scan experiments/, scaling-study/, requirements/, .github/ and root-level files; match grand/gpfs/lustre roots; count internal identities (usernames, old repo name) on the same ratchet. Scrub: bioreason/GRPO scripts, DAOS/interactive launch scripts, universal_evaluator, convert_parquet_to_webdataset, prism_designs.yaml, KEGG docs, the multi-agent skill and the _repo_paths clone hint now use PRISM_* variables or placeholders. Remove three study-record experiment files; mark the rest with TODO(release). Document PRISM_CONDA_ENV / PRISM_REPO_ROOT. Co-Authored-By: Claude Sonnet 5.5 --- .env.template | 11 + CHANGELOG.md | 9 + .../scripts/eval/submit_eval_kegg.sh | 14 +- .../scripts/test/verify_dna_dataset_mix.sh | 12 +- .../scripts/training/submit_bioreason_grpo.sh | 14 +- docs/applications/bioreason_grpo.md | 2 +- .../kegg_curation_implementation_status.md | 12 +- docs/applications/kegg_curation_pipeline.md | 6 +- docs/platforms/site_paths.md | 9 + .../prism-multi-agent-workflow/SKILL.md | 2 +- experiments/LLM_timeseries_scaling.yaml | 3 + experiments/ablation_catalog.yaml | 126 ----- experiments/prism_designs.yaml | 6 +- experiments/qwen3_siglip_2node_search.yaml | 3 + experiments/qwen3_siglip_scaling.yaml | 3 + experiments/qwen3_siglip_smokes.yaml | 47 -- experiments/stage_a_round1_olmo3_1b.yaml | 463 ------------------ scripts/build_stage_a_plan.py | 4 + scripts/convert_parquet_to_webdataset.py | 16 +- scripts/setup_daos_models.sh | 8 +- scripts/smoke_test_grpo.sh | 14 +- src/_repo_paths.py | 4 +- tests/test_isoflop_launcher_flag.py | 4 +- tests/test_site_path_gate.py | 20 + tools/ci/check_site_paths.py | 48 +- tools/ci/site_path_allowlist.json | 24 +- tools/hold_4node_debug_scaling.sh | 5 + tools/launch_polaris.py | 2 +- tools/run_agpt2b_interactive.sh | 10 +- tools/run_composite_interactive.sh | 8 +- tools/stage_a_round1_batch.sh | 7 +- tools/universal_evaluator.py | 15 +- 32 files changed, 205 insertions(+), 726 deletions(-) delete mode 100644 experiments/ablation_catalog.yaml delete mode 100644 experiments/qwen3_siglip_smokes.yaml delete mode 100644 experiments/stage_a_round1_olmo3_1b.yaml diff --git a/.env.template b/.env.template index 53b69ef..373527d 100644 --- a/.env.template +++ b/.env.template @@ -83,6 +83,17 @@ PRISM_ASSETS="/flare///assets" # the Stage 1 projector named by src/conf/training/bioreason_sft.yaml. PRISM_OUTPUT_ROOT="/flare///outputs" +# Shell-script variables (optional) +# Read by the bioreason / GRPO submit scripts and scripts/smoke_test_grpo.sh, +# which fail with a "set " message when they are unset. These are plain +# shell variables, not part of src/site_paths.py. + +# Conda environment those scripts activate. +# PRISM_CONDA_ENV="////conda_env/prism" + +# Checkout the scripts cd into (scripts/smoke_test_grpo.sh). +# PRISM_REPO_ROOT="/flare///PRISM" + # Model & Dataset Path Overrides (optional) # These override placeholder defaults baked into src/config.py and # src/encoders/geometry.py. Leave unset to use the placeholder paths — this diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b79aef..8b56a9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,12 @@ will make up v0.1.0. ### Changed +- Site-path gate now also flags internal identities (usernames, the old repo + name) and scans `experiments/`, `scaling-study/`, + `requirements/`, `.github/` and root-level files, and the `grand`, `gpfs` + and `lustre` roots; the bioreason/GRPO scripts and `experiments/prism_designs.yaml` + take locations from `PRISM_*` variables (`PRISM_CONDA_ENV` and `PRISM_REPO_ROOT` + are new, see `.env.template`) - Corrected the `src/libs/walrus` submodule URL to the upstream `PolymathicAI/walrus` organization (#186) - `requirements/base.txt` now installs `h5py`, which `src/data/multimodal.py` @@ -51,6 +57,9 @@ will make up v0.1.0. ### Removed +- `experiments/stage_a_round1_olmo3_1b.yaml`, `ablation_catalog.yaml` and + `qwen3_siglip_smokes.yaml` — study records tied to internal paths with no + code dependents. Remaining study files carry `TODO(release)` notes. - `docs/plans/` (9 files) and `docs/historical/` (5 files) — dated design plans and bring-up debug journals whose work has all landed. Plans are tracking artifacts and belong in the issue tracker rather than the source tree. diff --git a/applications/bioreason/scripts/eval/submit_eval_kegg.sh b/applications/bioreason/scripts/eval/submit_eval_kegg.sh index 249d6b9..eacec41 100755 --- a/applications/bioreason/scripts/eval/submit_eval_kegg.sh +++ b/applications/bioreason/scripts/eval/submit_eval_kegg.sh @@ -14,8 +14,8 @@ PROJ_DIR="${PROJ_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." 2>/dev/n if [ -n "${PBS_O_WORKDIR:-}" ] && [ ! -f "${PROJ_DIR}/src/train.py" ]; then PROJ_DIR="${PBS_O_WORKDIR}" fi -CHECKPOINT_DIR="/lus/eagle/projects/argonne_tpc/abalaji/modcon/genome/output/GENOME-SFT-1B/2026-06-29/07-41-10/checkpoints/step_8000" -# SFT with sample weigting and answer structure : /lus/eagle/projects/argonne_tpc/abalaji/modcon/genome/output/GENOME-SFT-1B/2026-06-29/07-41-10/checkpoints/step_3000 +CHECKPOINT_DIR="${PRISM_OUTPUT_ROOT:?set PRISM_OUTPUT_ROOT}/GENOME-SFT-1B//