diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml new file mode 100644 index 0000000..1b47c60 --- /dev/null +++ b/.github/workflows/book.yml @@ -0,0 +1,21 @@ +name: Jupyter Book +on: + push: + pull_request: + workflow_dispatch: +permissions: + contents: read +jobs: + book: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + - name: Install pinned Jupyter Book + run: | + python3 -m venv .venv + .venv/bin/python -m pip install --disable-pip-version-check -r requirements-book.txt + - name: Build the static book without execution + run: .venv/bin/jupyter book build --html --strict diff --git a/.gitignore b/.gitignore index e868d9f..3f8a17a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /build/ /dist/ +/_build/ .venv/ .ipynb_checkpoints/ **/.ipynb_checkpoints/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d1e1bc6..00f7282 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,8 @@ | Change | Edit | | --- | --- | | A build or verification operation | `notebooks/.ipynb` | -| Explanation or tutorial | `notebooks/01-*` to `04-*` or `docs/` | +| Guided OS-building chapters | `book/*.ipynb` and `book/*.md` | +| Compact operational notebooks | `notebooks/01-*` to `04-*` | | Docker or mkosi orchestration | `scripts/*.sh`, `Dockerfile` | | OS packages, boot settings or partition layout | `images/*/mkosi.conf` and `mkosi.repart/` | | Files installed in the OS | `images/*/overlay/` | @@ -16,7 +17,9 @@ The notebooks are the **canonical host-side Python sources**, not copies of `scripts/*.py`. The runner in `scripts/notebook.sh` executes them without Jupyter, so the exact notebook cells are used in CI. -Run interactive notebooks with `make lab`. It creates working copies in ignored `build/notebooks/` so execution counters and saved outputs never dirty the committed source. To contribute, copy reviewed edits back into `notebooks/`, clear cell outputs, and commit the canonical version. +Run canonical operational notebooks with `make lab`; run Jupyter Book chapters with `make book-lab` (ignored working copies). Render the book with `make book-html` without notebook execution. + +Run interactive operational notebooks with `make lab`. It creates working copies in ignored `build/notebooks/` so execution counters and saved outputs never dirty the committed source. To contribute, copy reviewed edits back into `notebooks/`, clear cell outputs, and commit the canonical version. Keep each notebook easy to read from top to bottom: goal, prerequisites, visible arguments, focused cells, and checks. Put import-safe helpers before the final guarded command cell. Do not commit generated outputs, secrets, private keys, kernel traces, or host credentials. Avoid hidden notebook state and unreviewed network access. diff --git a/Makefile b/Makefile index 590ee6a..e0f8313 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,15 @@ .DEFAULT_GOAL := help -.PHONY: help check builder images smoke repro lab +.PHONY: help check builder images smoke repro lab book-html book-lab book help: @printf '%s\n' \ 'SC Linux image factory' \ '' \ ' make check Run unit tests, source checks, and shell syntax checks' \ - ' make lab Open JupyterLab working copies' \ + ' make lab Open operational notebooks in JupyterLab' \ + ' make book-lab Open guided book chapters in JupyterLab' \ + ' make book-html Render the Jupyter Book site without execution' \ + ' make book Build the static Jupyter Book without executing cells' \ ' make builder Build the pinned Docker toolchain' \ ' make images Build the RAW disk and UEFI installer ISO' \ ' make smoke Boot RAW clones, install the ISO, and boot the result' \ @@ -36,3 +39,12 @@ repro: lab: bash scripts/lab.sh + +book-lab: + bash scripts/book-lab.sh + +book-html: + jupyter book build --html + +book: + jupyter book build --html --strict diff --git a/README.md b/README.md index d99b54d..ce50006 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,24 @@ make check The build requires an **amd64 Linux** machine with Docker, Git, Make, OpenSSL and at least **40 GiB free**. It uses privileged containers. Follow [the build guide](docs/building.md) before running it. +## Guided Jupyter Book: build an OS step by step + +The [SC Linux build book](book/index.md) separates the workflow into nine self-contained chapters: host prerequisites, image configuration, CA trust, builder, image construction, checksums/footprint, QEMU boot, host seed and reproducibility. + +Read the chapters on GitHub or open working copies in JupyterLab with `make book-lab`. + +Install the optional documentation tools in a development virtual environment: + +```sh +python3 -m venv .venv +. .venv/bin/activate +python -m pip install -r requirements-book.txt jupyterlab +make book +make book-lab +``` + +`make book` renders HTML in strict mode without running image-build cells. The book's code cells run local commands and some build privileged Docker images; the public HTML book must **not** execute them. To render the site, install Jupyter Book 2 and use `make book-html` (without `--execute`). The installed OS never includes Jupyter. + ## Learn and operate through notebooks Open [`notebooks/`](notebooks) in JupyterLab, starting Jupyter from the repository root: diff --git a/book/01-environment.ipynb b/book/01-environment.ipynb new file mode 100644 index 0000000..7620041 --- /dev/null +++ b/book/01-environment.ipynb @@ -0,0 +1,72 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# 1 · Prepare the build host\n\n## Goal\n\nConfirm that the host can build a **Debian 13 amd64** RAW image and boot-test it with QEMU. This chapter changes nothing.\n\n## Prerequisites\n\nUse a Linux x86-64 host with Docker, Git, Make, Python 3.11+, OpenSSL and at least 40 GiB of available space. Docker Desktop is not qualified for this workflow." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "from pathlib import Path\nimport os, subprocess, sys\nSTART = Path.cwd().resolve()\nROOT = next((p for p in (START, *START.parents) if (p/\"myst.yml\").is_file() and (p/\"scripts/build.sh\").is_file()), None)\nassert ROOT is not None, \"Run from the SC Linux checkout (make book-lab)\"\ndef run(*argv):\n print(\"+\", \" \".join(map(str, argv)), flush=True)\n return subprocess.run([str(a) for a in argv], cwd=ROOT, check=True)\nprint(\"Source:\", ROOT)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 1 · Identify the platform" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "import platform, shutil\nprint(\"OS:\", platform.system(), \"Architecture:\", platform.machine())\nassert platform.system()==\"Linux\", \"Use an amd64 Linux host\"\nassert platform.machine() in (\"x86_64\",\"AMD64\"), \"The current image target is amd64\"\nfor binary in (\"docker\",\"git\",\"make\",\"python3\",\"openssl\"):\n print(binary, shutil.which(binary) or \"MISSING\")\n assert shutil.which(binary), f\"Install {binary}\"" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 2 · Check storage and Docker" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "disk=shutil.disk_usage(ROOT)\nprint(f\"Free disk: {disk.free/2**30:.1f} GiB\")\nassert disk.free >= 40*2**30, \"Reserve at least 40 GiB\"\nrun(\"docker\",\"info\",\"--format\",\"{{.OSType}} {{.Architecture}}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 3 · Inspect Git state\n\nImage builds require every tracked change to be committed; artifacts belong under ignored `build/`." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "run(\"git\",\"status\",\"--short\")\nrun(\"git\",\"rev-parse\",\"--short\",\"HEAD\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Check\n\nThe preceding cells must all complete without errors. Host readiness alone does not imply valid images.\n\n## Next\n\nContinue to [review the build configuration](02-configuration.ipynb)." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "tags": [ + "skip-execution" + ] + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/book/02-configuration.ipynb b/book/02-configuration.ipynb new file mode 100644 index 0000000..f88706c --- /dev/null +++ b/book/02-configuration.ipynb @@ -0,0 +1,84 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# 2 · Understand the image inputs\n\n## Goal\n\nIdentify the pinned software inputs, image layout and packages **before** running a privileged build.\n\n## Prerequisites\n\nComplete the host preflight; all cells below are read-only." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "from pathlib import Path\nimport os, subprocess, sys\nSTART = Path.cwd().resolve()\nROOT = next((p for p in (START, *START.parents) if (p/\"myst.yml\").is_file() and (p/\"scripts/build.sh\").is_file()), None)\nassert ROOT is not None, \"Run from the SC Linux checkout (make book-lab)\"\ndef run(*argv):\n print(\"+\", \" \".join(map(str, argv)), flush=True)\n return subprocess.run([str(a) for a in argv], cwd=ROOT, check=True)\nprint(\"Source:\", ROOT)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 1 · Read pinned versions\n\nThe container uses a Debian snapshot and `mkosi` package defined in `config/build.json`." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "import json\nbuild_config=json.loads((ROOT/\"config/build.json\").read_text())\nfor key, value in sorted(build_config.items()):\n print(f\"{key:24} {value}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 2 · Review server packages\n\nThese are mkosi package inputs, not a promise of the exact installed footprint." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "import configparser\ndef read_mkosi(name):\n parser=configparser.ConfigParser(interpolation=None)\n parser.read(ROOT/\"images\"/name/\"mkosi.conf\")\n return parser\nserver=read_mkosi(\"server\")\ninstaller=read_mkosi(\"installer\")\nfor role,cfg in ((\"Server\",server),(\"Installer\",installer)):\n names=cfg.get(\"Content\",\"Packages\").split()\n print(f\"{role} packages ({len(names)}):\", \", \".join(names))" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 3 · Inspect partitions and boot command\n\nThe server contains an ESP and writable ext4 root. The UKI embeds the kernel and initrd." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "for path in sorted((ROOT/\"images/server/mkosi.repart\").glob(\"*.conf\")):\n print(\"\\n\", path.name)\n print(path.read_text())\nprint(\"Kernel command line:\",server.get(\"Content\",\"KernelCommandLine\"))" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 4 · Run static source checks\n\nThis executes the canonical checks without creating image artifacts." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "run(\"make\",\"check\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Check\n\nYou should know what is pinned, which packages go into each image, and why the disk has multiple partitions.\n\n## Next\n\n[Validate FreeIPA CA trust](03-trust.ipynb)." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "tags": [ + "skip-execution" + ] + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/book/03-trust.ipynb b/book/03-trust.ipynb new file mode 100644 index 0000000..f579259 --- /dev/null +++ b/book/03-trust.ipynb @@ -0,0 +1,72 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# 3 · Verify the FreeIPA trust anchor\n\n## Goal\n\nCheck that the embedded **public CA** is a valid CA certificate and its DER SHA-256 digest matches the local fingerprint file.\n\n## Prerequisites\n\nObtain the expected CA fingerprint independently from your FreeIPA administrator. A matching repository fingerprint alone is **not** independent authentication." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "from pathlib import Path\nimport os, subprocess, sys\nSTART = Path.cwd().resolve()\nROOT = next((p for p in (START, *START.parents) if (p/\"myst.yml\").is_file() and (p/\"scripts/build.sh\").is_file()), None)\nassert ROOT is not None, \"Run from the SC Linux checkout (make book-lab)\"\ndef run(*argv):\n print(\"+\", \" \".join(map(str, argv)), flush=True)\n return subprocess.run([str(a) for a in argv], cwd=ROOT, check=True)\nprint(\"Source:\", ROOT)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 1 · Inspect the checked-in certificate\n\nNo private key or one-time enrollment password belongs in the repository." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "CERT = ROOT/\"trust/ipa-ca.crt\"\nFINGERPRINT = ROOT/\"trust/ipa-ca.sha256\"\nassert CERT.is_file() and FINGERPRINT.is_file(), \"Import a reviewed public CA first\"\nprint(\"Certificate:\", CERT.relative_to(ROOT))\nprint(\"Local recorded DER SHA-256:\", FINGERPRINT.read_text().strip())\nrun(\"openssl\",\"x509\",\"-in\",CERT,\"-noout\",\"-subject\",\"-issuer\",\"-dates\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 2 · Check certificate constraints and digest\n\nThe shared runtime library verifies DER digest, validity dates and the CA constraint." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "sys.path.insert(0,str(ROOT/\"lib\"))\nfrom scos.core import verify_ca\nlocal_fingerprint = verify_ca(CERT, FINGERPRINT.read_text())\nprint(\"Local certificate check passed:\", local_fingerprint)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 3 · Compare with independently verified fingerprint\n\nEnter the fingerprint obtained **out-of-band**, not copied from `trust/ipa-ca.sha256`. This cell does not send the value anywhere." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "INDEPENDENT_DER_SHA256 = \"\"\nif INDEPENDENT_DER_SHA256:\n assert local_fingerprint == INDEPENDENT_DER_SHA256.strip().lower().replace(\":\",\"\"), \"Independent CA fingerprint mismatch\"\n print(\"Independent CA fingerprint matches\")\nelse:\n print(\"NOT VERIFIED INDEPENDENTLY — obtain the authoritative fingerprint before building\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Check\n\nA local certificate check passes only the structural and digest tests. **Do not build or distribute images until independent CA verification is complete.** If replacing the CA, use `notebooks/trust.ipynb` and commit the reviewed public files.\n\n## Next\n\n[Build the pinned toolchain](04-builder.ipynb)." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "tags": [ + "skip-execution" + ] + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/book/04-builder.ipynb b/book/04-builder.ipynb new file mode 100644 index 0000000..11ad857 --- /dev/null +++ b/book/04-builder.ipynb @@ -0,0 +1,84 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# 4. Construct the pinned builder\n\n## Goal\nCreate the Docker toolchain used by both image construction and disposable VM tests.\n\n## Prerequisites\nNative amd64 Linux, Docker access, a reviewed CA, and a clean repository. **This operation uses privileged Docker later in the image build. Review the source before running it.**" + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "from pathlib import Path\nimport os, json, shutil, subprocess, shlex\nSTART = Path.cwd().resolve()\nROOT = next((p for p in (START, *START.parents)\n if (p / \"config/build.json\").is_file()\n and (p / \"notebooks/check.ipynb\").is_file()), None)\nassert ROOT is not None, \"Open this notebook inside a clone of sc-linux\"\nprint(\"SC Linux:\", ROOT)\n\ndef command(*argv, enabled=False, env=None):\n args = [str(x) for x in argv]\n print(\"$\", shlex.join(args))\n if not enabled:\n print(\"Preview only; opt in with the RUN_* variable above.\")\n return None\n return subprocess.run(args, cwd=ROOT, env=env, check=True)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 1: Inspect the toolchain definition\n\nThe builder is separate from the installed operating system. Docker, QEMU and Jupyter are not installed in the guest by this action." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "dockerfile = (ROOT / \"Dockerfile\").read_text()\nprint(dockerfile)\nassert \"mkosi\" in dockerfile and \"xorriso\" in dockerfile and \"qemu\" in dockerfile" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 2: Identify the configured Debian snapshot" + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "config = json.loads((ROOT / \"config/build.json\").read_text())\nprint(\"Debian suite:\", config[\"release\"])\nprint(\"Snapshot:\", config[\"snapshot\"])\nprint(\"Expected mkosi package:\", config[\"mkosi_package\"])" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 3: Create the local Docker builder (opt-in)\n\n`make builder` resolves a base image to a digest, builds a pinned Debian toolchain and records its local identity in `build/`." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "RUN_BUILDER = False\ncommand(\"make\", \"builder\", enabled=RUN_BUILDER)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 4: Inspect recorded provenance\n\nIf the preceding build was not run, these files may not exist." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "for name in (\"builder.id\", \"builder-base.txt\"):\n path = ROOT / \"build\" / name\n print(f\"{path.relative_to(ROOT)}: {path.read_text().strip() if path.is_file() else 'NOT YET CREATED'}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Check\nThe builder image ID is local to this Docker daemon. The base image digest identifies a reviewed input; by itself it does not make the complete build hermetic.\n\n## Next\nContinue to [5. Build the system image](05-image.ipynb)." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "tags": [ + "skip-execution" + ] + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/book/05-image.ipynb b/book/05-image.ipynb new file mode 100644 index 0000000..db37df4 --- /dev/null +++ b/book/05-image.ipynb @@ -0,0 +1,130 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# 5 · Build the server disk and installer ISO\n\n## Goal\n\nConstruct a bootable server RAW disk, compress it, generate payload digests, build an installer UKI, and package the UEFI ISO. The tested builder runs these operations in one **atomic build transaction** to avoid mismatching server payloads and installer checksums.\n\n## Prerequisites\n\nThe committed source tree must be clean, the public FreeIPA CA verified out-of-band, the Docker builder available, and `build/output` absent or empty. The build uses privileged Docker." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Why the build stays atomic\n\nThe container entrypoint (`scripts/build-inside.sh`) now exposes **ten named build functions**, each emitting an `SCOS_BUILD_STAGE` marker: verify toolchain → prepare workspace → validate/stage inputs → configure EFI signing → build server RAW → zstd compression → record payload digests → build installer UKI → assemble ISO → record evidence. These are individual, reviewable source blocks but execute as one transaction: the installer must always describe exactly the RAW built in the same run. The checks below inspect each output independently." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "from pathlib import Path\nimport subprocess, sys\nSTART=Path.cwd().resolve()\nROOT=next((p for p in (START,*START.parents) if (p/\"myst.yml\").is_file() and (p/\"scripts/build.sh\").is_file()),None)\nassert ROOT is not None, \"Run from the SC Linux checkout (make book-lab)\"\ndef run(*argv):\n print(\"+\",\" \".join(map(str,argv)),flush=True)\n return subprocess.run([str(a) for a in argv],cwd=ROOT,check=True)\nprint(\"Source:\",ROOT)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 1 · Check that source and output are safe to use" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "status=subprocess.run([\"git\",\"status\",\"--porcelain\",\"--untracked-files=normal\"],cwd=ROOT,capture_output=True,text=True,check=True).stdout\nassert not status.strip(), f\"Commit source changes before build:\\n{status}\"\nOUTPUT=ROOT/\"build/output\"\nassert not OUTPUT.exists() or not any(OUTPUT.iterdir()), \"Choose a fresh output directory; current build/output is not empty\"\nassert (ROOT/\"build/builder.id\").is_file(), \"Run chapter 4 first\"\nprint(\"Image destination:\",OUTPUT)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 2 · Validate trust before the privileged build\n\nThis validates certificate structure and its checked-in digest. It cannot establish your independent verification." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 3 · Read the image-builder phases\n\n`build-inside.sh` has named stages in execution order. The server RAW is finalized before its payload digest is embedded in the installer. Keeping this order prevents assembling an ISO with stale payload metadata." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "STAGES = (\"verify_toolchain\", \"prepare_workspace\", \"validate_and_stage_inputs\",\n \"configure_efi_signing\", \"build_server_raw\", \"compress_server_raw\",\n \"record_installer_payload\", \"build_installer_uki\",\n \"assemble_installer_iso\", \"record_build_evidence\")\npipeline = (ROOT/\"scripts/build-inside.sh\").read_text()\nfor position, stage in enumerate(STAGES, 1):\n assert f\"{stage}()\" in pipeline, f\"Missing build stage: {stage}\"\n print(f\"{position:2}. {stage}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 4 · Inspect staged server files (optional)\n\nThis calls the **canonical preparation notebook** using the current Git revision. It writes only into ignored `build/` paths. The real build later stages its own fresh copy of committed source in the Docker container. This inspection is optional." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import os\nPREVIEW_STAGING = False\nif PREVIEW_STAGING:\n source_sha = subprocess.check_output([\"git\",\"rev-parse\",\"HEAD\"],cwd=ROOT,text=True).strip()\n source_epoch = subprocess.check_output([\"git\",\"show\",\"-s\",\"--format=%ct\",\"HEAD\"],cwd=ROOT,text=True).strip()\n stage_env = dict(os.environ, SOURCE_COMMIT=source_sha, SOURCE_DATE_EPOCH=source_epoch)\n subprocess.run([\"bash\",\"scripts/notebook.sh\",\"notebooks/prepare.ipynb\"],\n cwd=ROOT,env=stage_env,check=True)\n for image in (\"server\",\"installer\"):\n overlay = ROOT/\"build\"/f\"{image}-overlay\"\n assert overlay.is_dir(), f\"Staging failed: {overlay}\"\n print(image, \"staged files:\", sum(p.is_file() for p in overlay.rglob(\"*\")))\nelse:\n print(\"Optional overlay inspection skipped\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 5 · Build RAW and ISO in one reviewed transaction\n\n`make images` runs the checked stages inside a privileged Docker container. The output directory must be empty; image creation can use tens of GiB of temporary space." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "run(\"make\",\"images\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 6 · Confirm the server disk and its compressed payload\n\nThe uncompressed RAW is the whole GPT disk. The `.zst` file is the distributable compressed form of that same disk." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "raw=OUTPUT/\"raw/sc-linux.raw\"\ncompressed=OUTPUT/\"sc-linux.raw.zst\"\nfor name,artifact in ((\"Server RAW\",raw),(\"Compressed RAW\",compressed)):\n assert artifact.is_file() and artifact.stat().st_size > 1_048_576, f\"Missing {name}\"\n print(f\"{name}: {artifact.stat().st_size:,} bytes\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 7 · Inspect embedded payload metadata\n\nThe installer receives the size and SHA-256 of both RAW forms *before* its UKI is assembled. Chapter 6 independently re-hashes the artifacts." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "import json\npayload=OUTPUT/\"payload.json\"\nassert payload.is_file()\nrecord=json.loads(payload.read_text())\nassert set(record)=={\"raw\",\"compressed\"}\nassert record[\"raw\"][\"size\"]==raw.stat().st_size\nassert record[\"compressed\"][\"size\"]==compressed.stat().st_size\nfor name,item in record.items():\n print(name, \"bytes:\", item[\"size\"], \"SHA-256:\", item[\"sha256\"])" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 8 · Check the installer and recorded provenance\n\nDo not treat mere existence as boot verification. The QEMU acceptance test is a separate chapter." + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": "uki=OUTPUT/\"installer/sc-linux-installer.efi\"\niso=OUTPUT/\"sc-linux.iso\"\nfor label,path in ((\"Installer UKI\",uki),(\"Installer ISO\",iso)):\n assert path.is_file() and path.stat().st_size > 1_048_576, f\"Missing {label}\"\n print(label, f\"{path.stat().st_size:,} bytes\")\ncommit_file=OUTPUT/\"source-commit.txt\"\nassert commit_file.is_file()\nprint(\"Source revision:\", commit_file.read_text().strip())" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Check\n\nThe ordered stages produced a server RAW, its compressed payload, matching installer manifest, installer UKI and ISO. This is **image construction evidence**, not proof of a successful boot.\n\n## Next\n\n[Inspect the artifact checksums and footprint](06-artifacts.ipynb)." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "tags": [ + "skip-execution" + ] + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/book/06-artifacts.ipynb b/book/06-artifacts.ipynb new file mode 100644 index 0000000..ee00a46 --- /dev/null +++ b/book/06-artifacts.ipynb @@ -0,0 +1,84 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# 6 · Verify artifacts and measure size\n\n## Goal\n\nRecompute SHA-256 for each distributable artifact, compare embedded payload metadata, and identify size drivers without modifying any image.\n\n## Prerequisites\n\nComplete the image build in chapter 5. The precomputed hashes are integrity measurements, **not release signatures**." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "from pathlib import Path\nimport subprocess, sys\nSTART=Path.cwd().resolve()\nROOT=next((p for p in (START,*START.parents) if (p/\"myst.yml\").is_file() and (p/\"scripts/build.sh\").is_file()),None)\nassert ROOT is not None, \"Run from the SC Linux checkout (make book-lab)\"\ndef run(*argv):\n print(\"+\",\" \".join(map(str,argv)),flush=True)\n return subprocess.run([str(a) for a in argv],cwd=ROOT,check=True)\nprint(\"Source:\",ROOT)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 1 · Load build reports" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "import json, hashlib\nOUTPUT=ROOT/\"build/output\"\nmanifest=json.loads((OUTPUT/\"artifacts.json\").read_text())\nfootprint=json.loads((OUTPUT/\"footprint.json\").read_text())\npayload=json.loads((OUTPUT/\"payload.json\").read_text())\nprint(\"Distributable artifacts:\",\", \".join(manifest))" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 2 · Recompute hashes using bounded memory" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "def digest(path):\n h=hashlib.sha256()\n with path.open(\"rb\") as src:\n for chunk in iter(lambda:src.read(1024*1024),b\"\"):\n h.update(chunk)\n return h.hexdigest()\nfor name,record in manifest.items():\n file=OUTPUT/name\n assert file.stat().st_size==record[\"size\"], f\"Size mismatch: {name}\"\n assert digest(file)==record[\"sha256\"], f\"Digest mismatch: {name}\"\n print(\"Verified:\",name)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 3 · Compare the installer payload with the server image" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "for key,name in ((\"raw\",\"raw/sc-linux.raw\"),(\"compressed\",\"sc-linux.raw.zst\")):\n file=OUTPUT/name\n assert payload[key][\"size\"]==file.stat().st_size\n assert payload[key][\"sha256\"]==digest(file)\n print(\"Payload matches:\",name)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 4 · Review sizes and largest packages" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "for name,value in footprint[\"artifacts\"].items():\n print(f\"{name}: {value}\")\nprint(\"\\nLargest installed packages (KiB):\")\nfor pkg in footprint[\"packages\"][\"largest\"][:12]:\n print(f\"{pkg['installed_size_kib']:>9,} {pkg['package']}\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Check\n\nAll artifact hashes, sizes and embedded payload descriptors must match. The RAW includes partition and filesystem overhead; package Installed-Size is an estimate. See [footprint methodology](https://github.com/soft-cloud-dev/sc-linux/blob/main/docs/footprint.md).\n\n## Next\n\n[Boot the actual images](07-boot.ipynb)." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "tags": [ + "skip-execution" + ] + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/book/07-boot.ipynb b/book/07-boot.ipynb new file mode 100644 index 0000000..39f60ee --- /dev/null +++ b/book/07-boot.ipynb @@ -0,0 +1,84 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# 7 · Boot the image in disposable virtual machines\n\n## Goal\n\nVerify that two cloned RAW disks boot with different machine IDs, the installer installs to a blank VM disk, and the installed machine boots.\n\n## Prerequisites\n\nBuild both images first. QEMU, OVMF and swtpm are provided by the Docker builder. These tests can consume substantial CPU time." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "from pathlib import Path\nimport subprocess, json\nSTART=Path.cwd().resolve()\nROOT=next((p for p in (START,*START.parents) if (p/\"myst.yml\").is_file() and (p/\"scripts/build.sh\").is_file()),None)\nassert ROOT is not None, \"Run from the SC Linux checkout (make book-lab)\"\ndef run(*argv):\n print(\"+\",\" \".join(map(str,argv)),flush=True)\n return subprocess.run([str(a) for a in argv],cwd=ROOT,check=True)\nprint(\"Source:\",ROOT)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 1 · Check required inputs" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "OUTPUT=ROOT/\"build/output\"\nfor rel in (\"raw/sc-linux.raw\",\"sc-linux.iso\"):\n path=OUTPUT/rel\n assert path.is_file() and path.stat().st_size>1024*1024, f\"Build missing: {rel}\"\n print(rel, f\"{path.stat().st_size:,} bytes\")\nassert (ROOT/\"build/builder.id\").is_file(), \"Builder image ID missing\"" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 2 · Execute VM acceptance tests\n\nTests use temporary virtual disks, isolated firmware variables, a software TPM and **no network interface**." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "run(\"make\",\"smoke\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 3 · Inspect structured boot evidence" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "report_file=OUTPUT/\"tests/smoke.json\"\nreport=json.loads(report_file.read_text())\nassert report.get(\"status\")==\"passed\", \"Smoke report does not indicate success\"\nassert set(report.get(\"checks\",[]))=={\"two-raw-clones\",\"iso-install\",\"installed-boot\"}\nrecords=report.get(\"records\",[])\nassert len(records)==3 and len({r[\"machine_id\"] for r in records})==3\nprint(\"Checks:\", \", \".join(report[\"checks\"]))\nfor index,record in enumerate(records,1):\n print(index,\"machine_id=\",record[\"machine_id\"],\"authorization=\",record[\"authorization\"])" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 4 · Locate console logs" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "for path in sorted((OUTPUT/\"tests\").glob(\"*.log\")):\n print(path.name, f\"{path.stat().st_size:,} bytes\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Check\n\nPassing QEMU tests demonstrate selected boot and installation paths, but **not** Secure Boot enforcement, authenticated root, remote attestation, live FreeIPA enrollment, or root growth on arbitrary disks.\n\n## Next\n\n[Create a host-specific provisioning seed](08-provision.ipynb)." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "tags": [ + "skip-execution" + ] + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/book/08-provision.ipynb b/book/08-provision.ipynb new file mode 100644 index 0000000..a5ff822 --- /dev/null +++ b/book/08-provision.ipynb @@ -0,0 +1,72 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# 8 · Prepare a host for first boot\n\n## Goal\n\nCreate a **per-host NoCloud ISO** containing a unique hostname and a public SSH key. Do not include a FreeIPA OTP or a private key.\n\n## Prerequisites\n\nInstall `xorriso` and `ssh-keygen` on the host. Prepare an OpenSSH public key, and choose a client name under the configured IPA domain. Creating media is non-destructive; **installing an OS to a real disk is destructive**." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "from pathlib import Path\nimport subprocess, json\nSTART=Path.cwd().resolve()\nROOT=next((p for p in (START,*START.parents) if (p/\"myst.yml\").is_file() and (p/\"scripts/build.sh\").is_file()),None)\nassert ROOT is not None, \"Run from the SC Linux checkout (make book-lab)\"\ndef run(*argv):\n print(\"+\",\" \".join(map(str,argv)),flush=True)\n return subprocess.run([str(a) for a in argv],cwd=ROOT,check=True)\nprint(\"Source:\",ROOT)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 1 · Choose the hostname, key and output\n\nChange only the following values, then execute. The seed tool refuses to overwrite an existing ISO." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "import sys\nsys.path.insert(0, str(ROOT/'lib'))\nfrom scos.core import read_identity, validate_hostname\nidentity=read_identity(ROOT/\"config/identity.json\")\nHOSTNAME=\"node01.\"+identity[\"domain\"]\nPUBLIC_KEY=Path.home()/\".ssh/id_ed25519.pub\"\nSEED=ROOT/\"build/seeds\"/f\"{HOSTNAME}.iso\"\nvalidate_hostname(HOSTNAME,identity[\"domain\"])\nassert HOSTNAME!=identity[\"server\"]\nassert PUBLIC_KEY.is_file(), f\"Missing public key: {PUBLIC_KEY}\"\nassert not SEED.exists(), f\"Seed already exists: {SEED}\"\nprint(\"Hostname:\",HOSTNAME)\nprint(\"Seed target:\",SEED)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 2 · Build the NoCloud ISO\n\nOnly the public SSH key and hostname enter the seed. The host gets its own machine ID and host SSH keys at first boot." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "run(\"bash\",\"scripts/notebook.sh\",\"notebooks/seed.ipynb\",HOSTNAME,PUBLIC_KEY,SEED)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 3 · Verify the output and record the deployment command" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "assert SEED.is_file() and SEED.stat().st_size>0\nprint(\"Seed created:\",SEED.relative_to(ROOT),f\"({SEED.stat().st_size:,} bytes)\")\nprint(\"After boot: ssh scos@\"+HOSTNAME)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Check\n\nAttach the seed read-only to the VM **before first boot**. Follow [installation instructions](https://github.com/soft-cloud-dev/sc-linux/blob/main/docs/installing.md) to boot the installer or import RAW. Confirm disk selection before any destructive disk write. On the provisioned host, run `sudo scos-enroll HOSTNAME` and enter a host-scoped OTP **at its native prompt**, never in a notebook.\n\n## Next\n\nReview [verification boundaries](https://github.com/soft-cloud-dev/sc-linux/blob/main/docs/verification.md) and consider the optional [reproducibility comparison](09-reproducibility.ipynb)." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "tags": [ + "skip-execution" + ] + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/book/09-reproducibility.ipynb b/book/09-reproducibility.ipynb new file mode 100644 index 0000000..05ab166 --- /dev/null +++ b/book/09-reproducibility.ipynb @@ -0,0 +1,72 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": "# 9 · Compare two independent image builds\n\n## Goal\n\nTest whether two **unsigned** builds from the same reviewed commit produce identical RAW and ISO bytes.\n\n## Prerequisites\n\nFinish the standard image build and smoke test. This optional experiment runs **two more complete builds**, requires significant disk space, and cannot be performed with `SCOS_SIGNING_DIR` set." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "from pathlib import Path\nimport os, subprocess, hashlib\nSTART=Path.cwd().resolve()\nROOT=next((p for p in (START,*START.parents) if (p/\"myst.yml\").is_file() and (p/\"scripts/build.sh\").is_file()),None)\nassert ROOT is not None, \"Run from the SC Linux checkout (make book-lab)\"\ndef run(*argv):\n print(\"+\",\" \".join(map(str,argv)),flush=True)\n return subprocess.run([str(x) for x in argv],cwd=ROOT,check=True)\nprint(\"Source:\",ROOT)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 1 · Check prerequisites\n\nLeave previous comparisons intact; the script deliberately refuses non-empty output directories." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "assert not os.environ.get(\"SCOS_SIGNING_DIR\"), \"Reproducibility requires unsigned builds\"\nassert (ROOT/\"build/builder.id\").is_file(), \"Build toolchain missing\"\nfor name in (\"repro-a\",\"repro-b\"):\n folder=ROOT/\"build\"/name\n assert not folder.exists() or not any(folder.iterdir()), f\"Move or remove previous comparison: {folder}\"\nprint(\"Outputs will be written under build/repro-a/ and build/repro-b/\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 2 · Build both images twice\n\nThis command builds from the committed source twice and compares complete uncompressed RAW and ISO files byte-for-byte." + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "run(\"make\",\"repro\")" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Action 3 · Recheck byte equality" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": "import filecmp\nfor relative in (\"raw/sc-linux.raw\",\"sc-linux.iso\"):\n a=ROOT/\"build/repro-a\"/relative\n b=ROOT/\"build/repro-b\"/relative\n assert filecmp.cmp(a,b,shallow=False), f\"Reproducibility mismatch: {relative}\"\n print(\"Identical:\",relative)" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Check\n\nA passed comparison establishes byte equality **for these two builds** under the same builder. It does not prove universal byte-for-byte reproducibility across hosts, availability of pinned base images, root integrity, or production security.\n\n## Next\n\nRecord the tested commit and artifact hashes, and review [verification requirements](https://github.com/soft-cloud-dev/sc-linux/blob/main/docs/verification.md)." + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "tags": [ + "skip-execution" + ] + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/book/appendix.md b/book/appendix.md new file mode 100644 index 0000000..63fe2d3 --- /dev/null +++ b/book/appendix.md @@ -0,0 +1,22 @@ +# Reference: where the actual code lives + +| Area | Authoritative file | +| --- | --- | +| Source checks | [check notebook](https://github.com/soft-cloud-dev/sc-linux/blob/main/notebooks/check.ipynb) | +| Trust input validation | [trust notebook](https://github.com/soft-cloud-dev/sc-linux/blob/main/notebooks/trust.ipynb) | +| Host filesystem and apt snapshot staging | [prepare notebook](https://github.com/soft-cloud-dev/sc-linux/blob/main/notebooks/prepare.ipynb) | +| mkosi disk and installer recipes | [server configuration](https://github.com/soft-cloud-dev/sc-linux/blob/main/images/server/mkosi.conf), [installer configuration](https://github.com/soft-cloud-dev/sc-linux/blob/main/images/installer/mkosi.conf) | +| Container orchestration | [build-inside.sh](https://github.com/soft-cloud-dev/sc-linux/blob/main/scripts/build-inside.sh) | +| Installer ISO assembly | [make-iso notebook](https://github.com/soft-cloud-dev/sc-linux/blob/main/notebooks/make-iso.ipynb) | +| Artifact SHA-256 manifests | [artifacts notebook](https://github.com/soft-cloud-dev/sc-linux/blob/main/notebooks/artifacts.ipynb) | +| Disk usage report | [footprint notebook](https://github.com/soft-cloud-dev/sc-linux/blob/main/notebooks/footprint.ipynb) | +| QEMU smoke tests | [smoke notebook](https://github.com/soft-cloud-dev/sc-linux/blob/main/notebooks/smoke.ipynb) | +| NoCloud ISO | [seed notebook](https://github.com/soft-cloud-dev/sc-linux/blob/main/notebooks/seed.ipynb) | + +A kernel and userspace inside the deployed image must work independently of Jupyter. Guest runtime scripts and `lib/scos/core.py` therefore remain normal executables. + +## Book maintenance + +The chapters are teaching and operator interfaces. Avoid duplicating ISO assembly, cryptographic validation, or disk writes in educational cells: instead, call the tested underlying command and inspect its outputs. Never commit executed notebook outputs, secret material, TPM logs, seed media or enrollment passwords. + +Build documentation with `jupyter book build --html` (no execution). Live execution is opt-in in your local JupyterLab environment. For implementation detail and limits consult [architecture](https://github.com/soft-cloud-dev/sc-linux/blob/main/docs/architecture.md), [verification](https://github.com/soft-cloud-dev/sc-linux/blob/main/docs/verification.md) and [security](https://github.com/soft-cloud-dev/sc-linux/blob/main/SECURITY.md). diff --git a/book/index.md b/book/index.md new file mode 100644 index 0000000..dda91ec --- /dev/null +++ b/book/index.md @@ -0,0 +1,22 @@ +# Build SC Linux, one action at a time + +SC Linux builds a Debian 13 (`amd64`) UEFI server disk and an installer ISO using `mkosi`. This book explains each decision, runs a sequence of independently inspectable actions, and checks the result before moving on. + +**What you will produce:** `sc-linux.raw.zst` (deployable compressed disk) and `sc-linux.iso` (UEFI installer), plus size reports, hashes and disposable VM boot evidence. + +## How to use this book + +Follow the nine chapters in the left navigation. Each executable chapter has the same flow: **Goal → Prerequisites → Action → Check → Next**. Execute the code cells **locally in JupyterLab**, not on a public website. Use `make book-lab` for ignored working copies, so notebook autosaves cannot dirty the committed source. Notebook actions may access Docker, build privileged images or modify local outputs. + +The book is a **guided operator interface** to the tested build pipeline, not a second implementation. Generate documentation with `jupyter book build --html` **without `--execute`**. The existing `make` and `notebooks/*.ipynb` commands remain the canonical automation and CI implementation. + +## Safety and reproducibility + +- **Do not** run untrusted code using privileged Docker. +- The build requires a trusted public FreeIPA CA; verify its DER SHA-256 fingerprint via an independent channel. +- Source checks, completed image builds, VM boot tests and live FreeIPA enrollment are separate milestones. +- Individual cells print what they are about to do. The imaging and installer stages are destructive to their own output directories, and disk installation is destructive to the selected target. + +## Starting point + +Clone the repository, follow [Environment](01-environment.ipynb), then complete the remaining chapters in order. The book uses the existing build pipeline rather than redefining lower-level image assembly logic. diff --git a/docs/building.md b/docs/building.md index 4a3343b..4017191 100644 --- a/docs/building.md +++ b/docs/building.md @@ -1,5 +1,7 @@ # Build your first image +For an interactive, chapter-by-chapter explanation of the complete image factory, use the [SC Linux Jupyter Book](../book/index.md). The book separates environment checks, version pins, trust, filesystem staging, mkosi image construction, packaging, checksum verification and VM testing. The actual CI image pipeline remains the canonical source of build behavior. + [Home](../README.md) · [Install](installing.md) · [Verification](verification.md) This guide builds **two files from one server definition**: a compressed RAW disk for a hypervisor, and an ISO that installs the same disk image. diff --git a/myst.yml b/myst.yml new file mode 100644 index 0000000..57026fc --- /dev/null +++ b/myst.yml @@ -0,0 +1,34 @@ +version: 1 +project: + id: d08aee12-7bd3-4630-bf4a-0d739f97a82d + exclude: + - README.md + - CONTRIBUTING.md + - SECURITY.md + - docs/** + - notebooks/** + - trust/** + title: Building SC Linux + description: A step-by-step laboratory for building and verifying a Debian 13 operating system image + github: soft-cloud-dev/sc-linux + toc: + - file: book/index.md + - title: Prepare + children: + - file: book/01-environment.ipynb + - file: book/02-configuration.ipynb + - file: book/03-trust.ipynb + - title: Build + children: + - file: book/04-builder.ipynb + - file: book/05-image.ipynb + - title: Verify and deploy + children: + - file: book/06-artifacts.ipynb + - file: book/07-boot.ipynb + - file: book/08-provision.ipynb + - file: book/09-reproducibility.ipynb + - file: book/appendix.md +site: + template: book-theme + title: Building SC Linux diff --git a/requirements-book.txt b/requirements-book.txt new file mode 100644 index 0000000..84900d5 --- /dev/null +++ b/requirements-book.txt @@ -0,0 +1 @@ +jupyter-book==2.1.7 diff --git a/scripts/book-lab.sh b/scripts/book-lab.sh new file mode 100644 index 0000000..946f0f5 --- /dev/null +++ b/scripts/book-lab.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." + +if ! command -v jupyter >/dev/null 2>&1; then + printf '%s\n' 'JupyterLab is required. Install it in a local virtual environment.' >&2 + exit 1 +fi + +mkdir -p build/book-notebooks +for source in book/*.ipynb book/*.md; do + destination="build/book-notebooks/${source##*/}" + if [[ ! -e "$destination" ]]; then + cp "$source" "$destination" + fi +done + +printf '%s\n' 'Editing ignored working copies in build/book-notebooks/' >&2 +exec jupyter lab --ServerApp.root_dir="$PWD/build/book-notebooks" diff --git a/scripts/build-inside.sh b/scripts/build-inside.sh index 39fab93..00af059 100755 --- a/scripts/build-inside.sh +++ b/scripts/build-inside.sh @@ -2,45 +2,72 @@ # Container entrypoint: stage inputs, build both images, and record provenance. set -euo pipefail -[[ $(dpkg-query -W -f='${Version}' mkosi) == 25.3-7 ]] -: "${SOURCE_DATE_EPOCH:?}" "${SOURCE_COMMIT:?}" +stage() { + printf 'SCOS_BUILD_STAGE %s\n' "$1" +} + +verify_toolchain() { + stage verify-toolchain + [[ $(dpkg-query -W -f='${Version}' mkosi) == 25.3-7 ]] + : "${SOURCE_DATE_EPOCH:?}" "${SOURCE_COMMIT:?}" +} + +prepare_workspace() { + stage prepare-workspace + work_directory=$(mktemp -d /var/tmp/sc-linux.XXXXXX) + trap 'rm -rf "$work_directory"' EXIT -work_directory=$(mktemp -d /var/tmp/sc-linux.XXXXXX) -trap 'rm -rf "$work_directory"' EXIT + tar -C /src \ + --exclude=./.git --exclude=./build --exclude=./dist \ + --exclude=./_build --exclude=./.myst --exclude=./.venv \ + --exclude=__pycache__ --exclude='*.pyc' \ + --exclude=.ipynb_checkpoints -cf - . | + tar -C "$work_directory" -xf - + cd "$work_directory" +} -tar -C /src \ - --exclude=./.git --exclude=./build --exclude=./dist \ - --exclude=__pycache__ --exclude='*.pyc' -cf - . | - tar -C "$work_directory" -xf - -cd "$work_directory" +validate_and_stage_inputs() { + stage validate-and-stage-inputs + bash scripts/notebook.sh notebooks/check.ipynb --require-trust + bash scripts/notebook.sh notebooks/prepare.ipynb + partition_seed=$(python3 -c 'import json; print(json.load(open("config/build.json"))["seed"])') +} -bash scripts/notebook.sh notebooks/check.ipynb --require-trust -bash scripts/notebook.sh notebooks/prepare.ipynb -partition_seed=$(python3 -c 'import json; print(json.load(open("config/build.json"))["seed"])') +configure_efi_signing() { + stage configure-efi-signing + signing_arguments=() + if [[ -d /signing ]]; then + signing_arguments=( + --secure-boot=yes + --secure-boot-key=/signing/private.key + --secure-boot-certificate=/signing/certificate.pem + --secure-boot-auto-enroll=no + ) + fi +} -signing_arguments=() -if [[ -d /signing ]]; then - signing_arguments=( - --secure-boot=yes - --secure-boot-key=/signing/private.key - --secure-boot-certificate=/signing/certificate.pem - --secure-boot-auto-enroll=no - ) -fi +build_server_raw() { + stage build-server-raw + mkdir -p /out/raw /out/installer -mkdir -p /out/raw /out/installer + # The server is the payload installed by both deployment paths. + mkosi -C images/server \ + --output-directory=/out/raw \ + --source-date-epoch="$SOURCE_DATE_EPOCH" \ + --seed="$partition_seed" \ + "${signing_arguments[@]}" build + test -s /out/raw/sc-linux.raw +} -# The server is the payload installed by both deployment paths. -mkosi -C images/server \ - --output-directory=/out/raw \ - --source-date-epoch="$SOURCE_DATE_EPOCH" \ - --seed="$partition_seed" \ - "${signing_arguments[@]}" build -test -s /out/raw/sc-linux.raw -zstd -T1 -19 --no-progress /out/raw/sc-linux.raw -o /out/sc-linux.raw.zst +compress_server_raw() { + stage compress-server-raw + zstd -T1 -19 --no-progress /out/raw/sc-linux.raw -o /out/sc-linux.raw.zst +} -# Embed the expected payload digest in the installer before building its UKI. -PYTHONPATH=lib python3 - <<'PY' +record_installer_payload() { + stage record-installer-payload + # Embed the expected payload digest in the installer before building its UKI. + PYTHONPATH=lib python3 - <<'PY' import json from pathlib import Path from scos.core import describe_file @@ -53,18 +80,43 @@ text = json.dumps(manifest, sort_keys=True, indent=2) + '\n' Path('build/installer-overlay/usr/local/share/scos/payload.json').write_text(text) Path('/out/payload.json').write_text(text) PY +} + +build_installer_uki() { + stage build-installer-uki + mkosi -C images/installer \ + --output-directory=/out/installer \ + --source-date-epoch="$SOURCE_DATE_EPOCH" \ + "${signing_arguments[@]}" build + test -s /out/installer/sc-linux-installer.efi +} -mkosi -C images/installer \ - --output-directory=/out/installer \ - --source-date-epoch="$SOURCE_DATE_EPOCH" \ - "${signing_arguments[@]}" build -test -s /out/installer/sc-linux-installer.efi -bash scripts/notebook.sh notebooks/make-iso.ipynb \ - /out/installer/sc-linux-installer.efi /out/sc-linux.raw.zst /out/sc-linux.iso +assemble_installer_iso() { + stage assemble-installer-iso + bash scripts/notebook.sh notebooks/make-iso.ipynb \ + /out/installer/sc-linux-installer.efi /out/sc-linux.raw.zst /out/sc-linux.iso +} + +record_build_evidence() { + stage record-build-evidence + bash scripts/notebook.sh notebooks/artifacts.ipynb /out + bash scripts/notebook.sh notebooks/footprint.ipynb /out + cp config/build.json /out/build-config.json + dpkg-query -W -f='${binary:Package}\t${Version}\n' > /out/builder-packages.tsv + printf '%s\n' "$SOURCE_COMMIT" > /out/source-commit.txt + printf '%s\n' "$SOURCE_DATE_EPOCH" > /out/source-date-epoch.txt +} +main() { + verify_toolchain + prepare_workspace + validate_and_stage_inputs + configure_efi_signing + build_server_raw + compress_server_raw + record_installer_payload + build_installer_uki + assemble_installer_iso + record_build_evidence +} -bash scripts/notebook.sh notebooks/artifacts.ipynb /out -bash scripts/notebook.sh notebooks/footprint.ipynb /out -cp config/build.json /out/build-config.json -dpkg-query -W -f='${binary:Package}\t${Version}\n' > /out/builder-packages.tsv -printf '%s\n' "$SOURCE_COMMIT" > /out/source-commit.txt -printf '%s\n' "$SOURCE_DATE_EPOCH" > /out/source-date-epoch.txt +main "$@" diff --git a/tests/test_book_contract.py b/tests/test_book_contract.py new file mode 100644 index 0000000..e74737b --- /dev/null +++ b/tests/test_book_contract.py @@ -0,0 +1,81 @@ +"""Structural and safety checks for the Jupyter Book laboratory.""" + +import ast +import json +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BOOK = ROOT / "book" +CHAPTERS = ( + "01-environment", "02-configuration", "03-trust", + "04-builder", "05-image", "06-artifacts", + "07-boot", "08-provision", "09-reproducibility", +) + + +class BookContractTests(unittest.TestCase): + def test_project_toc_lists_every_chapter_in_order(self): + toc = (ROOT / "myst.yml").read_text() + files = re.findall(r"^\s+- file: ([^\s#]+)", toc, flags=re.MULTILINE) + expected = ["book/index.md", *[f"book/{n}.ipynb" for n in CHAPTERS], "book/appendix.md"] + self.assertEqual(files, expected) + for relative in files: + self.assertTrue((ROOT / relative).is_file(), relative) + + def test_chapters_are_actionable_and_not_executed_by_site(self): + for name in CHAPTERS: + with self.subTest(chapter=name): + path = BOOK / f"{name}.ipynb" + doc = json.loads(path.read_text()) + self.assertEqual(doc["nbformat"], 4) + self.assertIn("skip-execution", doc.get("metadata", {}).get("tags", [])) + markdown = "\n".join( + c["source"] if isinstance(c["source"], str) else "".join(c["source"]) + for c in doc["cells"] if c["cell_type"] == "markdown" + ) + for marker in ("## Goal", "## Prerequisites", "## Action", "## Check", "## Next"): + self.assertIn(marker, markdown) + code = [c for c in doc["cells"] if c["cell_type"] == "code"] + self.assertGreaterEqual(len(code), 2) + for index, cell in enumerate(code, 1): + source = cell["source"] + if isinstance(source, list): + source = "".join(source) + ast.parse(source, filename=f"{path}:{index}") + self.assertIsNone(cell["execution_count"]) + self.assertEqual(cell["outputs"], []) + + def test_internal_links_resolve(self): + files = [BOOK / "index.md", BOOK / "appendix.md"] + files.extend(BOOK.glob("*.ipynb")) + for path in files: + if path.suffix == ".ipynb": + doc = json.loads(path.read_text()) + sources = [ + c["source"] if isinstance(c["source"], str) else "".join(c["source"]) + for c in doc["cells"] if c["cell_type"] == "markdown" + ] + else: + sources = [path.read_text()] + for link in re.findall(r"\[[^\]]+\]\(([^)]+)\)", "\n".join(sources)): + relative = link.split("#", 1)[0].split("?", 1)[0] + if not relative or "://" in relative or relative.startswith(("/", "mailto:")): + continue + with self.subTest(page=path.name, link=link): + self.assertTrue((path.parent / relative).exists(), f"Missing link: {link}") + + def test_book_lab_does_not_write_to_source(self): + script = (ROOT / "scripts/book-lab.sh").read_text() + self.assertIn("build/book-notebooks", script) + self.assertIn('for source in book/*.ipynb', script) + self.assertIn("ServerApp.root_dir", script) + makefile = (ROOT / "Makefile").read_text() + self.assertIn("book-lab:", makefile) + self.assertIn("jupyter book build --html", makefile) + self.assertNotIn("jupyter book build --execute", makefile) + + +if __name__ == "__main__": + unittest.main()