diff --git a/.github/workflows/family-release.yml b/.github/workflows/family-release.yml new file mode 100644 index 0000000..9bbf1d5 --- /dev/null +++ b/.github/workflows/family-release.yml @@ -0,0 +1,128 @@ +name: Family release + +# The release of the version group this repository states (mcpp design +# 2026-11-07, §4.2.2, A5 and A6), run on every change of the group and on demand: +# +# group every member's tag and its GitCode mirror (the same bytes); the +# open issues of the members' repositories, for the release notes; +# the group's one index change, written into mcpp-index's main and +# kept as an artifact (`index-change.patch`), from which the +# maintainer opens the one index PR +# resolve over that candidate index, the released mcpp and the one under +# review resolve an image of the group and build it; and for a +# patch group, the C library and the C++ run-time objects the image +# links keep every global symbol of the previous patch's +# +# The repository holds no credential for mcpp-index: the PR is opened from the +# artifact (`tools/family_release.py index` does the same locally). + +on: + pull_request: + paths: [mcpp.toml, tools/family_release.py, .github/workflows/family-release.yml] + workflow_dispatch: + inputs: + also: + description: "Packages released with the group, as NAME@VERSION separated by spaces (os@0.3.1)" + required: false + default: "" + mcpp_ref: + description: "Branch of mcpp-community/mcpp to resolve with as the mcpp under review (empty = the released mcpp)" + required: false + default: "" + +permissions: + contents: read + +env: + MCPP_PREVIOUS: 2026.11.5.1 + MCPP_SOURCE_REF: ${{ github.event.inputs.mcpp_ref || vars.MCPP_SOURCE_REF || '' }} + XLINGS_VERSION: v2026.10.26.1 + XLINGS_NON_INTERACTIVE: '1' + GITHUB_TOKEN: ${{ github.token }} + +jobs: + group: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - name: Tags and mirrors + run: | + also=(); for spec in ${{ github.event.inputs.also }}; do also+=(--also "$spec"); done + echo "ALSO=${also[*]}" >> "$GITHUB_ENV" + python3 tools/family_release.py "${also[@]}" check | tee -a "$GITHUB_STEP_SUMMARY" + - name: Open issues of the members + run: python3 tools/family_release.py $ALSO defects | tee -a "$GITHUB_STEP_SUMMARY" + - name: The group's one index change + run: | + git clone --quiet --depth 1 https://github.com/mcpplibs/mcpp-index.git "$RUNNER_TEMP/index" + python3 tools/family_release.py $ALSO index "$RUNNER_TEMP/index" + git -C "$RUNNER_TEMP/index" diff > index-change.patch + git -C "$RUNNER_TEMP/index" diff --stat + - uses: actions/upload-artifact@v4 + with: + name: index-change + path: index-change.patch + retention-days: 30 + + resolve: + needs: group + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: index-change + - name: xlings, the released mcpp and the one under review + run: | + curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.sh | bash -s "$XLINGS_VERSION" + export PATH="$HOME/.xlings/subos/current/bin:$PATH" + echo "$HOME/.xlings/subos/current/bin" >> "$GITHUB_PATH" + for attempt in 1 2 3 4 5; do + xlings update > /dev/null 2>&1 || true + if xlings install "mcpp@$MCPP_PREVIOUS" -y -g; then break; fi + [ "$attempt" = 5 ] && exit 1 + sleep 60 + done + previous=$(command -v mcpp) + echo "MCPP_OLD=$previous" >> "$GITHUB_ENV" + current="$previous" + if [ -n "$MCPP_SOURCE_REF" ]; then + src="$RUNNER_TEMP/mcpp-src" + git clone --quiet --depth 1 --branch "$MCPP_SOURCE_REF" https://github.com/mcpp-community/mcpp.git "$src" + rm -f "$src/.xlings.json" + ( cd "$src" && mcpp build --release ) + current=$(find "$src/target" -type f -name mcpp -path '*/bin/*' | head -1) + else + xlings install mcpp -y -g > /dev/null 2>&1 || true + current=$(command -v mcpp) + fi + echo "MCPP_NEW=$current" >> "$GITHUB_ENV" + "$previous" --version; "$current" --version + "$current" self config --mirror GLOBAL > /dev/null + - name: The candidate index (the published one with the group's change) + run: | + git clone --quiet --depth 1 https://github.com/mcpplibs/mcpp-index.git "$RUNNER_TEMP/index" + git -C "$RUNNER_TEMP/index" apply "$GITHUB_WORKSPACE/index-change.patch" + python3 "$RUNNER_TEMP/index/tests/prepare_builtin_index.py" --home "$HOME/.mcpp" --source "$RUNNER_TEMP/index" + - name: Both mcpp releases resolve and build an image of the group + run: | + version=$(python3 -c 'import tomllib; print(tomllib.load(open("mcpp.toml","rb"))["package"]["version"])') + echo "PLATFORM=openkal-platform@$version" >> "$GITHUB_ENV" + for mcpp in "$MCPP_OLD" "$MCPP_NEW"; do + dir="$RUNNER_TEMP/probe-$(basename "$(dirname "$(dirname "$mcpp")")")" + mkdir -p "$dir/src" + printf '[package]\nname = "probe"\nversion = "0.1.0"\n' > "$dir/mcpp.toml" + printf 'import std;\nint main() { std::println("group ok"); }\n' > "$dir/src/main.cpp" + ( cd "$dir" && "$mcpp" build --target x86_64-openkal --platform "openkal-platform@$version" ) \ + || { echo "::error::$("$mcpp" --version) did not build an image of the group"; exit 1; } + done + - name: A patch keeps every global symbol of the previous patch + run: | + version=${PLATFORM#*@} + patch=${version##*.} + if [ "$patch" = 0 ]; then echo "a minor version: no previous patch to compare"; exit 0; fi + previous="openkal-platform@${version%.*}.$((patch - 1))" + nm=$(find "$HOME/.mcpp/registry/data/xpkgs" -path '*xim-x-llvm*' -name llvm-nm -type f | head -1) + python3 tools/family_release.py exports "$previous" "$PLATFORM" --mcpp "$MCPP_NEW" --nm "$nm" | tee -a "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index 5950139..b4552b0 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,11 @@ and image bindings 0.5.0 over the OpenKal 0.24 implementations. The C library calls the optional resolved-name operation directly, preserving its fallback, error and output-bound checks. +Platform 0.24.1 corrects that group: openkal-musl 0.28.1 keeps a directory's +whole name however long, openkal-llvm-runtime 0.19.1 reads a name beneath a +Windows volume as absolute, openkal-macos 0.22.1 sizes its confinement rules to +the grants of each start, and openkal-image 0.5.1 builds okrun over it. + A project overrides any group under `[target.]`, for example `[target.'cfg(platform = "openkal")'.layers] "c++-abi" = false`, and selects another platform, or none, with `platform = "..."` or `--platform`. @@ -143,9 +148,28 @@ architecture and run by okrun (`checks/host-graph`). ## Version group -`[platform.versions]` names one version of each package of the family. The -packages pin one another exactly; a graph holding another version of one of -them is refused, naming the group and the two remedies. +`[platform.versions]` names one version of each package of the family. From +0.24.1 the packages name one another by compatible ranges (`^0.28.0`), and the +group chooses: mcpp 2026.11.7.1 and later resolve each member to the group's +version, and an earlier mcpp refuses a graph holding another version of one of +them, naming the group and the two remedies. A patch version of a member keeps +every symbol its previous patch defines. + +A fix of one member is that member's release and a patch of this group. The +group is released as one index change: + +```bash +python3 tools/family_release.py check # tags, GitCode mirrors, the index +python3 tools/family_release.py defects # open issues, for the release notes +python3 tools/family_release.py index ../mcpp-index # the group's one index change +python3 tools/family_release.py exports openkal-platform@0.24.0 openkal-platform@0.24.1 +``` + +`.github/workflows/family-release.yml` runs them on every change of the group: +it keeps the index change as an artifact, resolves an image of the group with +the released mcpp and the one under review over the candidate index, and +compares the global symbols of the C library and the C++ run-time with the +previous patch's. ## Checks @@ -156,6 +180,7 @@ them is refused, naming the group and the two remedies. | `checks/hosted` | the hosted arrangement runs on Linux, macOS and Windows | | `checks/host-graph` | a build program runs as an ok image | | `checks/snapshot` | a C++ program starts from a snapshot of its initialisers, and a start whose named value differs does not | +| family release | the group's members are tagged and mirrored, both mcpp releases resolve an image of it, and a patch removes no symbol | ## License diff --git a/mcpp.toml b/mcpp.toml index 5273a55..f81fdd0 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -12,7 +12,7 @@ [package] namespace = "mcpplibs" name = "openkal-platform" -version = "0.24.0" +version = "0.24.1" description = "The platform of ok images (-openkal): their toolchain, C library and C++ run-time, tools, compile and link conventions, seal, runner and carrier, as one versioned statement mcpp reads." license = "Apache-2.0" authors = ["mcpplibs"] @@ -28,7 +28,7 @@ toolchain = "llvm@23.1.3" runner = { cmd = ["okrun", "{}"], host = "same-arch", hosts = ["linux-x86_64", "linux-aarch64", "macos-aarch64", "windows-x86_64"] } # `mcpp pack --mode self-contained`: a program of the packing machine that # carries the image in a section and runs it through libokrun (M3). -carrier = { package = "openkal-run@0.5.0", section = ".oke", main = "carrier/main.cpp" } +carrier = { package = "openkal-run@0.5.1", section = ".oke", main = "carrier/main.cpp" } # The `okb` package format (`mcpp pack --pkg-format okb`): the images of every # architecture as one file okrun runs (openkal-image docs/okb.md). plugin = "build.mcpp" @@ -40,18 +40,18 @@ debuggee = ["okrun", "--debugger", "{}"] # The C library and the C++ run-time inside an image. A package of the graph # that provides a layer takes its place (`provides`), and `false` leaves it out. [platform.layers] -compiler-runtime = "openkal-llvm-runtime@0.19.0" -"c++-abi" = "openkal-llvm-runtime@0.19.0" -c-abi = { name = "openkal-musl", version = "0.28.0", default-features = false, features = ["posix", "malloc-mimalloc"] } +compiler-runtime = "openkal-llvm-runtime@0.19.1" +"c++-abi" = "openkal-llvm-runtime@0.19.1" +c-abi = { name = "openkal-musl", version = "0.28.1", default-features = false, features = ["posix", "malloc-mimalloc"] } # Built for the building machine: `okld` writes and checks the notes, `okrun` # runs an image, `okprof` reads what a measured run recorded (0.23.0: the # order of a start's first executions, which a producer lays its code out by # and writes as the touch note). [platform.tools] -okld = "openkal-tools@0.5.0" -okrun = "openkal-tools@0.5.0" -okprof = "openkal-tools@0.5.0" +okld = "openkal-tools@0.5.1" +okrun = "openkal-tools@0.5.1" +okprof = "openkal-tools@0.5.1" # Every unit of an image (SPEC 12.2): # -fPIC the image is ET_DYN and has no text relocation; @@ -162,9 +162,9 @@ targets = ["x86_64-linux-gnu", "aarch64-linux-gnu", "x86_64-linux-musl", "aarc toolchain = "llvm@23.1.3" [platform.hosted.layers] -compiler-runtime = "openkal-llvm-runtime@0.19.0" -"c++-abi" = "openkal-llvm-runtime@0.19.0" -c-abi = { name = "openkal-musl", version = "0.28.0", default-features = false, features = ["posix", "malloc-mimalloc"] } +compiler-runtime = "openkal-llvm-runtime@0.19.1" +"c++-abi" = "openkal-llvm-runtime@0.19.1" +c-abi = { name = "openkal-musl", version = "0.28.1", default-features = false, features = ["posix", "malloc-mimalloc"] } # The family released together for openkal 0.17 (the 0.17.2 round: okb, the # runner's bundle and library interfaces, musl's renameat2; the 0.17.3 round: @@ -190,23 +190,30 @@ c-abi = { name = "openkal-musl", version = "0.28.0", default-features # 0.23.0, openkal 0.23: the system's null device at a started program's # position, which openkal-musl 0.27.0 maps `/dev/null` onto (openkal 0.22); an # image placed at its base, and realpath answered by the environment; the -# relative relocations of an image packed): -# one version of each in a build (their pins are exact). +# relative relocations of an image packed; 0.24.1: openkal-musl 0.28.1 keeps a +# directory's whole name, openkal-llvm-runtime 0.19.1 reads a name beneath a +# Windows volume as absolute, openkal-macos 0.22.1 sizes its confinement rules +# to the grants, and openkal-image 0.5.1 starts programs over it): +# one version of each in a build. From 0.24.1 the members name one another by +# compatible ranges (`^0.28.0`) and this group chooses the versions: mcpp +# 2026.11.7.1 resolves within it, and a fix of one member is that member's +# release and a patch of this group, released together through one index change +# (tools/family_release.py, .github/workflows/family-release.yml). [platform.versions] openkal = "0.24.0" openkal-linux = "0.25.0" -openkal-macos = "0.22.0" +openkal-macos = "0.22.1" openkal-windows = "0.20.0" -openkal-musl = "0.28.0" -openkal-llvm-runtime = "0.19.0" -openkal-ld = "0.5.0" -openkal-run = "0.5.0" -openkal-tools = "0.5.0" +openkal-musl = "0.28.1" +openkal-llvm-runtime = "0.19.1" +openkal-ld = "0.5.1" +openkal-run = "0.5.1" +openkal-tools = "0.5.1" # The plugin's tool, `okld bundle`, which writes the okb, and its library, the # provider of a package format (mcpp.plugins.platform). [build-dependencies] -openkal-tools = { version = "0.5.0", tools = ["okld"] } +openkal-tools = { version = "0.5.1", tools = ["okld"] } [build-dependencies.mcpp] plugins = { version = "0.20.0", features = ["plugins-platform"], host-module = true } diff --git a/tools/family_release.py b/tools/family_release.py new file mode 100644 index 0000000..f57b01e --- /dev/null +++ b/tools/family_release.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""The release of the openkal family's version group (mcpp design 2026-11-07, §4.2.2, A5 and A6). + +The group is `[platform.versions]` of this repository's mcpp.toml, with the +platform itself; `--also NAME@VERSION` adds a package released with it (os). A +member is released by its own repository: a tag equal to its version, the +GitHub archive of the tag, and the same bytes on GitCode (mcpp-res/). The +commands state facts about the group; only `index` writes, into a checkout of +mcpp-index it is given. + + check each member's tag, archive and mirror, and whether the + published index lists it already + index DIR adds the members the index lacks to the mcpp-index + checkout DIR: the whole group as one change, the content + of its one index PR + defects the open issues of each member's repository, for the + release notes + exports OLD NEW builds one image with the platform at OLD and at NEW and + compares the global symbols of the C library and the C++ + run-time objects it links: a patch version removes none + +Network reads use GITHUB_TOKEN when it is set. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import tomllib +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +# The packages one repository releases under one tag and one archive. +REPOSITORY = {"openkal-ld": "openkal-image", "openkal-run": "openkal-image", + "openkal-tools": "openkal-image", "openkal-bind": "openkal-image"} +INDEX_RAW = "https://raw.githubusercontent.com/mcpplibs/mcpp-index/main/pkgs/o/{}.lua" +# The layers whose objects an image links, by the directory mcpp builds them into. +LAYERS = ("mcpplibs_openkal-musl", "mcpplibs_openkal-llvm-runtime") + + +def group(also): + manifest = tomllib.loads((ROOT / "mcpp.toml").read_text(encoding="utf-8")) + members = dict(manifest["platform"]["versions"]) + members["openkal-platform"] = manifest["package"]["version"] + for spec in also: + name, version = spec.split("@", 1) + members[name] = version + return members + + +def fetch(url, binary=True): + request = urllib.request.Request(url, headers={"User-Agent": "openkal-family-release"}) + token = os.environ.get("GITHUB_TOKEN") + if token and url.startswith("https://api.github.com/"): + request.add_header("Authorization", f"Bearer {token}") + try: + with urllib.request.urlopen(request, timeout=120) as response: + data = response.read() + except (urllib.error.HTTPError, urllib.error.URLError): + return None + return data if binary else data.decode("utf-8") + + +def urls(name, version): + repo = REPOSITORY.get(name, name) + return (f"https://github.com/mcpplibs/{repo}/archive/refs/tags/{version}.tar.gz", + f"https://gitcode.com/mcpp-res/{repo}/releases/download/{version}/{repo}-{version}.tar.gz") + + +def listed(descriptor, version): + """The sha256 the descriptor states for `version`, or None.""" + m = re.search(r'\["' + re.escape(version) + r'"\]\s*=\s*\{.*?sha256\s*=\s*"([0-9a-f]{64})"', descriptor, re.S) + return m.group(1) if m else None + + +def check(members): + rows, problems = [], 0 + digests = {} + for name, version in members.items(): + github, gitcode = urls(name, version) + repo = REPOSITORY.get(name, name) + if repo in digests and digests[repo][0] == version: + a = b = digests[repo][1] + else: + data = fetch(github) + a = hashlib.sha256(data).hexdigest() if data else None + mirror = fetch(gitcode) + b = hashlib.sha256(mirror).hexdigest() if mirror else None + digests[repo] = (version, a) + descriptor = fetch(INDEX_RAW.format(name), binary=False) or "" + indexed = listed(descriptor, version) + state = ("no tag" if a is None else "no mirror" if b is None else "mirror differs" if a != b + else "indexed" if indexed == a else "index differs" if indexed else "not indexed") + problems += state in ("no tag", "no mirror", "mirror differs", "index differs") + rows.append((name, version, state, a or "-")) + print("| member | version | state | sha256 |\n|---|---|---|---|") + for row in rows: + print("| {} | {} | {} | `{}` |".format(*row)) + return 1 if problems else 0 + + +def entry(version, github, gitcode, sha): + return (f' ["{version}"] = {{\n' + f' url = {{\n' + f' GLOBAL = "{github}",\n' + f' CN = "{gitcode}",\n' + f' }},\n' + f' sha256 = "{sha}",\n' + f' }},\n') + + +def index(members, checkout): + added = [] + for name, version in members.items(): + path = checkout / "pkgs/o" / f"{name}.lua" + text = path.read_text(encoding="utf-8") + if listed(text, version): + continue + github, gitcode = urls(name, version) + data, mirror = fetch(github), fetch(gitcode) + if not data or not mirror or hashlib.sha256(data).digest() != hashlib.sha256(mirror).digest(): + sys.exit(f"{name} {version}: the tag or its mirror is missing, or they differ; run `check`") + sha = hashlib.sha256(data).hexdigest() + # Newest first in each system's table, as the descriptors are written. + text, n = re.subn(r'(\n\s*(?:linux|macosx|windows)\s*=\s*\{\n)', + lambda m: m.group(1) + entry(version, github, gitcode, sha), text) + if n == 0: + sys.exit(f"{path}: no system table to add {version} to") + path.write_text(text, encoding="utf-8") + added.append(f"{name} {version}") + print("added: " + (", ".join(added) if added else "nothing, the index lists the group")) + return 0 + + +def defects(members): + seen = set() + print("## Open issues of the group's repositories\n") + for name in members: + repo = REPOSITORY.get(name, name) + if repo in seen: + continue + seen.add(repo) + text = fetch(f"https://api.github.com/repos/mcpplibs/{repo}/issues?state=open&per_page=100", binary=False) + if text is None: + print(f"- {repo}: the issues could not be read") + continue + issues = [i for i in json.loads(text) if "pull_request" not in i] + for issue in issues: + print(f"- {repo}#{issue['number']}: {issue['title']}") + if not issues: + print(f"- {repo}: none") + return 0 + + +def symbols(mcpp, platform, target, work, nm): + project = work / platform.replace("@", "-") + (project / "src").mkdir(parents=True, exist_ok=True) + (project / "mcpp.toml").write_text('[package]\nname = "exports"\nversion = "0.1.0"\n', encoding="utf-8") + (project / "src/main.cpp").write_text( + 'import std;\nint main() { std::vector v{"a"}; std::println("{}", v.size()); }\n', + encoding="utf-8") + subprocess.run([mcpp, "build", "--target", target, "--platform", platform], cwd=project, check=True) + found = {} + for layer in LAYERS: + objects = [p for p in (project / "target").rglob("*.o") if layer in p.parts] + if not objects: + sys.exit(f"{platform}: the build linked no objects of {layer}") + out = subprocess.run([nm, "--defined-only", "--extern-only", "--format=just-symbols", *map(str, objects)], + check=True, capture_output=True, text=True).stdout + found[layer] = {s for s in out.split() if s and not s.endswith(":")} + return found + + +def exports(old, new, mcpp, target, nm): + with tempfile.TemporaryDirectory(prefix="openkal-exports-") as directory: + work = Path(directory) + before = symbols(mcpp, old, target, work, nm) + after = symbols(mcpp, new, target, work, nm) + problems = 0 + for layer in LAYERS: + removed = sorted(before[layer] - after[layer]) + gained = sorted(after[layer] - before[layer]) + print(f"{layer}: {len(before[layer])} -> {len(after[layer])} global symbols, " + f"{len(gained)} added, {len(removed)} removed") + for symbol in removed: + print(f" removed: {symbol}") + problems += len(removed) + return 1 if problems else 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--also", action="append", default=[], metavar="NAME@VERSION") + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("check") + p = sub.add_parser("index") + p.add_argument("checkout", type=Path) + sub.add_parser("defects") + p = sub.add_parser("exports") + p.add_argument("old", help="openkal-platform@ released before") + p.add_argument("new", help="openkal-platform@ of this group") + p.add_argument("--mcpp", default="mcpp") + p.add_argument("--target", default="x86_64-openkal") + p.add_argument("--nm", default=shutil.which("llvm-nm") or "llvm-nm") + args = parser.parse_args() + members = group(args.also) + if args.command == "check": + return check(members) + if args.command == "index": + return index(members, args.checkout.resolve()) + if args.command == "defects": + return defects(members) + return exports(args.old, args.new, args.mcpp, args.target, args.nm) + + +if __name__ == "__main__": + sys.exit(main())