From cafd827fcd3a859bb6f36b065ccc66e66998067b Mon Sep 17 00:00:00 2001 From: Andreas Trawoeger Date: Thu, 5 Nov 2026 20:48:10 +0100 Subject: [PATCH 1/4] fix(ensure): build into the run engine the caller asked for build:ensure's local fallback called runBoxBuild, whose driveConfig.RunEngine came from build:resolve's host-runtime reply. A caller that asked for a different run engine (a pod deploy authored engine: containerd) therefore had its locally-built image bridged into podman, and the alias step failed 'image "...": not found' in the containerd store. Thread the requested run_engine through an internal override so the BuildKit export lands in the store the deploy will actually run on (opencharly/charly#1109 / #809). Assisted-by: Claude Code ollama-cloud/deepseek-v4.1-flash:cloud (fully tested and validated) --- candy/plugin-build/drive.go | 20 +++++++++++++++++++- candy/plugin-build/ensure.go | 6 +++--- candy/plugin-build/ensure_alias_test.go | 25 +++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/candy/plugin-build/drive.go b/candy/plugin-build/drive.go index acf2445..ee911b6 100644 --- a/candy/plugin-build/drive.go +++ b/candy/plugin-build/drive.go @@ -78,12 +78,30 @@ func resolveBuild(ctx context.Context, ex *sdk.Executor, req spec.BuildRequest, // refs (the BuildReply.Written provenance). The engine RESOLVE runs plugin-side (resolveBuildEngine, // K3 U6); the layer MERGE crosses to verb:oci; the podman exec + the build-order orchestration live HERE. func runBoxBuild(ctx context.Context, ex *sdk.Executor, req spec.BuildRequest) ([]string, error) { + return runBoxBuildWithEngine(ctx, ex, req, "") +} + +// buildEngineFor picks the engine the BUILD bridges its OCI export into. `build:resolve` +// reports the host-wide runtime engine; a `build:ensure` caller that asked for a DIFFERENT +// run engine must win, or a locally-built image lands in the host engine's store and the +// deploy that asked for containerd never finds it (the alias step fails +// `image "...": not found`, and a pod bed cannot run on containerd). +func buildEngineFor(resolved, requested string) string { + if requested != "" { + return requested + } + return resolved +} + +// runBoxBuildWithEngine is runBoxBuild with an explicit run-engine override. `requested` +// empty preserves the pre-existing host-runtime resolution. +func runBoxBuildWithEngine(ctx context.Context, ex *sdk.Executor, req spec.BuildRequest, requested string) ([]string, error) { reply, err := resolveBuild(ctx, ex, req, false) if err != nil { return nil, err } cfg := driveConfig{ - RunEngine: reply.EngineName, + RunEngine: buildEngineFor(reply.EngineName, requested), Platform: reply.Platform, Cache: reply.Cache, Push: req.Push, diff --git a/candy/plugin-build/ensure.go b/candy/plugin-build/ensure.go index 3e18814..527c729 100644 --- a/candy/plugin-build/ensure.go +++ b/candy/plugin-build/ensure.go @@ -125,7 +125,7 @@ func ensureRemoteRef(ctx context.Context, ex *sdk.Executor, image, stripped, run } } fmt.Fprintf(os.Stderr, "ensure-image: building remote %s from cached source\n", image) - if _, berr := runBoxBuild(ctx, ex, spec.BuildRequest{Boxes: []string{rr.BoxName}, Dir: rr.CacheDir}); berr != nil { + if _, berr := runBoxBuildWithEngine(ctx, ex, spec.BuildRequest{Boxes: []string{rr.BoxName}, Dir: rr.CacheDir}, runEngine); berr != nil { return fmt.Errorf("ensure-image %q: pull failed and remote build failed: %w", image, berr) } return aliasBuiltImageOntoRequestedRef(ctx, image, runEngine, func() string { @@ -232,12 +232,12 @@ func ensureProjectRef(ctx context.Context, ex *sdk.Executor, image, dir, runEngi } fmt.Fprintf(os.Stderr, "ensure-image: building %s locally\n", fallbackShort) - if _, berr := runBoxBuild(ctx, ex, spec.BuildRequest{ + if _, berr := runBoxBuildWithEngine(ctx, ex, spec.BuildRequest{ Boxes: []string{fallbackShort}, Dir: dir, IncludeDisabled: true, Jobs: 4, - }); berr != nil { + }, runEngine); berr != nil { return fmt.Errorf("ensure-image %q: pull failed and local build failed: %w", image, berr) } diff --git a/candy/plugin-build/ensure_alias_test.go b/candy/plugin-build/ensure_alias_test.go index c470869..49e6de8 100644 --- a/candy/plugin-build/ensure_alias_test.go +++ b/candy/plugin-build/ensure_alias_test.go @@ -90,3 +90,28 @@ func TestAliasBuiltImage_FullRefPredicateSeparatesTheCases(t *testing.T) { t.Error("a bare short name must NOT read as a full ref") } } + +// TestBuildEngineFor pins the run-engine override the local build fallback needs: the +// build:resolve reply names the HOST runtime engine, but a build:ensure caller that asked +// for a different run engine must have the build bridge its OCI export into THAT engine's +// store. Without the override a locally-built image lands in podman and the deploy that +// asked for containerd never finds it (the alias fails `image "...": not found`). +func TestBuildEngineFor(t *testing.T) { + cases := []struct { + name string + resolved string + requested string + want string + }{ + {"requested run engine wins", "podman", "containerd", "containerd"}, + {"empty requested keeps the resolved engine", "podman", "", "podman"}, + {"both empty stays empty", "", "", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := buildEngineFor(tc.resolved, tc.requested); got != tc.want { + t.Fatalf("buildEngineFor(%q, %q) = %q, want %q", tc.resolved, tc.requested, got, tc.want) + } + }) + } +} From 99a978d02933714c5492aa9c7348b2c33d01dbf1 Mon Sep 17 00:00:00 2001 From: Andreas Trawoeger Date: Thu, 5 Nov 2026 20:54:17 +0100 Subject: [PATCH 2/4] fix(ensure): alias the build's OWN produced ref The override exposed a second assumption: the alias step re-resolved the produced ref from a local image store, which reads the DEFAULT engine's store (and can return the requested prefixed bed tag itself, since a check-- tag still carries a CalVer suffix). Use the build drive's returned FullTags as the authoritative produced ref, matched to the requested box. Unit-tested; fails on the store-resolve callback (opencharly/charly#1109). Assisted-by: Claude Code ollama-cloud/deepseek-v4.1-flash:cloud (fully tested and validated) --- candy/plugin-build/ensure.go | 52 ++++++++++++++++++------- candy/plugin-build/ensure_alias_test.go | 44 +++++++++++++++++++++ 2 files changed, 82 insertions(+), 14 deletions(-) diff --git a/candy/plugin-build/ensure.go b/candy/plugin-build/ensure.go index 527c729..09d92ed 100644 --- a/candy/plugin-build/ensure.go +++ b/candy/plugin-build/ensure.go @@ -125,11 +125,12 @@ func ensureRemoteRef(ctx context.Context, ex *sdk.Executor, image, stripped, run } } fmt.Fprintf(os.Stderr, "ensure-image: building remote %s from cached source\n", image) - if _, berr := runBoxBuildWithEngine(ctx, ex, spec.BuildRequest{Boxes: []string{rr.BoxName}, Dir: rr.CacheDir}, runEngine); berr != nil { + built, berr := runBoxBuildWithEngine(ctx, ex, spec.BuildRequest{Boxes: []string{rr.BoxName}, Dir: rr.CacheDir}, runEngine) + if berr != nil { return fmt.Errorf("ensure-image %q: pull failed and remote build failed: %w", image, berr) } return aliasBuiltImageOntoRequestedRef(ctx, image, runEngine, func() string { - return resolveRemoteImageRef(ctx, ex, rr.CacheDir, rr.BoxName) + return builtRefForBox(built, rr.BoxName) }) } @@ -138,15 +139,15 @@ func ensureRemoteRef(ctx context.Context, ex *sdk.Executor, image, stripped, run // build `image` ITSELF must resolve in local storage. A box build produces the project's // CalVer-tagged ref, so a full input ref (pinned to a different tag, or tagless) needs an alias. // -// resolveProduced is a CALLBACK, deliberately, because it must run AFTER the build: every producer -// of it bottoms out in ResolveShellImageRef with an empty tag, which READS THE LOCAL IMAGE STORE -// (spec/container/local_image_coneb.go — empty tag → ResolveNewestLocalCalVer). Resolved BEFORE -// the build, on a store that does not yet hold the image, it degrades to the bare -// `/` — which both EQUALS `image` (silently defeating a `produced != image` guard) -// and names a ref the build never created. That was a real cold-store defect: the first run on a -// fresh store left the requested ref absent and failed ~1000 log lines later with podman's -// `image not known`, while every subsequent run passed because run 1's build had seeded the store -// — so the bug reproduced only on the fresh-machine path R10 exists to cover. +// resolveProduced is a CALLBACK, deliberately, because it must run AFTER the build. It must +// return the ref the build drive actually produced (builtRefForBox over the drive's returned +// FullTags). Re-resolving it from a local image store instead reads the DEFAULT engine's store, +// which is the wrong store once the caller overrode the run engine (and a prefixed bed tag such +// as `check-engine-containerd-pod-2026.309.1951` still carries a CalVer suffix, so the wrong +// store can return the REQUESTED tag itself and defeat the `produced != image` guard). Resolving +// before the build is equally wrong: it degrades to the bare `/`, which both +// equals `image` and names a ref the build never created (the cold-store `image not known` +// defect R10 exists to cover). // // A failed alias is a HARD error, not a warning: the caller cannot proceed without the ref, and a // warning here only relocates the failure to a downstream message that names neither this build @@ -169,6 +170,28 @@ func aliasBuiltImageOntoRequestedRef(ctx context.Context, image, runEngine strin return nil } +// builtRefForBox picks the ref the build produced for box from the build drive's returned +// FullTags. The drive's OWN return value is the authoritative produced ref. Re-resolving it +// from a local image store (the former callback) reads the DEFAULT engine's store — which is +// exactly the wrong store once build:ensure has overridden the run engine — and can return the +// requested tag itself (defeating the `produced != image` guard) because a prefixed bed tag +// still carries a CalVer suffix. +func builtRefForBox(built []string, box string) string { + for _, ref := range built { + repo := ref + if colon := strings.LastIndex(repo, ":"); colon > strings.LastIndex(repo, "/") { + repo = repo[:colon] + } + if repo == box || strings.HasSuffix(repo, "/"+box) { + return ref + } + } + if len(built) == 1 { + return built[0] + } + return "" +} + // resolveRemoteImageRef computes the registry pull ref for a remote box PLUGIN-SIDE, loading the // cached repo's cfg via the K1 loader reverse legs (the host "remote-image-resolve" seam now does // ONLY the git clone/cache — K1 loader wave, deploykit.ResolveSpecBox shed from charly core). It @@ -232,17 +255,18 @@ func ensureProjectRef(ctx context.Context, ex *sdk.Executor, image, dir, runEngi } fmt.Fprintf(os.Stderr, "ensure-image: building %s locally\n", fallbackShort) - if _, berr := runBoxBuildWithEngine(ctx, ex, spec.BuildRequest{ + built, berr := runBoxBuildWithEngine(ctx, ex, spec.BuildRequest{ Boxes: []string{fallbackShort}, Dir: dir, IncludeDisabled: true, Jobs: 4, - }, runEngine); berr != nil { + }, runEngine) + if berr != nil { return fmt.Errorf("ensure-image %q: pull failed and local build failed: %w", image, berr) } return aliasBuiltImageOntoRequestedRef(ctx, image, runEngine, func() string { - return resolveImageRefPlugin(fallbackShort, cfg) + return builtRefForBox(built, fallbackShort) }) } diff --git a/candy/plugin-build/ensure_alias_test.go b/candy/plugin-build/ensure_alias_test.go index 49e6de8..3496c01 100644 --- a/candy/plugin-build/ensure_alias_test.go +++ b/candy/plugin-build/ensure_alias_test.go @@ -115,3 +115,47 @@ func TestBuildEngineFor(t *testing.T) { }) } } + +// TestBuiltRefForBox pins the produced-ref source: the build drive's returned FullTags are +// authoritative. Re-resolving from a local store reads the default engine's store (wrong once +// the run engine is overridden) and can return the requested prefixed bed tag itself. +func TestBuiltRefForBox(t *testing.T) { + cases := []struct { + name string + built []string + box string + want string + }{ + { + name: "single built ref for the requested box", + built: []string{"ghcr.io/opencharly/probe-engine-image:2026.309.1953"}, + box: "probe-engine-image", + want: "ghcr.io/opencharly/probe-engine-image:2026.309.1953", + }, + { + name: "picks the matching box out of several", + built: []string{"ghcr.io/opencharly/other:2026.309.1953", "ghcr.io/opencharly/probe-engine-image:2026.309.1953"}, + box: "probe-engine-image", + want: "ghcr.io/opencharly/probe-engine-image:2026.309.1953", + }, + { + name: "single ref with a registry-namespaced leaf still matches", + built: []string{"ghcr.io/opencharly/probe-engine-image:2026.309.1953"}, + box: "opencharly/probe-engine-image", + want: "ghcr.io/opencharly/probe-engine-image:2026.309.1953", + }, + { + name: "nothing built", + built: nil, + box: "probe-engine-image", + want: "", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := builtRefForBox(tc.built, tc.box); got != tc.want { + t.Fatalf("builtRefForBox(%v, %q) = %q, want %q", tc.built, tc.box, got, tc.want) + } + }) + } +} From 1a23225148dfddaaaa4d213d3b29ae77d75b8d11 Mon Sep 17 00:00:00 2001 From: Andreas Trawoeger Date: Thu, 5 Nov 2026 20:59:26 +0100 Subject: [PATCH 3/4] fix(ensure): keep a handled pull miss out of the gate log A pull miss is an expected fallback (the caller rebuilds locally), but the engine's own level=fatal line escaped into the config log, so the R10 zero-warning gate counted it even when config exited 0 after a successful fallback. Capture the engine output; the caller already reports the concise failure (opencharly/charly#1109). Assisted-by: Claude Code ollama-cloud/deepseek-v4.1-flash:cloud (fully tested and validated) --- candy/plugin-build/ensure.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/candy/plugin-build/ensure.go b/candy/plugin-build/ensure.go index 09d92ed..d2bac7c 100644 --- a/candy/plugin-build/ensure.go +++ b/candy/plugin-build/ensure.go @@ -34,6 +34,7 @@ package build // and the podman pull/tag exec are pure sdk/kit + os/exec. import ( + "bytes" "context" "encoding/json" "fmt" @@ -383,8 +384,13 @@ func remoteImageResolve(ctx context.Context, ex *sdk.Executor, ref, tag string) // back to a local build. func enginePull(ctx context.Context, runEngine, ref string) error { cmd := exec.CommandContext(ctx, container.EngineBinary(runEngine), "pull", ref) - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr + // CAPTURE the engine's output. A pull miss is an EXPECTED fallback here — the caller + // rebuilds the image locally — so the engine's own `level=fatal` line must not escape into + // a bed's log as an unhandled error (the R10 zero-warning gate counts it, even though the + // config path exits 0 after the fallback). The caller reports the concise failure. + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out return cmd.Run() } From 2fd947775c373636faad009a8729210789499e80 Mon Sep 17 00:00:00 2001 From: Andreas Trawoeger Date: Thu, 5 Nov 2026 21:15:28 +0100 Subject: [PATCH 4/4] fix(ensure): keep a real pull failure's diagnostic in the error enginePull now captures the engine output and returns it on the error path, so a genuine pull failure (network/auth) keeps the engine's reason instead of collapsing to 'exit status 1', while a handled fallback miss still does not print an unhandled level=fatal line into the bed log (opencharly/charly#1109). --- candy/plugin-build/ensure.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/candy/plugin-build/ensure.go b/candy/plugin-build/ensure.go index d2bac7c..cb75a9b 100644 --- a/candy/plugin-build/ensure.go +++ b/candy/plugin-build/ensure.go @@ -387,11 +387,19 @@ func enginePull(ctx context.Context, runEngine, ref string) error { // CAPTURE the engine's output. A pull miss is an EXPECTED fallback here — the caller // rebuilds the image locally — so the engine's own `level=fatal` line must not escape into // a bed's log as an unhandled error (the R10 zero-warning gate counts it, even though the - // config path exits 0 after the fallback). The caller reports the concise failure. + // config path exits 0 after the fallback). The caller reports the concise failure, and the + // captured reason rides the returned ERROR so a genuine pull failure (network/auth) keeps + // its diagnostic instead of collapsing to "exit status 1". var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out - return cmd.Run() + if err := cmd.Run(); err != nil { + if detail := strings.TrimSpace(out.String()); detail != "" { + return fmt.Errorf("%w: %s", err, detail) + } + return err + } + return nil } // engineTag adds a second tag to an existing local image. Used to satisfy a tag-pinned input ref