From 71564db79efb0f52294b363c4594dab97042701d Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Wed, 4 Nov 2026 09:33:05 +0000 Subject: [PATCH 1/3] feat(models): add llmman as a models provider Add llmman (https://github.com/llmmanorg/llmman) as an alternative backend for the top-level models element. It is selected explicitly with COMPOSE_MODEL_PROVIDER=llmman (shell or .env), and is also used when COMPOSE_MODEL_PROVIDER is unset, the docker model plugin is not installed and llmman is found in PATH. Docker Model Runner remains the default and is unchanged whenever its plugin is installed. modelAPI becomes a small internal interface implemented by the existing Docker Model Runner code and by a new llmman backend that drives the llmman CLI: llmman ps checks the daemon first, llmman list finds stored models, llmman pull fetches missing ones. context_size and runtime_flags have no per-model equivalent in llmman and are ignored with a warning. Signed-off-by: Eric Curtin --- pkg/compose/model.go | 119 ++++++- pkg/compose/model_llmman.go | 318 ++++++++++++++++++ pkg/compose/model_llmman_test.go | 547 +++++++++++++++++++++++++++++++ 3 files changed, 970 insertions(+), 14 deletions(-) create mode 100644 pkg/compose/model_llmman.go create mode 100644 pkg/compose/model_llmman_test.go diff --git a/pkg/compose/model.go b/pkg/compose/model.go index ed7f7f77bb..74467ea3fc 100644 --- a/pkg/compose/model.go +++ b/pkg/compose/model.go @@ -31,6 +31,7 @@ import ( "github.com/containerd/errdefs" "github.com/docker/cli/cli-plugins/manager" "github.com/moby/moby/client/pkg/versions" + "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/docker/compose/v5/internal/concurrency" @@ -48,7 +49,11 @@ func (s *composeService) ensureModels(ctx context.Context, project *types.Projec return err } defer mdlAPI.Close() - availableModels, err := mdlAPI.ListModels(ctx) + if err := mdlAPI.Check(ctx); err != nil { + return err + } + // best effort: when the models cannot be listed, they are all pulled + availableModels, _ := mdlAPI.ListModels(ctx) eg, ctx := concurrency.NewErrgroup(ctx, s.maxConcurrency) eg.Go(func() error { @@ -60,7 +65,11 @@ func (s *composeService) ensureModels(ctx context.Context, project *types.Projec config.Name = name } eg.Go(func() error { - if !slices.Contains(availableModels, config.Model) { + available, err := mdlAPI.HasModel(ctx, availableModels, config.Model) + if err != nil { + return err + } + if !available { err = mdlAPI.PullModel(ctx, config, quietPull, s.events) if err != nil { return err @@ -72,7 +81,74 @@ func (s *composeService) ensureModels(ctx context.Context, project *types.Projec return eg.Wait() } -type modelAPI struct { +// modelAPI is a backend able to serve the top-level `models` element. +// Docker Model Runner is the default one, llmman can be selected with +// COMPOSE_MODEL_PROVIDER or is used as a fallback when the Docker Model +// plugin is not installed (see model_llmman.go). +type modelAPI interface { + // Check verifies the backend is usable before any model is listed or pulled. + Check(ctx context.Context) error + // ListModels returns the models already available locally. Its error is not fatal. + ListModels(ctx context.Context) ([]string, error) + // HasModel tells whether model is available, given the result of ListModels. + HasModel(ctx context.Context, available []string, model string) (bool, error) + // PullModel fetches a model which is not available yet. + PullModel(ctx context.Context, model types.ModelConfig, quietPull bool, events api.EventProcessor) error + // ConfigureModel applies the model settings (context size, runtime flags). + ConfigureModel(ctx context.Context, config types.ModelConfig, events api.EventProcessor) error + // SetModelVariables injects the model and endpoint variables into services. + SetModelVariables(ctx context.Context, project *types.Project) error + // Close releases resources held by the backend. + Close() +} + +const ( + // modelProviderEnv selects the models backend, read from the project environment. + // When unset or empty, Docker Model Runner is used, or llmman if the Docker Model + // plugin is not installed and llmman is found in PATH. Setting it explicitly + // disables that fallback. + modelProviderEnv = "COMPOSE_MODEL_PROVIDER" + // modelProviderDMR is the default models backend, Docker Model Runner. + modelProviderDMR = "docker-model-runner" + // modelProviderLLMMan selects llmman, see model_llmman.go. + modelProviderLLMMan = "llmman" +) + +// errDMRPluginNotFound is returned when the `docker model` CLI plugin is not installed. +var errDMRPluginNotFound = errors.New("'models' support requires Docker Model plugin") + +// newModelAPI returns the backend selected by COMPOSE_MODEL_PROVIDER. When the +// variable is unset or empty, Docker Model Runner is used. Only if its plugin +// is not installed (a broken plugin is an error) and an llmman binary is found +// in PATH is llmman used instead. An explicit value never falls back. +func (s *composeService) newModelAPI(project *types.Project) (modelAPI, error) { + switch provider := strings.TrimSpace(project.Environment[modelProviderEnv]); provider { + case "", modelProviderDMR: + m, err := s.newDMRModelAPI(project) + if err != nil { + if provider == "" && errors.Is(err, errDMRPluginNotFound) { + if path, lookErr := exec.LookPath(llmmanBinary); lookErr == nil { + logrus.Debugf("Docker Model plugin not found, using llmman (%s) for models", path) + return s.newLLMManModelAPI(project) + } + } + return nil, err + } + return m, nil + case modelProviderLLMMan: + m, err := s.newLLMManModelAPI(project) + if err != nil { + return nil, err + } + return m, nil + default: + return nil, fmt.Errorf("unsupported %s %q: supported values are %q (default) and %q", + modelProviderEnv, provider, modelProviderDMR, modelProviderLLMMan) + } +} + +// dmrModelAPI is the Docker Model Runner backend, driven through the `docker model` CLI plugin. +type dmrModelAPI struct { path string env []string prepare func(ctx context.Context, cmd *exec.Cmd) error @@ -81,11 +157,11 @@ type modelAPI struct { dryRun bool } -func (s *composeService) newModelAPI(project *types.Project) (*modelAPI, error) { +func (s *composeService) newDMRModelAPI(project *types.Project) (*dmrModelAPI, error) { dockerModel, err := manager.GetPlugin("model", s.dockerCli, &cobra.Command{}) if err != nil { if errdefs.IsNotFound(err) { - return nil, errors.New("'models' support requires Docker Model plugin") + return nil, errDMRPluginNotFound } return nil, err } @@ -96,7 +172,7 @@ func (s *composeService) newModelAPI(project *types.Project) (*modelAPI, error) if err != nil { return nil, err } - return &modelAPI{ + return &dmrModelAPI{ path: dockerModel.Path, version: dockerModel.Version, prepare: func(ctx context.Context, cmd *exec.Cmd) error { @@ -108,11 +184,20 @@ func (s *composeService) newModelAPI(project *types.Project) (*modelAPI, error) }, nil } -func (m *modelAPI) Close() { +func (m *dmrModelAPI) Close() { m.cleanup() } -func (m *modelAPI) PullModel(ctx context.Context, model types.ModelConfig, quietPull bool, events api.EventProcessor) error { +// Check is a no-op: the plugin is validated when loaded. +func (m *dmrModelAPI) Check(context.Context) error { + return nil +} + +func (m *dmrModelAPI) HasModel(_ context.Context, available []string, model string) (bool, error) { + return slices.Contains(available, model), nil +} + +func (m *dmrModelAPI) PullModel(ctx context.Context, model types.ModelConfig, quietPull bool, events api.EventProcessor) error { events.On(api.Resource{ ID: model.Name, Status: api.Working, @@ -167,7 +252,7 @@ func (m *modelAPI) PullModel(ctx context.Context, model types.ModelConfig, quiet return err } -func (m *modelAPI) ConfigureModel(ctx context.Context, config types.ModelConfig, events api.EventProcessor) error { +func (m *dmrModelAPI) ConfigureModel(ctx context.Context, config types.ModelConfig, events api.EventProcessor) error { events.On(api.Resource{ ID: config.Name, Status: api.Working, @@ -206,7 +291,7 @@ func (m *modelAPI) ConfigureModel(ctx context.Context, config types.ModelConfig, return nil } -func (m *modelAPI) SetModelVariables(ctx context.Context, project *types.Project) error { +func (m *dmrModelAPI) SetModelVariables(ctx context.Context, project *types.Project) error { cmd := exec.CommandContext(ctx, m.path, "status", "--json") err := m.prepare(ctx, cmd) if err != nil { @@ -227,6 +312,13 @@ func (m *modelAPI) SetModelVariables(ctx context.Context, project *types.Project return err } + setModelVariables(project, status.Endpoint) + return nil +} + +// setModelVariables injects, for each model bound to a service, the model name and +// endpoint variables (_MODEL and _URL, or the model_var and endpoint_var overrides). +func setModelVariables(project *types.Project, endpoint string) { for _, service := range project.Services { for ref, modelConfig := range service.Models { model := project.Models[ref] @@ -244,10 +336,9 @@ func (m *modelAPI) SetModelVariables(ctx context.Context, project *types.Project } else { variable = varPrefix + "_URL" } - service.Environment[variable] = &status.Endpoint + service.Environment[variable] = &endpoint } } - return nil } type Model struct { @@ -263,7 +354,7 @@ type Model struct { } `json:"config"` } -func (m *modelAPI) ListModels(ctx context.Context) ([]string, error) { +func (m *dmrModelAPI) ListModels(ctx context.Context) ([]string, error) { cmd := exec.CommandContext(ctx, m.path, "ls", "--json") err := m.prepare(ctx, cmd) if err != nil { @@ -295,7 +386,7 @@ func (m *modelAPI) ListModels(ctx context.Context) ([]string, error) { // supportsRuntimeFlags checks if the docker model version supports runtime flags // Runtime flags are supported in version >= v1.0.6 -func (m *modelAPI) supportsRuntimeFlags() bool { +func (m *dmrModelAPI) supportsRuntimeFlags() bool { // If version is not cached, don't append runtime flags to be safe if m.version == "" { return false diff --git a/pkg/compose/model_llmman.go b/pkg/compose/model_llmman.go new file mode 100644 index 0000000000..05b3d4568d --- /dev/null +++ b/pkg/compose/model_llmman.go @@ -0,0 +1,318 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "net" + "os/exec" + "slices" + "strconv" + "strings" + + "github.com/compose-spec/compose-go/v2/types" + + "github.com/docker/compose/v5/internal/progress" + "github.com/docker/compose/v5/pkg/api" +) + +// llmman backend for the top-level `models` element. +// +// llmman (https://github.com/llmmanorg/llmman) stores models as OCI images and +// serves them through Ollama, OpenAI (/v1) and Anthropic compatible APIs. +// It is selected, instead of Docker Model Runner, by setting +// COMPOSE_MODEL_PROVIDER=llmman in the shell or in the project's .env file. +// It is also used automatically, as a fallback, when COMPOSE_MODEL_PROVIDER is +// unset or empty, the Docker Model plugin is not installed and an llmman binary +// is found in PATH. Docker Model Runner always wins when its plugin is installed, +// and an explicit COMPOSE_MODEL_PROVIDER=docker-model-runner never falls back. +// +// Compose drives the `llmman` CLI found in PATH, with the project environment +// (so LLMMAN_HOST, LLMMAN_API_KEY, LLMMAN_MODELS... are honored): +// +// - `llmman ps` checks the daemon is reachable (and the API key accepted). +// It never starts the daemon: run `llmman serve` first. It is checked +// before anything else because `llmman pull` would otherwise start one. +// - `llmman list --format ...` reads the local store to find which models are +// already pulled. References are normalized by llmman itself +// (`llmman list `), so only missing models are pulled. +// - `llmman pull ` fetches missing models. It is skipped on --dry-run. +// +// Models references follow llmman rules, not Docker Model Runner ones: a bare +// name such as `smollm2` is Docker Hub's `ai/` namespace, `docker.io/ai/smollm2` +// is spelled in full, but `ai/smollm2` (owner/repo without a registry) is a +// Hugging Face repository (`hf.co/ai/smollm2`). +// +// llmman has no per-model configuration: the context size is daemon-wide +// (LLMMAN_CONTEXT_LENGTH) and so are the engine arguments (e.g. LLMMAN_VLLM_ARGS), +// both read when `llmman serve` starts. `context_size` and `runtime_flags` are +// therefore ignored, with a warning. +// +// Services get _MODEL (the reference as written in the Compose file) and +// _URL (or the model_var and endpoint_var overrides), where the URL is +// the OpenAI compatible base URL of the daemon, as seen from a container: +// scheme://host:port/v1 built from LLMMAN_HOST (default 127.0.0.1:17434). A +// loopback or wildcard host is replaced by host.docker.internal. llmman binds +// 127.0.0.1 by default, which containers cannot reach on a Linux engine: bind +// the daemon on an address reachable from containers (LLMMAN_HOST=0.0.0.0:17434, +// which requires LLMMAN_API_KEYS or LLMMAN_AUTH=off) and map the name with +// `extra_hosts: ["host.docker.internal:host-gateway"]`. When the daemon requires +// an API key, services must be given it explicitly, Compose does not inject it. + +const ( + llmmanBinary = "llmman" + llmmanHostEnv = "LLMMAN_HOST" + llmmanDefaultPort = "17434" + llmmanContainerHost = "host.docker.internal" + llmmanInstallURL = "https://github.com/llmmanorg/llmman" +) + +type llmmanModelAPI struct { + path string + prepare func(ctx context.Context, cmd *exec.Cmd) error + endpoint string + dryRun bool +} + +func (s *composeService) newLLMManModelAPI(project *types.Project) (*llmmanModelAPI, error) { + path, err := exec.LookPath(llmmanBinary) + if err != nil { + if errors.Is(err, exec.ErrNotFound) { + return nil, fmt.Errorf("'models' support with %s=%s requires the llmman CLI in PATH, see %s", + modelProviderEnv, modelProviderLLMMan, llmmanInstallURL) + } + return nil, fmt.Errorf("failed to locate llmman: %w", err) + } + return &llmmanModelAPI{ + path: path, + prepare: func(ctx context.Context, cmd *exec.Cmd) error { + return s.prepareShellOut(ctx, project.Environment, cmd) + }, + endpoint: llmmanEndpoint(project.Environment[llmmanHostEnv]), + dryRun: s.dryRun, + }, nil +} + +func (m *llmmanModelAPI) Close() {} + +// run executes llmman with args and returns its stdout. stderr is only used to +// report a failure, as llmman prints warnings and progress there. +func (m *llmmanModelAPI) run(ctx context.Context, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, m.path, args...) + if err := m.prepare(ctx, cmd); err != nil { + return "", err + } + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if msg := strings.TrimSpace(stderr.String() + stdout.String()); msg != "" { + return "", fmt.Errorf("%w: %s", err, msg) + } + return "", err + } + return stdout.String(), nil +} + +// Check verifies the llmman daemon is running, without starting it. +func (m *llmmanModelAPI) Check(ctx context.Context) error { + if _, err := m.run(ctx, "ps"); err != nil { + return fmt.Errorf("the llmman daemon is not reachable, start it with `llmman serve` (LLMMAN_HOST and LLMMAN_API_KEY are read from the project environment): %w", err) + } + return nil +} + +// ListModels returns the references of the models in llmman's local store. +func (m *llmmanModelAPI) ListModels(ctx context.Context) ([]string, error) { + out, err := m.run(ctx, "list", "--format", "{{.Name}}") + if err != nil { + return nil, fmt.Errorf("error checking available models: %w", err) + } + return nonEmptyLines(out), nil +} + +// HasModel tells whether model is in the local store. An exact match with a stored +// reference is enough. Otherwise llmman is asked to resolve the reference, as the +// stored one is fully qualified (e.g. docker.io/ai/smollm2:latest) while the Compose +// file may use any shorthand llmman accepts. A false negative only costs a redundant pull. +func (m *llmmanModelAPI) HasModel(ctx context.Context, available []string, model string) (bool, error) { + if slices.Contains(available, model) { + return true, nil + } + if len(available) == 0 || strings.Contains(model, "@") { + return false, nil + } + out, err := m.run(ctx, "list", model, "--format", "{{.Name}}|{{.Tag}}") + if err != nil { + return false, fmt.Errorf("error checking model %s: %w", model, err) + } + // Without an explicit tag llmman matches every tag of the repository, while + // pulling the reference fetches :latest. + explicitTag := strings.Contains(model[strings.LastIndex(model, "/")+1:], ":") + for _, line := range nonEmptyLines(out) { + _, tag, _ := strings.Cut(line, "|") + if explicitTag || tag == "latest" { + return true, nil + } + } + return false, nil +} + +func (m *llmmanModelAPI) PullModel(ctx context.Context, model types.ModelConfig, quietPull bool, events api.EventProcessor) error { + events.On(api.Resource{ + ID: model.Name, + Status: api.Working, + Text: api.StatusPulling, + }) + if m.dryRun { + // llmman is an external process the dry-run API client cannot intercept + events.On(api.Resource{ID: model.Name, Status: api.Working, Text: api.StatusPulled}) + return nil + } + + cmd := exec.CommandContext(ctx, m.path, "pull", model.Model) + if err := m.prepare(ctx, cmd); err != nil { + return err + } + stream, err := cmd.StdoutPipe() + if err != nil { + return err + } + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + return err + } + + scanner := bufio.NewScanner(stream) + for scanner.Scan() { + if scanner.Text() == "" || quietPull { + continue + } + events.On(api.Resource{ + ID: model.Name, + Status: api.Working, + Text: api.StatusPulling, + }) + } + + if err := cmd.Wait(); err != nil { + if msg := strings.TrimSpace(stderr.String()); msg != "" { + err = fmt.Errorf("%w: %s", err, msg) + } + if rest, ok := strings.CutPrefix(model.Model, "ai/"); ok { + // the reference of Docker Model Runner for Docker Hub's ai namespace + err = fmt.Errorf("%w (llmman resolves %q as a Hugging Face repository, use %q or %q for Docker Hub)", + err, model.Model, rest, "docker.io/"+model.Model) + } + events.On(progress.Error(model.Name, err.Error())) + return err + } + events.On(api.Resource{ + ID: model.Name, + Status: api.Working, + Text: api.StatusPulled, + }) + return nil +} + +// ConfigureModel has nothing to apply: llmman has no per-model settings, so the +// unsupported ones are reported and ignored rather than approximated. +func (m *llmmanModelAPI) ConfigureModel(_ context.Context, config types.ModelConfig, events api.EventProcessor) error { + events.On(api.Resource{ + ID: config.Name, + Status: api.Working, + Text: api.StatusConfiguring, + }) + if config.ContextSize > 0 { + events.On(progress.New(config.Name, api.Warning, "Ignored context_size", + "llmman has no per-model context size, set LLMMAN_CONTEXT_LENGTH when starting llmman serve")) + } + if len(config.RuntimeFlags) > 0 { + events.On(progress.New(config.Name, api.Warning, "Ignored runtime_flags", + "llmman has no per-model runtime flags, see the engine settings (e.g. LLMMAN_VLLM_ARGS) of llmman serve")) + } + events.On(api.Resource{ + ID: config.Name, + Status: api.Done, + Text: api.StatusConfigured, + }) + return nil +} + +func (m *llmmanModelAPI) SetModelVariables(_ context.Context, project *types.Project) error { + setModelVariables(project, m.endpoint) + return nil +} + +func nonEmptyLines(out string) []string { + var lines []string + for line := range strings.SplitSeq(out, "\n") { + if line = strings.TrimSpace(line); line != "" { + lines = append(lines, line) + } + } + return lines +} + +// llmmanEndpoint returns the OpenAI compatible base URL of the llmman daemon as +// seen from a container. host is parsed like llmman does for LLMMAN_HOST +// ([scheme://][host][:port]): the scheme is https only when spelled so, a missing +// host is 127.0.0.1 and a missing port is 17434, or 80/443 when a scheme is given. +func llmmanEndpoint(host string) string { + host = strings.Trim(strings.TrimSpace(host), `"'`) + scheme, port := "http", llmmanDefaultPort + if s, rest, ok := strings.Cut(host, "://"); ok { + host = rest + switch strings.ToLower(s) { + case "https": + scheme, port = "https", "443" + case "http": + port = "80" + } + } + // a trailing path is discarded + host, _, _ = strings.Cut(host, "/") + if h, p, err := net.SplitHostPort(host); err == nil { + host = h + if n, err := strconv.ParseUint(p, 10, 16); err == nil { + port = strconv.FormatUint(n, 10) + } + } else { + // no port, or a bare IPv6 literal + host = strings.TrimSuffix(strings.TrimPrefix(host, "["), "]") + } + if isLoopbackOrWildcard(host) { + host = llmmanContainerHost + } + return scheme + "://" + net.JoinHostPort(host, port) + "/v1" +} + +// isLoopbackOrWildcard tells whether host (or the wildcard llmman rewrites to +// loopback when connecting) designates the machine running the daemon, which +// is not what a container sees when it uses that name. +func isLoopbackOrWildcard(host string) bool { + if host == "" || strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && (ip.IsLoopback() || ip.IsUnspecified()) +} diff --git a/pkg/compose/model_llmman_test.go b/pkg/compose/model_llmman_test.go new file mode 100644 index 0000000000..29539dd04f --- /dev/null +++ b/pkg/compose/model_llmman_test.go @@ -0,0 +1,547 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "sync" + "testing" + + "github.com/compose-spec/compose-go/v2/types" + "github.com/containerd/errdefs" + "github.com/docker/cli/cli-plugins/manager" + "github.com/docker/cli/cli/config/configfile" + "github.com/docker/cli/cli/context/docker" + "github.com/spf13/cobra" + "go.uber.org/mock/gomock" + "gotest.tools/v3/assert" + + "github.com/docker/compose/v5/pkg/api" + "github.com/docker/compose/v5/pkg/mocks" +) + +// fakeLLMMan is a shell script standing in for the llmman CLI. State is driven +// by files next to it: +// - store: references `llmman list` prints, one per line +// - resolved: lines "=|" answered by `llmman list ` +// - down: when present, `llmman ps` fails as if no daemon was running +// - pullfail: when present, `llmman pull` fails +// +// Every invocation is appended to calls.log, with the LLMMAN_* variables seen. +type fakeLLMMan struct { + dir string + calls string +} + +func newFakeLLMMan(t *testing.T) *fakeLLMMan { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("the fake llmman is a shell script") + } + dir := t.TempDir() + f := &fakeLLMMan{dir: dir, calls: filepath.Join(dir, "calls.log")} + script := `#!/bin/sh +dir="` + dir + `" +echo "$* [host=$LLMMAN_HOST key=$LLMMAN_API_KEY]" >> "` + f.calls + `" +case "$1" in +ps) + if [ -f "$dir/down" ]; then + echo "llmman serve is not running (nothing is loaded) - start it with llmman serve" >&2 + exit 1 + fi + echo "NAME ID SIZE PROCESSOR CONTEXT STARTED UNTIL" + ;; +list) + if [ "$2" = "--format" ]; then + [ -f "$dir/store" ] && cat "$dir/store" + exit 0 + fi + [ -f "$dir/resolved" ] && while IFS='=' read -r ref out; do + [ "$ref" = "$2" ] && echo "$out" + done < "$dir/resolved" + exit 0 + ;; +pull) + if [ -f "$dir/pullfail" ]; then + echo "registry exploded" >&2 + exit 1 + fi + echo "pulling $2" + echo "Pulled $2" + ;; +*) + echo "unexpected command $*" >&2 + exit 2 + ;; +esac +` + assert.NilError(t, os.WriteFile(filepath.Join(dir, "llmman"), []byte(script), 0o700)) + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + return f +} + +func (f *fakeLLMMan) set(t *testing.T, name, content string) { + t.Helper() + assert.NilError(t, os.WriteFile(filepath.Join(f.dir, name), []byte(content), 0o600)) +} + +// log returns the commands run, without the environment suffix. +func (f *fakeLLMMan) log(t *testing.T) []string { + t.Helper() + data, err := os.ReadFile(f.calls) + if os.IsNotExist(err) { + return nil + } + assert.NilError(t, err) + var cmds []string + for line := range strings.SplitSeq(strings.TrimSpace(string(data)), "\n") { + cmd, _, _ := strings.Cut(line, " [host=") + cmds = append(cmds, cmd) + } + return cmds +} + +type recordedEvents struct { + mu sync.Mutex + events []api.Resource +} + +func (r *recordedEvents) Start(context.Context, string) {} +func (r *recordedEvents) Done(string, bool) {} +func (r *recordedEvents) On(events ...api.Resource) { + r.mu.Lock() + defer r.mu.Unlock() + r.events = append(r.events, events...) +} + +// has tells whether an event with the given status and text was recorded for id. +func (r *recordedEvents) has(id string, status api.EventStatus, text string) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, e := range r.events { + if e.ID == id && e.Status == status && e.Text == text { + return true + } + } + return false +} + +func (r *recordedEvents) details(id string, status api.EventStatus) []string { + r.mu.Lock() + defer r.mu.Unlock() + var details []string + for _, e := range r.events { + if e.ID == id && e.Status == status { + details = append(details, e.Details) + } + } + return details +} + +// llmmanProject returns a project with one service "app" bound to the given models. +func llmmanProject(env types.Mapping, models types.Models, bindings map[string]*types.ServiceModelConfig) *types.Project { + // Environment and Models are promoted fields, which struct literals + // cannot set before go1.27 + var app types.ServiceConfig + app.Environment = types.MappingWithEquals{} + app.Models = bindings + return &types.Project{ + Environment: env, + Models: models, + Services: types.Services{"app": app}, + } +} + +func llmmanEnv(extra ...string) types.Mapping { + env := types.Mapping{modelProviderEnv: modelProviderLLMMan} + for i := 0; i+1 < len(extra); i += 2 { + env[extra[i]] = extra[i+1] + } + return env +} + +func TestModelProviderSelection(t *testing.T) { + fake := newFakeLLMMan(t) + s := &composeService{events: &recordedEvents{}} + + models := types.Models{"llm": {Model: "smollm2"}} + bindings := map[string]*types.ServiceModelConfig{"llm": nil} + + t.Run("llmman is selected by COMPOSE_MODEL_PROVIDER", func(t *testing.T) { + m, err := s.newModelAPI(llmmanProject(llmmanEnv(), models, bindings)) + assert.NilError(t, err) + _, ok := m.(*llmmanModelAPI) + assert.Assert(t, ok, "got %T", m) + }) + + for _, value := range []string{"LLMMAN", "ollama", "llmman,docker-model-runner"} { + t.Run("unknown provider "+value, func(t *testing.T) { + env := types.Mapping{modelProviderEnv: value} + err := s.ensureModels(t.Context(), llmmanProject(env, models, bindings), false) + assert.ErrorContains(t, err, "unsupported COMPOSE_MODEL_PROVIDER") + assert.ErrorContains(t, err, value) + assert.ErrorContains(t, err, modelProviderDMR) + }) + } + + t.Run("the default is Docker Model Runner", func(t *testing.T) { + // llmman is on PATH, yet Docker Model Runner wins when its plugin is installed + dir := t.TempDir() + script := "#!/bin/sh\ncase \"$1\" in docker-cli-plugin-metadata) echo '{\"SchemaVersion\":\"0.1.0\",\"Vendor\":\"test\",\"Version\":\"v1.2.6\"}' ;; esac\n" + assert.NilError(t, os.WriteFile(filepath.Join(dir, "docker-model"), []byte(script), 0o700)) + cli := mocks.NewMockCli(gomock.NewController(t)) + cli.EXPECT().ConfigFile().Return(&configfile.ConfigFile{CLIPluginsExtraDirs: []string{dir}}).AnyTimes() + cli.EXPECT().CurrentContext().Return("default").AnyTimes() + cli.EXPECT().DockerEndpoint().Return(docker.Endpoint{}).AnyTimes() + dmr := &composeService{dockerCli: cli} + + for _, env := range []types.Mapping{{}, {modelProviderEnv: ""}, {modelProviderEnv: " " + modelProviderDMR + " "}} { + m, err := dmr.newModelAPI(llmmanProject(env, models, bindings)) + assert.NilError(t, err) + _, ok := m.(*dmrModelAPI) + assert.Assert(t, ok, "got %T for %v", m, env) + m.Close() + } + assert.Equal(t, len(fake.log(t)), 0, "llmman must not be run when not selected") + }) + + t.Run("nothing is checked without models", func(t *testing.T) { + env := types.Mapping{modelProviderEnv: "bogus"} + assert.NilError(t, s.ensureModels(t.Context(), llmmanProject(env, nil, nil), false)) + }) +} + +// fallbackService returns a service whose Docker CLI looks for plugins in a +// single directory, which holds a fake docker-model plugin running metadata +// when it is not empty. The test is skipped if a real plugin is installed +// in a system directory and the plugin is expected to be absent. +func fallbackService(t *testing.T, metadata string) *composeService { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("the fake plugin is a shell script") + } + dir := t.TempDir() + if metadata != "" { + script := "#!/bin/sh\ncase \"$1\" in docker-cli-plugin-metadata) " + metadata + " ;; esac\n" + assert.NilError(t, os.WriteFile(filepath.Join(dir, "docker-model"), []byte(script), 0o700)) + } + cli := mocks.NewMockCli(gomock.NewController(t)) + cli.EXPECT().ConfigFile().Return(&configfile.ConfigFile{CLIPluginsExtraDirs: []string{dir}}).AnyTimes() + cli.EXPECT().CurrentContext().Return("default").AnyTimes() + cli.EXPECT().DockerEndpoint().Return(docker.Endpoint{}).AnyTimes() + if metadata == "" { + if _, err := manager.GetPlugin("model", cli, &cobra.Command{}); !errdefs.IsNotFound(err) { + t.Skip("a Docker Model plugin is installed on this system") + } + } + return &composeService{dockerCli: cli, events: &recordedEvents{}} +} + +const ( + validPluginMetadata = `echo '{"SchemaVersion":"0.1.0","Vendor":"test","Version":"v1.2.6"}'` + brokenPluginMetadata = `echo 'not json'` + dmrPluginMissingText = "'models' support requires Docker Model plugin" +) + +func TestLLMManFallbackWhenPluginIsMissing(t *testing.T) { + models := types.Models{"llm": {Model: "smollm2"}} + bindings := map[string]*types.ServiceModelConfig{"llm": nil} + + for name, env := range map[string]types.Mapping{ + "unset": {}, + "empty": {modelProviderEnv: ""}, + "whitespace": {modelProviderEnv: " "}, + } { + t.Run("llmman is selected and used when the provider is "+name, func(t *testing.T) { + fake := newFakeLLMMan(t) + s := fallbackService(t, "") + project := llmmanProject(env, models, bindings) + + m, err := s.newModelAPI(project) + assert.NilError(t, err) + _, ok := m.(*llmmanModelAPI) + assert.Assert(t, ok, "got %T", m) + + assert.NilError(t, s.ensureModels(t.Context(), project, false)) + assert.DeepEqual(t, fake.log(t), []string{"ps", "list --format {{.Name}}", "pull smollm2"}) + url := project.Services["app"].Environment["LLM_URL"] + assert.Assert(t, url != nil && *url == "http://host.docker.internal:17434/v1") + }) + } + + t.Run("explicit docker-model-runner never falls back", func(t *testing.T) { + fake := newFakeLLMMan(t) + s := fallbackService(t, "") + project := llmmanProject(types.Mapping{modelProviderEnv: modelProviderDMR}, models, bindings) + + err := s.ensureModels(t.Context(), project, false) + assert.Error(t, err, dmrPluginMissingText) + assert.ErrorIs(t, err, errDMRPluginNotFound) + assert.Equal(t, len(fake.log(t)), 0, "llmman must not be run") + }) + + t.Run("no plugin and no llmman keeps the original error", func(t *testing.T) { + s := fallbackService(t, "") + t.Setenv("PATH", t.TempDir()) + err := s.ensureModels(t.Context(), llmmanProject(types.Mapping{}, models, bindings), false) + assert.Error(t, err, dmrPluginMissingText) + }) + + t.Run("Docker Model Runner wins when its plugin is installed", func(t *testing.T) { + fake := newFakeLLMMan(t) + s := fallbackService(t, validPluginMetadata) + m, err := s.newModelAPI(llmmanProject(types.Mapping{}, models, bindings)) + assert.NilError(t, err) + _, ok := m.(*dmrModelAPI) + assert.Assert(t, ok, "got %T", m) + m.Close() + assert.Equal(t, len(fake.log(t)), 0, "llmman must not be run") + }) + + t.Run("a broken plugin does not fall back", func(t *testing.T) { + fake := newFakeLLMMan(t) + s := fallbackService(t, brokenPluginMetadata) + _, err := s.newModelAPI(llmmanProject(types.Mapping{}, models, bindings)) + assert.ErrorContains(t, err, "failed to load Docker Model plugin") + assert.Assert(t, !errors.Is(err, errDMRPluginNotFound)) + assert.Equal(t, len(fake.log(t)), 0, "llmman must not be run") + }) +} + +func TestLLMManMissingBinary(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("PATH lookup differs on windows") + } + t.Setenv("PATH", t.TempDir()) + s := &composeService{events: &recordedEvents{}} + project := llmmanProject(llmmanEnv(), types.Models{"llm": {Model: "smollm2"}}, map[string]*types.ServiceModelConfig{"llm": nil}) + err := s.ensureModels(t.Context(), project, false) + assert.ErrorContains(t, err, "requires the llmman CLI in PATH") + assert.ErrorContains(t, err, "https://github.com/llmmanorg/llmman") +} + +func TestLLMManDaemonUnreachable(t *testing.T) { + fake := newFakeLLMMan(t) + fake.set(t, "down", "") + s := &composeService{events: &recordedEvents{}} + project := llmmanProject(llmmanEnv(), types.Models{"llm": {Model: "smollm2"}}, map[string]*types.ServiceModelConfig{"llm": nil}) + + err := s.ensureModels(t.Context(), project, false) + assert.ErrorContains(t, err, "start it with `llmman serve`") + assert.ErrorContains(t, err, "llmman serve is not running") + // the daemon is checked first: `llmman pull` would otherwise start one + assert.DeepEqual(t, fake.log(t), []string{"ps"}) + assert.Assert(t, project.Services["app"].Environment["LLM_URL"] == nil, "no variable is set on failure") +} + +func TestLLMManPullsOnlyMissingModels(t *testing.T) { + fake := newFakeLLMMan(t) + fake.set(t, "store", "docker.io/ai/smollm2:latest\nhf.co/unsloth/qwen:Q4_K_M\ndocker.io/ai/other:360M\n") + fake.set(t, "resolved", strings.Join([]string{ + "smollm2=docker.io/ai/smollm2:latest|latest", + "hf.co/unsloth/qwen:Q4_K_M=hf.co/unsloth/qwen:Q4_K_M|Q4_K_M", + // no tag asked for and only another tag stored: :latest is what a pull fetches + "other=docker.io/ai/other:360M|360M", + // the explicit tag is stored under the fully qualified name + "ai/new:1b=", + }, "\n")+"\n") + + events := &recordedEvents{} + s := &composeService{events: events} + project := llmmanProject(llmmanEnv(), types.Models{ + "short": {Model: "smollm2"}, + "exact": {Model: "hf.co/unsloth/qwen:Q4_K_M"}, + "tagged": {Model: "docker.io/ai/other:360M"}, + "latest": {Model: "other"}, + "new": {Model: "ai/new:1b"}, + }, map[string]*types.ServiceModelConfig{"short": nil}) + assert.NilError(t, s.ensureModels(t.Context(), project, false)) + + var pulls []string + for _, call := range fake.log(t) { + if strings.HasPrefix(call, "pull ") { + pulls = append(pulls, call) + } + } + slices.Sort(pulls) + assert.DeepEqual(t, pulls, []string{"pull ai/new:1b", "pull other"}) + + assert.Assert(t, events.has("latest", api.Working, api.StatusPulling)) + assert.Assert(t, events.has("latest", api.Working, api.StatusPulled)) + assert.Assert(t, events.has("latest", api.Done, api.StatusConfigured)) + assert.Assert(t, !events.has("short", api.Working, api.StatusPulling), "an available model is not pulled") + assert.Assert(t, events.has("short", api.Done, api.StatusConfigured)) +} + +func TestLLMManEmptyStoreSkipsResolution(t *testing.T) { + fake := newFakeLLMMan(t) + s := &composeService{events: &recordedEvents{}} + project := llmmanProject(llmmanEnv(), types.Models{"llm": {Model: "smollm2"}}, map[string]*types.ServiceModelConfig{"llm": nil}) + assert.NilError(t, s.ensureModels(t.Context(), project, false)) + assert.DeepEqual(t, fake.log(t), []string{"ps", "list --format {{.Name}}", "pull smollm2"}) +} + +func TestLLMManPullFailure(t *testing.T) { + fake := newFakeLLMMan(t) + fake.set(t, "pullfail", "") + events := &recordedEvents{} + s := &composeService{events: events} + project := llmmanProject(llmmanEnv(), types.Models{"llm": {Model: "smollm2"}}, map[string]*types.ServiceModelConfig{"llm": nil}) + + err := s.ensureModels(t.Context(), project, false) + assert.ErrorContains(t, err, "registry exploded") + assert.Assert(t, events.has("llm", api.Error, api.StatusError), "pull error is reported") + assert.Assert(t, !events.has("llm", api.Working, api.StatusPulled)) + assert.Assert(t, !events.has("llm", api.Done, api.StatusConfigured)) +} + +// TestLLMManPullFailureHintsAtDockerHubReferences: `ai/` is how Docker Model +// Runner spells Docker Hub's ai namespace, which llmman reads as a Hugging Face repository. +func TestLLMManPullFailureHintsAtDockerHubReferences(t *testing.T) { + fake := newFakeLLMMan(t) + fake.set(t, "pullfail", "") + s := &composeService{events: &recordedEvents{}} + + project := llmmanProject(llmmanEnv(), types.Models{"llm": {Model: "ai/smollm2"}}, map[string]*types.ServiceModelConfig{"llm": nil}) + err := s.ensureModels(t.Context(), project, false) + assert.ErrorContains(t, err, "registry exploded") + assert.ErrorContains(t, err, `use "smollm2" or "docker.io/ai/smollm2" for Docker Hub`) + + project = llmmanProject(llmmanEnv(), types.Models{"llm": {Model: "unsloth/qwen"}}, map[string]*types.ServiceModelConfig{"llm": nil}) + err = s.ensureModels(t.Context(), project, false) + assert.ErrorContains(t, err, "registry exploded") + assert.Assert(t, !strings.Contains(err.Error(), "Docker Hub"), err.Error()) +} + +func TestLLMManVariables(t *testing.T) { + newFakeLLMMan(t) + s := &composeService{events: &recordedEvents{}} + + tests := []struct { + name string + host string + url string + }{ + {"default", "", "http://host.docker.internal:17434/v1"}, + {"loopback with port", "127.0.0.1:8080", "http://host.docker.internal:8080/v1"}, + {"wildcard", "0.0.0.0:17434", "http://host.docker.internal:17434/v1"}, + {"localhost", "localhost", "http://host.docker.internal:17434/v1"}, + {"ipv6 loopback", "[::1]:9000", "http://host.docker.internal:9000/v1"}, + {"port only", ":9000", "http://host.docker.internal:9000/v1"}, + {"lan address", "192.168.1.20:17434", "http://192.168.1.20:17434/v1"}, + {"remote host with scheme", "http://gpu-box", "http://gpu-box:80/v1"}, + {"tls", "https://gpu.example.com", "https://gpu.example.com:443/v1"}, + {"quoted and with path", `"gpu.example.com:1234/ignored"`, "http://gpu.example.com:1234/v1"}, + {"ipv6 remote", "[2001:db8::1]:17434", "http://[2001:db8::1]:17434/v1"}, + {"invalid port", "gpu:nope", "http://gpu:17434/v1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + env := llmmanEnv() + if tt.host != "" { + env[llmmanHostEnv] = tt.host + } + project := llmmanProject(env, + types.Models{"my-llm": {Model: "smollm2"}, "embed": {Model: "hf.co/org/embed-GGUF"}}, + map[string]*types.ServiceModelConfig{ + "my-llm": nil, + "embed": {ModelVariable: "EMBED_NAME", EndpointVariable: "EMBED_BASE_URL"}, + }) + assert.NilError(t, s.ensureModels(t.Context(), project, false)) + envs := project.Services["app"].Environment + assert.Equal(t, *envs["MY_LLM_MODEL"], "smollm2") + assert.Equal(t, *envs["MY_LLM_URL"], tt.url) + assert.Equal(t, *envs["EMBED_NAME"], "hf.co/org/embed-GGUF") + assert.Equal(t, *envs["EMBED_BASE_URL"], tt.url) + assert.Assert(t, envs["EMBED_MODEL"] == nil && envs["EMBED_URL"] == nil, "overridden names are not set") + }) + } +} + +func TestLLMManPropagatesProjectEnvironment(t *testing.T) { + fake := newFakeLLMMan(t) + s := &composeService{events: &recordedEvents{}} + project := llmmanProject(llmmanEnv("LLMMAN_HOST", "10.0.0.5:9999", "LLMMAN_API_KEY", "s3cret"), + types.Models{"llm": {Model: "smollm2"}}, map[string]*types.ServiceModelConfig{"llm": nil}) + assert.NilError(t, s.ensureModels(t.Context(), project, false)) + + data, err := os.ReadFile(fake.calls) + assert.NilError(t, err) + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + assert.Assert(t, len(lines) >= 3) + for _, line := range lines { + assert.Assert(t, strings.HasSuffix(line, "[host=10.0.0.5:9999 key=s3cret]"), line) + } + // the key is for the CLI only: it is not handed to services + for _, v := range project.Services["app"].Environment { + assert.Assert(t, v == nil || *v != "s3cret") + } +} + +// TestLLMManUnsupportedSettingsAreIgnored: llmman has no per-model context size +// nor runtime flags, they are reported and nothing is run for them. +func TestLLMManUnsupportedSettingsAreIgnored(t *testing.T) { + fake := newFakeLLMMan(t) + fake.set(t, "store", "docker.io/ai/smollm2:latest\n") + events := &recordedEvents{} + s := &composeService{events: events} + project := llmmanProject(llmmanEnv(), types.Models{ + "tuned": {Model: "docker.io/ai/smollm2:latest", ContextSize: 4096, RuntimeFlags: []string{"--no-mmap"}}, + "plain": {Model: "docker.io/ai/smollm2:latest"}, + }, map[string]*types.ServiceModelConfig{"tuned": nil, "plain": nil}) + assert.NilError(t, s.ensureModels(t.Context(), project, false)) + + assert.Assert(t, events.has("tuned", api.Warning, "Ignored context_size")) + assert.Assert(t, events.has("tuned", api.Warning, "Ignored runtime_flags")) + details := strings.Join(events.details("tuned", api.Warning), "\n") + assert.Assert(t, strings.Contains(details, "LLMMAN_CONTEXT_LENGTH"), details) + assert.Equal(t, len(events.details("plain", api.Warning)), 0) + assert.Assert(t, events.has("tuned", api.Done, api.StatusConfigured), "the model is still usable") + + for _, call := range fake.log(t) { + assert.Assert(t, !strings.Contains(call, "4096") && !strings.Contains(call, "no-mmap"), call) + } + assert.DeepEqual(t, fake.log(t), []string{"ps", "list --format {{.Name}}"}) +} + +// TestLLMManDryRunDoesNotPull: llmman is an external process the dry-run API +// client cannot intercept, so only its read-only commands (ps, list) may run. +func TestLLMManDryRunDoesNotPull(t *testing.T) { + fake := newFakeLLMMan(t) + events := &recordedEvents{} + s := &composeService{events: events, dryRun: true} + project := llmmanProject(llmmanEnv(), types.Models{"llm": {Model: "smollm2"}}, map[string]*types.ServiceModelConfig{"llm": nil}) + assert.NilError(t, s.ensureModels(t.Context(), project, false)) + + for _, call := range fake.log(t) { + assert.Assert(t, strings.HasPrefix(call, "ps") || strings.HasPrefix(call, "list"), "%q is not read-only", call) + } + assert.Assert(t, events.has("llm", api.Working, api.StatusPulled), "the pull is still reported") + url := project.Services["app"].Environment["LLM_URL"] + assert.Assert(t, url != nil && *url == "http://host.docker.internal:17434/v1", "model variables are still computed") + + // a daemon which is down is reported in dry-run mode too + fake.set(t, "down", "") + err := s.ensureModels(t.Context(), project, false) + assert.ErrorContains(t, err, "start it with `llmman serve`") +} From 3787bdeae6b5edee4355aa413c593832b1579ab6 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Wed, 4 Nov 2026 23:16:47 +0000 Subject: [PATCH 2/3] fix(models): close the pull pipe on start failure and drain long output A line past the scanner's limit stopped it reading, which could block the child. Signed-off-by: Eric Curtin --- pkg/compose/model_llmman.go | 7 +++++++ pkg/compose/model_llmman_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/pkg/compose/model_llmman.go b/pkg/compose/model_llmman.go index 05b3d4568d..96869fadf2 100644 --- a/pkg/compose/model_llmman.go +++ b/pkg/compose/model_llmman.go @@ -22,6 +22,7 @@ import ( "context" "errors" "fmt" + "io" "net" "os/exec" "slices" @@ -199,6 +200,7 @@ func (m *llmmanModelAPI) PullModel(ctx context.Context, model types.ModelConfig, var stderr bytes.Buffer cmd.Stderr = &stderr if err := cmd.Start(); err != nil { + _ = stream.Close() return err } @@ -213,6 +215,11 @@ func (m *llmmanModelAPI) PullModel(ctx context.Context, model types.ModelConfig, Text: api.StatusPulling, }) } + if scanner.Err() != nil { + // A line too long for the scanner stops it reading: drain the rest so + // the child does not block on a full pipe. + _, _ = io.Copy(io.Discard, stream) + } if err := cmd.Wait(); err != nil { if msg := strings.TrimSpace(stderr.String()); msg != "" { diff --git a/pkg/compose/model_llmman_test.go b/pkg/compose/model_llmman_test.go index 29539dd04f..5cc5995c65 100644 --- a/pkg/compose/model_llmman_test.go +++ b/pkg/compose/model_llmman_test.go @@ -26,6 +26,7 @@ import ( "strings" "sync" "testing" + "time" "github.com/compose-spec/compose-go/v2/types" "github.com/containerd/errdefs" @@ -46,6 +47,7 @@ import ( // - resolved: lines "=|" answered by `llmman list ` // - down: when present, `llmman ps` fails as if no daemon was running // - pullfail: when present, `llmman pull` fails +// - longline: when present, `llmman pull` prints a line longer than a scanner reads // // Every invocation is appended to calls.log, with the LLMMAN_* variables seen. type fakeLLMMan struct { @@ -86,6 +88,10 @@ pull) echo "registry exploded" >&2 exit 1 fi + if [ -f "$dir/longline" ]; then + head -c 300000 /dev/zero | tr '\0' x + echo + fi echo "pulling $2" echo "Pulled $2" ;; @@ -415,6 +421,24 @@ func TestLLMManPullFailure(t *testing.T) { assert.Assert(t, !events.has("llm", api.Done, api.StatusConfigured)) } +// TestLLMManPullDrainsOutputTooLongToScan: a line past the scanner's limit stops +// it reading, and the child must not then block on a full pipe. +func TestLLMManPullDrainsOutputTooLongToScan(t *testing.T) { + fake := newFakeLLMMan(t) + fake.set(t, "longline", "") + s := &composeService{events: &recordedEvents{}} + project := llmmanProject(llmmanEnv(), types.Models{"llm": {Model: "smollm2"}}, map[string]*types.ServiceModelConfig{"llm": nil}) + + done := make(chan error, 1) + go func() { done <- s.ensureModels(t.Context(), project, false) }() + select { + case err := <-done: + assert.NilError(t, err) + case <-time.After(30 * time.Second): + t.Fatal("pull blocked: the output nobody reads filled the pipe") + } +} + // TestLLMManPullFailureHintsAtDockerHubReferences: `ai/` is how Docker Model // Runner spells Docker Hub's ai namespace, which llmman reads as a Hugging Face repository. func TestLLMManPullFailureHintsAtDockerHubReferences(t *testing.T) { From f84297cd1e23280a8e34bc294b790f531513921f Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Fri, 6 Nov 2026 10:41:08 +0000 Subject: [PATCH 3/3] refactor(models): share the pull loop and test llmman errors Signed-off-by: Eric Curtin --- pkg/compose/model.go | 48 +++++++------- pkg/compose/model_llmman.go | 30 +-------- pkg/compose/model_llmman_test.go | 104 +++++++++++++++++++++++++++++-- 3 files changed, 126 insertions(+), 56 deletions(-) diff --git a/pkg/compose/model.go b/pkg/compose/model.go index 74467ea3fc..2e21b3ec67 100644 --- a/pkg/compose/model.go +++ b/pkg/compose/model.go @@ -22,6 +22,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os/exec" "slices" "strconv" @@ -214,42 +215,43 @@ func (m *dmrModelAPI) PullModel(ctx context.Context, model types.ModelConfig, qu if err != nil { return err } - stream, err := cmd.StdoutPipe() + err = streamPull(cmd, model.Name, quietPull, events) if err != nil { - return err + events.On(progress.Error(model.Name, err.Error())) } + events.On(api.Resource{ + ID: model.Name, + Status: api.Working, + Text: api.StatusPulled, + }) + return err +} - err = cmd.Start() +// streamPull runs a pull command and reports a pulling event for each line it +// prints on stdout, unless quiet. +func streamPull(cmd *exec.Cmd, name string, quiet bool, events api.EventProcessor) error { + stream, err := cmd.StdoutPipe() if err != nil { return err } + if err := cmd.Start(); err != nil { + _ = stream.Close() + return err + } scanner := bufio.NewScanner(stream) for scanner.Scan() { - msg := scanner.Text() - if msg == "" { + if scanner.Text() == "" || quiet { continue } - - if !quietPull { - events.On(api.Resource{ - ID: model.Name, - Status: api.Working, - Text: api.StatusPulling, - }) - } + events.On(api.Resource{ID: name, Status: api.Working, Text: api.StatusPulling}) } - - err = cmd.Wait() - if err != nil { - events.On(progress.Error(model.Name, err.Error())) + if scanner.Err() != nil { + // A line too long for the scanner stops it reading: drain the rest so + // the child does not block on a full pipe. + _, _ = io.Copy(io.Discard, stream) } - events.On(api.Resource{ - ID: model.Name, - Status: api.Working, - Text: api.StatusPulled, - }) - return err + return cmd.Wait() } func (m *dmrModelAPI) ConfigureModel(ctx context.Context, config types.ModelConfig, events api.EventProcessor) error { diff --git a/pkg/compose/model_llmman.go b/pkg/compose/model_llmman.go index 96869fadf2..3b797bbd30 100644 --- a/pkg/compose/model_llmman.go +++ b/pkg/compose/model_llmman.go @@ -17,12 +17,10 @@ package compose import ( - "bufio" "bytes" "context" "errors" "fmt" - "io" "net" "os/exec" "slices" @@ -193,35 +191,9 @@ func (m *llmmanModelAPI) PullModel(ctx context.Context, model types.ModelConfig, if err := m.prepare(ctx, cmd); err != nil { return err } - stream, err := cmd.StdoutPipe() - if err != nil { - return err - } var stderr bytes.Buffer cmd.Stderr = &stderr - if err := cmd.Start(); err != nil { - _ = stream.Close() - return err - } - - scanner := bufio.NewScanner(stream) - for scanner.Scan() { - if scanner.Text() == "" || quietPull { - continue - } - events.On(api.Resource{ - ID: model.Name, - Status: api.Working, - Text: api.StatusPulling, - }) - } - if scanner.Err() != nil { - // A line too long for the scanner stops it reading: drain the rest so - // the child does not block on a full pipe. - _, _ = io.Copy(io.Discard, stream) - } - - if err := cmd.Wait(); err != nil { + if err := streamPull(cmd, model.Name, quietPull, events); err != nil { if msg := strings.TrimSpace(stderr.String()); msg != "" { err = fmt.Errorf("%w: %s", err, msg) } diff --git a/pkg/compose/model_llmman_test.go b/pkg/compose/model_llmman_test.go index 5cc5995c65..8681474632 100644 --- a/pkg/compose/model_llmman_test.go +++ b/pkg/compose/model_llmman_test.go @@ -19,7 +19,10 @@ package compose import ( "context" "errors" + "io" + "io/fs" "os" + "os/exec" "path/filepath" "runtime" "slices" @@ -46,6 +49,8 @@ import ( // - store: references `llmman list` prints, one per line // - resolved: lines "=|" answered by `llmman list ` // - down: when present, `llmman ps` fails as if no daemon was running +// - listfail: when present, `llmman list --format` fails +// - resolvefail: when present, `llmman list ` fails // - pullfail: when present, `llmman pull` fails // - longline: when present, `llmman pull` prints a line longer than a scanner reads // @@ -75,9 +80,17 @@ ps) ;; list) if [ "$2" = "--format" ]; then + if [ -f "$dir/listfail" ]; then + echo "store unreadable" >&2 + exit 1 + fi [ -f "$dir/store" ] && cat "$dir/store" exit 0 fi + if [ -f "$dir/resolvefail" ]; then + echo "cannot resolve" >&2 + exit 1 + fi [ -f "$dir/resolved" ] && while IFS='=' read -r ref out; do [ "$ref" = "$2" ] && echo "$out" done < "$dir/resolved" @@ -140,16 +153,22 @@ func (r *recordedEvents) On(events ...api.Resource) { r.events = append(r.events, events...) } -// has tells whether an event with the given status and text was recorded for id. -func (r *recordedEvents) has(id string, status api.EventStatus, text string) bool { +// count returns how many events with the given status and text were recorded for id. +func (r *recordedEvents) count(id string, status api.EventStatus, text string) int { r.mu.Lock() defer r.mu.Unlock() + n := 0 for _, e := range r.events { if e.ID == id && e.Status == status && e.Text == text { - return true + n++ } } - return false + return n +} + +// has tells whether an event with the given status and text was recorded for id. +func (r *recordedEvents) has(id string, status api.EventStatus, text string) bool { + return r.count(id, status, text) > 0 } func (r *recordedEvents) details(id string, status api.EventStatus) []string { @@ -457,6 +476,83 @@ func TestLLMManPullFailureHintsAtDockerHubReferences(t *testing.T) { assert.Assert(t, !strings.Contains(err.Error(), "Docker Hub"), err.Error()) } +// TestLLMManListFailureIsNotFatal: when the store cannot be listed, every model is pulled. +func TestLLMManListFailureIsNotFatal(t *testing.T) { + fake := newFakeLLMMan(t) + fake.set(t, "listfail", "") + s := &composeService{events: &recordedEvents{}} + project := llmmanProject(llmmanEnv(), types.Models{"llm": {Model: "smollm2"}}, map[string]*types.ServiceModelConfig{"llm": nil}) + assert.NilError(t, s.ensureModels(t.Context(), project, false)) + assert.DeepEqual(t, fake.log(t), []string{"ps", "list --format {{.Name}}", "pull smollm2"}) +} + +func TestLLMManResolveFailure(t *testing.T) { + fake := newFakeLLMMan(t) + fake.set(t, "store", "docker.io/ai/other:latest\n") + fake.set(t, "resolvefail", "") + s := &composeService{events: &recordedEvents{}} + project := llmmanProject(llmmanEnv(), types.Models{"llm": {Model: "smollm2"}}, map[string]*types.ServiceModelConfig{"llm": nil}) + err := s.ensureModels(t.Context(), project, false) + assert.ErrorContains(t, err, "error checking model smollm2") + assert.ErrorContains(t, err, "cannot resolve") +} + +func TestLLMManQuietPull(t *testing.T) { + newFakeLLMMan(t) + model := types.ModelConfig{Name: "llm", Model: "smollm2"} + for quiet, want := range map[bool]int{false: 3, true: 1} { + events := &recordedEvents{} + s := &composeService{events: events} + m, err := s.newLLMManModelAPI(llmmanProject(llmmanEnv(), nil, nil)) + assert.NilError(t, err) + assert.NilError(t, m.PullModel(t.Context(), model, quiet, events)) + // one event when the pull starts, then one per line llmman prints + assert.Equal(t, events.count("llm", api.Working, api.StatusPulling), want, "quiet=%v", quiet) + } +} + +func TestLLMManCommandFailures(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("relies on the false command") + } + ctx := t.Context() + events := &recordedEvents{} + model := types.ModelConfig{Name: "llm", Model: "smollm2"} + noop := func(context.Context, *exec.Cmd) error { return nil } + errBoom := errors.New("boom") + + t.Run("failure without output", func(t *testing.T) { + // false exits 1 with nothing on stdout and stderr + m := &llmmanModelAPI{path: "false", prepare: noop} + assert.ErrorContains(t, m.Check(ctx), "exit status 1") + _, err := m.ListModels(ctx) + assert.ErrorContains(t, err, "error checking available models: exit status 1") + _, err = m.HasModel(ctx, []string{"other"}, "smollm2") + assert.ErrorContains(t, err, "error checking model smollm2: exit status 1") + assert.Error(t, m.PullModel(ctx, model, false, events), "exit status 1") + }) + + t.Run("prepare failure", func(t *testing.T) { + m := &llmmanModelAPI{path: "false", prepare: func(context.Context, *exec.Cmd) error { return errBoom }} + _, err := m.run(ctx, "ps") + assert.ErrorIs(t, err, errBoom) + assert.ErrorIs(t, m.PullModel(ctx, model, false, events), errBoom) + }) + + t.Run("stdout cannot be piped", func(t *testing.T) { + m := &llmmanModelAPI{path: "false", prepare: func(_ context.Context, cmd *exec.Cmd) error { + cmd.Stdout = io.Discard + return nil + }} + assert.ErrorContains(t, m.PullModel(ctx, model, false, events), "Stdout already set") + }) + + t.Run("binary cannot start", func(t *testing.T) { + m := &llmmanModelAPI{path: filepath.Join(t.TempDir(), "missing"), prepare: noop} + assert.ErrorIs(t, m.PullModel(ctx, model, false, events), fs.ErrNotExist) + }) +} + func TestLLMManVariables(t *testing.T) { newFakeLLMMan(t) s := &composeService{events: &recordedEvents{}}