feat(projectconfig): load configuration without lock files

Teach the project configuration loader the lock-file-free mode selected by
'--without-lockfile'. In that mode component definitions merge with
override semantics rather than additively, so a generated upstream-commit
TOML can replace a component's configured pin, and the config file that
supplied the pin is recorded separately from the component's primary TOML
so synthetic history can follow it.

Because a single file may now hold a partial component definition,
component validation is deferred until the whole project is assembled.
The project's lock directory is left unset in this mode; a project may
still declare 'lock-dir' for compatibility with the default mode, where
the value keeps its existing meaning.

The default mode is unchanged: components merge additively, each config
file is validated on its own, and the lock directory keeps its default.
The load-time modes are carried in an internal loadOptions value instead of
adding more positional booleans to every loader helper.

Refs: microsoft#323

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: ddstreet

internal/app/azldev/app.go

@@ -604,6 +604,7 @@ func (a *App) findAndLoadConfig(tempDirPath string, extraConfigFiles []string) (
 		tempDirPath,
 		extraConfigFiles,
 		a.permissiveConfigParsing,
+		a.withoutLockfile,
 	)
 	if err != nil {
 		return projectDir, config, fmt.Errorf("failed to load project configuration:\n%w", err)

internal/app/azldev/cmds/component/history_internal_test.go

@@ -157,6 +157,12 @@ func TestCustomizationCollectorsCoverEveryFingerprintableField(t *testing.T) {
 			field := st.Field(i)
 			key := st.Name() + "." + field.Name
 
+			// Unexported fields are never fingerprinted: hashstructure skips
+			// them because it cannot read them by reflection.
+			if field.PkgPath != "" {
+				continue
+			}
+
 			// Fields excluded from the fingerprint are operational
 			// metadata (publish channels, build hints, maintenance
 			// markers, etc.), not modifications to upstream. Skip them.

internal/projectconfig/component.go

@@ -368,7 +368,13 @@ type ComponentConfig struct {
 
 	// Reference to the source config file that this definition came from; not present
 	// in serialized files.
-	SourceConfigFile *ConfigFile `toml:"-" json:"-" table:"-" fingerprint:"-"`
+	SourceConfigFile *ConfigFile `toml:"-" json:"-" table:"-" validate:"-" fingerprint:"-"`
+
+	// upstreamCommitConfigFile references the config file that supplied the
+	// component's upstream commit pin. Populated in lock-file-free mode, where
+	// synthetic history must follow the file that actually changed the pin even
+	// when another partial component definition is merged later.
+	upstreamCommitConfigFile *ConfigFile
 
 	// RenderedSpecDir is the output directory for this component's rendered spec files.
 	// Derived at resolve time from the project's rendered-specs-dir setting; not present
@@ -458,6 +464,71 @@ func (c *ComponentConfig) MergeUpdatesFrom(other *ComponentConfig) error {
 	return nil
 }
 
+// MergeOverridesFrom mutates the component config so that values present in other
+// replace the existing ones, instead of being merged additively into them.
+//
+// Used in lock-file-free mode, where a generated upstream-commit config file holds
+// a partial component definition whose 'spec' block must override the component's
+// primary definition rather than merge with it. Slices in the build config are
+// still appended, matching [ComponentConfig.MergeUpdatesFrom].
+func (c *ComponentConfig) MergeOverridesFrom(other *ComponentConfig) error {
+	otherOverlayFiles := slices.Clone(other.OverlayFiles)
+
+	// Merge the nested config blocks separately so that mergo does not descend
+	// into them with slice-appending semantics.
+	otherTopLevel := *other
+	otherTopLevel.Spec = SpecSource{}
+	otherTopLevel.Release = ReleaseConfig{}
+	otherTopLevel.Build = ComponentBuildConfig{}
+	otherTopLevel.Render = ComponentRenderConfig{}
+	otherTopLevel.Publish = ComponentPublishConfig{}
+
+	err := mergo.Merge(c, &otherTopLevel, mergo.WithOverride, mergo.WithAppendSlice)
+	if err != nil {
+		return fmt.Errorf("failed to merge project info:\n%w", err)
+	}
+
+	for destination, source := range map[any]any{
+		&c.Spec:    &other.Spec,
+		&c.Release: &other.Release,
+		&c.Render:  &other.Render,
+		&c.Publish: &other.Publish,
+	} {
+		if err := mergo.Merge(destination, source, mergo.WithOverride); err != nil {
+			return fmt.Errorf("failed to merge component config:\n%w", err)
+		}
+	}
+
+	if err := mergo.Merge(&c.Build, &other.Build, mergo.WithOverride, mergo.WithAppendSlice); err != nil {
+		return fmt.Errorf("failed to merge component build config:\n%w", err)
+	}
+
+	if other.SourceConfigFile != nil {
+		c.SourceConfigFile = other.SourceConfigFile
+	}
+
+	if other.upstreamCommitConfigFile != nil {
+		c.upstreamCommitConfigFile = other.upstreamCommitConfigFile
+	}
+
+	if other.OverlayFiles != nil {
+		c.OverlayFiles = otherOverlayFiles
+	}
+
+	return nil
+}
+
+// UpstreamCommitConfigFile returns the config file that supplied the component's
+// effective upstream commit pin, or nil when no commit is pinned. Only meaningful
+// in lock-file-free mode, where the pin lives in generated component config.
+func (c *ComponentConfig) UpstreamCommitConfigFile() *ConfigFile {
+	if c.upstreamCommitConfigFile == nil && c.Spec.UpstreamCommit != "" {
+		return c.SourceConfigFile
+	}
+
+	return c.upstreamCommitConfigFile
+}
+
 // EffectiveUpstreamCommit returns the commit to use for upstream operations.
 // Prefers the locked commit (resolved reality) over the config pin (user intent).
 // Falls back to Spec.UpstreamCommit for SkipLockValidation paths (update, list,
@@ -518,17 +589,18 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi
 	// the SourceConfigFile, as we *do* want to alias that pointer, sharing it across
 	// all configs that came from that source config file.
 	result := &ComponentConfig{
-		Name:             c.Name,
-		SourceConfigFile: c.SourceConfigFile,
-		RenderedSpecDir:  c.RenderedSpecDir,
-		Locked:           deep.MustCopy(c.Locked),
-		Release:          c.Release,
-		Spec:             deep.MustCopy(c.Spec),
-		Build:            deep.MustCopy(c.Build),
-		Render:           c.Render,
-		SourceFiles:      deep.MustCopy(c.SourceFiles),
-		Packages:         deep.MustCopy(c.Packages),
-		Publish:          deep.MustCopy(c.Publish),
+		Name:                     c.Name,
+		SourceConfigFile:         c.SourceConfigFile,
+		upstreamCommitConfigFile: c.upstreamCommitConfigFile,
+		RenderedSpecDir:          c.RenderedSpecDir,
+		Locked:                   deep.MustCopy(c.Locked),
+		Release:                  c.Release,
+		Spec:                     deep.MustCopy(c.Spec),
+		Build:                    deep.MustCopy(c.Build),
+		Render:                   c.Render,
+		SourceFiles:              deep.MustCopy(c.SourceFiles),
+		Packages:                 deep.MustCopy(c.Packages),
+		Publish:                  deep.MustCopy(c.Publish),
 		// OverlayFiles is consumed after component config resolution; preserve it verbatim
 		// here so inherited patterns can be interpreted relative to the concrete component
 		// config file.

internal/projectconfig/config.go

@@ -17,6 +17,10 @@ import (
 // may make use of the provided temporary directory, with the expectation that the caller is responsible
 // for cleaning it up -- but not until after it is done using the loaded configuration. The loaded
 // configuration may implicitly depend on the contents of the temporary directory.
+//
+// When withoutLockfile is set, component definitions merge with override semantics,
+// are validated only once the whole project is assembled, and the project's lock
+// directory is left unset.
 func LoadProjectConfig(
 	fs opctx.FS,
 	osEnv opctx.OSEnv,
@@ -25,6 +29,7 @@ func LoadProjectConfig(
 	tempDirPath string,
 	extraConfigFilePaths []string,
 	permissiveConfigParsing bool,
+	withoutLockfile bool,
 ) (projectDir string, config *ProjectConfig, err error) {
 	// Look for project root and azldev.toml file.
 	projectDir, projectFilePath, err := FindProjectRootAndConfigFile(fs, referenceDir)
@@ -79,7 +84,12 @@ func LoadProjectConfig(
 	//
 	// NOTE: We don't wrap the error returned back here (if one is returned) because we already have
 	// a decent one coming from this function.
-	config, err = loadAndResolveProjectConfig(fs, permissiveConfigParsing, configFilePaths...)
+	options := loadOptions{
+		permissiveConfigParsing: permissiveConfigParsing,
+		withoutLockfile:         withoutLockfile,
+	}
+
+	config, err = loadAndResolveProjectConfig(fs, options, configFilePaths...)
 	if err != nil {
 		return "", nil, err
 	}
@@ -90,5 +100,12 @@ func LoadProjectConfig(
 	// Apply project-relative defaults for any unset path fields.
 	config.Project.ApplyProjectDefaults(projectDir)
 
+	// Lock-file-free mode never reads or writes lock files, so the lock directory
+	// is left unset. A project may still declare 'lock-dir' for compatibility with
+	// azldev's default mode; the value is simply ignored here.
+	if withoutLockfile {
+		config.Project.LockDir = ""
+	}
+
 	return projectDir, config, nil
 }

internal/projectconfig/config_test.go

@@ -51,7 +51,7 @@ description = "`+testProjectDesc+`"
 `)
 
 	_, config, err := projectconfig.LoadProjectConfig(
-		ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil, false,
+		ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil, false, false,
 	)
 	require.NoError(t, err)
 	require.NotNil(t, config)
@@ -66,7 +66,7 @@ func TestLoadProjectConfig_WithDefaultConfig(t *testing.T) {
 	require.NoError(t, fileutils.MkdirAll(ctx.FS(), tempDir))
 
 	_, config, err := projectconfig.LoadProjectConfig(
-		ctx.FS(), ctx.OSEnv(), testProjectDir, false /*disableDefaultConfig*/, tempDir, nil, false,
+		ctx.FS(), ctx.OSEnv(), testProjectDir, false /*disableDefaultConfig*/, tempDir, nil, false, false,
 	)
 	require.NoError(t, err)
 	require.NotNil(t, config)
@@ -94,7 +94,7 @@ output-dir = "/from/user/out"
 `), fileperms.PublicFile))
 
 	_, config, err := projectconfig.LoadProjectConfig(
-		ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil, false,
+		ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil, false, false,
 	)
 	require.NoError(t, err)
 	require.NotNil(t, config)
@@ -136,7 +136,7 @@ description = "`+testUserDesc+`"
 
 	_, config, err := projectconfig.LoadProjectConfig(
 		ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(),
-		[]string{extraConfigPath}, false,
+		[]string{extraConfigPath}, false, false,
 	)
 	require.NoError(t, err)
 	require.NotNil(t, config)
@@ -166,9 +166,41 @@ description = "`+testUserDesc+`"
 `), fileperms.PublicFile))
 
 	_, config, err := projectconfig.LoadProjectConfig(
-		ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil, false,
+		ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil, false, false,
 	)
 	require.NoError(t, err)
 	require.NotNil(t, config)
 	assert.Equal(t, testUserDesc, config.Project.Description)
 }
+
+// TestLoadProjectConfig_LockDirByMode verifies that the project's lock directory is
+// defaulted in azldev's default mode and left unset in lock-file-free mode, where an
+// explicitly configured 'lock-dir' is accepted but ignored.
+func TestLoadProjectConfig_LockDirByMode(t *testing.T) {
+	testCases := []struct {
+		name            string
+		withoutLockfile bool
+		expected        string
+	}{
+		{name: "default mode", withoutLockfile: false, expected: filepath.Join(testProjectDir, "legacy-locks")},
+		{name: "lock-file-free mode", withoutLockfile: true, expected: ""},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			ctx := newTestCtxWithXDGConfigHome()
+			writeProjectConfig(t, ctx, `
+[project]
+lock-dir = "legacy-locks"
+`)
+
+			_, config, err := projectconfig.LoadProjectConfig(
+				ctx.FS(), ctx.OSEnv(), testProjectDir, true /*disableDefaultConfig*/, t.TempDir(), nil,
+				false, testCase.withoutLockfile,
+			)
+			require.NoError(t, err)
+			require.NotNil(t, config)
+			assert.Equal(t, testCase.expected, config.Project.LockDir)
+		})
+	}
+}

internal/projectconfig/configfile.go

@@ -108,14 +108,41 @@ func (f ConfigFile) Validate() error {
 		return err
 	}
 
-	// Per-component snapshot timestamps are not allowed. Components inherit
-	// the snapshot from the distro/group default-component-config or the
-	// project's default-distro. Per-component snapshots would create
-	// non-deterministic builds that the lock file cannot reliably track.
-	// Use an explicit 'upstream-commit' pin instead.
-
-	// Validate overlay configurations for each component.
-	for componentName, component := range f.Components {
+	if err := validateComponentConfigs(f.Components); err != nil {
+		return err
+	}
+
+	if err := validateTestSuites(f.TestSuites); err != nil {
+		return err
+	}
+
+	if err := validateTestDefinitions(f.Tests); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// validateNonComponentFields validates every field except the component
+// definitions. Lock-file-free mode merges component definitions across config
+// files with override semantics, so a single file may legitimately be
+// incomplete; components are validated once the whole project is assembled.
+func (f ConfigFile) validateNonComponentFields() error {
+	f.Components = nil
+
+	return f.Validate()
+}
+
+// validateComponentConfigs validates the parts of a component definition that
+// the struct validator cannot express.
+//
+// Per-component snapshot timestamps are not allowed. Components inherit the
+// snapshot from the distro/group default-component-config or the project's
+// default-distro. Per-component snapshots would create non-deterministic builds
+// that the lock file cannot reliably track. Use an explicit 'upstream-commit'
+// pin instead.
+func validateComponentConfigs(components map[string]ComponentConfig) error {
+	for componentName, component := range components {
 		for i, overlay := range component.Overlays {
 			err := overlay.Validate()
 			if err != nil {
@@ -142,14 +169,6 @@ func (f ConfigFile) Validate() error {
 		}
 	}
 
-	if err := validateTestSuites(f.TestSuites); err != nil {
-		return err
-	}
-
-	if err := validateTestDefinitions(f.Tests); err != nil {
-		return err
-	}
-
 	return nil
 }
 
@@ -652,3 +671,19 @@ func (f ConfigFile) Serialize(fs opctx.FS, filePath string) error {
 
 	return nil
 }
+
+// validateComponentStructs runs the struct validator over each component
+// definition. Config files declare components with a 'dive' validation tag, so
+// this is only needed when component validation is deferred until the whole
+// project has been merged (lock-file-free mode).
+func validateComponentStructs(components map[string]ComponentConfig) error {
+	validate := validator.New()
+
+	for componentName, component := range components {
+		if err := validate.Struct(&component); err != nil {
+			return fmt.Errorf("invalid component %#q:\n%w", componentName, err)
+		}
+	}
+
+	return nil
+}