Write enhanced fragment loading diagnostics and expose to container in the security context

internal/gcs-sidecar/handlers.go

@@ -22,6 +22,7 @@ import (
 	"github.com/Microsoft/hcsshim/internal/guestpath"
 	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
 	"github.com/Microsoft/hcsshim/internal/log"
+	"github.com/Microsoft/hcsshim/internal/logfields"
 	oci "github.com/Microsoft/hcsshim/internal/oci"
 	"github.com/Microsoft/hcsshim/internal/ot"
 	"github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
@@ -163,8 +164,10 @@ func (b *Bridge) createContainer(req *request) (err error) {
 			}
 		}()
 
+		var securityContextDir string
+
 		if oci.ParseAnnotationsBool(ctx, spec.Annotations, annotations.WCOWSecurityPolicyEnv, true) {
-			securityContextDir, err := b.hostState.securityOptions.WriteSecurityContextDir(&spec)
+			securityContextDir, err = b.hostState.securityOptions.WriteSecurityContextDir(&spec)
 			if err != nil {
 				return fmt.Errorf("failed to write security context dir: %w", err)
 			}
@@ -181,6 +184,21 @@ func (b *Bridge) createContainer(req *request) (err error) {
 			cwcowHostedSystemConfig.Spec = spec
 		}
 
+		// Add this fragments.info mount after policy enforcement and
+		// reconcile checks so the policy does not have to explicitly
+		// allow it.
+		if securityContextDir != "" {
+			if err := b.hostState.securityOptions.EnsureFragmentDiagnosticsDir(); err != nil {
+				log.G(ctx).WithError(err).WithField(logfields.Path, guestpath.WCOWFragmentsPath).Warn("failed to prepare fragments.info mount path in uVM")
+			} else {
+				container.MappedDirectories = append(container.MappedDirectories, hcsschema.MappedDirectory{
+					HostPath:      guestpath.WCOWFragmentsPath,
+					ContainerPath: filepath.Join(`C:\`, filepath.Base(securityContextDir), "fragments.info"),
+					ReadOnly:      true,
+				})
+			}
+		}
+
 		// Strip the spec field
 		hostedSystemBytes, err := json.Marshal(cwcowHostedSystem)
 

internal/guest/runtime/hcsv2/uvm.go

@@ -826,9 +826,24 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
 	}
 
 	if oci.ParseAnnotationsBool(ctx, settings.OCISpecification.Annotations, annotations.LCOWSecurityPolicyEnv, true) {
-		if _, err := h.securityOptions.WriteSecurityContextDir(settings.OCISpecification); err != nil {
+		securityContextDir, err := h.securityOptions.WriteSecurityContextDir(settings.OCISpecification)
+		if err != nil {
 			return nil, fmt.Errorf("failed to write security context dir: %w", err)
 		}
+		// Add this special fragments.info mount after policy enforcement
+		// so the policy does not have to explicitly allow it.
+		if securityContextDir != "" {
+			if err := h.securityOptions.EnsureFragmentDiagnosticsDir(); err != nil {
+				log.G(ctx).WithError(err).WithField(logfields.Path, guestpath.LCOWFragmentsPath).Warn("failed to prepare fragments.info mount path in uVM")
+			} else {
+				settings.OCISpecification.Mounts = append(settings.OCISpecification.Mounts, specs.Mount{
+					Destination: path.Join("/", filepath.Base(securityContextDir), "fragments.info"),
+					Type:        "bind",
+					Source:      guestpath.LCOWFragmentsPath,
+					Options:     []string{"bind", "ro"},
+				})
+			}
+		}
 	}
 
 	// Determine hostNetwork mode. For sandbox/standalone containers, check their

internal/guestpath/paths.go

@@ -14,6 +14,12 @@ const (
 	// WCOWSandboxMountPath is the path inside the UVM where WCOW sandbox mounts
 	// are created.
 	WCOWSandboxMountPath = `C:\SandboxMounts`
+	// LCOWFragmentsPath is the path inside the UVM where injected security
+	// policy fragments are stored.
+	LCOWFragmentsPath = "/tmp/fragments"
+	// WCOWFragmentsPath is the path inside the UVM where injected security
+	// policy fragments are stored.
+	WCOWFragmentsPath = `C:\InjectedFragments`
 	// SandboxMountPrefix is mount prefix used in container spec to mark a
 	// sandbox-mount
 	SandboxMountPrefix = "sandbox://"

pkg/securitypolicy/fragments_info_README

@@ -0,0 +1 @@
+The content of this directory is for informational purposes and should not be relied upon for security.

pkg/securitypolicy/securitypolicy_linux.go

@@ -15,7 +15,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-//nolint:unused
 const osType = "linux"
 
 func ExtendPolicyWithNetworkingMounts(sandboxRoot string, enforcer SecurityPolicyEnforcer, spec *oci.Spec) error {

pkg/securitypolicy/securitypolicy_options.go

@@ -3,17 +3,21 @@ package securitypolicy
 import (
 	"context"
 	"crypto/sha256"
+	_ "embed"
 	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"math"
 	"os"
 	"path/filepath"
 	"sync"
-	"time"
+	"sync/atomic"
 
 	"github.com/Microsoft/cosesign1go/pkg/cosesign1"
 	didx509resolver "github.com/Microsoft/didx509go/pkg/did-x509-resolver"
+	"github.com/Microsoft/hcsshim/internal/guestpath"
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/ot"
 	"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
@@ -41,6 +45,31 @@ type SecurityOptions struct {
 	logWriter io.Writer
 }
 
+// Global counter for fragment injection requests, used to create unique
+// deny / success marker files even when the same fragment is injected
+// multiple times.
+var fragmentRequestID atomic.Uint64
+
+//go:embed fragments_info_README
+var fragmentsInfoREADME []byte
+
+func fragmentsPath() string {
+	if osType == "windows" {
+		return guestpath.WCOWFragmentsPath
+	}
+	return guestpath.LCOWFragmentsPath
+}
+
+// EnsureFragmentDiagnosticsDir creates the fragment diagnostics directory and
+// its informational README.
+func (s *SecurityOptions) EnsureFragmentDiagnosticsDir() error {
+	dir := fragmentsPath()
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return err
+	}
+	return writeFileIfNotExists(filepath.Join(dir, "README"), fragmentsInfoREADME, 0644)
+}
+
 func NewSecurityOptions(enforcer SecurityPolicyEnforcer, enforcerSet bool, uvmReferenceInfo string, uvmHashEnvelopeReferenceInfo string, logWriter io.Writer) *SecurityOptions {
 	return &SecurityOptions{
 		PolicyEnforcer:               enforcer,
@@ -65,6 +94,14 @@ func (s *SecurityOptions) SetConfidentialOptions(ctx context.Context, enforcerTy
 		return errors.New("security policy has already been set")
 	}
 
+	// Pre-create this directory so that we can mount this dir into
+	// containers even if no fragments have been injected yet when the
+	// container starts.
+	if err := s.EnsureFragmentDiagnosticsDir(); err != nil {
+		// This is not fatal, don't fail here.
+		log.G(ctx).WithError(err).Error("failed to prepare injected fragments directory")
+	}
+
 	hostData, err := NewSecurityPolicyDigest(encodedSecurityPolicy)
 	if err != nil {
 		return err
@@ -152,6 +189,42 @@ func asInt64(v interface{}) (int64, error) {
 	}
 }
 
+func writeFileIfNotExists(filename string, data []byte, perm os.FileMode) error {
+	file, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm)
+	if os.IsExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	if _, err := file.Write(data); err != nil {
+		_ = file.Close()
+		return err
+	}
+	return file.Close()
+}
+
+func writeFragmentMetadata(ctx context.Context, filename, issuer, feed string, headerSVN *int64) {
+	metadata := struct {
+		Issuer    string `json:"issuer"`
+		Feed      string `json:"feed"`
+		HeaderSVN *int64 `json:"headerSvn"`
+	}{
+		Issuer:    issuer,
+		Feed:      feed,
+		HeaderSVN: headerSVN,
+	}
+	contents, err := json.Marshal(metadata)
+	if err != nil {
+		log.G(ctx).WithError(err).Warn("failed to marshal injected fragment metadata")
+		return
+	}
+	if err := writeFileIfNotExists(filename, contents, 0644); err != nil {
+		log.G(ctx).WithError(err).Warn("failed to write injected fragment metadata")
+	}
+}
+
 // Fragment extends current security policy with additional constraints
 // from the incoming fragment. Note that it is base64 encoded over the bridge/
 //
@@ -167,13 +240,40 @@ func (s *SecurityOptions) InjectFragment(ctx context.Context, fragment *guestres
 	defer span.End()
 	defer func() { ot.SetSpanStatus(span, err) }()
 	span.SetAttributes(attribute.String("fragment", fmt.Sprintf("%+v", fragment)))
+	currReqID := fragmentRequestID.Add(1)
 
 	// An empty media type defaults to a Rego policy fragment, for backward
 	// compatibility with older hosts that do not set the field.
 	mediaType := fragment.MediaType
 	if mediaType == "" {
 		mediaType = mediaTypeFragment
 	}
+
+	raw, err := base64.StdEncoding.DecodeString(fragment.Fragment)
+	if err != nil {
+		return fmt.Errorf("failed to decode fragment: %w", err)
+	}
+	sha := sha256.Sum256(raw)
+	shaHex := hex.EncodeToString(sha[:])
+	thisFragmentDir := filepath.Join(fragmentsPath(), shaHex)
+	defer func() {
+		markerName := fmt.Sprintf("%d.succeed", currReqID)
+		var markerContents []byte
+		if err != nil {
+			markerName = fmt.Sprintf("%d.deny", currReqID)
+			markerContents = []byte(err.Error())
+		}
+		if markerErr := os.WriteFile(filepath.Join(thisFragmentDir, markerName), markerContents, 0644); markerErr != nil {
+			log.G(ctx).WithError(markerErr).Warnf("failed to write injected fragment outcome marker %s", markerName)
+		}
+	}()
+
+	if err := os.MkdirAll(thisFragmentDir, 0755); err != nil {
+		return fmt.Errorf("failed to create injected fragment directory: %w", err)
+	}
+	if err := writeFileIfNotExists(filepath.Join(thisFragmentDir, "fragment.cose"), raw, 0644); err != nil {
+		return fmt.Errorf("failed to write fragment.cose: %w", err)
+	}
 	switch mediaType {
 	case mediaTypeFragment, mediaTypeTransparencyTrustList, mediaTypeTCBReferenceInfo, mediaTypePlatformReferenceInfo:
 	default:
@@ -184,24 +284,14 @@ func (s *SecurityOptions) InjectFragment(ctx context.Context, fragment *guestres
 		return fmt.Errorf("cannot inject fragment blob with unsupported media type %q", mediaType)
 	}
 
-	raw, err := base64.StdEncoding.DecodeString(fragment.Fragment)
-	if err != nil {
-		return fmt.Errorf("failed to decode fragment: %w", err)
-	}
-	blob := []byte(fragment.Fragment)
-	// keep a copy of the fragment, so we can manually figure out what went wrong
-	// will be removed eventually. Give it a unique name to avoid any potential
-	// race conditions.
-	sha := sha256.New()
-	sha.Write(blob)
-	timestamp := time.Now()
-	fragmentPath := fmt.Sprintf("fragment-%x-%d.blob", sha.Sum(nil), timestamp.UnixMilli())
-	_ = os.WriteFile(filepath.Join(os.TempDir(), fragmentPath), blob, 0644)
-
 	unpacked, err := cosesign1.UnpackAndValidateCOSE1CertChain(raw)
 	if err != nil {
 		return fmt.Errorf("InjectFragment failed COSE validation: %w", err)
 	}
+	if err := writeFileIfNotExists(filepath.Join(thisFragmentDir, "fragment"), unpacked.Payload, 0644); err != nil {
+		return fmt.Errorf("failed to write injected fragment payload: %w", err)
+	}
+
 	// We do not need to validate this.
 	if mediaType == mediaTypeTCBReferenceInfo || mediaType == mediaTypePlatformReferenceInfo {
 		s.platformReferenceInfoMutex.Lock()
@@ -264,6 +354,7 @@ func (s *SecurityOptions) InjectFragment(ctx context.Context, fragment *guestres
 			svnFromCwt = &svn
 		}
 	}
+	writeFragmentMetadata(ctx, filepath.Join(thisFragmentDir, "metadata.json"), issuer, feed, svnFromCwt)
 
 	switch mediaType {
 	case mediaTypeTransparencyTrustList:

pkg/securitypolicy/securitypolicy_windows.go

@@ -7,7 +7,6 @@ import (
 	oci "github.com/opencontainers/runtime-spec/specs-go"
 )
 
-//nolint:unused
 const osType = "windows"
 
 // SandboxMountsDir returns sandbox mounts directory inside UVM/host.