From 8fc94e23a0cbe4cae0bf8404a137f807ba7ba030 Mon Sep 17 00:00:00 2001 From: Mistium <92952823+Mistium@users.noreply.github.com> Date: Sat, 21 Nov 2026 06:43:59 +0000 Subject: [PATCH 1/2] Serve commit files without walking the whole tree, and share opened history archives A burst of per-file requests for one commit (history restore, sprite texts) stalled the API: every request reopened the archive, walked the history and decompressed every blob in the tree to find one path. Requests now share an open archive with decoded commits, trees, blob sizes and reachability, look a file up along its path, and run at most half the cores at a time. The result cache never wrote anything because its file names were over 255 bytes; names are now hashed and kept in their own directory so they cannot evict workspace copies. The workspace cache is capped at 1 GiB. --- native/gitinspection/archive_cache.go | 277 ++++++++++++++++++ native/gitinspection/archive_cache_test.go | 274 +++++++++++++++++ native/gitinspection/inspection.go | 264 +++++++++++++---- native/gitinspection/inspection_bench_test.go | 80 +++++ native/gitinspection/inspection_test.go | 2 +- native/gitinspection/last_commits.go | 36 +-- native/storagefs/files.go | 2 +- opal.json | 2 +- scripts/e2e_smoke.py | 26 ++ src/git/git-inspection.osl | 33 +-- 10 files changed, 891 insertions(+), 105 deletions(-) create mode 100644 native/gitinspection/archive_cache.go create mode 100644 native/gitinspection/archive_cache_test.go create mode 100644 native/gitinspection/inspection_bench_test.go diff --git a/native/gitinspection/archive_cache.go b/native/gitinspection/archive_cache.go new file mode 100644 index 0000000..81d4c0a --- /dev/null +++ b/native/gitinspection/archive_cache.go @@ -0,0 +1,277 @@ +package gitinspection + +import ( + "archive/zip" + "os" + "runtime" + "sync" + "time" +) + +const ( + sharedArchiveLimit = 6 + sharedArchiveIdle = 5 * time.Minute + memoCommitLimit = 20_000 + memoBlobLimit = 100_000 + memoTreeBytesLimit = 16 << 20 + memoListingFileLimit = 100_000 + memoReachLimit = 16 +) + +type blobInfo struct { + size int + binary bool +} + +type reachSet struct { + once sync.Once + visited map[string]bool +} + +type objectMemo struct { + mu sync.Mutex + commits map[string]gitCommit + blobs map[string]blobInfo + trees map[string][]byte + treeBytes int + listings map[string][]gitFile + listingFiles int + reach map[string]*reachSet +} + +func newObjectMemo() *objectMemo { + return &objectMemo{ + commits: make(map[string]gitCommit), + blobs: make(map[string]blobInfo), + trees: make(map[string][]byte), + listings: make(map[string][]gitFile), + reach: make(map[string]*reachSet), + } +} + +func cloneCommit(commit gitCommit) gitCommit { + commit.Parents = append([]string{}, commit.Parents...) + return commit +} + +func (m *objectMemo) commit(oid string) (gitCommit, bool) { + m.mu.Lock() + defer m.mu.Unlock() + commit, ok := m.commits[oid] + if !ok { + return gitCommit{}, false + } + return cloneCommit(commit), true +} + +func (m *objectMemo) storeCommit(oid string, commit gitCommit) { + m.mu.Lock() + defer m.mu.Unlock() + if len(m.commits) < memoCommitLimit { + m.commits[oid] = cloneCommit(commit) + } +} + +func (m *objectMemo) blob(oid string) (blobInfo, bool) { + m.mu.Lock() + defer m.mu.Unlock() + info, ok := m.blobs[oid] + return info, ok +} + +func (m *objectMemo) storeBlob(oid string, info blobInfo) { + m.mu.Lock() + defer m.mu.Unlock() + if len(m.blobs) < memoBlobLimit { + m.blobs[oid] = info + } +} + +func (m *objectMemo) tree(oid string) ([]byte, bool) { + m.mu.Lock() + defer m.mu.Unlock() + body, ok := m.trees[oid] + return body, ok +} + +func (m *objectMemo) storeTree(oid string, body []byte) { + m.mu.Lock() + defer m.mu.Unlock() + if _, exists := m.trees[oid]; exists || m.treeBytes+len(body) > memoTreeBytesLimit { + return + } + m.trees[oid] = body + m.treeBytes += len(body) +} + +func (m *objectMemo) listing(treeOID string) ([]gitFile, bool) { + m.mu.Lock() + defer m.mu.Unlock() + files, ok := m.listings[treeOID] + if !ok { + return nil, false + } + return append([]gitFile{}, files...), true +} + +func (m *objectMemo) storeListing(treeOID string, files []gitFile) { + m.mu.Lock() + defer m.mu.Unlock() + if _, exists := m.listings[treeOID]; exists || m.listingFiles+len(files) > memoListingFileLimit { + return + } + m.listings[treeOID] = append([]gitFile{}, files...) + m.listingFiles += len(files) +} + +func (m *objectMemo) reachableFrom(allowedHeads string, walk func() map[string]bool) map[string]bool { + m.mu.Lock() + set := m.reach[allowedHeads] + if set == nil { + set = &reachSet{} + if len(m.reach) < memoReachLimit { + m.reach[allowedHeads] = set + } + } + m.mu.Unlock() + set.once.Do(func() { set.visited = walk() }) + return set.visited +} + +type sharedArchive struct { + path string + file *os.File + info os.FileInfo + archive archive + ready chan struct{} + err error + refs int + evicted bool + lastUsed time.Time +} + +var sharedArchives = struct { + sync.Mutex + entries map[string]*sharedArchive +}{entries: make(map[string]*sharedArchive)} + +var inspectionSlots = make(chan struct{}, max(2, runtime.NumCPU()/2)) + +func acquireInspectionSlot() func() { + inspectionSlots <- struct{}{} + return func() { <-inspectionSlots } +} + +func sameArchiveFile(a, b os.FileInfo) bool { + return os.SameFile(a, b) && a.Size() == b.Size() && a.ModTime().Equal(b.ModTime()) +} + +func (s *sharedArchive) closeLocked() { + if s.file != nil { + s.file.Close() + s.file = nil + } +} + +func dropSharedArchiveLocked(entry *sharedArchive) { + if sharedArchives.entries[entry.path] == entry { + delete(sharedArchives.entries, entry.path) + } + entry.evicted = true + if entry.refs == 0 { + entry.closeLocked() + } +} + +func evictSharedArchivesLocked(now time.Time) { + for _, entry := range sharedArchives.entries { + if entry.refs == 0 && now.Sub(entry.lastUsed) > sharedArchiveIdle { + dropSharedArchiveLocked(entry) + } + } + for len(sharedArchives.entries) > sharedArchiveLimit { + var oldest *sharedArchive + for _, entry := range sharedArchives.entries { + if oldest == nil || entry.lastUsed.Before(oldest.lastUsed) { + oldest = entry + } + } + dropSharedArchiveLocked(oldest) + } +} + +func (s *sharedArchive) release() { + sharedArchives.Lock() + defer sharedArchives.Unlock() + s.refs-- + if s.evicted && s.refs == 0 { + s.closeLocked() + } +} + +func openSharedArchiveFile(path string) (*os.File, os.FileInfo, archive, error) { + file, err := os.Open(path) + if err != nil { + return nil, nil, archive{}, err + } + info, err := file.Stat() + if err != nil { + file.Close() + return nil, nil, archive{}, err + } + reader, err := zip.NewReader(file, info.Size()) + if err != nil { + file.Close() + return nil, nil, archive{}, err + } + opened := indexArchive(reader.File) + opened.memo = newObjectMemo() + return file, info, opened, nil +} + +func acquireSharedArchive(path string) (*sharedArchive, error) { + info, err := os.Stat(path) + if err != nil { + return nil, err + } + now := time.Now() + sharedArchives.Lock() + entry := sharedArchives.entries[path] + if entry != nil && !sameArchiveFile(entry.info, info) { + dropSharedArchiveLocked(entry) + entry = nil + } + opener := entry == nil + if opener { + entry = &sharedArchive{path: path, info: info, ready: make(chan struct{})} + sharedArchives.entries[path] = entry + } + entry.refs++ + entry.lastUsed = now + evictSharedArchivesLocked(now) + sharedArchives.Unlock() + + if opener { + file, openedInfo, opened, openErr := openSharedArchiveFile(path) + sharedArchives.Lock() + entry.err = openErr + if openErr != nil { + dropSharedArchiveLocked(entry) + } else { + entry.file = file + entry.info = openedInfo + entry.archive = opened + if entry.evicted && entry.refs == 0 { + entry.closeLocked() + } + } + close(entry.ready) + sharedArchives.Unlock() + } + <-entry.ready + if entry.err != nil { + entry.release() + return nil, entry.err + } + return entry, nil +} diff --git a/native/gitinspection/archive_cache_test.go b/native/gitinspection/archive_cache_test.go new file mode 100644 index 0000000..30154e0 --- /dev/null +++ b/native/gitinspection/archive_cache_test.go @@ -0,0 +1,274 @@ +package gitinspection + +import ( + "encoding/base64" + "os" + "strings" + "sync" + "testing" +) + +func resetArchiveCacheForTest() { + sharedArchives.Lock() + defer sharedArchives.Unlock() + for _, entry := range sharedArchives.entries { + dropSharedArchiveLocked(entry) + } +} + +func nestedRepository(t *testing.T) (string, string) { + repository := newTestRepository() + files := map[string]string{ + "project.json": repository.object("blob", []byte("{}\n")), + "sprites/cat/code.fractch": repository.object("blob", []byte("say \"meow\"\n")), + "sprites/cat/costume.png": repository.object("blob", []byte("\x89PNG\x00data")), + "sprites/dog/code.fractch": repository.object("blob", []byte("say \"woof\"\n")), + } + first := repository.commit(repository.nestedTree(files), "", "First") + files["sprites/cat/code.fractch"] = repository.object("blob", []byte("say \"purr\"\n")) + head := repository.commit(repository.nestedTree(files), first, "Second") + return repository.write(t), head +} + +func TestInspectFileFindsNestedPaths(t *testing.T) { + resetArchiveCacheForTest() + path, head := nestedRepository(t) + result := decodeResult(t, Inspect(path, head, "file", "sprites/cat/code.fractch", head)) + if result["content"] != base64.StdEncoding.EncodeToString([]byte("say \"purr\"\n")) { + t.Fatalf("unexpected nested file: %#v", result) + } + file := result["file"].(map[string]any) + if file["path"] != "sprites/cat/code.fractch" || file["size"] != float64(11) || file["binary"] != false || file["mode"] != "100644" { + t.Fatalf("unexpected file metadata: %#v", file) + } + binary := decodeResult(t, Inspect(path, head, "file", "sprites/cat/costume.png", head)) + if binary["file"].(map[string]any)["binary"] != true { + t.Fatalf("binary file not flagged: %#v", binary) + } +} + +func TestInspectFileMatchesTreeListing(t *testing.T) { + resetArchiveCacheForTest() + path, head := nestedRepository(t) + tree := decodeResult(t, Inspect(path, head, "tree", "", head)) + for _, item := range tree["files"].([]any) { + listed := item.(map[string]any) + result := decodeResult(t, Inspect(path, head, "file", listed["path"].(string), head)) + file := result["file"].(map[string]any) + for _, key := range []string{"path", "oid", "mode", "size", "binary"} { + if file[key] != listed[key] { + t.Fatalf("file %s differs on %s: %#v vs %#v", listed["path"], key, file, listed) + } + } + } +} + +func TestInspectFileRejectsDirectoriesAndMissingPaths(t *testing.T) { + resetArchiveCacheForTest() + path, head := nestedRepository(t) + for _, requested := range []string{"sprites", "sprites/cat", "sprites/cow/code.fractch", "project.json/x", "missing.txt"} { + result := decodeResult(t, Inspect(path, head, "file", requested, head)) + if result["ok"] != false || result["error"] != "file was not found at this commit" { + t.Fatalf("%s: expected not found, got %#v", requested, result) + } + } +} + +func TestInspectFileReportsMissingBlob(t *testing.T) { + resetArchiveCacheForTest() + repository := newTestRepository() + blob := repository.object("blob", []byte("code\n")) + head := repository.commit(repository.tree("main.fractch", blob), "", "Initial") + delete(repository.objects, blob) + path := repository.write(t) + result := decodeResult(t, Inspect(path, head, "file", "main.fractch", head)) + if result["error"] != "file object is unavailable" { + t.Fatalf("expected missing blob, got %#v", result) + } +} + +func TestInspectSeesReplacedArchive(t *testing.T) { + resetArchiveCacheForTest() + path, head := nestedRepository(t) + if decodeResult(t, Inspect(path, head, "file", "project.json", head))["ok"] != true { + t.Fatal("first archive unreadable") + } + repository := newTestRepository() + blob := repository.object("blob", []byte("replaced\n")) + replacedHead := repository.commit(repository.tree("project.json", blob), "", "Replaced") + replacement := repository.write(t) + if err := os.Rename(replacement, path); err != nil { + t.Fatal(err) + } + result := decodeResult(t, Inspect(path, replacedHead, "file", "project.json", replacedHead)) + if result["content"] != base64.StdEncoding.EncodeToString([]byte("replaced\n")) { + t.Fatalf("stale archive served: %#v", result) + } + if decodeResult(t, Inspect(path, head, "file", "project.json", head))["ok"] != false { + t.Fatal("old commit still readable after the archive was replaced") + } +} + +func TestInspectReportsRemovedArchive(t *testing.T) { + resetArchiveCacheForTest() + path, head := nestedRepository(t) + Inspect(path, head, "tree", "", head) + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if result := decodeResult(t, Inspect(path, head, "tree", "", head)); result["error"] != "workspace archive is unavailable" { + t.Fatalf("expected unavailable archive, got %#v", result) + } +} + +func TestInspectConcurrentRequestsAgree(t *testing.T) { + resetArchiveCacheForTest() + path, head := nestedRepository(t) + want := map[string]string{} + for _, requested := range []string{"project.json", "sprites/cat/code.fractch", "sprites/dog/code.fractch"} { + want[requested] = Inspect(path, head, "file", requested, head) + } + wantTree := Inspect(path, head, "tree", "", head) + resetArchiveCacheForTest() + var group sync.WaitGroup + for i := 0; i < 64; i++ { + group.Add(1) + go func(i int) { + defer group.Done() + if i%4 == 0 { + if got := Inspect(path, head, "tree", "", head); got != wantTree { + t.Errorf("tree differs under concurrency") + } + return + } + for requested, expected := range want { + if got := Inspect(path, head, "file", requested, head); got != expected { + t.Errorf("%s differs under concurrency", requested) + } + } + }(i) + } + group.Wait() +} + +func TestSharedArchivesStayBounded(t *testing.T) { + resetArchiveCacheForTest() + for i := 0; i < sharedArchiveLimit+3; i++ { + path, head := nestedRepository(t) + Inspect(path, head, "tree", "", head) + } + sharedArchives.Lock() + count := len(sharedArchives.entries) + for _, entry := range sharedArchives.entries { + if entry.refs != 0 { + t.Errorf("archive %s still referenced", entry.path) + } + } + sharedArchives.Unlock() + if count > sharedArchiveLimit { + t.Fatalf("%d archives kept open, limit %d", count, sharedArchiveLimit) + } +} + +func TestMemoReachabilityMatchesWalk(t *testing.T) { + repository := newTestRepository() + blob := repository.object("blob", []byte("a\n")) + tree := repository.tree("a.txt", blob) + first := repository.commit(tree, "", "First") + second := repository.commit(tree, first, "Second") + side := repository.commit(tree, "", "Side") + path := repository.write(t) + reader, plain, err := openArchive(path) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + memoized := plain + memoized.memo = newObjectMemo() + for _, sha := range []string{first, second, side, strings.Repeat("0", 40)} { + if plain.reachable(sha, second) != memoized.reachable(sha, second) { + t.Fatalf("reachability of %s differs with the memo", sha) + } + } + if !memoized.reachable(first, second) || memoized.reachable(side, second) { + t.Fatal("memoized reachability is wrong") + } +} + +func TestInspectionError(t *testing.T) { + cases := map[string]string{ + `{"ok":true,"content":"eA=="}`: "", + `{"error":"commit tree is invalid","ok":false}`: "commit tree is invalid", + `{"ok":false}`: "commit not found", + `not json`: "could not inspect project history", + } + for input, want := range cases { + if got := InspectionError(input); got != want { + t.Fatalf("InspectionError(%q) = %q, want %q", input, got, want) + } + } +} + +func plainInspect(t *testing.T, path, sha, operation, requestedPath, allowedHeads string) string { + t.Helper() + reader, plain, err := openArchive(path) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + commit, ok := plain.parseCommit(sha) + if !ok || !plain.reachable(sha, allowedHeads) { + t.Fatalf("plain archive cannot read %s", sha) + } + switch operation { + case "tree": + files, ok := plain.walkTree(commit.Tree) + if !ok { + t.Fatal("plain tree invalid") + } + return encode(map[string]any{"ok": true, "sha": sha, "commit": commit, "files": files}, "") + case "commit-inline": + return plain.inspectCommit(sha, commit, true) + case "changes": + return plain.inspectCommitChanges(sha, commit) + case "summary": + return plain.inspectCommitSummary(sha, commit) + case "last-commits": + return plain.inspectLastCommits(sha, commit) + } + return plain.inspectCommit(sha, commit, false) +} + +func TestSharedArchiveMatchesPlainArchive(t *testing.T) { + resetArchiveCacheForTest() + repository := newTestRepository() + files := map[string]string{ + "project.json": repository.object("blob", []byte("{}\n")), + "sprites/cat/code.fractch": repository.object("blob", []byte("sprite \"Cat\";\nwhen flag {\n say \"a\";\n}\n")), + "sprites/dog/code.fractch": repository.object("blob", []byte("sprite \"Dog\";\n")), + "notes.txt": repository.object("blob", []byte("one\n")), + } + heads := []string{} + head := "" + for i := 0; i < 12; i++ { + switch i % 3 { + case 0: + files["notes.txt"] = repository.object("blob", []byte(strings.Repeat("line\n", i+1))) + case 1: + files["sprites/cat/code.fractch"] = repository.object("blob", []byte("sprite \"Cat\";\nwhen flag {\n say \""+strings.Repeat("b", i)+"\";\n}\n")) + default: + files["sprites/new"+string(rune('a'+i))+"/code.fractch"] = repository.object("blob", []byte("sprite \"N\";\n")) + } + head = repository.commit(repository.nestedTree(files), head, "Save "+string(rune('a'+i))) + heads = append(heads, head) + } + path := repository.write(t) + for _, sha := range heads { + for _, operation := range []string{"tree", "commit", "commit-inline", "changes", "summary", "last-commits"} { + shared := Inspect(path, sha, operation, "", head) + if plain := plainInspect(t, path, sha, operation, "", head); shared != plain { + t.Fatalf("%s of %s differs:\nshared %s\nplain %s", operation, sha, shared, plain) + } + } + } +} diff --git a/native/gitinspection/inspection.go b/native/gitinspection/inspection.go index 63ff6a1..10dd59d 100644 --- a/native/gitinspection/inspection.go +++ b/native/gitinspection/inspection.go @@ -41,6 +41,13 @@ type gitCommit struct { type archive struct { objects map[string]*zip.File trusted map[string]bool + memo *objectMemo +} + +type treeItem struct { + mode string + name string + oid string } func failure(message string) string { @@ -53,13 +60,17 @@ func openArchive(path string) (*zip.ReadCloser, archive, error) { if err != nil { return nil, archive{}, err } + return reader, indexArchive(reader.File), nil +} + +func indexArchive(files []*zip.File) archive { result := archive{objects: make(map[string]*zip.File)} - for _, item := range reader.File { + for _, item := range files { if strings.HasPrefix(item.Name, ".git/objects/") && !item.FileInfo().IsDir() { result.objects[item.Name] = item } } - return reader, result, nil + return result } func (a archive) readObject(oid string) (string, []byte, bool) { @@ -138,6 +149,11 @@ func (a archive) objectKind(oid string) string { } func (a archive) parseCommit(oid string) (gitCommit, bool) { + if a.memo != nil { + if commit, ok := a.memo.commit(oid); ok { + return commit, true + } + } kind, body, ok := a.readObject(oid) if !ok || kind != "commit" { return gitCommit{}, false @@ -173,10 +189,135 @@ func (a archive) parseCommit(oid string) (gitCommit, bool) { if len(result.Tree) != 40 { return gitCommit{}, false } + if a.memo != nil { + a.memo.storeCommit(oid, result) + } return result, true } +func (a archive) treeObject(oid string) ([]byte, bool) { + if a.memo != nil { + if body, ok := a.memo.tree(oid); ok { + return body, true + } + } + kind, body, ok := a.readObject(oid) + if !ok || kind != "tree" { + return nil, false + } + if a.memo != nil { + a.memo.storeTree(oid, body) + } + return body, true +} + +func parseTreeEntries(body []byte) ([]treeItem, bool) { + entries := []treeItem{} + position := 0 + for position < len(body) { + space := bytes.IndexByte(body[position:], ' ') + if space < 1 { + return nil, false + } + space += position + nul := bytes.IndexByte(body[space+1:], 0) + if nul < 1 { + return nil, false + } + nul += space + 1 + if nul+21 > len(body) { + return nil, false + } + name := string(body[space+1 : nul]) + if name == "" || name == "." || name == ".." || strings.ContainsAny(name, "/\\") { + return nil, false + } + entries = append(entries, treeItem{ + mode: string(body[position:space]), name: name, oid: hex.EncodeToString(body[nul+1 : nul+21]), + }) + position = nul + 21 + } + return entries, true +} + +func treeMode(mode string) bool { + return mode == "40000" || mode == "040000" +} + +func blobSummary(blob []byte) blobInfo { + probe := blob + if len(probe) > 8000 { + probe = probe[:8000] + } + return blobInfo{size: len(blob), binary: bytes.IndexByte(probe, 0) >= 0} +} + +func (a archive) blobInfo(oid string) (blobInfo, bool) { + if a.memo != nil { + if info, ok := a.memo.blob(oid); ok { + return info, true + } + } + kind, blob, ok := a.readObject(oid) + if !ok || kind != "blob" { + return blobInfo{}, false + } + info := blobSummary(blob) + if a.memo != nil { + a.memo.storeBlob(oid, info) + } + return info, true +} + +func (a archive) findFile(rootOID, requestedPath string) (gitFile, []byte, string) { + treeOID := rootOID + parts := strings.Split(requestedPath, "/") + for depth, part := range parts { + if depth > maxTreeDepth { + return gitFile{}, nil, "commit tree is invalid" + } + body, ok := a.treeObject(treeOID) + if !ok { + return gitFile{}, nil, "commit tree is invalid" + } + entries, ok := parseTreeEntries(body) + if !ok { + return gitFile{}, nil, "commit tree is invalid" + } + last := depth == len(parts)-1 + next := "" + for _, entry := range entries { + if entry.name != part || treeMode(entry.mode) == last { + continue + } + if !last { + next = entry.oid + break + } + kind, content, found := a.readObject(entry.oid) + if !found || kind != "blob" { + return gitFile{}, nil, "file object is unavailable" + } + info := blobSummary(content) + if a.memo != nil { + a.memo.storeBlob(entry.oid, info) + } + return gitFile{Path: requestedPath, OID: entry.oid, Mode: entry.mode, Size: info.size, Binary: info.binary}, content, "" + } + if next == "" { + return gitFile{}, nil, "file was not found at this commit" + } + treeOID = next + } + return gitFile{}, nil, "file was not found at this commit" +} + func (a archive) walkTree(rootOID string) ([]gitFile, bool) { + if a.memo != nil { + if files, ok := a.memo.listing(rootOID); ok { + return files, true + } + } files := []gitFile{} active := make(map[string]bool) var walk func(string, string, int) bool @@ -186,70 +327,69 @@ func (a archive) walkTree(rootOID string) ([]gitFile, bool) { } active[treeOID] = true defer delete(active, treeOID) - kind, body, ok := a.readObject(treeOID) - if !ok || kind != "tree" { + body, ok := a.treeObject(treeOID) + if !ok { return false } - position := 0 - for position < len(body) { - space := bytes.IndexByte(body[position:], ' ') - if space < 1 { - return false - } - space += position - nul := bytes.IndexByte(body[space+1:], 0) - if nul < 1 { - return false - } - nul += space + 1 - if nul+21 > len(body) { - return false - } - mode := string(body[position:space]) - name := string(body[space+1 : nul]) - if name == "" || name == "." || name == ".." || strings.ContainsAny(name, "/\\") { - return false - } - childOID := hex.EncodeToString(body[nul+1 : nul+21]) - fullPath := name + entries, ok := parseTreeEntries(body) + if !ok { + return false + } + for _, entry := range entries { + fullPath := entry.name if prefix != "" { - fullPath = prefix + "/" + name + fullPath = prefix + "/" + entry.name } - position = nul + 21 - if mode == "40000" || mode == "040000" { - if !walk(childOID, fullPath, depth+1) { + if treeMode(entry.mode) { + if !walk(entry.oid, fullPath, depth+1) { return false } continue } - blobKind, blob, ok := a.readObject(childOID) - if !ok || blobKind != "blob" { + info, ok := a.blobInfo(entry.oid) + if !ok { return false } - probe := blob - if len(probe) > 8000 { - probe = probe[:8000] - } files = append(files, gitFile{ - Path: fullPath, OID: childOID, Mode: mode, Size: len(blob), Binary: bytes.IndexByte(probe, 0) >= 0, + Path: fullPath, OID: entry.oid, Mode: entry.mode, Size: info.size, Binary: info.binary, }) if len(files) > maxTreeFiles { return false } } - return position == len(body) + return true } - return files, walk(rootOID, "", 0) + if !walk(rootOID, "", 0) { + return files, false + } + if a.memo != nil { + a.memo.storeListing(rootOID, files) + } + return files, true } func (a archive) reachable(sha, allowedHeads string) bool { + if a.memo == nil { + return a.walkReachable(sha, allowedHeads) + } + return a.memo.reachableFrom(allowedHeads, func() map[string]bool { + visited := make(map[string]bool) + a.walkHistory(allowedHeads, visited, "") + return visited + })[sha] +} + +func (a archive) walkReachable(sha, allowedHeads string) bool { + return a.walkHistory(allowedHeads, make(map[string]bool), sha) +} + +func (a archive) walkHistory(allowedHeads string, visited map[string]bool, target string) bool { queue := []string{} for _, candidate := range strings.Split(allowedHeads, ",") { if len(candidate) == 40 { queue = append(queue, candidate) } } - visited := make(map[string]bool) for len(queue) > 0 && len(visited) <= maxReachable { candidate := queue[0] queue = queue[1:] @@ -257,7 +397,7 @@ func (a archive) reachable(sha, allowedHeads string) bool { continue } visited[candidate] = true - if candidate == sha { + if target != "" && candidate == target { return true } commit, ok := a.parseCommit(candidate) @@ -308,11 +448,14 @@ func CommitTreesDiffer(basePath, baseHead, incomingPath, incomingHead string) st // Inspect reads a commit, tree, or file from the loose Git objects stored in a // MistWarp project archive. It returns the JSON contract consumed by the OSL API. func Inspect(workspacePath, sha, operation, requestedPath, allowedHeads string) string { - reader, archive, err := openArchive(workspacePath) + releaseSlot := acquireInspectionSlot() + defer releaseSlot() + shared, err := acquireSharedArchive(workspacePath) if err != nil { return failure("workspace archive is unavailable") } - defer reader.Close() + defer shared.release() + archive := shared.archive commit, ok := archive.parseCommit(sha) if !ok { @@ -330,21 +473,11 @@ func Inspect(workspacePath, sha, operation, requestedPath, allowedHeads string) } return encode(map[string]any{"ok": true, "sha": sha, "commit": commit, "files": files}, "could not encode commit tree") case "file": - files, ok := archive.walkTree(commit.Tree) - if !ok { - return failure("commit tree is invalid") + file, content, problem := archive.findFile(commit.Tree, requestedPath) + if problem != "" { + return failure(problem) } - for _, file := range files { - if file.Path != requestedPath { - continue - } - kind, content, found := archive.readObject(file.OID) - if !found || kind != "blob" { - return failure("file object is unavailable") - } - return encode(map[string]any{"ok": true, "sha": sha, "file": file, "content": content}, "could not encode file") - } - return failure("file was not found at this commit") + return encode(map[string]any{"ok": true, "sha": sha, "file": file, "content": content}, "could not encode file") case "commit-inline": return archive.inspectCommit(sha, commit, true) case "changes": @@ -526,3 +659,20 @@ func compareCommitTrees(baseArchive archive, baseSHA string, baseCommit gitCommi "additions": additions, "deletions": deletions, "legacy": legacy, }, "could not encode commit inspection") } + +func InspectionError(result string) string { + var status struct { + OK bool `json:"ok"` + Error string `json:"error"` + } + if err := json.Unmarshal([]byte(result), &status); err != nil { + return "could not inspect project history" + } + if status.OK { + return "" + } + if status.Error == "" { + return "commit not found" + } + return status.Error +} diff --git a/native/gitinspection/inspection_bench_test.go b/native/gitinspection/inspection_bench_test.go new file mode 100644 index 0000000..c0ce995 --- /dev/null +++ b/native/gitinspection/inspection_bench_test.go @@ -0,0 +1,80 @@ +package gitinspection + +import ( + "math/rand" + "strconv" + "strings" + "sync" + "testing" +) + +type benchProject struct { + path string + head string + files []string +} + +func buildBenchProject(t testing.TB, textFiles, assetFiles, commits int) benchProject { + random := rand.New(rand.NewSource(1)) + repository := newTestRepository() + files := map[string]string{} + paths := []string{} + for i := 0; i < textFiles; i++ { + path := "sprites/sprite" + strconv.Itoa(i) + "/code.fractch" + size := 300 + random.Intn(30000) + files[path] = repository.object("blob", []byte(strings.Repeat("say \"hello\"\n", size/12))) + paths = append(paths, path) + } + for i := 0; i < assetFiles; i++ { + path := "assets/asset" + strconv.Itoa(i) + ".png" + data := make([]byte, 64<<10+random.Intn(512<<10)) + random.Read(data) + files[path] = repository.object("blob", data) + paths = append(paths, path) + } + head := "" + for i := 0; i < commits; i++ { + edited := paths[random.Intn(textFiles)] + files[edited] = repository.object("blob", []byte("say \"version "+strconv.Itoa(i)+"\"\n")) + head = repository.commit(repository.nestedTree(files), head, "Save "+strconv.Itoa(i)) + } + file := repository.write(t) + return benchProject{path: file, head: head, files: paths} +} + +func benchmarkFileBurst(b *testing.B, project benchProject) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + resetArchiveCacheForTest() + var group sync.WaitGroup + for _, path := range project.files { + group.Add(1) + go func(path string) { + defer group.Done() + result := Inspect(project.path, project.head, "file", path, project.head) + if !strings.Contains(result, `"ok":true`) { + b.Errorf("file %s failed: %.120s", path, result) + } + }(path) + } + group.Wait() + } +} + +func BenchmarkInspectFileBurst(b *testing.B) { + benchmarkFileBurst(b, buildBenchProject(b, 180, 40, 300)) +} + +func BenchmarkInspectTreeRepeated(b *testing.B) { + project := buildBenchProject(b, 180, 40, 300) + resetArchiveCacheForTest() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + result := Inspect(project.path, project.head, "tree", "", project.head) + if !strings.Contains(result, `"ok":true`) { + b.Fatalf("tree failed: %.120s", result) + } + } +} diff --git a/native/gitinspection/inspection_test.go b/native/gitinspection/inspection_test.go index e824d83..6191254 100644 --- a/native/gitinspection/inspection_test.go +++ b/native/gitinspection/inspection_test.go @@ -48,7 +48,7 @@ func (r *testRepository) commit(tree, parent, message string) string { return r.object("commit", []byte(body)) } -func (r *testRepository) write(t *testing.T) string { +func (r *testRepository) write(t testing.TB) string { t.Helper() file, err := os.CreateTemp(t.TempDir(), "workspace-*.mwp") if err != nil { diff --git a/native/gitinspection/last_commits.go b/native/gitinspection/last_commits.go index 9dd77ab..4c38ceb 100644 --- a/native/gitinspection/last_commits.go +++ b/native/gitinspection/last_commits.go @@ -1,8 +1,6 @@ package gitinspection import ( - "bytes" - "encoding/hex" "strings" ) @@ -16,33 +14,17 @@ type treeEntry struct { } func (a archive) readTreeEntries(oid string) (map[string]treeEntry, bool) { - kind, body, ok := a.readObject(oid) - if !ok || kind != "tree" { + body, ok := a.treeObject(oid) + if !ok { + return nil, false + } + items, ok := parseTreeEntries(body) + if !ok { return nil, false } - entries := map[string]treeEntry{} - position := 0 - for position < len(body) { - space := bytes.IndexByte(body[position:], ' ') - if space < 1 { - return nil, false - } - space += position - nul := bytes.IndexByte(body[space+1:], 0) - if nul < 1 { - return nil, false - } - nul += space + 1 - if nul+21 > len(body) { - return nil, false - } - mode := string(body[position:space]) - name := string(body[space+1 : nul]) - if name == "" || name == "." || name == ".." || strings.ContainsAny(name, "/\\") { - return nil, false - } - entries[name] = treeEntry{OID: hex.EncodeToString(body[nul+1 : nul+21]), Dir: mode == "40000" || mode == "040000"} - position = nul + 21 + entries := make(map[string]treeEntry, len(items)) + for _, item := range items { + entries[item.name] = treeEntry{OID: item.oid, Dir: treeMode(item.mode)} } return entries, true } diff --git a/native/storagefs/files.go b/native/storagefs/files.go index ffacd4e..13726df 100644 --- a/native/storagefs/files.go +++ b/native/storagefs/files.go @@ -14,7 +14,7 @@ import ( const ( cacheMaxAge = 24 * time.Hour cacheMaxFiles = 256 - cacheMaxBytes = 2147483648 + cacheMaxBytes = 1 << 30 ) func Missing(path string) bool { diff --git a/opal.json b/opal.json index 125615f..a504779 100644 --- a/opal.json +++ b/opal.json @@ -1,6 +1,6 @@ { "name": "mistwarp-api", - "version": "1.4.0", + "version": "1.4.1", "main": "main.osl", "scripts": { "dev": ["osl", "run", "main.osl"], diff --git a/scripts/e2e_smoke.py b/scripts/e2e_smoke.py index c1de57e..d6a310f 100644 --- a/scripts/e2e_smoke.py +++ b/scripts/e2e_smoke.py @@ -1,4 +1,5 @@ import argparse +import concurrent.futures import difflib import gzip import hashlib @@ -648,6 +649,31 @@ def head_and_edited(): step("commit changes", "GET", path + "/commits/" + head + "/changes", "sam") step("commit changes signed out", "GET", path + "/commits/" + head + "/changes") step("changes of unknown commit", "GET", path + "/commits/" + "1" * 40 + "/changes", "sam") + step("commit inline", "GET", path + "/commits/" + head + "?inline=1", "sam") + status, last = server.call("GET", "/v1/projects/%s/commits/%s/last-commits" % (real_id(ids, ""), head), "sam")[:2] + last_commits = last.get("commits") or {} if isinstance(last, dict) else {} + steps.append({"step": "commit last commits", "status": status, "body": { + "complete": last.get("complete") if isinstance(last, dict) else None, + "paths": {name: (last_commits.get(sha) or {}).get("message") for name, sha in ((last.get("paths") or {}) if isinstance(last, dict) else {}).items()}, + }}) + status, tree = step("commit tree", "GET", path + "/commits/" + head + "/tree", "sam") + tree_paths = sorted(str(item.get("path", "")) for item in (tree.get("files") or [])) if isinstance(tree, dict) else [] + for tree_path in tree_paths: + step("commit file " + tree_path, "GET", path + "/commits/" + head + "/file?path=" + urllib.parse.quote(tree_path), "sam") + if tree_paths: + step("commit file signed out", "GET", path + "/commits/" + head + "/file?path=" + urllib.parse.quote(tree_paths[0])) + directory = tree_paths[-1].rpartition("/")[0] + if directory: + step("commit file of a directory", "GET", path + "/commits/" + head + "/file?path=" + urllib.parse.quote(directory), "sam") + step("commit file missing", "GET", path + "/commits/" + head + "/file?path=missing.txt", "sam") + step("commit file invalid path", "GET", path + "/commits/" + head + "/file?path=../x", "sam") + step("commit file of unknown commit", "GET", path + "/commits/" + "1" * 40 + "/file?path=project.json", "sam") + burst = [server.call("GET", "/v1/projects/%s/commits/%s/file?path=%s" % (real_id(ids, ""), head, urllib.parse.quote(tree_path)), "sam") for tree_path in tree_paths] + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as pool: + parallel = list(pool.map(lambda tree_path: server.call("GET", "/v1/projects/%s/commits/%s/file?path=%s" % (real_id(ids, ""), head, urllib.parse.quote(tree_path)), "sam"), tree_paths * 8)) + steps.append({"step": "commit files in parallel match", "status": 0, "body": all( + result[0] == burst[index % len(burst)][0] and result[1] == burst[index % len(burst)][1] for index, result in enumerate(parallel) + ) if burst else True}) step("commit summaries", "GET", path + "/commit-summaries?shas=" + head + ",nothex," + "1" * 40 + "," + head, "sam") step("commit summaries signed out", "GET", path + "/commit-summaries?shas=" + head) step("commit summaries from cache", "GET", path + "/commit-summaries?shas=" + head, "sam") diff --git a/src/git/git-inspection.osl b/src/git/git-inspection.osl index 91b31b5..cb7a10f 100644 --- a/src/git/git-inspection.osl +++ b/src/git/git-inspection.osl @@ -169,23 +169,21 @@ def validGitFilePath(string path) boolean ( ) def cachedWorkspaceGitInspectionJSON(object project, string workspacePath, string sha, string operation, string requestedPath, string allowedHeads) string ( - string cacheDir = tmpDir ++ "git-inspection/" + string cacheDir = tmpDir ++ "git-inspection-results/" void fs.mkdir(cacheDir, {recursive: true}) - string cacheName = gitInspectionCacheVersion() ++ "." ++ projectWorkspaceFingerprint(project) ++ "." ++ allowedHeads.hashSHA256() ++ "." ++ sha ++ "." ++ operation ++ "." ++ requestedPath.hashSHA256() ++ ".json" + string cacheKey = projectWorkspaceFingerprint(project) ++ "." ++ allowedHeads.hashSHA256() ++ "." ++ sha ++ "." ++ operation ++ "." ++ requestedPath.hashSHA256() + string cacheName = gitInspectionCacheVersion() ++ "." ++ cacheKey.hashSHA256() ++ ".json" pruneGitInspectionCache(cacheDir, cacheName) string cachePath = cacheDir ++ cacheName if fs.exists(cachePath) and fs.size(cachePath).unwrapOr(0) <= 33554432 ( return fs.readFile(cachePath).unwrapOr("") ) string resultJson = gitinspection.Inspect(workspacePath, sha, operation, requestedPath, allowedHeads) - if resultJson.toBytes().len <= 33554432 ( - auto parsed = json.parse(resultJson, {type: "object"}) - if !parsed.isErr() and parsed.unwrap().assert(object).ok == true ( - string temporaryPath = cachePath ++ ".tmp." ++ randomString(8) - if fs.writeFile(temporaryPath, resultJson) ( - if !fs.rename(temporaryPath, cachePath) ( - void fs.remove(temporaryPath) - ) + if resultJson.toBytes().len <= 33554432 and gitinspection.InspectionError(resultJson) == "" ( + string temporaryPath = cachePath ++ ".tmp." ++ randomString(8) + if fs.writeFile(temporaryPath, resultJson) ( + if !fs.rename(temporaryPath, cachePath) ( + void fs.remove(temporaryPath) ) ) ) @@ -235,18 +233,17 @@ def gitInspectionResponse(*serve.Context c, object project, string sha, string o c.notFound("workspace not found") return ) - string resultJson = cachedWorkspaceGitInspectionJSON(project, workspacePath, sha, operation, requestedPath, allowedHeads) - auto parsed = json.parse(resultJson, {type: "object"}) - if parsed.isErr() ( - c.internalError("could not inspect project history") + string resultJson = if operation == "file" then gitinspection.Inspect(workspacePath, sha, operation, requestedPath, allowedHeads) else cachedWorkspaceGitInspectionJSON(project, workspacePath, sha, operation, requestedPath, allowedHeads) + string problem = gitinspection.InspectionError(resultJson) + if problem == "could not inspect project history" ( + c.internalError(problem) return ) - object res = parsed.unwrap().assert(object) - if !res.contains("ok") or res.ok != true ( - c.notFound((res.error ?? "commit not found").toStr()) + if problem != "" ( + c.notFound(problem) return ) - c.json(200, res) + c.send(200, "application/json; charset=utf-8", resultJson) ) def handleGetCommitInspection(*serve.Context c) ( From 7fe7cc2633a07358906bde929a0a6d4760f0f36e Mon Sep 17 00:00:00 2001 From: Mistium <92952823+Mistium@users.noreply.github.com> Date: Sat, 21 Nov 2026 06:53:08 +0000 Subject: [PATCH 2/2] Free deleted archives, bound the result cache and clean up failed writes --- native/gitinspection/archive_cache.go | 31 ++++++++++++++++++---- native/gitinspection/archive_cache_test.go | 26 ++++++++++++++++++ native/gitinspection/inspection.go | 8 ++++-- src/git/git-inspection.osl | 8 +++--- 4 files changed, 61 insertions(+), 12 deletions(-) diff --git a/native/gitinspection/archive_cache.go b/native/gitinspection/archive_cache.go index 81d4c0a..d9147b4 100644 --- a/native/gitinspection/archive_cache.go +++ b/native/gitinspection/archive_cache.go @@ -124,18 +124,20 @@ func (m *objectMemo) storeListing(treeOID string, files []gitFile) { m.listingFiles += len(files) } -func (m *objectMemo) reachableFrom(allowedHeads string, walk func() map[string]bool) map[string]bool { +func (m *objectMemo) reachableFrom(allowedHeads string, walk func() map[string]bool) (map[string]bool, bool) { m.mu.Lock() set := m.reach[allowedHeads] if set == nil { - set = &reachSet{} - if len(m.reach) < memoReachLimit { - m.reach[allowedHeads] = set + if len(m.reach) >= memoReachLimit { + m.mu.Unlock() + return nil, false } + set = &reachSet{} + m.reach[allowedHeads] = set } m.mu.Unlock() set.once.Do(func() { set.visited = walk() }) - return set.visited + return set.visited, true } type sharedArchive struct { @@ -155,6 +157,8 @@ var sharedArchives = struct { entries map[string]*sharedArchive }{entries: make(map[string]*sharedArchive)} +var sharedArchiveJanitor sync.Once + var inspectionSlots = make(chan struct{}, max(2, runtime.NumCPU()/2)) func acquireInspectionSlot() func() { @@ -204,6 +208,7 @@ func (s *sharedArchive) release() { sharedArchives.Lock() defer sharedArchives.Unlock() s.refs-- + s.lastUsed = time.Now() if s.evicted && s.refs == 0 { s.closeLocked() } @@ -229,9 +234,25 @@ func openSharedArchiveFile(path string) (*os.File, os.FileInfo, archive, error) return file, info, opened, nil } +func startSharedArchiveJanitor() { + go func() { + for range time.Tick(time.Minute) { + sharedArchives.Lock() + evictSharedArchivesLocked(time.Now()) + sharedArchives.Unlock() + } + }() +} + func acquireSharedArchive(path string) (*sharedArchive, error) { + sharedArchiveJanitor.Do(startSharedArchiveJanitor) info, err := os.Stat(path) if err != nil { + sharedArchives.Lock() + if entry := sharedArchives.entries[path]; entry != nil { + dropSharedArchiveLocked(entry) + } + sharedArchives.Unlock() return nil, err } now := time.Now() diff --git a/native/gitinspection/archive_cache_test.go b/native/gitinspection/archive_cache_test.go index 30154e0..049ce2d 100644 --- a/native/gitinspection/archive_cache_test.go +++ b/native/gitinspection/archive_cache_test.go @@ -119,6 +119,32 @@ func TestInspectReportsRemovedArchive(t *testing.T) { if result := decodeResult(t, Inspect(path, head, "tree", "", head)); result["error"] != "workspace archive is unavailable" { t.Fatalf("expected unavailable archive, got %#v", result) } + sharedArchives.Lock() + _, kept := sharedArchives.entries[path] + sharedArchives.Unlock() + if kept { + t.Fatal("removed archive is still held open") + } +} + +func TestReachabilityWorksWhenMemoIsFull(t *testing.T) { + repository := newTestRepository() + tree := repository.tree("a.txt", repository.object("blob", []byte("a\n"))) + first := repository.commit(tree, "", "First") + second := repository.commit(tree, first, "Second") + path := repository.write(t) + reader, memoized, err := openArchive(path) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + memoized.memo = newObjectMemo() + for i := 0; i < memoReachLimit; i++ { + memoized.reachable(first, strings.Repeat(string(rune('a'+i)), 40)) + } + if !memoized.reachable(first, second) || memoized.reachable(second, first) { + t.Fatal("reachability is wrong once the memo is full") + } } func TestInspectConcurrentRequestsAgree(t *testing.T) { diff --git a/native/gitinspection/inspection.go b/native/gitinspection/inspection.go index 10dd59d..70abb75 100644 --- a/native/gitinspection/inspection.go +++ b/native/gitinspection/inspection.go @@ -372,11 +372,15 @@ func (a archive) reachable(sha, allowedHeads string) bool { if a.memo == nil { return a.walkReachable(sha, allowedHeads) } - return a.memo.reachableFrom(allowedHeads, func() map[string]bool { + visited, memoized := a.memo.reachableFrom(allowedHeads, func() map[string]bool { visited := make(map[string]bool) a.walkHistory(allowedHeads, visited, "") return visited - })[sha] + }) + if !memoized { + return a.walkReachable(sha, allowedHeads) + } + return visited[sha] } func (a archive) walkReachable(sha, allowedHeads string) bool { diff --git a/src/git/git-inspection.osl b/src/git/git-inspection.osl index cb7a10f..297d008 100644 --- a/src/git/git-inspection.osl +++ b/src/git/git-inspection.osl @@ -179,12 +179,10 @@ def cachedWorkspaceGitInspectionJSON(object project, string workspacePath, strin return fs.readFile(cachePath).unwrapOr("") ) string resultJson = gitinspection.Inspect(workspacePath, sha, operation, requestedPath, allowedHeads) - if resultJson.toBytes().len <= 33554432 and gitinspection.InspectionError(resultJson) == "" ( + if resultJson.toBytes().len <= 1048576 and gitinspection.InspectionError(resultJson) == "" ( string temporaryPath = cachePath ++ ".tmp." ++ randomString(8) - if fs.writeFile(temporaryPath, resultJson) ( - if !fs.rename(temporaryPath, cachePath) ( - void fs.remove(temporaryPath) - ) + if !fs.writeFile(temporaryPath, resultJson) or !fs.rename(temporaryPath, cachePath) ( + void fs.remove(temporaryPath) ) ) return resultJson