diff --git a/native/gitinspection/archive_cache.go b/native/gitinspection/archive_cache.go new file mode 100644 index 0000000..d9147b4 --- /dev/null +++ b/native/gitinspection/archive_cache.go @@ -0,0 +1,298 @@ +package gitinspection + +import ( + "archive/zip" + "os" + "runtime" + "sync" + "time" +) + +const ( + sharedArchiveLimit = 6 + sharedArchiveIdle = 5 * time.Minute + memoCommitLimit = 20_000 + memoBlobLimit = 100_000 + memoTreeBytesLimit = 16 << 20 + memoListingFileLimit = 100_000 + memoReachLimit = 16 +) + +type blobInfo struct { + size int + binary bool +} + +type reachSet struct { + once sync.Once + visited map[string]bool +} + +type objectMemo struct { + mu sync.Mutex + commits map[string]gitCommit + blobs map[string]blobInfo + trees map[string][]byte + treeBytes int + listings map[string][]gitFile + listingFiles int + reach map[string]*reachSet +} + +func newObjectMemo() *objectMemo { + return &objectMemo{ + commits: make(map[string]gitCommit), + blobs: make(map[string]blobInfo), + trees: make(map[string][]byte), + listings: make(map[string][]gitFile), + reach: make(map[string]*reachSet), + } +} + +func cloneCommit(commit gitCommit) gitCommit { + commit.Parents = append([]string{}, commit.Parents...) + return commit +} + +func (m *objectMemo) commit(oid string) (gitCommit, bool) { + m.mu.Lock() + defer m.mu.Unlock() + commit, ok := m.commits[oid] + if !ok { + return gitCommit{}, false + } + return cloneCommit(commit), true +} + +func (m *objectMemo) storeCommit(oid string, commit gitCommit) { + m.mu.Lock() + defer m.mu.Unlock() + if len(m.commits) < memoCommitLimit { + m.commits[oid] = cloneCommit(commit) + } +} + +func (m *objectMemo) blob(oid string) (blobInfo, bool) { + m.mu.Lock() + defer m.mu.Unlock() + info, ok := m.blobs[oid] + return info, ok +} + +func (m *objectMemo) storeBlob(oid string, info blobInfo) { + m.mu.Lock() + defer m.mu.Unlock() + if len(m.blobs) < memoBlobLimit { + m.blobs[oid] = info + } +} + +func (m *objectMemo) tree(oid string) ([]byte, bool) { + m.mu.Lock() + defer m.mu.Unlock() + body, ok := m.trees[oid] + return body, ok +} + +func (m *objectMemo) storeTree(oid string, body []byte) { + m.mu.Lock() + defer m.mu.Unlock() + if _, exists := m.trees[oid]; exists || m.treeBytes+len(body) > memoTreeBytesLimit { + return + } + m.trees[oid] = body + m.treeBytes += len(body) +} + +func (m *objectMemo) listing(treeOID string) ([]gitFile, bool) { + m.mu.Lock() + defer m.mu.Unlock() + files, ok := m.listings[treeOID] + if !ok { + return nil, false + } + return append([]gitFile{}, files...), true +} + +func (m *objectMemo) storeListing(treeOID string, files []gitFile) { + m.mu.Lock() + defer m.mu.Unlock() + if _, exists := m.listings[treeOID]; exists || m.listingFiles+len(files) > memoListingFileLimit { + return + } + m.listings[treeOID] = append([]gitFile{}, files...) + m.listingFiles += len(files) +} + +func (m *objectMemo) reachableFrom(allowedHeads string, walk func() map[string]bool) (map[string]bool, bool) { + m.mu.Lock() + set := m.reach[allowedHeads] + if set == nil { + if len(m.reach) >= memoReachLimit { + m.mu.Unlock() + return nil, false + } + set = &reachSet{} + m.reach[allowedHeads] = set + } + m.mu.Unlock() + set.once.Do(func() { set.visited = walk() }) + return set.visited, true +} + +type sharedArchive struct { + path string + file *os.File + info os.FileInfo + archive archive + ready chan struct{} + err error + refs int + evicted bool + lastUsed time.Time +} + +var sharedArchives = struct { + sync.Mutex + entries map[string]*sharedArchive +}{entries: make(map[string]*sharedArchive)} + +var sharedArchiveJanitor sync.Once + +var inspectionSlots = make(chan struct{}, max(2, runtime.NumCPU()/2)) + +func acquireInspectionSlot() func() { + inspectionSlots <- struct{}{} + return func() { <-inspectionSlots } +} + +func sameArchiveFile(a, b os.FileInfo) bool { + return os.SameFile(a, b) && a.Size() == b.Size() && a.ModTime().Equal(b.ModTime()) +} + +func (s *sharedArchive) closeLocked() { + if s.file != nil { + s.file.Close() + s.file = nil + } +} + +func dropSharedArchiveLocked(entry *sharedArchive) { + if sharedArchives.entries[entry.path] == entry { + delete(sharedArchives.entries, entry.path) + } + entry.evicted = true + if entry.refs == 0 { + entry.closeLocked() + } +} + +func evictSharedArchivesLocked(now time.Time) { + for _, entry := range sharedArchives.entries { + if entry.refs == 0 && now.Sub(entry.lastUsed) > sharedArchiveIdle { + dropSharedArchiveLocked(entry) + } + } + for len(sharedArchives.entries) > sharedArchiveLimit { + var oldest *sharedArchive + for _, entry := range sharedArchives.entries { + if oldest == nil || entry.lastUsed.Before(oldest.lastUsed) { + oldest = entry + } + } + dropSharedArchiveLocked(oldest) + } +} + +func (s *sharedArchive) release() { + sharedArchives.Lock() + defer sharedArchives.Unlock() + s.refs-- + s.lastUsed = time.Now() + if s.evicted && s.refs == 0 { + s.closeLocked() + } +} + +func openSharedArchiveFile(path string) (*os.File, os.FileInfo, archive, error) { + file, err := os.Open(path) + if err != nil { + return nil, nil, archive{}, err + } + info, err := file.Stat() + if err != nil { + file.Close() + return nil, nil, archive{}, err + } + reader, err := zip.NewReader(file, info.Size()) + if err != nil { + file.Close() + return nil, nil, archive{}, err + } + opened := indexArchive(reader.File) + opened.memo = newObjectMemo() + return file, info, opened, nil +} + +func startSharedArchiveJanitor() { + go func() { + for range time.Tick(time.Minute) { + sharedArchives.Lock() + evictSharedArchivesLocked(time.Now()) + sharedArchives.Unlock() + } + }() +} + +func acquireSharedArchive(path string) (*sharedArchive, error) { + sharedArchiveJanitor.Do(startSharedArchiveJanitor) + info, err := os.Stat(path) + if err != nil { + sharedArchives.Lock() + if entry := sharedArchives.entries[path]; entry != nil { + dropSharedArchiveLocked(entry) + } + sharedArchives.Unlock() + return nil, err + } + now := time.Now() + sharedArchives.Lock() + entry := sharedArchives.entries[path] + if entry != nil && !sameArchiveFile(entry.info, info) { + dropSharedArchiveLocked(entry) + entry = nil + } + opener := entry == nil + if opener { + entry = &sharedArchive{path: path, info: info, ready: make(chan struct{})} + sharedArchives.entries[path] = entry + } + entry.refs++ + entry.lastUsed = now + evictSharedArchivesLocked(now) + sharedArchives.Unlock() + + if opener { + file, openedInfo, opened, openErr := openSharedArchiveFile(path) + sharedArchives.Lock() + entry.err = openErr + if openErr != nil { + dropSharedArchiveLocked(entry) + } else { + entry.file = file + entry.info = openedInfo + entry.archive = opened + if entry.evicted && entry.refs == 0 { + entry.closeLocked() + } + } + close(entry.ready) + sharedArchives.Unlock() + } + <-entry.ready + if entry.err != nil { + entry.release() + return nil, entry.err + } + return entry, nil +} diff --git a/native/gitinspection/archive_cache_test.go b/native/gitinspection/archive_cache_test.go new file mode 100644 index 0000000..049ce2d --- /dev/null +++ b/native/gitinspection/archive_cache_test.go @@ -0,0 +1,300 @@ +package gitinspection + +import ( + "encoding/base64" + "os" + "strings" + "sync" + "testing" +) + +func resetArchiveCacheForTest() { + sharedArchives.Lock() + defer sharedArchives.Unlock() + for _, entry := range sharedArchives.entries { + dropSharedArchiveLocked(entry) + } +} + +func nestedRepository(t *testing.T) (string, string) { + repository := newTestRepository() + files := map[string]string{ + "project.json": repository.object("blob", []byte("{}\n")), + "sprites/cat/code.fractch": repository.object("blob", []byte("say \"meow\"\n")), + "sprites/cat/costume.png": repository.object("blob", []byte("\x89PNG\x00data")), + "sprites/dog/code.fractch": repository.object("blob", []byte("say \"woof\"\n")), + } + first := repository.commit(repository.nestedTree(files), "", "First") + files["sprites/cat/code.fractch"] = repository.object("blob", []byte("say \"purr\"\n")) + head := repository.commit(repository.nestedTree(files), first, "Second") + return repository.write(t), head +} + +func TestInspectFileFindsNestedPaths(t *testing.T) { + resetArchiveCacheForTest() + path, head := nestedRepository(t) + result := decodeResult(t, Inspect(path, head, "file", "sprites/cat/code.fractch", head)) + if result["content"] != base64.StdEncoding.EncodeToString([]byte("say \"purr\"\n")) { + t.Fatalf("unexpected nested file: %#v", result) + } + file := result["file"].(map[string]any) + if file["path"] != "sprites/cat/code.fractch" || file["size"] != float64(11) || file["binary"] != false || file["mode"] != "100644" { + t.Fatalf("unexpected file metadata: %#v", file) + } + binary := decodeResult(t, Inspect(path, head, "file", "sprites/cat/costume.png", head)) + if binary["file"].(map[string]any)["binary"] != true { + t.Fatalf("binary file not flagged: %#v", binary) + } +} + +func TestInspectFileMatchesTreeListing(t *testing.T) { + resetArchiveCacheForTest() + path, head := nestedRepository(t) + tree := decodeResult(t, Inspect(path, head, "tree", "", head)) + for _, item := range tree["files"].([]any) { + listed := item.(map[string]any) + result := decodeResult(t, Inspect(path, head, "file", listed["path"].(string), head)) + file := result["file"].(map[string]any) + for _, key := range []string{"path", "oid", "mode", "size", "binary"} { + if file[key] != listed[key] { + t.Fatalf("file %s differs on %s: %#v vs %#v", listed["path"], key, file, listed) + } + } + } +} + +func TestInspectFileRejectsDirectoriesAndMissingPaths(t *testing.T) { + resetArchiveCacheForTest() + path, head := nestedRepository(t) + for _, requested := range []string{"sprites", "sprites/cat", "sprites/cow/code.fractch", "project.json/x", "missing.txt"} { + result := decodeResult(t, Inspect(path, head, "file", requested, head)) + if result["ok"] != false || result["error"] != "file was not found at this commit" { + t.Fatalf("%s: expected not found, got %#v", requested, result) + } + } +} + +func TestInspectFileReportsMissingBlob(t *testing.T) { + resetArchiveCacheForTest() + repository := newTestRepository() + blob := repository.object("blob", []byte("code\n")) + head := repository.commit(repository.tree("main.fractch", blob), "", "Initial") + delete(repository.objects, blob) + path := repository.write(t) + result := decodeResult(t, Inspect(path, head, "file", "main.fractch", head)) + if result["error"] != "file object is unavailable" { + t.Fatalf("expected missing blob, got %#v", result) + } +} + +func TestInspectSeesReplacedArchive(t *testing.T) { + resetArchiveCacheForTest() + path, head := nestedRepository(t) + if decodeResult(t, Inspect(path, head, "file", "project.json", head))["ok"] != true { + t.Fatal("first archive unreadable") + } + repository := newTestRepository() + blob := repository.object("blob", []byte("replaced\n")) + replacedHead := repository.commit(repository.tree("project.json", blob), "", "Replaced") + replacement := repository.write(t) + if err := os.Rename(replacement, path); err != nil { + t.Fatal(err) + } + result := decodeResult(t, Inspect(path, replacedHead, "file", "project.json", replacedHead)) + if result["content"] != base64.StdEncoding.EncodeToString([]byte("replaced\n")) { + t.Fatalf("stale archive served: %#v", result) + } + if decodeResult(t, Inspect(path, head, "file", "project.json", head))["ok"] != false { + t.Fatal("old commit still readable after the archive was replaced") + } +} + +func TestInspectReportsRemovedArchive(t *testing.T) { + resetArchiveCacheForTest() + path, head := nestedRepository(t) + Inspect(path, head, "tree", "", head) + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if result := decodeResult(t, Inspect(path, head, "tree", "", head)); result["error"] != "workspace archive is unavailable" { + t.Fatalf("expected unavailable archive, got %#v", result) + } + sharedArchives.Lock() + _, kept := sharedArchives.entries[path] + sharedArchives.Unlock() + if kept { + t.Fatal("removed archive is still held open") + } +} + +func TestReachabilityWorksWhenMemoIsFull(t *testing.T) { + repository := newTestRepository() + tree := repository.tree("a.txt", repository.object("blob", []byte("a\n"))) + first := repository.commit(tree, "", "First") + second := repository.commit(tree, first, "Second") + path := repository.write(t) + reader, memoized, err := openArchive(path) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + memoized.memo = newObjectMemo() + for i := 0; i < memoReachLimit; i++ { + memoized.reachable(first, strings.Repeat(string(rune('a'+i)), 40)) + } + if !memoized.reachable(first, second) || memoized.reachable(second, first) { + t.Fatal("reachability is wrong once the memo is full") + } +} + +func TestInspectConcurrentRequestsAgree(t *testing.T) { + resetArchiveCacheForTest() + path, head := nestedRepository(t) + want := map[string]string{} + for _, requested := range []string{"project.json", "sprites/cat/code.fractch", "sprites/dog/code.fractch"} { + want[requested] = Inspect(path, head, "file", requested, head) + } + wantTree := Inspect(path, head, "tree", "", head) + resetArchiveCacheForTest() + var group sync.WaitGroup + for i := 0; i < 64; i++ { + group.Add(1) + go func(i int) { + defer group.Done() + if i%4 == 0 { + if got := Inspect(path, head, "tree", "", head); got != wantTree { + t.Errorf("tree differs under concurrency") + } + return + } + for requested, expected := range want { + if got := Inspect(path, head, "file", requested, head); got != expected { + t.Errorf("%s differs under concurrency", requested) + } + } + }(i) + } + group.Wait() +} + +func TestSharedArchivesStayBounded(t *testing.T) { + resetArchiveCacheForTest() + for i := 0; i < sharedArchiveLimit+3; i++ { + path, head := nestedRepository(t) + Inspect(path, head, "tree", "", head) + } + sharedArchives.Lock() + count := len(sharedArchives.entries) + for _, entry := range sharedArchives.entries { + if entry.refs != 0 { + t.Errorf("archive %s still referenced", entry.path) + } + } + sharedArchives.Unlock() + if count > sharedArchiveLimit { + t.Fatalf("%d archives kept open, limit %d", count, sharedArchiveLimit) + } +} + +func TestMemoReachabilityMatchesWalk(t *testing.T) { + repository := newTestRepository() + blob := repository.object("blob", []byte("a\n")) + tree := repository.tree("a.txt", blob) + first := repository.commit(tree, "", "First") + second := repository.commit(tree, first, "Second") + side := repository.commit(tree, "", "Side") + path := repository.write(t) + reader, plain, err := openArchive(path) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + memoized := plain + memoized.memo = newObjectMemo() + for _, sha := range []string{first, second, side, strings.Repeat("0", 40)} { + if plain.reachable(sha, second) != memoized.reachable(sha, second) { + t.Fatalf("reachability of %s differs with the memo", sha) + } + } + if !memoized.reachable(first, second) || memoized.reachable(side, second) { + t.Fatal("memoized reachability is wrong") + } +} + +func TestInspectionError(t *testing.T) { + cases := map[string]string{ + `{"ok":true,"content":"eA=="}`: "", + `{"error":"commit tree is invalid","ok":false}`: "commit tree is invalid", + `{"ok":false}`: "commit not found", + `not json`: "could not inspect project history", + } + for input, want := range cases { + if got := InspectionError(input); got != want { + t.Fatalf("InspectionError(%q) = %q, want %q", input, got, want) + } + } +} + +func plainInspect(t *testing.T, path, sha, operation, requestedPath, allowedHeads string) string { + t.Helper() + reader, plain, err := openArchive(path) + if err != nil { + t.Fatal(err) + } + defer reader.Close() + commit, ok := plain.parseCommit(sha) + if !ok || !plain.reachable(sha, allowedHeads) { + t.Fatalf("plain archive cannot read %s", sha) + } + switch operation { + case "tree": + files, ok := plain.walkTree(commit.Tree) + if !ok { + t.Fatal("plain tree invalid") + } + return encode(map[string]any{"ok": true, "sha": sha, "commit": commit, "files": files}, "") + case "commit-inline": + return plain.inspectCommit(sha, commit, true) + case "changes": + return plain.inspectCommitChanges(sha, commit) + case "summary": + return plain.inspectCommitSummary(sha, commit) + case "last-commits": + return plain.inspectLastCommits(sha, commit) + } + return plain.inspectCommit(sha, commit, false) +} + +func TestSharedArchiveMatchesPlainArchive(t *testing.T) { + resetArchiveCacheForTest() + repository := newTestRepository() + files := map[string]string{ + "project.json": repository.object("blob", []byte("{}\n")), + "sprites/cat/code.fractch": repository.object("blob", []byte("sprite \"Cat\";\nwhen flag {\n say \"a\";\n}\n")), + "sprites/dog/code.fractch": repository.object("blob", []byte("sprite \"Dog\";\n")), + "notes.txt": repository.object("blob", []byte("one\n")), + } + heads := []string{} + head := "" + for i := 0; i < 12; i++ { + switch i % 3 { + case 0: + files["notes.txt"] = repository.object("blob", []byte(strings.Repeat("line\n", i+1))) + case 1: + files["sprites/cat/code.fractch"] = repository.object("blob", []byte("sprite \"Cat\";\nwhen flag {\n say \""+strings.Repeat("b", i)+"\";\n}\n")) + default: + files["sprites/new"+string(rune('a'+i))+"/code.fractch"] = repository.object("blob", []byte("sprite \"N\";\n")) + } + head = repository.commit(repository.nestedTree(files), head, "Save "+string(rune('a'+i))) + heads = append(heads, head) + } + path := repository.write(t) + for _, sha := range heads { + for _, operation := range []string{"tree", "commit", "commit-inline", "changes", "summary", "last-commits"} { + shared := Inspect(path, sha, operation, "", head) + if plain := plainInspect(t, path, sha, operation, "", head); shared != plain { + t.Fatalf("%s of %s differs:\nshared %s\nplain %s", operation, sha, shared, plain) + } + } + } +} diff --git a/native/gitinspection/inspection.go b/native/gitinspection/inspection.go index 63ff6a1..70abb75 100644 --- a/native/gitinspection/inspection.go +++ b/native/gitinspection/inspection.go @@ -41,6 +41,13 @@ type gitCommit struct { type archive struct { objects map[string]*zip.File trusted map[string]bool + memo *objectMemo +} + +type treeItem struct { + mode string + name string + oid string } func failure(message string) string { @@ -53,13 +60,17 @@ func openArchive(path string) (*zip.ReadCloser, archive, error) { if err != nil { return nil, archive{}, err } + return reader, indexArchive(reader.File), nil +} + +func indexArchive(files []*zip.File) archive { result := archive{objects: make(map[string]*zip.File)} - for _, item := range reader.File { + for _, item := range files { if strings.HasPrefix(item.Name, ".git/objects/") && !item.FileInfo().IsDir() { result.objects[item.Name] = item } } - return reader, result, nil + return result } func (a archive) readObject(oid string) (string, []byte, bool) { @@ -138,6 +149,11 @@ func (a archive) objectKind(oid string) string { } func (a archive) parseCommit(oid string) (gitCommit, bool) { + if a.memo != nil { + if commit, ok := a.memo.commit(oid); ok { + return commit, true + } + } kind, body, ok := a.readObject(oid) if !ok || kind != "commit" { return gitCommit{}, false @@ -173,10 +189,135 @@ func (a archive) parseCommit(oid string) (gitCommit, bool) { if len(result.Tree) != 40 { return gitCommit{}, false } + if a.memo != nil { + a.memo.storeCommit(oid, result) + } return result, true } +func (a archive) treeObject(oid string) ([]byte, bool) { + if a.memo != nil { + if body, ok := a.memo.tree(oid); ok { + return body, true + } + } + kind, body, ok := a.readObject(oid) + if !ok || kind != "tree" { + return nil, false + } + if a.memo != nil { + a.memo.storeTree(oid, body) + } + return body, true +} + +func parseTreeEntries(body []byte) ([]treeItem, bool) { + entries := []treeItem{} + position := 0 + for position < len(body) { + space := bytes.IndexByte(body[position:], ' ') + if space < 1 { + return nil, false + } + space += position + nul := bytes.IndexByte(body[space+1:], 0) + if nul < 1 { + return nil, false + } + nul += space + 1 + if nul+21 > len(body) { + return nil, false + } + name := string(body[space+1 : nul]) + if name == "" || name == "." || name == ".." || strings.ContainsAny(name, "/\\") { + return nil, false + } + entries = append(entries, treeItem{ + mode: string(body[position:space]), name: name, oid: hex.EncodeToString(body[nul+1 : nul+21]), + }) + position = nul + 21 + } + return entries, true +} + +func treeMode(mode string) bool { + return mode == "40000" || mode == "040000" +} + +func blobSummary(blob []byte) blobInfo { + probe := blob + if len(probe) > 8000 { + probe = probe[:8000] + } + return blobInfo{size: len(blob), binary: bytes.IndexByte(probe, 0) >= 0} +} + +func (a archive) blobInfo(oid string) (blobInfo, bool) { + if a.memo != nil { + if info, ok := a.memo.blob(oid); ok { + return info, true + } + } + kind, blob, ok := a.readObject(oid) + if !ok || kind != "blob" { + return blobInfo{}, false + } + info := blobSummary(blob) + if a.memo != nil { + a.memo.storeBlob(oid, info) + } + return info, true +} + +func (a archive) findFile(rootOID, requestedPath string) (gitFile, []byte, string) { + treeOID := rootOID + parts := strings.Split(requestedPath, "/") + for depth, part := range parts { + if depth > maxTreeDepth { + return gitFile{}, nil, "commit tree is invalid" + } + body, ok := a.treeObject(treeOID) + if !ok { + return gitFile{}, nil, "commit tree is invalid" + } + entries, ok := parseTreeEntries(body) + if !ok { + return gitFile{}, nil, "commit tree is invalid" + } + last := depth == len(parts)-1 + next := "" + for _, entry := range entries { + if entry.name != part || treeMode(entry.mode) == last { + continue + } + if !last { + next = entry.oid + break + } + kind, content, found := a.readObject(entry.oid) + if !found || kind != "blob" { + return gitFile{}, nil, "file object is unavailable" + } + info := blobSummary(content) + if a.memo != nil { + a.memo.storeBlob(entry.oid, info) + } + return gitFile{Path: requestedPath, OID: entry.oid, Mode: entry.mode, Size: info.size, Binary: info.binary}, content, "" + } + if next == "" { + return gitFile{}, nil, "file was not found at this commit" + } + treeOID = next + } + return gitFile{}, nil, "file was not found at this commit" +} + func (a archive) walkTree(rootOID string) ([]gitFile, bool) { + if a.memo != nil { + if files, ok := a.memo.listing(rootOID); ok { + return files, true + } + } files := []gitFile{} active := make(map[string]bool) var walk func(string, string, int) bool @@ -186,70 +327,73 @@ func (a archive) walkTree(rootOID string) ([]gitFile, bool) { } active[treeOID] = true defer delete(active, treeOID) - kind, body, ok := a.readObject(treeOID) - if !ok || kind != "tree" { + body, ok := a.treeObject(treeOID) + if !ok { return false } - position := 0 - for position < len(body) { - space := bytes.IndexByte(body[position:], ' ') - if space < 1 { - return false - } - space += position - nul := bytes.IndexByte(body[space+1:], 0) - if nul < 1 { - return false - } - nul += space + 1 - if nul+21 > len(body) { - return false - } - mode := string(body[position:space]) - name := string(body[space+1 : nul]) - if name == "" || name == "." || name == ".." || strings.ContainsAny(name, "/\\") { - return false - } - childOID := hex.EncodeToString(body[nul+1 : nul+21]) - fullPath := name + entries, ok := parseTreeEntries(body) + if !ok { + return false + } + for _, entry := range entries { + fullPath := entry.name if prefix != "" { - fullPath = prefix + "/" + name + fullPath = prefix + "/" + entry.name } - position = nul + 21 - if mode == "40000" || mode == "040000" { - if !walk(childOID, fullPath, depth+1) { + if treeMode(entry.mode) { + if !walk(entry.oid, fullPath, depth+1) { return false } continue } - blobKind, blob, ok := a.readObject(childOID) - if !ok || blobKind != "blob" { + info, ok := a.blobInfo(entry.oid) + if !ok { return false } - probe := blob - if len(probe) > 8000 { - probe = probe[:8000] - } files = append(files, gitFile{ - Path: fullPath, OID: childOID, Mode: mode, Size: len(blob), Binary: bytes.IndexByte(probe, 0) >= 0, + Path: fullPath, OID: entry.oid, Mode: entry.mode, Size: info.size, Binary: info.binary, }) if len(files) > maxTreeFiles { return false } } - return position == len(body) + return true + } + if !walk(rootOID, "", 0) { + return files, false + } + if a.memo != nil { + a.memo.storeListing(rootOID, files) } - return files, walk(rootOID, "", 0) + return files, true } func (a archive) reachable(sha, allowedHeads string) bool { + if a.memo == nil { + return a.walkReachable(sha, allowedHeads) + } + visited, memoized := a.memo.reachableFrom(allowedHeads, func() map[string]bool { + visited := make(map[string]bool) + a.walkHistory(allowedHeads, visited, "") + return visited + }) + if !memoized { + return a.walkReachable(sha, allowedHeads) + } + return visited[sha] +} + +func (a archive) walkReachable(sha, allowedHeads string) bool { + return a.walkHistory(allowedHeads, make(map[string]bool), sha) +} + +func (a archive) walkHistory(allowedHeads string, visited map[string]bool, target string) bool { queue := []string{} for _, candidate := range strings.Split(allowedHeads, ",") { if len(candidate) == 40 { queue = append(queue, candidate) } } - visited := make(map[string]bool) for len(queue) > 0 && len(visited) <= maxReachable { candidate := queue[0] queue = queue[1:] @@ -257,7 +401,7 @@ func (a archive) reachable(sha, allowedHeads string) bool { continue } visited[candidate] = true - if candidate == sha { + if target != "" && candidate == target { return true } commit, ok := a.parseCommit(candidate) @@ -308,11 +452,14 @@ func CommitTreesDiffer(basePath, baseHead, incomingPath, incomingHead string) st // Inspect reads a commit, tree, or file from the loose Git objects stored in a // MistWarp project archive. It returns the JSON contract consumed by the OSL API. func Inspect(workspacePath, sha, operation, requestedPath, allowedHeads string) string { - reader, archive, err := openArchive(workspacePath) + releaseSlot := acquireInspectionSlot() + defer releaseSlot() + shared, err := acquireSharedArchive(workspacePath) if err != nil { return failure("workspace archive is unavailable") } - defer reader.Close() + defer shared.release() + archive := shared.archive commit, ok := archive.parseCommit(sha) if !ok { @@ -330,21 +477,11 @@ func Inspect(workspacePath, sha, operation, requestedPath, allowedHeads string) } return encode(map[string]any{"ok": true, "sha": sha, "commit": commit, "files": files}, "could not encode commit tree") case "file": - files, ok := archive.walkTree(commit.Tree) - if !ok { - return failure("commit tree is invalid") - } - for _, file := range files { - if file.Path != requestedPath { - continue - } - kind, content, found := archive.readObject(file.OID) - if !found || kind != "blob" { - return failure("file object is unavailable") - } - return encode(map[string]any{"ok": true, "sha": sha, "file": file, "content": content}, "could not encode file") + file, content, problem := archive.findFile(commit.Tree, requestedPath) + if problem != "" { + return failure(problem) } - return failure("file was not found at this commit") + return encode(map[string]any{"ok": true, "sha": sha, "file": file, "content": content}, "could not encode file") case "commit-inline": return archive.inspectCommit(sha, commit, true) case "changes": @@ -526,3 +663,20 @@ func compareCommitTrees(baseArchive archive, baseSHA string, baseCommit gitCommi "additions": additions, "deletions": deletions, "legacy": legacy, }, "could not encode commit inspection") } + +func InspectionError(result string) string { + var status struct { + OK bool `json:"ok"` + Error string `json:"error"` + } + if err := json.Unmarshal([]byte(result), &status); err != nil { + return "could not inspect project history" + } + if status.OK { + return "" + } + if status.Error == "" { + return "commit not found" + } + return status.Error +} diff --git a/native/gitinspection/inspection_bench_test.go b/native/gitinspection/inspection_bench_test.go new file mode 100644 index 0000000..c0ce995 --- /dev/null +++ b/native/gitinspection/inspection_bench_test.go @@ -0,0 +1,80 @@ +package gitinspection + +import ( + "math/rand" + "strconv" + "strings" + "sync" + "testing" +) + +type benchProject struct { + path string + head string + files []string +} + +func buildBenchProject(t testing.TB, textFiles, assetFiles, commits int) benchProject { + random := rand.New(rand.NewSource(1)) + repository := newTestRepository() + files := map[string]string{} + paths := []string{} + for i := 0; i < textFiles; i++ { + path := "sprites/sprite" + strconv.Itoa(i) + "/code.fractch" + size := 300 + random.Intn(30000) + files[path] = repository.object("blob", []byte(strings.Repeat("say \"hello\"\n", size/12))) + paths = append(paths, path) + } + for i := 0; i < assetFiles; i++ { + path := "assets/asset" + strconv.Itoa(i) + ".png" + data := make([]byte, 64<<10+random.Intn(512<<10)) + random.Read(data) + files[path] = repository.object("blob", data) + paths = append(paths, path) + } + head := "" + for i := 0; i < commits; i++ { + edited := paths[random.Intn(textFiles)] + files[edited] = repository.object("blob", []byte("say \"version "+strconv.Itoa(i)+"\"\n")) + head = repository.commit(repository.nestedTree(files), head, "Save "+strconv.Itoa(i)) + } + file := repository.write(t) + return benchProject{path: file, head: head, files: paths} +} + +func benchmarkFileBurst(b *testing.B, project benchProject) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + resetArchiveCacheForTest() + var group sync.WaitGroup + for _, path := range project.files { + group.Add(1) + go func(path string) { + defer group.Done() + result := Inspect(project.path, project.head, "file", path, project.head) + if !strings.Contains(result, `"ok":true`) { + b.Errorf("file %s failed: %.120s", path, result) + } + }(path) + } + group.Wait() + } +} + +func BenchmarkInspectFileBurst(b *testing.B) { + benchmarkFileBurst(b, buildBenchProject(b, 180, 40, 300)) +} + +func BenchmarkInspectTreeRepeated(b *testing.B) { + project := buildBenchProject(b, 180, 40, 300) + resetArchiveCacheForTest() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + result := Inspect(project.path, project.head, "tree", "", project.head) + if !strings.Contains(result, `"ok":true`) { + b.Fatalf("tree failed: %.120s", result) + } + } +} diff --git a/native/gitinspection/inspection_test.go b/native/gitinspection/inspection_test.go index e824d83..6191254 100644 --- a/native/gitinspection/inspection_test.go +++ b/native/gitinspection/inspection_test.go @@ -48,7 +48,7 @@ func (r *testRepository) commit(tree, parent, message string) string { return r.object("commit", []byte(body)) } -func (r *testRepository) write(t *testing.T) string { +func (r *testRepository) write(t testing.TB) string { t.Helper() file, err := os.CreateTemp(t.TempDir(), "workspace-*.mwp") if err != nil { diff --git a/native/gitinspection/last_commits.go b/native/gitinspection/last_commits.go index 9dd77ab..4c38ceb 100644 --- a/native/gitinspection/last_commits.go +++ b/native/gitinspection/last_commits.go @@ -1,8 +1,6 @@ package gitinspection import ( - "bytes" - "encoding/hex" "strings" ) @@ -16,33 +14,17 @@ type treeEntry struct { } func (a archive) readTreeEntries(oid string) (map[string]treeEntry, bool) { - kind, body, ok := a.readObject(oid) - if !ok || kind != "tree" { + body, ok := a.treeObject(oid) + if !ok { + return nil, false + } + items, ok := parseTreeEntries(body) + if !ok { return nil, false } - entries := map[string]treeEntry{} - position := 0 - for position < len(body) { - space := bytes.IndexByte(body[position:], ' ') - if space < 1 { - return nil, false - } - space += position - nul := bytes.IndexByte(body[space+1:], 0) - if nul < 1 { - return nil, false - } - nul += space + 1 - if nul+21 > len(body) { - return nil, false - } - mode := string(body[position:space]) - name := string(body[space+1 : nul]) - if name == "" || name == "." || name == ".." || strings.ContainsAny(name, "/\\") { - return nil, false - } - entries[name] = treeEntry{OID: hex.EncodeToString(body[nul+1 : nul+21]), Dir: mode == "40000" || mode == "040000"} - position = nul + 21 + entries := make(map[string]treeEntry, len(items)) + for _, item := range items { + entries[item.name] = treeEntry{OID: item.oid, Dir: treeMode(item.mode)} } return entries, true } diff --git a/native/storagefs/files.go b/native/storagefs/files.go index ffacd4e..13726df 100644 --- a/native/storagefs/files.go +++ b/native/storagefs/files.go @@ -14,7 +14,7 @@ import ( const ( cacheMaxAge = 24 * time.Hour cacheMaxFiles = 256 - cacheMaxBytes = 2147483648 + cacheMaxBytes = 1 << 30 ) func Missing(path string) bool { diff --git a/opal.json b/opal.json index 125615f..a504779 100644 --- a/opal.json +++ b/opal.json @@ -1,6 +1,6 @@ { "name": "mistwarp-api", - "version": "1.4.0", + "version": "1.4.1", "main": "main.osl", "scripts": { "dev": ["osl", "run", "main.osl"], diff --git a/scripts/e2e_smoke.py b/scripts/e2e_smoke.py index c1de57e..d6a310f 100644 --- a/scripts/e2e_smoke.py +++ b/scripts/e2e_smoke.py @@ -1,4 +1,5 @@ import argparse +import concurrent.futures import difflib import gzip import hashlib @@ -648,6 +649,31 @@ def head_and_edited(): step("commit changes", "GET", path + "/commits/" + head + "/changes", "sam") step("commit changes signed out", "GET", path + "/commits/" + head + "/changes") step("changes of unknown commit", "GET", path + "/commits/" + "1" * 40 + "/changes", "sam") + step("commit inline", "GET", path + "/commits/" + head + "?inline=1", "sam") + status, last = server.call("GET", "/v1/projects/%s/commits/%s/last-commits" % (real_id(ids, ""), head), "sam")[:2] + last_commits = last.get("commits") or {} if isinstance(last, dict) else {} + steps.append({"step": "commit last commits", "status": status, "body": { + "complete": last.get("complete") if isinstance(last, dict) else None, + "paths": {name: (last_commits.get(sha) or {}).get("message") for name, sha in ((last.get("paths") or {}) if isinstance(last, dict) else {}).items()}, + }}) + status, tree = step("commit tree", "GET", path + "/commits/" + head + "/tree", "sam") + tree_paths = sorted(str(item.get("path", "")) for item in (tree.get("files") or [])) if isinstance(tree, dict) else [] + for tree_path in tree_paths: + step("commit file " + tree_path, "GET", path + "/commits/" + head + "/file?path=" + urllib.parse.quote(tree_path), "sam") + if tree_paths: + step("commit file signed out", "GET", path + "/commits/" + head + "/file?path=" + urllib.parse.quote(tree_paths[0])) + directory = tree_paths[-1].rpartition("/")[0] + if directory: + step("commit file of a directory", "GET", path + "/commits/" + head + "/file?path=" + urllib.parse.quote(directory), "sam") + step("commit file missing", "GET", path + "/commits/" + head + "/file?path=missing.txt", "sam") + step("commit file invalid path", "GET", path + "/commits/" + head + "/file?path=../x", "sam") + step("commit file of unknown commit", "GET", path + "/commits/" + "1" * 40 + "/file?path=project.json", "sam") + burst = [server.call("GET", "/v1/projects/%s/commits/%s/file?path=%s" % (real_id(ids, ""), head, urllib.parse.quote(tree_path)), "sam") for tree_path in tree_paths] + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as pool: + parallel = list(pool.map(lambda tree_path: server.call("GET", "/v1/projects/%s/commits/%s/file?path=%s" % (real_id(ids, ""), head, urllib.parse.quote(tree_path)), "sam"), tree_paths * 8)) + steps.append({"step": "commit files in parallel match", "status": 0, "body": all( + result[0] == burst[index % len(burst)][0] and result[1] == burst[index % len(burst)][1] for index, result in enumerate(parallel) + ) if burst else True}) step("commit summaries", "GET", path + "/commit-summaries?shas=" + head + ",nothex," + "1" * 40 + "," + head, "sam") step("commit summaries signed out", "GET", path + "/commit-summaries?shas=" + head) step("commit summaries from cache", "GET", path + "/commit-summaries?shas=" + head, "sam") diff --git a/src/git/git-inspection.osl b/src/git/git-inspection.osl index 91b31b5..297d008 100644 --- a/src/git/git-inspection.osl +++ b/src/git/git-inspection.osl @@ -169,24 +169,20 @@ def validGitFilePath(string path) boolean ( ) def cachedWorkspaceGitInspectionJSON(object project, string workspacePath, string sha, string operation, string requestedPath, string allowedHeads) string ( - string cacheDir = tmpDir ++ "git-inspection/" + string cacheDir = tmpDir ++ "git-inspection-results/" void fs.mkdir(cacheDir, {recursive: true}) - string cacheName = gitInspectionCacheVersion() ++ "." ++ projectWorkspaceFingerprint(project) ++ "." ++ allowedHeads.hashSHA256() ++ "." ++ sha ++ "." ++ operation ++ "." ++ requestedPath.hashSHA256() ++ ".json" + string cacheKey = projectWorkspaceFingerprint(project) ++ "." ++ allowedHeads.hashSHA256() ++ "." ++ sha ++ "." ++ operation ++ "." ++ requestedPath.hashSHA256() + string cacheName = gitInspectionCacheVersion() ++ "." ++ cacheKey.hashSHA256() ++ ".json" pruneGitInspectionCache(cacheDir, cacheName) string cachePath = cacheDir ++ cacheName if fs.exists(cachePath) and fs.size(cachePath).unwrapOr(0) <= 33554432 ( return fs.readFile(cachePath).unwrapOr("") ) string resultJson = gitinspection.Inspect(workspacePath, sha, operation, requestedPath, allowedHeads) - if resultJson.toBytes().len <= 33554432 ( - auto parsed = json.parse(resultJson, {type: "object"}) - if !parsed.isErr() and parsed.unwrap().assert(object).ok == true ( - string temporaryPath = cachePath ++ ".tmp." ++ randomString(8) - if fs.writeFile(temporaryPath, resultJson) ( - if !fs.rename(temporaryPath, cachePath) ( - void fs.remove(temporaryPath) - ) - ) + if resultJson.toBytes().len <= 1048576 and gitinspection.InspectionError(resultJson) == "" ( + string temporaryPath = cachePath ++ ".tmp." ++ randomString(8) + if !fs.writeFile(temporaryPath, resultJson) or !fs.rename(temporaryPath, cachePath) ( + void fs.remove(temporaryPath) ) ) return resultJson @@ -235,18 +231,17 @@ def gitInspectionResponse(*serve.Context c, object project, string sha, string o c.notFound("workspace not found") return ) - string resultJson = cachedWorkspaceGitInspectionJSON(project, workspacePath, sha, operation, requestedPath, allowedHeads) - auto parsed = json.parse(resultJson, {type: "object"}) - if parsed.isErr() ( - c.internalError("could not inspect project history") + string resultJson = if operation == "file" then gitinspection.Inspect(workspacePath, sha, operation, requestedPath, allowedHeads) else cachedWorkspaceGitInspectionJSON(project, workspacePath, sha, operation, requestedPath, allowedHeads) + string problem = gitinspection.InspectionError(resultJson) + if problem == "could not inspect project history" ( + c.internalError(problem) return ) - object res = parsed.unwrap().assert(object) - if !res.contains("ok") or res.ok != true ( - c.notFound((res.error ?? "commit not found").toStr()) + if problem != "" ( + c.notFound(problem) return ) - c.json(200, res) + c.send(200, "application/json; charset=utf-8", resultJson) ) def handleGetCommitInspection(*serve.Context c) (