diff --git a/GVFS/GVFS.Common/MissingTreeTracker.cs b/GVFS/GVFS.Common/MissingTreeTracker.cs new file mode 100644 index 000000000..3d5ca78a1 --- /dev/null +++ b/GVFS/GVFS.Common/MissingTreeTracker.cs @@ -0,0 +1,318 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using GVFS.Common.Tracing; + +namespace GVFS.Common +{ + /// + /// Tracks missing trees per commit to support batching tree downloads. + /// Maintains LRU eviction based on commits (not individual trees). + /// A single tree SHA may be shared across multiple commits. + /// + public class MissingTreeTracker + { + private const string EtwArea = nameof(MissingTreeTracker); + + private readonly int treeCapacity; + private readonly ITracer tracer; + private readonly object syncLock = new object(); + + // Primary storage: commit -> set of missing trees + private readonly Dictionary> missingTreesByCommit; + + // Reverse lookup: tree -> set of commits (for fast lookups) + private readonly Dictionary> commitsByTree; + + // LRU ordering based on commits + private readonly LinkedList commitOrder; + private readonly Dictionary> commitNodes; + + public MissingTreeTracker(ITracer tracer, int treeCapacity) + { + this.tracer = tracer; + this.treeCapacity = treeCapacity; + this.missingTreesByCommit = new Dictionary>(StringComparer.OrdinalIgnoreCase); + this.commitsByTree = new Dictionary>(StringComparer.OrdinalIgnoreCase); + this.commitOrder = new LinkedList(); + this.commitNodes = new Dictionary>(StringComparer.OrdinalIgnoreCase); + } + + /// + /// Records a missing root tree for a commit. Marks the commit as recently used. + /// A tree may be associated with multiple commits. + /// + public void AddMissingRootTree(string treeSha, string commitSha) + { + lock (this.syncLock) + { + this.EnsureCommitTracked(commitSha); + this.AddTreeToCommit(treeSha, commitSha); + } + } + + /// + /// Records missing sub-trees discovered while processing a parent tree. + /// Each sub-tree is associated with all commits currently tracking the parent tree. + /// + public void AddMissingSubTrees(string parentTreeSha, string[] subTreeShas) + { + lock (this.syncLock) + { + if (!this.commitsByTree.TryGetValue(parentTreeSha, out var commits)) + { + return; + } + + // Snapshot the set because AddTreeToCommit may modify commitsByTree indirectly + string[] commitSnapshot = commits.ToArray(); + foreach (string subTreeSha in subTreeShas) + { + foreach (string commitSha in commitSnapshot) + { + /* Ensure it wasn't evicted earlier in the loop. */ + if (!this.missingTreesByCommit.ContainsKey(commitSha)) + { + continue; + } + /* Ensure we don't evict this commit while trying to add a tree to it. */ + this.MarkCommitAsUsed(commitSha); + this.AddTreeToCommit(subTreeSha, commitSha); + } + } + } + } + + /// + /// Tries to get all commits associated with a tree SHA. + /// Marks all found commits as recently used. + /// + public bool TryGetCommits(string treeSha, out string[] commitShas) + { + lock (this.syncLock) + { + if (this.commitsByTree.TryGetValue(treeSha, out var commits)) + { + commitShas = commits.ToArray(); + foreach (string commitSha in commitShas) + { + this.MarkCommitAsUsed(commitSha); + } + + return true; + } + + commitShas = null; + return false; + } + } + + /// + /// Given a set of commits, finds the one with the most missing trees. + /// + public int GetHighestMissingTreeCount(string[] commitShas, out string highestCountCommitSha) + { + lock (this.syncLock) + { + highestCountCommitSha = null; + int highestCount = 0; + + foreach (string commitSha in commitShas) + { + if (this.missingTreesByCommit.TryGetValue(commitSha, out var trees) + && trees.Count > highestCount) + { + highestCount = trees.Count; + highestCountCommitSha = commitSha; + } + } + + return highestCount; + } + } + + /// + /// Marks a commit as complete (e.g. its pack was downloaded successfully). + /// Because the trees are now available, they are also removed from tracking + /// for any other commits that shared them, and those commits are cleaned up + /// if they become empty. + /// + public void MarkCommitComplete(string commitSha) + { + lock (this.syncLock) + { + this.RemoveCommitWithCascade(commitSha); + + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + metadata.Add("CompletedCommit", commitSha); + metadata.Add("RemainingCommits", this.commitNodes.Count); + metadata.Add("RemainingTrees", this.commitsByTree.Count); + this.tracer.RelatedEvent(EventLevel.Informational, nameof(this.MarkCommitComplete), metadata, Keywords.Telemetry); + } + } + + private void EnsureCommitTracked(string commitSha) + { + if (!this.missingTreesByCommit.TryGetValue(commitSha, out _)) + { + this.missingTreesByCommit[commitSha] = new HashSet(StringComparer.OrdinalIgnoreCase); + var node = this.commitOrder.AddFirst(commitSha); + this.commitNodes[commitSha] = node; + } + else + { + this.MarkCommitAsUsed(commitSha); + } + } + + private void AddTreeToCommit(string treeSha, string commitSha) + { + if (!this.commitsByTree.ContainsKey(treeSha)) + { + // Evict LRU commits until there is room for the new tree + while (this.commitsByTree.Count >= this.treeCapacity) + { + // If evict fails it means we only have one commit left. + if (!this.EvictLruCommit()) + { + break; + } + } + + this.commitsByTree[treeSha] = new HashSet(StringComparer.OrdinalIgnoreCase); + } + + this.missingTreesByCommit[commitSha].Add(treeSha); + this.commitsByTree[treeSha].Add(commitSha); + } + + private void MarkCommitAsUsed(string commitSha) + { + if (this.commitNodes.TryGetValue(commitSha, out var node)) + { + this.commitOrder.Remove(node); + var newNode = this.commitOrder.AddFirst(commitSha); + this.commitNodes[commitSha] = newNode; + } + } + + private bool EvictLruCommit() + { + var last = this.commitOrder.Last; + if (last != null && last.Value != this.commitOrder.First.Value) + { + string lruCommit = last.Value; + var treeCountBefore = this.commitsByTree.Count; + this.RemoveCommitNoCache(lruCommit); + + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + metadata.Add("EvictedCommit", lruCommit); + metadata.Add("TreeCountBefore", treeCountBefore); + metadata.Add("TreeCountAfter", this.commitsByTree.Count); + this.tracer.RelatedEvent(EventLevel.Informational, nameof(this.EvictLruCommit), metadata, Keywords.Telemetry); + + return true; + } + + EventMetadata warnMetadata = new EventMetadata(); + warnMetadata.Add("Area", EtwArea); + warnMetadata.Add("TreeCount", this.commitsByTree.Count); + warnMetadata.Add("CommitCount", this.commitNodes.Count); + this.tracer.RelatedEvent(EventLevel.Warning, $"{nameof(this.EvictLruCommit)}CouldNotEvict", warnMetadata, Keywords.Telemetry); + + return false; + } + + /// + /// Removes a commit without cascading tree removal to other commits. + /// Used during LRU eviction: the trees are still missing, so other commits + /// that share those trees should continue to track them. + /// + private void RemoveCommitNoCache(string commitSha) + { + if (!this.missingTreesByCommit.TryGetValue(commitSha, out var trees)) + { + return; + } + + foreach (string treeSha in trees) + { + if (this.commitsByTree.TryGetValue(treeSha, out var commits)) + { + commits.Remove(commitSha); + if (commits.Count == 0) + { + this.commitsByTree.Remove(treeSha); + } + } + } + + this.missingTreesByCommit.Remove(commitSha); + this.RemoveFromLruOrder(commitSha); + } + + /// + /// Removes a commit and cascades: trees that were in this commit's set are + /// also removed from all other commits that shared them. Any commit that + /// becomes empty as a result is also removed (without further cascade). + /// + private void RemoveCommitWithCascade(string commitSha) + { + if (!this.missingTreesByCommit.TryGetValue(commitSha, out var trees)) + { + return; + } + + // Collect commits that may become empty after we remove the shared trees. + // We don't cascade further than one level. + var commitsToCheck = new HashSet(); + + foreach (string treeSha in trees) + { + if (this.commitsByTree.TryGetValue(treeSha, out var sharingCommits)) + { + sharingCommits.Remove(commitSha); + + foreach (string otherCommit in sharingCommits) + { + if (this.missingTreesByCommit.TryGetValue(otherCommit, out var otherTrees)) + { + otherTrees.Remove(treeSha); + if (otherTrees.Count == 0) + { + commitsToCheck.Add(otherCommit); + } + } + } + + sharingCommits.Clear(); + this.commitsByTree.Remove(treeSha); + } + } + + this.missingTreesByCommit.Remove(commitSha); + this.RemoveFromLruOrder(commitSha); + + // Clean up any commits that became empty due to the cascade + foreach (string emptyCommit in commitsToCheck) + { + if (this.missingTreesByCommit.TryGetValue(emptyCommit, out var remaining) && remaining.Count == 0) + { + this.missingTreesByCommit.Remove(emptyCommit); + this.RemoveFromLruOrder(emptyCommit); + } + } + } + + private void RemoveFromLruOrder(string commitSha) + { + if (this.commitNodes.TryGetValue(commitSha, out var node)) + { + this.commitOrder.Remove(node); + this.commitNodes.Remove(commitSha); + } + } + } +} diff --git a/GVFS/GVFS.Mount/InProcessMount.cs b/GVFS/GVFS.Mount/InProcessMount.cs index e6d43a842..c5126bac0 100644 --- a/GVFS/GVFS.Mount/InProcessMount.cs +++ b/GVFS/GVFS.Mount/InProcessMount.cs @@ -33,6 +33,12 @@ public class InProcessMount // all the trees in a commit to ~2-3 seconds. private const int MissingTreeThresholdForDownloadingCommitPack = 200; + // Number of unique missing trees to track with LRU eviction. Eviction is commit-based: + // when capacity is reached, the LRU commit and all its unique trees are dropped to make room. + // Set to 20x the threshold so that enough trees can accumulate for the heuristic to + // reliably trigger a commit pack download. + private const int TrackedTreeCapacity = MissingTreeThresholdForDownloadingCommitPack * 20; + private readonly bool showDebugWindow; private FileSystemCallbacks fileSystemCallbacks; @@ -52,7 +58,7 @@ public class InProcessMount private HeartbeatThread heartbeat; private ManualResetEvent unmountEvent; - private readonly Dictionary treesWithDownloadedCommits = new Dictionary(); + private readonly MissingTreeTracker missingTreeTracker; // True if InProcessMount is calling git reset as part of processing // a folder dehydrate request @@ -67,6 +73,7 @@ public InProcessMount(ITracer tracer, GVFSEnlistment enlistment, CacheServerInfo this.enlistment = enlistment; this.showDebugWindow = showDebugWindow; this.unmountEvent = new ManualResetEvent(false); + this.missingTreeTracker = new MissingTreeTracker(tracer, TrackedTreeCapacity); } private enum MountState @@ -590,19 +597,20 @@ private void HandleDownloadObjectRequest(NamedPipeMessages.Message message, Name this.context.Repository.GVFSLock.Stats.RecordObjectDownload(objectType == Native.ObjectTypes.Blob, downloadTime.ElapsedMilliseconds); if (objectType == Native.ObjectTypes.Commit - && !this.PrefetchHasBeenDone() && !this.context.Repository.CommitAndRootTreeExists(objectSha, out var treeSha) && !string.IsNullOrEmpty(treeSha)) { /* If a commit is downloaded, it wasn't prefetched. - * If any prefetch has been done, there is probably a commit in the prefetch packs that is close enough that - * loose object download of missing trees will be faster than downloading a pack of all the trees for the commit. - * Otherwise, the trees for the commit may be needed soon depending on the context. + * The trees for the commit may be needed soon depending on the context. * e.g. git log (without a pathspec) doesn't need trees, but git checkout does. * + * If any prefetch has been done there is probably a similar commit/tree in the graph, + * but in case there isn't (such as if the cache server repack maintenance job is failing) + * we should still try to avoid downloading an excessive number of loose trees for a commit. + * * Save the tree/commit so if more trees are requested we can download all the trees for the commit in a batch. */ - this.treesWithDownloadedCommits[treeSha] = objectSha; + this.missingTreeTracker.AddMissingRootTree(treeSha: treeSha, commitSha: objectSha); } } } @@ -610,22 +618,11 @@ private void HandleDownloadObjectRequest(NamedPipeMessages.Message message, Name connection.TrySendResponse(response.CreateMessage()); } - private bool PrefetchHasBeenDone() - { - var prefetchPacks = this.gitObjects.ReadPackFileNames(this.enlistment.GitPackRoot, GVFSConstants.PrefetchPackPrefix); - var result = prefetchPacks.Length > 0; - if (result) - { - this.treesWithDownloadedCommits.Clear(); - } - return result; - } - private bool ShouldDownloadCommitPack(string objectSha, out string commitSha) { - if (!this.treesWithDownloadedCommits.TryGetValue(objectSha, out commitSha) - || this.PrefetchHasBeenDone()) + if (!this.missingTreeTracker.TryGetCommits(objectSha, out string[] commitShas)) { + commitSha = null; return false; } @@ -634,8 +631,8 @@ private bool ShouldDownloadCommitPack(string objectSha, out string commitSha) * Conversely, if we know (from previously downloaded missing trees) that a commit has a lot of missing * trees left, we'll probably need to download many more trees for the commit so we should download the pack. */ - var commitShaLocal = commitSha; // can't use out parameter in lambda - int missingTreeCount = this.treesWithDownloadedCommits.Where(x => x.Value == commitShaLocal).Count(); + int missingTreeCount = this.missingTreeTracker.GetHighestMissingTreeCount(commitShas, out commitSha); + return missingTreeCount > MissingTreeThresholdForDownloadingCommitPack; } @@ -646,8 +643,7 @@ private void UpdateTreesForDownloadedCommits(string objectSha) * as a heuristic to decide whether to batch download all the trees for the commit the * next time a missing one is requested. */ - if (!this.treesWithDownloadedCommits.TryGetValue(objectSha, out var commitSha) - || this.PrefetchHasBeenDone()) + if (!this.missingTreeTracker.TryGetCommits(objectSha, out _)) { return; } @@ -660,20 +656,13 @@ private void UpdateTreesForDownloadedCommits(string objectSha) if (this.context.Repository.TryGetMissingSubTrees(objectSha, out var missingSubTrees)) { - foreach (var missingSubTree in missingSubTrees) - { - this.treesWithDownloadedCommits[missingSubTree] = commitSha; - } + this.missingTreeTracker.AddMissingSubTrees(objectSha, missingSubTrees); } } private void DownloadedCommitPack(string commitSha) { - var toRemove = this.treesWithDownloadedCommits.Where(x => x.Value == commitSha).ToList(); - foreach (var tree in toRemove) - { - this.treesWithDownloadedCommits.Remove(tree.Key); - } + this.missingTreeTracker.MarkCommitComplete(commitSha); } private void HandlePostFetchJobRequest(NamedPipeMessages.Message message, NamedPipeServer.Connection connection) diff --git a/GVFS/GVFS.UnitTests/Common/MissingTreeTrackerTests.cs b/GVFS/GVFS.UnitTests/Common/MissingTreeTrackerTests.cs new file mode 100644 index 000000000..a13a34554 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Common/MissingTreeTrackerTests.cs @@ -0,0 +1,517 @@ +using GVFS.Common; +using GVFS.Tests.Should; +using GVFS.UnitTests.Mock.Common; +using NUnit.Framework; + +namespace GVFS.UnitTests.Common +{ + [TestFixture] + public class MissingTreeTrackerTests + { + private static MissingTreeTracker CreateTracker(int treeCapacity) + { + return new MissingTreeTracker(new MockTracer(), treeCapacity); + } + + // ------------------------------------------------------------------------- + // AddMissingRootTree + // ------------------------------------------------------------------------- + + [TestCase] + public void AddMissingRootTree_SingleTreeAndCommit() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + tracker.AddMissingRootTree("tree1", "commit1"); + + tracker.TryGetCommits("tree1", out string[] commits).ShouldEqual(true); + commits.Length.ShouldEqual(1); + commits[0].ShouldEqual("commit1"); + tracker.GetHighestMissingTreeCount(commits, out _).ShouldEqual(1); + } + + [TestCase] + public void AddMissingRootTree_MultipleTreesForSameCommit() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree2", "commit1"); + tracker.AddMissingRootTree("tree3", "commit1"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(3); + + tracker.TryGetCommits("tree1", out string[] c1).ShouldEqual(true); + c1[0].ShouldEqual("commit1"); + + tracker.TryGetCommits("tree2", out string[] c2).ShouldEqual(true); + c2[0].ShouldEqual("commit1"); + + tracker.TryGetCommits("tree3", out string[] c3).ShouldEqual(true); + c3[0].ShouldEqual("commit1"); + } + + [TestCase] + public void AddMissingRootTree_SameTreeAddedTwiceToSameCommit() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree1", "commit1"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(1); + } + + [TestCase] + public void AddMissingRootTree_SameTreeAddedToMultipleCommits() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree1", "commit2"); + + // tree1 is now tracked under both commits + tracker.TryGetCommits("tree1", out string[] commits).ShouldEqual(true); + commits.Length.ShouldEqual(2); + + // Both commits each have 1 tree + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(1); + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(1); + } + + [TestCase] + public void AddMissingRootTree_MultipleTrees_ChecksCount() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(1); + + tracker.AddMissingRootTree("tree2", "commit1"); + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(2); + + tracker.AddMissingRootTree("tree3", "commit1"); + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(3); + + tracker.AddMissingRootTree("tree4", "commit1"); + tracker.AddMissingRootTree("tree5", "commit1"); + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(5); + } + + // ------------------------------------------------------------------------- + // AddMissingSubTrees + // ------------------------------------------------------------------------- + + [TestCase] + public void AddMissingSubTrees_AddsSubTreesUnderParentsCommits() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + tracker.AddMissingRootTree("rootTree", "commit1"); + tracker.AddMissingSubTrees("rootTree", new[] { "sub1", "sub2" }); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(3); + + tracker.TryGetCommits("sub1", out string[] c1).ShouldEqual(true); + c1[0].ShouldEqual("commit1"); + + tracker.TryGetCommits("sub2", out string[] c2).ShouldEqual(true); + c2[0].ShouldEqual("commit1"); + } + + [TestCase] + public void AddMissingSubTrees_PropagatesAcrossAllSharingCommits() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + // Two commits share the same root tree + tracker.AddMissingRootTree("rootTree", "commit1"); + tracker.AddMissingRootTree("rootTree", "commit2"); + + tracker.AddMissingSubTrees("rootTree", new[] { "sub1" }); + + // sub1 should be tracked under both commits + tracker.TryGetCommits("sub1", out string[] commits).ShouldEqual(true); + commits.Length.ShouldEqual(2); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(2); + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(2); + } + + [TestCase] + public void AddMissingSubTrees_NoOp_WhenParentNotTracked() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + // Should not throw; parent is not tracked + tracker.AddMissingSubTrees("unknownParent", new[] { "sub1" }); + + tracker.TryGetCommits("sub1", out _).ShouldEqual(false); + } + + [TestCase] + public void AddMissingSubTrees_SkipsCommitEvictedDuringLoop() + { + // treeCapacity = 2: rootTree fills slot 1, rootTree2 fills slot 2. + // commit1 and commit2 both share rootTree (1 unique tree so far). + // commit3 holds rootTree2 (2 unique trees, at capacity). + // AddMissingSubTrees(rootTree, [sub1]) must add sub1 to commit1 then commit2. + // Adding sub1 for commit1 fills the 3rd slot, which evicts the LRU commit. + // commit2 is LRU (added to the tracker last among commit1/commit2 and then not used + // again, while commit1 just got used), so it is evicted before we process commit2. + // The loop must skip commit2 rather than crashing. + MissingTreeTracker tracker = CreateTracker(treeCapacity: 2); + + tracker.AddMissingRootTree("rootTree", "commit1"); + tracker.AddMissingRootTree("rootTree", "commit2"); + tracker.AddMissingRootTree("rootTree2", "commit3"); + + // Does not throw, and sub1 ends up under whichever commit survived eviction + tracker.AddMissingSubTrees("rootTree", new[] { "sub1" }); + + // Exactly one of commit1/commit2 was evicted; sub1 exists under the survivor + bool commit1HasSub1 = tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _) == 2; + bool commit2HasSub1 = tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _) == 2; + (commit1HasSub1 || commit2HasSub1).ShouldEqual(true); + (commit1HasSub1 && commit2HasSub1).ShouldEqual(false); + } + + [TestCase] + public void AddMissingSubTrees_DoesNotEvictIfOnlyOneCommit() + { + /* This shouldn't be possible if user has a proper threshold and is marking commits + * as completed, but test to be safe. */ + MissingTreeTracker tracker = CreateTracker(treeCapacity: 2); + tracker.AddMissingRootTree("rootTree", "commit1"); + tracker.AddMissingSubTrees("rootTree", new[] { "sub1" }); + tracker.AddMissingSubTrees("rootTree", new[] { "sub2" }); + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(3); + } + + // ------------------------------------------------------------------------- + // TryGetCommits + // ------------------------------------------------------------------------- + + [TestCase] + public void TryGetCommits_NonExistentTree() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + tracker.TryGetCommits("nonexistent", out string[] commits).ShouldEqual(false); + commits.ShouldBeNull(); + } + + [TestCase] + public void TryGetCommits_MarksAllCommitsAsRecentlyUsed() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 3); + + tracker.AddMissingRootTree("sharedTree", "commit1"); + tracker.AddMissingRootTree("sharedTree", "commit2"); + tracker.AddMissingRootTree("tree2", "commit3"); + tracker.AddMissingRootTree("tree3", "commit4"); + + // Access commit1 and commit2 via TryGetCommits + tracker.TryGetCommits("sharedTree", out _); + + // Adding a fourth tree should evict commit3 (oldest unused) + tracker.AddMissingRootTree("tree4", "commit5"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(1); + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(1); + tracker.GetHighestMissingTreeCount(new[] { "commit3" }, out _).ShouldEqual(0); + tracker.GetHighestMissingTreeCount(new[] { "commit4" }, out _).ShouldEqual(1); + tracker.GetHighestMissingTreeCount(new[] { "commit5" }, out _).ShouldEqual(1); + } + + // ------------------------------------------------------------------------- + // GetHighestMissingTreeCount + // ------------------------------------------------------------------------- + + [TestCase] + public void GetHighestMissingTreeCount_NonExistentCommit() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + tracker.GetHighestMissingTreeCount(new[] { "nonexistent" }, out string highest).ShouldEqual(0); + highest.ShouldBeNull(); + } + + [TestCase] + public void GetHighestMissingTreeCount_ReturnsCommitWithMostTrees() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree2", "commit1"); + tracker.AddMissingRootTree("tree3", "commit2"); + + int count = tracker.GetHighestMissingTreeCount(new[] { "commit1", "commit2" }, out string highest); + count.ShouldEqual(2); + highest.ShouldEqual("commit1"); + } + + [TestCase] + public void GetHighestMissingTreeCount_DoesNotUpdateLru() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 3); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree2", "commit2"); + tracker.AddMissingRootTree("tree3", "commit3"); + + // Query commit1's count (should not update LRU) + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _); + + // Adding a fourth commit should still evict commit1 (oldest) + tracker.AddMissingRootTree("tree4", "commit4"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(0); + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(1); + tracker.GetHighestMissingTreeCount(new[] { "commit3" }, out _).ShouldEqual(1); + tracker.GetHighestMissingTreeCount(new[] { "commit4" }, out _).ShouldEqual(1); + } + + // ------------------------------------------------------------------------- + // MarkCommitComplete (cascade removal) + // ------------------------------------------------------------------------- + + [TestCase] + public void MarkCommitComplete_RemovesAllTreesForCommit() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree2", "commit1"); + tracker.AddMissingRootTree("tree3", "commit1"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(3); + + tracker.MarkCommitComplete("commit1"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(0); + tracker.TryGetCommits("tree1", out _).ShouldEqual(false); + tracker.TryGetCommits("tree2", out _).ShouldEqual(false); + tracker.TryGetCommits("tree3", out _).ShouldEqual(false); + } + + [TestCase] + public void MarkCommitComplete_NonExistentCommit() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + // Should not throw + tracker.MarkCommitComplete("nonexistent"); + } + + [TestCase] + public void MarkCommitComplete_CascadesSharedTreesToOtherCommits() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + // commit1 and commit2 share tree1; commit2 also has tree2 + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree1", "commit2"); + tracker.AddMissingRootTree("tree2", "commit2"); + + tracker.MarkCommitComplete("commit1"); + + // tree1 was in commit1, so it should be removed from commit2 as well + tracker.TryGetCommits("tree1", out _).ShouldEqual(false); + + // tree2 is unrelated to commit1, so commit2 still has it + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(1); + tracker.TryGetCommits("tree2", out string[] c2).ShouldEqual(true); + c2[0].ShouldEqual("commit2"); + } + + [TestCase] + public void MarkCommitComplete_RemovesOtherCommitWhenItBecomesEmpty() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + // commit2's only tree is shared with commit1 + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree1", "commit2"); + + tracker.MarkCommitComplete("commit1"); + + // commit2 had only tree1, which was cascaded away, so commit2 should be gone too + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(0); + tracker.TryGetCommits("tree1", out _).ShouldEqual(false); + } + + [TestCase] + public void MarkCommitComplete_DoesNotAffectUnrelatedCommits() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 10); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree2", "commit2"); + + tracker.MarkCommitComplete("commit1"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(0); + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(1); + tracker.TryGetCommits("tree2", out string[] c).ShouldEqual(true); + c[0].ShouldEqual("commit2"); + } + + // ------------------------------------------------------------------------- + // LRU eviction (no cascade) + // ------------------------------------------------------------------------- + + [TestCase] + public void LruEviction_EvictsOldestCommit() + { + // treeCapacity = 3 trees; one tree per commit + MissingTreeTracker tracker = CreateTracker(treeCapacity: 3); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree2", "commit2"); + tracker.AddMissingRootTree("tree3", "commit3"); + + // Adding a fourth tree exceeds treeCapacity, so commit1 (LRU) is evicted + tracker.AddMissingRootTree("tree4", "commit4"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(0); + tracker.TryGetCommits("tree1", out _).ShouldEqual(false); + + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(1); + tracker.GetHighestMissingTreeCount(new[] { "commit3" }, out _).ShouldEqual(1); + tracker.GetHighestMissingTreeCount(new[] { "commit4" }, out _).ShouldEqual(1); + } + + [TestCase] + public void LruEviction_DoesNotCascadeSharedTreesToOtherCommits() + { + // treeCapacity = 3 trees; tree1 is shared so only 2 unique trees + tree3 = 3 total + MissingTreeTracker tracker = CreateTracker(treeCapacity: 3); + + // tree1 is shared between commit1 and commit2 (counts as 1 unique tree) + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree2", "commit1"); + tracker.AddMissingRootTree("tree1", "commit2"); + tracker.AddMissingRootTree("tree3", "commit3"); + + // tree4 is the 4th unique tree, exceeding treeCapacity; evicts commit1 (LRU) + // which removes tree2, freeing up capacity. + tracker.AddMissingRootTree("tree4", "commit4"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(0); + + // tree1 is still missing (not yet downloaded), so commit2 retains it + tracker.TryGetCommits("tree1", out string[] commits).ShouldEqual(true); + commits.Length.ShouldEqual(1); + commits[0].ShouldEqual("commit2"); + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(1); + } + + [TestCase] + public void LruEviction_AddingTreeToExistingCommitUpdatesLru() + { + // treeCapacity = 4 trees; tree1, tree2, tree3 fill it, then tree1b re-uses commit1 + MissingTreeTracker tracker = CreateTracker(treeCapacity: 4); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree2", "commit2"); + tracker.AddMissingRootTree("tree3", "commit3"); + + // Adding tree1b to commit1 marks commit1 as recently used (it's a new unique tree) + tracker.AddMissingRootTree("tree1b", "commit1"); + + // tree4 is the 5th unique tree, exceeding treeCapacity; commit2 is now LRU + tracker.AddMissingRootTree("tree4", "commit4"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(2); + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(0); + tracker.GetHighestMissingTreeCount(new[] { "commit3" }, out _).ShouldEqual(1); + tracker.GetHighestMissingTreeCount(new[] { "commit4" }, out _).ShouldEqual(1); + } + + [TestCase] + public void LruEviction_MultipleTreesPerCommit_EvictsEntireCommit() + { + // treeCapacity = 4 trees; commit1 holds 3, commit2 holds 1 + MissingTreeTracker tracker = CreateTracker(treeCapacity: 4); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree2", "commit1"); + tracker.AddMissingRootTree("tree3", "commit1"); + tracker.AddMissingRootTree("tree4", "commit2"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(3); + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(1); + + // tree5 is the 5th unique tree; evict LRU (commit1) freeing 3 slots, then add tree5 + tracker.AddMissingRootTree("tree5", "commit3"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(0); + tracker.TryGetCommits("tree1", out _).ShouldEqual(false); + tracker.TryGetCommits("tree2", out _).ShouldEqual(false); + tracker.TryGetCommits("tree3", out _).ShouldEqual(false); + + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(1); + tracker.GetHighestMissingTreeCount(new[] { "commit3" }, out _).ShouldEqual(1); + } + + [TestCase] + public void LruEviction_CapacityOne() + { + MissingTreeTracker tracker = CreateTracker(treeCapacity: 1); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(1); + + tracker.AddMissingRootTree("tree2", "commit2"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(0); + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(1); + } + + [TestCase] + public void LruEviction_ManyTreesOneCommit_ExceedsCapacity() + { + // treeCapacity = 3 trees; all trees belong to commit1 + // Adding a 4th tree must evict commit1 (the only commit) to make room + MissingTreeTracker tracker = CreateTracker(treeCapacity: 3); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree2", "commit1"); + tracker.AddMissingRootTree("tree3", "commit1"); + + // tree4 exceeds the tree treeCapacity; the LRU commit (commit1) is evicted + // and then commit2 with tree4 is added fresh + tracker.AddMissingRootTree("tree4", "commit2"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(0); + tracker.TryGetCommits("tree1", out _).ShouldEqual(false); + tracker.TryGetCommits("tree2", out _).ShouldEqual(false); + tracker.TryGetCommits("tree3", out _).ShouldEqual(false); + + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(1); + } + + [TestCase] + public void LruEviction_TryGetCommitsUpdatesLru() + { + // treeCapacity = 3 trees, one per commit + MissingTreeTracker tracker = CreateTracker(treeCapacity: 3); + + tracker.AddMissingRootTree("tree1", "commit1"); + tracker.AddMissingRootTree("tree2", "commit2"); + tracker.AddMissingRootTree("tree3", "commit3"); + + // Access commit1 via TryGetCommits (marks it as recently used) + tracker.TryGetCommits("tree1", out _); + + // tree4 exceeds treeCapacity; commit2 is now LRU + tracker.AddMissingRootTree("tree4", "commit4"); + + tracker.GetHighestMissingTreeCount(new[] { "commit1" }, out _).ShouldEqual(1); + tracker.GetHighestMissingTreeCount(new[] { "commit2" }, out _).ShouldEqual(0); + tracker.GetHighestMissingTreeCount(new[] { "commit3" }, out _).ShouldEqual(1); + tracker.GetHighestMissingTreeCount(new[] { "commit4" }, out _).ShouldEqual(1); + } + } +}