index improvements

LogGrokX.Benchmarks/DataPortBenchmark.cs

@@ -0,0 +1,81 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.InteropServices;
+using BenchmarkDotNet.Attributes;
+using LogGrokX.Data;
+using LogGrokX.Data.Index;
+using LogGrokX.Data.IndexTree;
+using LogGrokX.Data.Monikers;
+
+namespace LogGrokX.Benchmarks;
+
+[MemoryDiagnoser]
+[InProcess]
+public class DataPortBenchmark
+{
+    private CountIndex<IndexTree<int, SimpleLeaf<int>>> _countIndex = null!;
+    private BenchmarkIndexer _indexer = null!;
+
+    private sealed class BenchmarkIndexer : Indexer
+    {
+        public int GetUncachedCount(int componentIndex, string componentValue) => Indices
+            .Where(pair => NumbersToKeys[pair.Key].GetComponent(componentIndex).SequenceEqual(componentValue.AsSpan()))
+            .Sum(pair => pair.Value.Count);
+    }
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        var indices = new Dictionary<IndexKeyNum, IndexTree<int, SimpleLeaf<int>>>();
+        for (var key = 0; key < 256; key++)
+        {
+            var tree = new IndexTree<int, SimpleLeaf<int>>(16, value => new SimpleLeaf<int>(value, 0));
+            for (var line = 0; line < 64; line++)
+                tree.Add(line);
+            indices.Add(new IndexKeyNum { KeyNum = key }, tree);
+        }
+
+        _countIndex = new CountIndex<IndexTree<int, SimpleLeaf<int>>>(indices);
+        _countIndex.Add(1, indices);
+
+        _indexer = new BenchmarkIndexer();
+        var meta = new LogMetaInformation(new LogFormat
+        {
+            Regex = @"^(?'Value'\w+)$",
+            IndexedFields = ["Value"]
+        });
+        var parser = new RegexBasedLineParser(meta, true);
+        for (var key = 0; key < 256; key++)
+        {
+            var line = $"value{key}";
+            var placeholder = new int[LineMetaInformation.GetSizeInts(1)];
+            var lineMeta = new LineMetaInformation(placeholder.AsSpan(), 1);
+            if (!parser.TryParse(line, 0, line.Length, lineMeta.ParsedLineComponents, out _))
+                throw new System.InvalidOperationException();
+            var metaChars = MemoryMarshal.Cast<int, char>(placeholder.AsSpan());
+            var chars = new char[metaChars.Length + line.Length];
+            metaChars.CopyTo(chars.AsSpan());
+            line.AsSpan().CopyTo(chars.AsSpan(metaChars.Length));
+            _indexer.Add(new IndexKey(new string(chars), 0, 1), key);
+        }
+    }
+
+    [Benchmark]
+    public int ReadLiveCounts() => _countIndex.Counts[^1].Count;
+
+    [Benchmark]
+    public int ReadCachedComponentCount() => _indexer.GetIndexCountForComponent(0, "value128");
+
+    [Benchmark]
+    public int ReadUncachedComponentCount() => _indexer.GetUncachedCount(0, "value128");
+
+    [Benchmark]
+    public int AppendIndexLines()
+    {
+        var tree = new IndexTree<int, SimpleLeaf<int>>(16, value => new SimpleLeaf<int>(value, 0));
+        for (var line = 0; line < 100_000; line++)
+            tree.Add(line);
+        return tree.Count;
+    }
+}

LogGrokX.Data.Tests/ConcurrentIndexReadTests.cs

@@ -0,0 +1,106 @@
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using LogGrokX.Data.Index;
+using LogGrokX.Data.IndexTree;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace LogGrokX.Data.Tests;
+
+[TestClass]
+public class ConcurrentIndexReadTests
+{
+    [TestMethod]
+    public void IndexTreeCanBeReadWhileGrowing()
+    {
+        const int total = 100_000;
+        var tree = new IndexTree<int, SimpleLeaf<int>>(16, value => new SimpleLeaf<int>(value, 0));
+        var ready = new CountdownEvent(2);
+        var done = false;
+
+        var readers = Enumerable.Range(0, 2).Select(_ => Task.Factory.StartNew(() =>
+        {
+            ready.Signal();
+            while (!Volatile.Read(ref done))
+            {
+                var count = tree.Count;
+                if (count == 0)
+                    continue;
+                var index = count / 2;
+                Assert.AreEqual(index, tree[index]);
+                Assert.AreEqual(index, tree.FindIndexByValue(index));
+                Assert.AreEqual(index, tree.GetEnumerableFromIndex(index).First());
+            }
+        }, TaskCreationOptions.LongRunning)).ToArray();
+
+        ready.Wait();
+        try
+        {
+            for (var i = 0; i < total; i++)
+            {
+                tree.Add(i);
+                if (i % 256 == 0)
+                    Thread.Yield();
+            }
+        }
+        finally
+        {
+            Volatile.Write(ref done, true);
+            Task.WaitAll(readers);
+        }
+
+        Assert.AreEqual(total, tree.Count);
+        Assert.AreEqual(total - 1, tree[total - 1]);
+    }
+
+    [TestMethod]
+    public void ChunkedListCanBeReadWhileGrowing()
+    {
+        const int total = 100_000;
+        var list = new ChunkedList<int>(64);
+        var ready = new CountdownEvent(2);
+        var done = false;
+
+        var readers = Enumerable.Range(0, 2).Select(_ => Task.Factory.StartNew(() =>
+        {
+            ready.Signal();
+            while (!Volatile.Read(ref done))
+            {
+                var count = list.Count;
+                if (count > 0)
+                    Assert.AreEqual(count - 1, list[count - 1]);
+            }
+        }, TaskCreationOptions.LongRunning)).ToArray();
+
+        ready.Wait();
+        try
+        {
+            for (var i = 0; i < total; i++)
+            {
+                list.Add(i);
+                if (i % 256 == 0)
+                    Thread.Yield();
+            }
+        }
+        finally
+        {
+            Volatile.Write(ref done, true);
+            Task.WaitAll(readers);
+        }
+
+        Assert.AreEqual(total, list.Count);
+        Assert.AreEqual(total - 1, list[total - 1]);
+    }
+
+    [TestMethod]
+    public void LongsLeafFindAndEnumeratePreserveOffset()
+    {
+        var tree = new IndexTree<long, LongsLeaf>(16, value => new LongsLeaf(value, 0));
+        for (var i = 0; i < 70_000; i++)
+            tree.Add(1_000_000L + i);
+
+        Assert.AreEqual(65_536, tree.FindIndexByValue(1_065_536));
+        Assert.AreEqual(1_065_536L, tree.GetEnumerableFromIndex(65_536).First());
+    }
+}

LogGrokX.Data.Tests/CountIndexCacheTests.cs

@@ -0,0 +1,35 @@
+using System.Collections.Generic;
+using LogGrokX.Data.Index;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace LogGrokX.Data.Tests;
+
+[TestClass]
+public class CountIndexCacheTests
+{
+    [TestMethod]
+    public void CountsReuseSnapshotUntilNextLineAndFinish()
+    {
+        var key = new IndexKeyNum { KeyNum = 1 };
+        var index = new LogGrokX.Data.Index.Index(4);
+        var indices = new Dictionary<IndexKeyNum, LogGrokX.Data.Index.Index> { [key] = index };
+        var counts = new CountIndex<LogGrokX.Data.Index.Index>(indices);
+
+        index.Add(0);
+        counts.Add(0, indices);
+        var first = counts.Counts;
+        Assert.AreSame(first, counts.Counts);
+        Assert.AreEqual(1, first[^1][0].Item2);
+
+        index.Add(1);
+        counts.Add(1, indices);
+        var second = counts.Counts;
+        Assert.AreNotSame(first, second);
+        Assert.AreEqual(2, second[^1][0].Item2);
+
+        counts.Finish(indices);
+        var finished = counts.Counts;
+        Assert.AreSame(finished, counts.Counts);
+        Assert.AreEqual(2, finished[^1][0].Item2);
+    }
+}

LogGrokX.Data.Tests/IndexTreeEdgeTests.cs

@@ -23,6 +23,16 @@ public void LongsLeafFindByValue()
         Assert.AreEqual(2, missingIndex);
     }
 
+    [TestMethod]
+    public void LongsLeafUsesFirstValueWhenIndexDiffers()
+    {
+        var leaf = new LongsLeaf(1_000_000, 5);
+        leaf.Add(1_000_004, 6);
+
+        Assert.AreEqual(6, leaf.FindByValue(1_000_004).index);
+        CollectionAssert.AreEqual(new[] { 1_000_000L, 1_000_004L }, new List<long>(leaf));
+    }
+
     [TestMethod]
     public void LongsLeafChainsWhenFull()
     {

LogGrokX.Data.Tests/IndexerTests.cs

@@ -77,6 +77,24 @@ public void GetIndexCountForComponentCountsLines()
         Assert.AreEqual(0, indexer.GetIndexCountForComponent(0, "missing"));
     }
 
+    [TestMethod]
+    public void GetIndexCountForComponentRefreshesCachedKeysAndLiveCounts()
+    {
+        using var indexer = new Indexer();
+        indexer.Add(Key("alpha beta one"), 0);
+        Assert.AreEqual(1, indexer.GetIndexCountForComponent(0, "alpha"));
+
+        indexer.Add(Key("alpha beta one"), 1);
+        Assert.AreEqual(2, indexer.GetIndexCountForComponent(0, "alpha"));
+
+        indexer.Add(Key("gamma delta two"), 2);
+        Assert.AreEqual(2, indexer.GetIndexCountForComponent(0, "alpha"));
+
+        indexer.Add(Key("alpha beta three"), 3);
+        Assert.AreEqual(3, indexer.GetIndexCountForComponent(0, "alpha"));
+        Assert.AreEqual(1, indexer.GetIndexCountForComponent(0, "gamma"));
+    }
+
     [TestMethod]
     public void GetIndexKeyNumAndIndexRoundTrip()
     {

LogGrokX.Data.Tests/LineIndexEdgeTests.cs

@@ -52,4 +52,35 @@ public async Task FetchRangesRefreshesCountAfterDelay()
 
         CollectionAssert.AreEqual(new[] { (0, 1) }, ranges);
     }
+
+    [TestMethod]
+    public async Task FetchRangesWakesWhenEnoughLinesArrive()
+    {
+        var index = new LineIndex();
+        using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(2));
+        await using var ranges = index.FetchRanges(cancellation.Token).GetAsyncEnumerator();
+        var next = ranges.MoveNextAsync().AsTask();
+
+        for (var i = 0; i <= 256; i++)
+            index.Add(i * 10L);
+
+        Assert.IsTrue(await next.WaitAsync(TimeSpan.FromMilliseconds(200)));
+        Assert.AreEqual((0, 256), ranges.Current);
+        index.Finish(10);
+    }
+
+    [TestMethod]
+    public async Task FetchRangesWakesWhenFinishedBelowBatchSize()
+    {
+        var index = new LineIndex();
+        index.Add(0);
+        using var cancellation = new CancellationTokenSource(TimeSpan.FromSeconds(2));
+        await using var ranges = index.FetchRanges(cancellation.Token).GetAsyncEnumerator();
+        var next = ranges.MoveNextAsync().AsTask();
+
+        index.Finish(10);
+
+        Assert.IsTrue(await next.WaitAsync(TimeSpan.FromMilliseconds(200)));
+        Assert.AreEqual((0, 1), ranges.Current);
+    }
 }

LogGrokX.Data.Tests/LineProviderRangeTests.cs

@@ -0,0 +1,35 @@
+using System;
+using System.IO;
+using LogGrokX.Data;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace LogGrokX.Data.Tests;
+
+[TestClass]
+public class LineProviderRangeTests
+{
+    private sealed class HugeRangeLineIndex : ILineIndex
+    {
+        public int Count => 2;
+        public (long offset, int length) GetLine(int index) =>
+            index == 0 ? (0, 0) : (3_000_000_000L, 10);
+        public void Fetch(int start, Span<(long offset, int length)> values) =>
+            throw new InvalidOperationException("Unexpected read");
+    }
+
+    [TestMethod]
+    public void FetchRejectsRangeLargerThanInt32()
+    {
+        var path = Path.GetTempFileName();
+        try
+        {
+            File.WriteAllText(path, "sample");
+            var provider = new LineProvider(new HugeRangeLineIndex(), new LogFile(path, 0));
+            Assert.ThrowsExactly<InvalidOperationException>(() => provider.Fetch(0, new (int, string)[2]));
+        }
+        finally
+        {
+            File.Delete(path);
+        }
+    }
+}

LogGrokX.Data.Tests/StringPoolLimitTests.cs

@@ -0,0 +1,22 @@
+using LogGrokX.Data;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace LogGrokX.Data.Tests;
+
+[TestClass]
+public class StringPoolLimitTests
+{
+    [TestMethod]
+    public void ReturnCapsEachBucketAndRentDecrementsCount()
+    {
+        var pool = new StringPool();
+        _ = pool.Rent(50);
+        for (var i = 0; i < 200; i++)
+            pool.Return(new string('\0', 64));
+
+        Assert.AreEqual(64, pool.GetPooledCount(50));
+        _ = pool.Rent(50);
+        Assert.AreEqual(63, pool.GetPooledCount(50));
+        Assert.AreEqual(0, pool.GetPooledCount(100));
+    }
+}