fix(#345): make SingleFile metadata region sizes configurable and file extension byte-based

Author: MPCoreDeveloper

docs/serialization/SERIALIZATION_AND_STORAGE_GUIDE.md

@@ -532,8 +532,10 @@ public ulong AllocatePages(int count)
         if (startPage == ulong.MaxValue)
         {
             // 2. No space found? Extend file exponentially
+            // Minimum extension is byte-based (issue #345): ~10 MB regardless of PageSize.
+            var minExtensionPages = Math.Max(1, (int)(MIN_EXTENSION_BYTES / _pageSize));
             var extensionSize = Math.Max(
-                MIN_EXTENSION_PAGES,           // 2560 pages = 10MB (Phase 3)
+                minExtensionPages,
                 Math.Max(count, currentSize / EXTENSION_GROWTH_FACTOR)
             );
 

src/SharpCoreDB/DatabaseOptions.cs

@@ -97,6 +97,26 @@ public sealed class DatabaseOptions
     /// </summary>
     public int CompressionThreshold { get; set; } = 256;
 
+    /// <summary>
+    /// Number of pages allocated for the SingleFile Block Registry.
+    /// Each block entry is 96 bytes; the default 4 pages @ 4KB supports ~170 blocks.
+    /// Increase for databases that store many named blocks (default: 4, backward compatible).
+    /// </summary>
+    public int BlockRegistrySizePages { get; set; } = 4;
+
+    /// <summary>
+    /// Number of pages allocated for the SingleFile Free Space Map (FSM).
+    /// The FSM tracks one bit per data page; the default 4 pages @ 4KB supports ~512 MB
+    /// of file data. Increase for databases expected to grow beyond that (default: 4, backward compatible).
+    /// </summary>
+    public int FsmSizePages { get; set; } = 4;
+
+    /// <summary>
+    /// Number of pages allocated for the SingleFile table directory (schema metadata).
+    /// Default: 4 (backward compatible).
+    /// </summary>
+    public int TableDirectorySizePages { get; set; } = 4;
+
     /// <summary>
     /// Gets or sets whether to enable memory-mapped I/O for reads.
     /// Default: true (enables zero-copy reads).
@@ -204,6 +224,25 @@ public void Validate()
                 $"WalBufferSizePages must be between 64 and 65536. Got: {WalBufferSizePages}");
         }
 
+        // Validate metadata region sizes (SingleFile storage layout, issue #345)
+        if (BlockRegistrySizePages < 1 || BlockRegistrySizePages > 65536)
+        {
+            throw new ArgumentException(
+                $"BlockRegistrySizePages must be between 1 and 65536. Got: {BlockRegistrySizePages}");
+        }
+
+        if (FsmSizePages < 1 || FsmSizePages > 65536)
+        {
+            throw new ArgumentException(
+                $"FsmSizePages must be between 1 and 65536. Got: {FsmSizePages}");
+        }
+
+        if (TableDirectorySizePages < 1 || TableDirectorySizePages > 65536)
+        {
+            throw new ArgumentException(
+                $"TableDirectorySizePages must be between 1 and 65536. Got: {TableDirectorySizePages}");
+        }
+
         // Validate fragmentation threshold
         if (FragmentationThreshold < 0 || FragmentationThreshold > 100)
         {

src/SharpCoreDB/Storage/FreeSpaceManager.cs

@@ -44,7 +44,9 @@ internal sealed class FreeSpaceManager : IDisposable
     private ulong _freePages;
 
     // ✅ C# 14: Pre-allocation settings for optimal file growth - Phase 3 optimized
-    private const int MIN_EXTENSION_PAGES = 2560;      // 10 MB @ 4KB pages (Phase 3: increased from 512 = 2MB)
+    // Minimum file extension is byte-based so it stays ~10 MB regardless of PageSize;
+    // a fixed page count would scale linearly (40 MB @ 16 KB, 80 MB @ 32 KB) - issue #345.
+    private const long MIN_EXTENSION_BYTES = 10L * 1024 * 1024;
     private const int EXTENSION_GROWTH_FACTOR = 2;     // Double size each time (exponential growth)
     private ulong _preallocatedPages = 0;
 
@@ -200,8 +202,10 @@ public ulong AllocatePages(int count)
                 // ✅ Calculate extension size (grow exponentially)
                 var requiredPages = (ulong)count;
                 var currentSize = _totalPages;
+                // Minimum extension is byte-based (issue #345): ~10 MB regardless of PageSize.
+                var minExtensionPages = (ulong)Math.Max(1, (int)(MIN_EXTENSION_BYTES / _pageSize));
                 var extensionSize = Math.Max(
-                    MIN_EXTENSION_PAGES,
+                    minExtensionPages,
                     Math.Max(requiredPages, currentSize / EXTENSION_GROWTH_FACTOR)
                 );
 

src/SharpCoreDB/Storage/SingleFileStorageProvider.cs

@@ -1522,22 +1522,21 @@ static ulong AlignToPage(ulong value, int pageSize)
         }
 
         // Initialize block registry at next page boundary
-        // ✅ Phase 3: Allocate 4 pages (16KB) to support up to ~250 block entries
-        // Calculation: (16384 - 64 header) / 64 per entry = 255 max entries
+        // Configurable region size (issue #345): BlockRegistrySizePages pages.
         header.BlockRegistryOffset = AlignToPage(ScdbFileHeader.HEADER_SIZE, options.PageSize);
-        header.BlockRegistryLength = (ulong)options.PageSize * 4;
+        header.BlockRegistryLength = (ulong)options.PageSize * (ulong)options.BlockRegistrySizePages;
 
-        // Initialize FSM at pages 5-8 (4 pages)
+        // Initialize FSM right after the registry (configurable region size)
         header.FsmOffset = header.BlockRegistryOffset + header.BlockRegistryLength;
-        header.FsmLength = (ulong)options.PageSize * 4; // 4 pages for FSM
+        header.FsmLength = (ulong)options.PageSize * (ulong)options.FsmSizePages; // FSM tracks 1 bit/page
 
-        // Initialize WAL at pages 9+ (configurable)
+        // Initialize WAL at the next page boundary (configurable)
         header.WalOffset = header.FsmOffset + header.FsmLength;
         header.WalLength = (ulong)options.PageSize * (ulong)options.WalBufferSizePages;
 
-        // Initialize table directory after WAL
+        // Initialize table directory after WAL (configurable region size)
         header.TableDirOffset = header.WalOffset + header.WalLength;
-        header.TableDirLength = (ulong)options.PageSize * 4; // 4 pages for table directory
+        header.TableDirLength = (ulong)options.PageSize * (ulong)options.TableDirectorySizePages;
 
         // Allocate space for metadata structures
         var totalMetadataSize = header.TableDirOffset + header.TableDirLength;

tests/SharpCoreDB.Tests/Storage/MetadataSizingTests.cs

@@ -0,0 +1,150 @@
+// tests/SharpCoreDB.Tests/Storage/MetadataSizingTests.cs
+// Copyright (c) 2025-2026 MPCoreDeveloper and GitHub Copilot. All rights reserved.
+// Licensed under the MIT License. See LICENSE file in the project root for full license information.
+
+namespace SharpCoreDB.Tests.Storage;
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using SharpCoreDB;
+using SharpCoreDB.Storage;
+using Xunit;
+
+/// <summary>
+/// REGRESSION TESTS: SingleFile metadata region sizing (issue #345).
+/// Verifies the FSM / Block Registry / Table Directory regions are no longer
+/// hard-coded to 4 pages and that the minimum file extension is byte-based
+/// (stays ~10 MB regardless of PageSize).
+/// </summary>
+public sealed class MetadataSizingTests : IDisposable
+{
+    private readonly string _testDbPath;
+
+    public MetadataSizingTests()
+    {
+        _testDbPath = Path.Combine(Path.GetTempPath(), $"metadata_sizing_{Guid.NewGuid():N}.scdb");
+    }
+
+    public void Dispose()
+    {
+        try
+        {
+            if (File.Exists(_testDbPath)) File.Delete(_testDbPath);
+            if (File.Exists(_testDbPath + ".wal")) File.Delete(_testDbPath + ".wal");
+            if (File.Exists(_testDbPath + ".vacuum.tmp")) File.Delete(_testDbPath + ".vacuum.tmp");
+            if (File.Exists(_testDbPath + ".backup")) File.Delete(_testDbPath + ".backup");
+        }
+        catch
+        {
+            // Ignore cleanup errors
+        }
+    }
+
+    [Fact]
+    public void ConfiguredRegionSizes_AreWrittenToHeader()
+    {
+        var options = DatabaseOptions.CreateSingleFileDefault();
+        options.PageSize = 4096;
+        options.WalBufferSizePages = 64;
+        options.EnableMemoryMapping = false;
+        options.BlockRegistrySizePages = 8;
+        options.FsmSizePages = 16;
+        options.TableDirectorySizePages = 12;
+
+        using (SingleFileStorageProvider.Open(_testDbPath, options))
+        {
+            // Provider writes the header immediately on create; dispose closes the file.
+        }
+
+        using var fs = new FileStream(_testDbPath, FileMode.Open, FileAccess.Read);
+        using var reader = new BinaryReader(fs);
+
+        fs.Position = 0x28; // BlockRegistryLength
+        var registryLength = reader.ReadUInt64();
+        fs.Position = 0x38; // FsmLength
+        var fsmLength = reader.ReadUInt64();
+        fs.Position = 0x58; // TableDirLength
+        var tableDirLength = reader.ReadUInt64();
+
+        Assert.Equal((ulong)(4096 * 8), registryLength);
+        Assert.Equal((ulong)(4096 * 16), fsmLength);
+        Assert.Equal((ulong)(4096 * 12), tableDirLength);
+    }
+
+    [Fact]
+    public async Task ManyBlocks_WithLargerRegistry_ShouldRoundtrip()
+    {
+        var options = DatabaseOptions.CreateSingleFileDefault();
+        options.PageSize = 4096;
+        options.WalBufferSizePages = 64;
+        options.EnableMemoryMapping = false;
+        options.BlockRegistrySizePages = 16; // 64 KB registry ≈ 680 entries
+
+        const int blockCount = 300; // exceeds the default 4-page (~170) capacity
+        var payload = new byte[64];
+        for (var i = 0; i < payload.Length; i++)
+        {
+            payload[i] = (byte)(i % 251);
+        }
+
+        using (var provider = SingleFileStorageProvider.Open(_testDbPath, options))
+        {
+            for (var i = 0; i < blockCount; i++)
+            {
+                await provider.WriteBlockAsync($"blk_{i}", payload);
+            }
+
+            // ForceFlushAsync = registry force flush + WAL checkpoint (full durability),
+            // avoiding a race between the periodic registry flusher and the reopen below.
+            await provider.ForceFlushAsync();
+        }
+
+        using var reopened = SingleFileStorageProvider.Open(_testDbPath, options);
+        for (var i = 0; i < blockCount; i++)
+        {
+            var data = await reopened.ReadBlockAsync($"blk_{i}");
+            Assert.NotNull(data);
+            Assert.Equal(payload, data);
+        }
+    }
+
+    [Fact]
+    public async Task MinExtension_IsByteBased_NotPageBased()
+    {
+        var options = DatabaseOptions.CreateSingleFileDefault();
+        options.PageSize = 16384; // 16 KB pages
+        options.WalBufferSizePages = 64; // keep the WAL small for the assertion
+        options.EnableMemoryMapping = false;
+
+        long fileSize;
+        using (var provider = SingleFileStorageProvider.Open(_testDbPath, options))
+        {
+            await provider.WriteBlockAsync("blob", new byte[8192]);
+            await provider.FlushAsync();
+            fileSize = new FileInfo(_testDbPath).Length;
+        }
+
+        // Old behavior: MIN_EXTENSION_PAGES=2560 * 16 KB = 40 MB -> file ~41 MB.
+        // Byte-based behavior: ~10 MB extension -> file ~11-12 MB.
+        Assert.True(
+            fileSize < 20L * 1024 * 1024,
+            $"File grew to {fileSize} bytes; expected a ~10 MB byte-based extension, not a page-count-based one.");
+    }
+
+    [Fact]
+    public void InvalidRegionSizes_ShouldThrowOnValidate()
+    {
+        var fsmZero = DatabaseOptions.CreateSingleFileDefault();
+        fsmZero.FsmSizePages = 0;
+        Assert.Throws<ArgumentException>(() => fsmZero.Validate());
+
+        var registryNegative = DatabaseOptions.CreateSingleFileDefault();
+        registryNegative.BlockRegistrySizePages = -1;
+        Assert.Throws<ArgumentException>(() => registryNegative.Validate());
+
+        var tableDirTooBig = DatabaseOptions.CreateSingleFileDefault();
+        tableDirTooBig.TableDirectorySizePages = 70000;
+        Assert.Throws<ArgumentException>(() => tableDirTooBig.Validate());
+    }
+}