Add Eytzinger layout option for key digests

src/DryDB/BTree/EytzingerLayout.cs

@@ -0,0 +1,68 @@
+using System;
+using System.Runtime.CompilerServices;
+
+namespace DryDB.BTree;
+
+/// <summary>
+/// Digest array stored as a complete binary tree in Eytzinger (BFS) order.
+/// </summary>
+/// <remarks>
+/// The array is padded with <see cref="ulong.MaxValue"/> up to 2^k - 1 slots so the
+/// tree is complete, which makes the descent fully branch-free — each level is
+/// <c>i = 2i + (digest &lt; key)</c> with no mispredictable branch — and gives a
+/// closed-form rank: after the descent, <c>rank = i - size - 1</c> counts exactly the
+/// digests below the probe. The padding acts as +infinity (a real digest can also be
+/// MaxValue; it then simply lands at the end of the order, which is still correct).
+/// The top of the tree sits in the first cache line, so the levels that a sorted
+/// binary search scatters across the page are clustered here.
+/// </remarks>
+static class EytzingerLayout
+{
+    /// <summary>
+    /// The smallest complete-tree slot count (2^k - 1) holding <paramref name="count"/> digests.
+    /// </summary>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public static int CompleteSize(int count)
+    {
+        var m = 1;
+        while (m < count + 1) m <<= 1;
+        return m - 1;
+    }
+
+    /// <summary>
+    /// Returns the number of digests strictly less than <paramref name="keyDigest"/>
+    /// (the lower-bound rank in sorted order). <paramref name="completeSize"/> is the
+    /// padded slot count; slot i (1-indexed) lives at
+    /// <paramref name="digestBase"/> + (i - 1) * 8.
+    /// </summary>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public static int LowerBoundRank(ref byte pageReference, int digestBase, int completeSize, ulong keyDigest)
+    {
+        var i = 1;
+        while (i <= completeSize)
+        {
+            var digest = Unsafe.ReadUnaligned<ulong>(
+                ref Unsafe.Add(ref pageReference, digestBase + (i - 1) * sizeof(ulong)));
+            i = 2 * i + (digest < keyDigest ? 1 : 0);
+        }
+        return i - completeSize - 1;
+    }
+
+    /// <summary>
+    /// Scatters sorted digests into Eytzinger slots (length must be
+    /// <see cref="CompleteSize"/> of the digest count); unused slots become MaxValue.
+    /// </summary>
+    public static void Scatter(ReadOnlySpan<ulong> sortedDigests, Span<ulong> slots)
+    {
+        var k = 0;
+        FillInOrder(sortedDigests, slots, 1, ref k);
+    }
+
+    static void FillInOrder(ReadOnlySpan<ulong> sortedDigests, Span<ulong> slots, int i, ref int k)
+    {
+        if (i > slots.Length) return;
+        FillInOrder(sortedDigests, slots, 2 * i, ref k);
+        slots[i - 1] = k < sortedDigests.Length ? sortedDigests[k++] : ulong.MaxValue;
+        FillInOrder(sortedDigests, slots, 2 * i + 1, ref k);
+    }
+}

src/DryDB/BTree/InternalNodeReader.cs

@@ -12,7 +12,7 @@ namespace DryDB.BTree;
 /// </summary>
 /// <remarks>
 /// </remarks>
-readonly ref struct InternalNodeReader(ReadOnlySpan<byte> page, int entryCount, bool hasKeyDigests)
+readonly ref struct InternalNodeReader(ReadOnlySpan<byte> page, int entryCount, bool hasKeyDigests, bool hasEytzingerDigests)
 {
     [StructLayout(LayoutKind.Explicit, Size = 6, Pack = 1)]
     struct NodeEntryMeta
@@ -31,8 +31,11 @@ struct NodeEntryMeta
 #else
     readonly ref byte pageReference = ref MemoryMarshal.GetReference(page);
 #endif
-    readonly int metaBase = DigestBase + (hasKeyDigests ? entryCount * sizeof(ulong) : 0);
+    readonly int metaBase = DigestBase + (hasKeyDigests
+        ? (hasEytzingerDigests ? EytzingerLayout.CompleteSize(entryCount) : entryCount) * sizeof(ulong)
+        : 0);
     readonly bool hasKeyDigests = hasKeyDigests;
+    readonly bool hasEytzingerDigests = hasEytzingerDigests;
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     public void GetAt(int index, out ReadOnlySpan<byte> key, out PageNumber childPageNumber)
@@ -60,12 +63,33 @@ public bool TrySearch<TComparer>(
 #if NETSTANDARD
         ref var pageReference = ref MemoryMarshal.GetReference(page);
 #endif
+        NodeEntryMeta meta;
+        if (hasEytzingerDigests && hasKeyDigest)
+        {
+            // Branch-free descent to the rank of the first entry with digest >=
+            // keyDigest, then advance through the run of equal digests with full
+            // comparisons to reach the upper bound (first entry > key).
+            var i = EytzingerLayout.LowerBoundRank(
+                ref pageReference, DigestBase, (metaBase - DigestBase) / sizeof(ulong), keyDigest);
+            while (i < entryCount && CompareFull(ref pageReference, i, key, comparer) <= 0)
+            {
+                i++;
+            }
+
+            var childIndex = i == 0 ? 0 : i - 1;
+            meta = GetMeta(childIndex);
+            childPageNumber = Unsafe.ReadUnaligned<PageNumber>(
+                ref Unsafe.Add(
+                    ref pageReference,
+                    meta.PageOffset + meta.KeyLength));
+            return true;
+        }
+
         var useDigest = hasKeyDigests && hasKeyDigest;
 
         var min = 0;
         var max = entryCount;
 
-        NodeEntryMeta meta;
         while (min < max)
         {
             var mid = min + ((max - min) >> 1);

src/DryDB/BTree/LeafNodeReader.cs

@@ -12,7 +12,7 @@ namespace DryDB.BTree;
 /// </summary>>
 /// <remarks>
 /// </remarks>
-readonly ref struct LeafNodeReader(ReadOnlySpan<byte> page, int entryCount, bool hasKeyDigests)
+readonly ref struct LeafNodeReader(ReadOnlySpan<byte> page, int entryCount, bool hasKeyDigests, bool hasEytzingerDigests)
 {
     internal const ushort OverflowSentinel = ushort.MaxValue; // 0xFFFF
 
@@ -39,8 +39,11 @@ struct NodeEntryMeta
 #else
     readonly ref byte pageReference = ref MemoryMarshal.GetReference(page);
 #endif
-    readonly int metaBase = DigestBase + (hasKeyDigests ? entryCount * sizeof(ulong) : 0);
+    readonly int metaBase = DigestBase + (hasKeyDigests
+        ? (hasEytzingerDigests ? EytzingerLayout.CompleteSize(entryCount) : entryCount) * sizeof(ulong)
+        : 0);
     readonly bool hasKeyDigests = hasKeyDigests;
+    readonly bool hasEytzingerDigests = hasEytzingerDigests;
 
     public void GetAt(int index, out ReadOnlySpan<byte> key, out ReadOnlySpan<byte> value)
     {
@@ -82,6 +85,34 @@ public bool TryFindValue<TComparer>(
 #if NETSTANDARD
         ref var pageReference = ref MemoryMarshal.GetReference(page);
 #endif
+        if (hasEytzingerDigests && hasKeyDigest)
+        {
+            // Branch-free descent yields the rank of the first entry whose digest is
+            // >= keyDigest; entries below the rank are < key. A match can only sit in
+            // the run of equal digests starting there, resolved with full comparisons
+            // (the digests are in Eytzinger order, so the run is walked via the keys).
+            var i = EytzingerLayout.LowerBoundRank(
+                ref pageReference, DigestBase, (metaBase - DigestBase) / sizeof(ulong), keyDigest);
+            for (; i < entryCount; i++)
+            {
+                var compared = CompareFull(ref pageReference, i, key, comparer);
+                if (compared == 0)
+                {
+                    var meta = GetMeta(i);
+                    index = i;
+                    valueOffset = meta.PageOffset + meta.KeyLength;
+                    valueLength = meta.ValueLength;
+                    return true;
+                }
+                if (compared > 0) break;
+            }
+
+            index = default;
+            valueOffset = default;
+            valueLength = default;
+            return false;
+        }
+
         var useDigest = hasKeyDigests && hasKeyDigest;
 
         var min = 0;
@@ -143,6 +174,59 @@ public bool TrySearch<TComparer>(
 #if  NETSTANDARD
         ref var pageReference = ref MemoryMarshal.GetReference(page);
 #endif
+        if (hasEytzingerDigests && hasKeyDigest)
+        {
+            // Branch-free descent to the rank of the first entry with digest >=
+            // keyDigest, then resolve the bound inside the run of equal digests with
+            // full comparisons. Entries past the run compare > key, which already
+            // satisfies both bound operators and terminates the walk.
+            var i = EytzingerLayout.LowerBoundRank(
+                ref pageReference, DigestBase, (metaBase - DigestBase) / sizeof(ulong), keyDigest);
+            switch (op)
+            {
+                case SearchOperator.Equal:
+                    for (; i < entryCount; i++)
+                    {
+                        var compared = CompareFull(ref pageReference, i, key, comparer);
+                        if (compared == 0)
+                        {
+                            index = i;
+                            return true;
+                        }
+                        if (compared > 0) break;
+                    }
+                    index = default;
+                    return false;
+
+                case SearchOperator.LowerBound:
+                    // first entry >= key
+                    while (i < entryCount && CompareFull(ref pageReference, i, key, comparer) < 0)
+                    {
+                        i++;
+                    }
+                    break;
+
+                case SearchOperator.UpperBound:
+                    // first entry > key
+                    while (i < entryCount && CompareFull(ref pageReference, i, key, comparer) <= 0)
+                    {
+                        i++;
+                    }
+                    break;
+
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(op), op, null);
+            }
+
+            if (i >= entryCount)
+            {
+                index = default;
+                return false;
+            }
+            index = i;
+            return true;
+        }
+
         var useDigest = hasKeyDigests && hasKeyDigest;
 
         var min = 0;

src/DryDB/BTree/NodeHeader.cs

@@ -21,6 +21,14 @@ static class NodeFlags
     /// entry metadata.
     /// </summary>
     public const int HasKeyDigests = 1 << 8;
+
+    /// <summary>
+    /// The digest array is stored as a complete binary tree in Eytzinger (BFS) order,
+    /// padded with <see cref="ulong.MaxValue"/> to 2^k - 1 slots, instead of sorted
+    /// order. Only valid together with <see cref="HasKeyDigests"/>. Introduced in
+    /// format 1.2.
+    /// </summary>
+    public const int EytzingerDigests = 1 << 9;
 }
 
 static class NodeHeaderExtensions
@@ -59,6 +67,12 @@ public bool HasKeyDigests
         get => ((int)Kind & NodeFlags.HasKeyDigests) != 0;
     }
 
+    public bool HasEytzingerDigests
+    {
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        get => ((int)Kind & NodeFlags.EytzingerDigests) != 0;
+    }
+
     [FieldOffset(4)]
     public fixed byte EntryCountBytes[4];