From d8893800193d146ce1d99ae7600026f4c4d70029 Mon Sep 17 00:00:00 2001 From: Syed Suhail Ahmed <114918598+suhailopensource@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:15:00 +0530 Subject: [PATCH] Add internal/nodepage: the on-disk byte layout of a B+Tree node Every link in the Phase A tree is a Go pointer, which is a RAM address, which means the tree cannot survive the process that built it. This adds a byte-for-byte page format where a link is a page number instead: page 12 is at byte 12*4096, in this process and in the next one. The layout is a slotted page, the shape Postgres and SQLite use. A 16-byte page header, then an 8-byte node header holding the next-leaf or first-child link, the level, and two bytes of padding that keep the slot array 4-byte aligned. Slots grow up from byte 24 at 4 bytes each, cells grow down from byte 4095, and free space is whatever lies between. The point of the indirection is that the slot array is sorted while the cells are not. Inserting a key in the middle shifts a few 4-byte slots and never moves a key byte, so binary search stays O(log n) while writes stay cheap. Growing the two regions toward each other also avoids having to decide a split between them up front, which would be wrong for every key length but one. A leaf cell is key bytes followed by a 6-byte row id; an internal cell is key bytes followed by a 4-byte child. The key length is never stored because the suffix is fixed per kind. An internal node with n keys has n+1 children and the extra one has no slot to sit in, so it lives in the node header, sharing four bytes with the leaf's next pointer and disambiguated by the kind byte. Fanout is now computed rather than estimated. With 4096-byte pages and 8-byte keys an internal node holds 254 keys and 255 children, a leaf holds 226, and three levels reach 14,695,650 keys. This corrects the README, which said roughly 340 per internal node from 4080/(8+4). That ignored the node header and the 4-byte slot each entry needs; overhead is 25% of fanout. MaxKeys and Fanout compute it, and a test fills a real page for every key length from 1 to 64 to confirm the arithmetic. Validate checks twelve structural facts, the strongest being that the cells exactly tile the region from free end to the end of the page with no gaps and no overlaps, and that slots are in strictly ascending key order. Eleven deliberate corruptions are asserted to fail it. TestGoldenLeafLayout pins fourteen exact byte ranges so the format cannot drift silently. FuzzNodePageSurvivesArbitraryKeys ran 2,382,065 executions, and a round trip through a real page.File asserts the bytes read back are identical to the bytes written. The B+Tree does not use this yet. 116 tests, 10 fuzz targets, race clean. --- Makefile | 3 + README.md | 150 +++++++++- cmd/scutedb-demo/main.go | 4 +- cmd/scutedb-demo/nodepage.go | 220 ++++++++++++++ internal/nodepage/describe.go | 179 +++++++++++ internal/nodepage/fuzz_test.go | 141 +++++++++ internal/nodepage/nodepage.go | 273 +++++++++++++++++ internal/nodepage/nodepage_test.go | 456 +++++++++++++++++++++++++++++ 8 files changed, 1421 insertions(+), 5 deletions(-) create mode 100644 cmd/scutedb-demo/nodepage.go create mode 100644 internal/nodepage/describe.go create mode 100644 internal/nodepage/fuzz_test.go create mode 100644 internal/nodepage/nodepage.go create mode 100644 internal/nodepage/nodepage_test.go diff --git a/Makefile b/Makefile index a87f783..6a95e3b 100644 --- a/Makefile +++ b/Makefile @@ -70,3 +70,6 @@ demo-range: demo-delete: go run ./cmd/scutedb-demo delete + +demo-nodepage: + go run ./cmd/scutedb-demo nodepage diff --git a/README.md b/README.md index 927f955..a386a7a 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ A *scute* is one of the bony plates that make up a turtle's shell. A shell is made of plates; a database file is made of pages. -**Status:** Phases 0 and A complete. The B+Tree searches, inserts, splits, -scans ranges and deletes. All of it still lives in memory — Phase B puts it on -disk. +**Status:** Phases 0 and A complete. Phase B has started: a B+Tree node now has +a byte-for-byte on-disk format, and a tree can be walked straight out of a file +using nothing but page numbers. --- @@ -20,6 +20,7 @@ make demo # list the runnable experiments make demo-scan # watch a file-based database degrade make hexdump # write real pages and look at the bytes make demo-delete # borrow, merge and root collapse, traced live +make demo-nodepage # decode a real B+Tree page out of a real file ``` --- @@ -30,7 +31,7 @@ make demo-delete # borrow, merge and root collapse, traced live |-------|------|--------| | **0** | Foundations — interfaces, the naive database, pages | **done** | | **A** | Bytes & the B+Tree | **done** | -| B | Persistence — storage manager, buffer pool, locking | **next** | +| B | Persistence — storage manager, buffer pool, locking | **in progress** | | C | A real data store — schema, rows, indexes | | | D | Transactions — WAL, recovery, 2PL, MVCC | | | E | Beyond — LSM engine, Raft, server, query planner | | @@ -661,6 +662,146 @@ Deliberate, each one is a later step: --- +## Phase B — Persistence + +### `0x09` Nodes as pages + +Every node in `0x06` to `0x08` was a Go struct held together by pointers. A +pointer is a RAM address: it is meaningless in the next process, so the entire +tree evaporates on exit. This step gives a node a byte layout and swaps every +pointer for a **page number**. + +``` +in memory on disk +children []*node 8 bytes each child core.PageID 4 bytes each + a RAM address a page number + valid until exit valid forever +``` + +Page 12 is at byte `12 * 4096 = 49152` — in this process, in the next one, and on +another machine. That single swap is the whole step; everything below follows +from it. + +**The layout.** `internal/nodepage` is a *slotted page*, the same shape Postgres +and SQLite use: + +``` +region bytes holds +page header 0..15 id, kind, key count, free start, free end +node header 16..23 next leaf / first child, level, 2 pad +slot array 24.. 4 bytes per entry, in KEY order +free space shrinks from both ends +cells ..4095 key + payload, in INSERTION order +``` + +Slots grow up from byte 24, cells grow down from byte 4095, and free space is +whatever is left between them. The trick worth noticing: **the slot array is +sorted, the cells are not.** A slot is 4 bytes (`offset`, `length`) and inserting +a key in the middle means shifting a few 4-byte slots, never the key bytes +themselves. Binary search reads the slot array, so lookups stay `O(log n)` while +writes stay cheap. + +**Two cell shapes**, and the key length never needs storing because the suffix is +fixed: + +``` +leaf cell key bytes || row page (4) || row slot (2) key = len - 6 +internal cell key bytes || child page (4) key = len - 4 +``` + +An internal node with *n* keys has *n+1* children. The extra child has no slot to +live in, so it lives in the node header — that is what `first child` is. + +**Fanout is computed, not chosen.** With a 4096-byte page and an 8-byte key: + +``` + page header 16 + node header 8 + left for entries 4072 + + internal entry = 4 slot + 8 key + 4 child = 16 bytes -> 254 keys, 255 children + leaf entry = 4 slot + 8 key + 6 row id = 18 bytes -> 226 keys + + levels reads per get keys it holds + 1 1 226 + 2 2 57,630 + 3 3 14,695,650 + 4 4 3,747,390,750 +``` + +This corrects a number stated earlier in this file. `0x06` estimated ~340 keys +per internal node from `4080 / (8 + 4)`, which quietly ignored the node header +and the 4-byte slot every entry needs. The real figure is **254** — the +bookkeeping costs 25% of the fanout. The estimate was optimistic in the usual +direction: it counted the data and forgot the structure that makes the data +findable. + +**A real page, out of a real file.** Three pages written to disk, then read back +with `xxd` — no ScuteDB code involved in the reading: + +``` +$ xxd -s 4096 -l 32 tree.db +00001000: 0000 0001 0300 0002 0020 0fe4 0000 0000 ......... ...... +00001010: 0000 0002 0000 0000 0ff2 000e 0fe4 000e ................ +``` + +Decoding it by hand, left to right: + +| bytes | value | meaning | +|---|---|---| +| `0000 0001` | 1 | page id | +| `03` | 3 | kind, `btree-leaf` | +| `00` | 0 | flags | +| `0002` | 2 | key count | +| `0020` | 32 | free start, `24 + 2 slots x 4` | +| `0fe4` | 4068 | free end, where the lowest cell begins | +| `0000 0000` | — | reserved for the checksum in `0x13` | +| `0000 0002` | 2 | next leaf is page 2 | +| `0000` | 0 | level, 0 means leaf | +| `0000` | — | padding, keeps slots 4-byte aligned | +| `0ff2 000e` | 4082, 14 | slot 0 points at a 14-byte cell | +| `0fe4 000e` | 4068, 14 | slot 1 points at a 14-byte cell | + +Slot 0 says byte 4082 of page 1, which is file offset `4096 + 4082 = 8178`: + +``` +$ xxd -s 8178 -l 14 tree.db +00001ff2: 7fff ffff ffff ffff 0000 0064 0000 ...........d.. +``` + +`7F FF FF FF FF FF FF FF` is the key **-1**, sign bit flipped by the ordered +encoding from `0x03`. Then `0000 0064` is row page 100, and `0000` is slot 0. +Every layer built so far is visible in those fourteen bytes. + +**Walking the tree with no pointers at all**, reading pages back from the file: + +``` +looking up 30, starting at page 0 + read page 0 at byte offset 0, kind btree-internal, 1 keys + not a leaf, so follow a page id: next is page 2 + read page 2 at byte offset 8192, kind btree-leaf, 2 keys + found at slot 1 -> row id page 200 slot 30 +``` + +**How the real ones do it.** Postgres calls a page number a `BlockNumber`, a +`uint32` index into the relation file, and its B-Tree pages carry a `btpo_level` +exactly like the level field here. SQLite uses 1-based 32-bit page numbers, with +page 1 always holding the schema. InnoDB uses 32-bit page numbers inside a +tablespace with 16 KB pages. Nobody stores an address. + +**How it is verified.** `Validate` checks twelve structural facts, including that +the cells exactly tile the region from `free end` to the end of the page with no +gaps and no overlaps, and that the slot array is in strictly ascending key order. +Eleven separate corruptions are asserted to fail it. `TestGoldenLeafLayout` +pins fourteen exact byte ranges so the format cannot drift silently, and +`TestMaxKeysMatchesWhatActuallyFits` fills a page for every key length from 1 to +64 and checks the count against the arithmetic. A fuzz target built pages from +arbitrary keys for 2.38M executions. + +Run it: `make demo-nodepage`. + +--- + ## Layout ``` @@ -676,6 +817,7 @@ internal/ nullbits/ null bitmaps and SQL three-valued logic slots/ fixed-size, aligned record slots inside a page btree/ in-memory B+Tree: search, insert, split, range scans, delete + nodepage/ the on-disk byte layout of a B+Tree node ``` ## Conventions diff --git a/cmd/scutedb-demo/main.go b/cmd/scutedb-demo/main.go index 51f85a5..26af582 100644 --- a/cmd/scutedb-demo/main.go +++ b/cmd/scutedb-demo/main.go @@ -27,7 +27,7 @@ import ( func main() { if len(os.Args) < 2 { - fmt.Println("usage: scutedb-demo scan|update|race|crash|pages|header|encode|nulls|align|btree|range|delete") + fmt.Println("usage: scutedb-demo scan|update|race|crash|pages|header|encode|nulls|align|btree|range|delete|nodepage") os.Exit(2) } switch os.Args[1] { @@ -55,6 +55,8 @@ func main() { expRange() case "delete": expDelete() + case "nodepage": + expNodePage() case "crash-child": crashChild(os.Args[2]) default: diff --git a/cmd/scutedb-demo/nodepage.go b/cmd/scutedb-demo/nodepage.go new file mode 100644 index 0000000..ca8e9c7 --- /dev/null +++ b/cmd/scutedb-demo/nodepage.go @@ -0,0 +1,220 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "unsafe" + + "github.com/scute-db/scutedb/internal/codec" + "github.com/scute-db/scutedb/internal/core" + "github.com/scute-db/scutedb/internal/nodepage" + "github.com/scute-db/scutedb/internal/page" +) + +func expNodePage() { + fmt.Print("STEP 0x09 Nodes as pages\n\n") + + section1() + section2() + section3() + section4() + section5() +} + +func nodeKey(v int64) []byte { return codec.AppendKeyInt64(nil, v) } + +func section1() { + fmt.Print("1. WHY A POINTER CANNOT GO ON DISK\n\n") + + type inMemoryNode struct { + children []*int + next *int + } + var n inMemoryNode + + fmt.Printf(" an in-memory child link is a Go pointer: %d bytes\n", unsafe.Sizeof(n.next)) + fmt.Print(" it holds a RAM address, valid only while this process lives.\n") + fmt.Print(" write it to disk, restart, and it points at someone else's memory.\n\n") + fmt.Printf(" an on-disk child link is a core.PageID: %d bytes\n", unsafe.Sizeof(core.PageID(0))) + fmt.Print(" it holds a page number. page 12 is at byte 12 * 4096 = 49152,\n") + fmt.Print(" in this process, in the next one, and on another machine.\n\n") + fmt.Print(" that swap is the whole step. everything else follows from it.\n\n") +} + +func section2() { + fmt.Print("2. FANOUT IS COMPUTED, NOT CHOSEN\n\n") + fmt.Print(indent(nodepage.FanoutTable(8), " ")) + + old := page.Usable / (8 + 4) + real := nodepage.MaxKeys(page.KindBTreeInternal, 8) + fmt.Printf(" an earlier estimate in this repo said %d keys per internal node:\n", old) + fmt.Printf(" %d usable bytes / (8 key + 4 child) = %d.\n", page.Usable, old) + fmt.Printf(" the real answer is %d, because that estimate forgot the %d-byte\n", real, nodepage.NodeHeaderSize) + fmt.Printf(" node header and the %d-byte slot per entry. overhead is %d%% of fanout.\n\n", + nodepage.SlotSize, (old-real)*100/old) +} + +func section3() { + fmt.Print("3. A REAL LEAF PAGE, BYTE FOR BYTE\n\n") + + leaf := nodepage.NewLeaf(1) + leaf.SetNext(2) + for _, v := range []int64{-1, 10, 20} { + if err := leaf.AppendLeaf(nodeKey(v), core.RowID{Page: core.PageID(v + 2), Slot: uint16(v + 1)}); err != nil { + fail(err) + } + } + if err := leaf.Validate(); err != nil { + fail(err) + } + fmt.Print(indent(leaf.Describe(), " ")) + + fmt.Print(" read the first cell yourself:\n") + lo := int(leaf.FreeEnd()) + 2*(8+nodepage.LeafSuffix) + fmt.Printf(" slot 0 says the cell is at %d. bytes %d..%d are\n", lo, lo, lo+13) + fmt.Printf(" % 02X\n", []byte(leaf.Page[lo:lo+14])) + fmt.Print(" first 8 bytes = the key, big-endian with the sign bit flipped\n") + fmt.Print(" 7F FF FF FF FF FF FF FF is -1 from step 0x03\n") + fmt.Print(" next 4 bytes = row id page number\n") + fmt.Print(" last 2 bytes = row id slot number\n\n") +} + +func section4() { + fmt.Print("4. AN INTERNAL PAGE HOLDS N KEYS AND N+1 CHILDREN\n\n") + + root, err := nodepage.NewInternal(0, 1) + if err != nil { + fail(err) + } + root.SetFirstChild(1) + if err := root.AppendInternal(nodeKey(20), 2); err != nil { + fail(err) + } + if err := root.Validate(); err != nil { + fail(err) + } + fmt.Print(indent(root.Describe(), " ")) + fmt.Print(" one key, two children. the extra child has nowhere to sit in the\n") + fmt.Print(" slot array, so it lives in the node header instead.\n\n") +} + +func section5() { + fmt.Print("5. WALKING A TREE THAT IS ONLY BYTES ON DISK\n\n") + + dir, err := os.MkdirTemp("", "scutedb-nodepage") + if err != nil { + fail(err) + } + defer os.RemoveAll(dir) + path := filepath.Join(dir, "tree.db") + + pf, err := page.Open(path) + if err != nil { + fail(err) + } + + root, err := nodepage.NewInternal(0, 1) + if err != nil { + fail(err) + } + root.SetFirstChild(1) + if err := root.AppendInternal(nodeKey(20), 2); err != nil { + fail(err) + } + + left := nodepage.NewLeaf(1) + left.SetNext(2) + for _, v := range []int64{-1, 10} { + if err := left.AppendLeaf(nodeKey(v), core.RowID{Page: 100, Slot: uint16(v + 1)}); err != nil { + fail(err) + } + } + + right := nodepage.NewLeaf(2) + for _, v := range []int64{20, 30} { + if err := right.AppendLeaf(nodeKey(v), core.RowID{Page: 200, Slot: uint16(v)}); err != nil { + fail(err) + } + } + + for _, n := range []nodepage.Node{root, left, right} { + if err := n.Validate(); err != nil { + fail(err) + } + if err := pf.Write(n.Page); err != nil { + fail(err) + } + } + if err := pf.Sync(); err != nil { + fail(err) + } + if err := pf.Close(); err != nil { + fail(err) + } + + info, err := os.Stat(path) + if err != nil { + fail(err) + } + fmt.Printf(" wrote %s: %d bytes = %d pages of %d\n\n", filepath.Base(path), info.Size(), + info.Size()/page.Size, page.Size) + + reopened, err := page.Open(path) + if err != nil { + fail(err) + } + defer reopened.Close() + + for _, want := range []int64{10, 30} { + fmt.Printf(" looking up %d, starting at page 0\n", want) + id := core.PageID(0) + for { + raw, err := reopened.Read(id) + if err != nil { + fail(err) + } + n, err := nodepage.Load(raw) + if err != nil { + fail(err) + } + fmt.Printf(" read page %d at byte offset %d, kind %s, %d keys\n", + id, page.Offset(id), n.Kind(), n.KeyCount()) + if n.Leaf() { + at, found := n.Search(nodeKey(want)) + if !found { + fmt.Printf(" %d is not in this leaf\n\n", want) + break + } + r := n.Row(at) + fmt.Printf(" found at slot %d -> row id page %d slot %d\n\n", at, r.Page, r.Slot) + break + } + id = n.ChildFor(nodeKey(want)) + fmt.Printf(" not a leaf, so follow a page id: next is page %d\n", id) + } + } + + fmt.Print(" nothing above was a pointer. every hop was a page number\n") + fmt.Print(" turned into a byte offset by multiplying by 4096.\n\n") + fmt.Print(" this is how the real ones do it too:\n") + fmt.Print(" postgres BlockNumber, a uint32 page index into the relation file\n") + fmt.Print(" sqlite 32-bit page numbers, 1-based, page 1 is the schema root\n") + fmt.Print(" innodb 32-bit page numbers inside a tablespace, 16 KB pages\n\n") +} + +func indent(s string, with string) string { + out := with + for i := 0; i < len(s); i++ { + out += string(s[i]) + if s[i] == '\n' && i != len(s)-1 { + out += with + } + } + return out + "\n" +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, "demo failed:", err) + os.Exit(1) +} diff --git a/internal/nodepage/describe.go b/internal/nodepage/describe.go new file mode 100644 index 0000000..93179f4 --- /dev/null +++ b/internal/nodepage/describe.go @@ -0,0 +1,179 @@ +package nodepage + +import ( + "fmt" + "strings" + + "github.com/scute-db/scutedb/internal/page" +) + +func Hexdump(b []byte, base int) string { + var s strings.Builder + for off := 0; off < len(b); off += 16 { + end := off + 16 + if end > len(b) { + end = len(b) + } + line := b[off:end] + fmt.Fprintf(&s, "%04X ", base+off) + for i := 0; i < 16; i++ { + if i == 8 { + s.WriteByte(' ') + } + if i < len(line) { + fmt.Fprintf(&s, "%02X ", line[i]) + } else { + s.WriteString(" ") + } + } + s.WriteString(" |") + for _, c := range line { + if c >= 0x20 && c < 0x7F { + s.WriteByte(c) + } else { + s.WriteByte('.') + } + } + s.WriteString("|\n") + } + return s.String() +} + +func (n Node) Map() string { + var s strings.Builder + count := n.KeyCount() + fmt.Fprintf(&s, "%-14s %-13s %s\n", "region", "bytes", "holds") + fmt.Fprintln(&s, strings.Repeat("-", 62)) + fmt.Fprintf(&s, "%-14s %-13s %s\n", "page header", + fmt.Sprintf("0..%d", page.HeaderSize-1), "id, kind, item count, free start, free end") + fmt.Fprintf(&s, "%-14s %-13s %s\n", "node header", + fmt.Sprintf("%d..%d", page.HeaderSize, SlotArrayStart-1), n.linkMeaning()+", level, 2 pad") + fmt.Fprintf(&s, "%-14s %-13s %s\n", "slot array", + fmt.Sprintf("%d..%d", SlotArrayStart, int(n.FreeStart())-1), + fmt.Sprintf("%d slots x %d bytes, in key order", count, SlotSize)) + fmt.Fprintf(&s, "%-14s %-13s %s\n", "free space", + fmt.Sprintf("%d..%d", n.FreeStart(), int(n.FreeEnd())-1), + fmt.Sprintf("%d bytes, grows shut from both ends", n.FreeSpace())) + fmt.Fprintf(&s, "%-14s %-13s %s\n", "cells", + fmt.Sprintf("%d..%d", n.FreeEnd(), page.Size-1), + fmt.Sprintf("%d cells, packed from the end, insertion order", count)) + return s.String() +} + +func (n Node) linkMeaning() string { + if n.Leaf() { + return "next leaf" + } + return "first child" +} + +func (n Node) Describe() string { + var s strings.Builder + + fmt.Fprintf(&s, "page %d %s level %d %d keys\n\n", n.ID(), n.Kind(), n.Level(), n.KeyCount()) + s.WriteString(n.Map()) + + fmt.Fprintf(&s, "\nheaders, byte for byte\n\n") + fmt.Fprint(&s, Hexdump(n.Page[:SlotArrayStart], 0)) + fmt.Fprintf(&s, "\n%-9s %-14s %s\n", "offset", "bytes", "meaning") + fmt.Fprintln(&s, strings.Repeat("-", 62)) + row := func(off, size int, meaning string) { + fmt.Fprintf(&s, "%-9s %-14s %s\n", + fmt.Sprintf("%d..%d", off, off+size-1), + fmt.Sprintf("% 02X", n.Page[off:off+size]), meaning) + } + row(0, 4, fmt.Sprintf("page id = %d", n.ID())) + row(4, 1, fmt.Sprintf("kind = %d (%s)", n.Page[4], n.Kind())) + row(5, 1, fmt.Sprintf("flags = %08b", n.Flags())) + row(6, 2, fmt.Sprintf("key count = %d", n.KeyCount())) + row(8, 2, fmt.Sprintf("free start = %d (slot array ends here)", n.FreeStart())) + row(10, 2, fmt.Sprintf("free end = %d (lowest cell starts here)", n.FreeEnd())) + row(12, 4, "reserved for the checksum in 0x13") + row(offLink, 4, fmt.Sprintf("%s = page %d", n.linkMeaning(), n.link())) + row(offLevel, 2, fmt.Sprintf("level = %d (0 means leaf)", n.Level())) + row(offPad, 2, "padding, keeps the slot array 4-byte aligned") + + count := n.KeyCount() + if count == 0 { + fmt.Fprintf(&s, "\nno entries\n") + return s.String() + } + + fmt.Fprintf(&s, "\nslot array at %d, %d slots of %d bytes\n\n", SlotArrayStart, count, SlotSize) + fmt.Fprint(&s, Hexdump(n.Page[SlotArrayStart:n.FreeStart()], SlotArrayStart)) + fmt.Fprintf(&s, "\n%-6s %-9s %-12s %-9s %s\n", "slot", "at", "bytes", "cell at", "length") + fmt.Fprintln(&s, strings.Repeat("-", 62)) + for i := 0; i < count; i++ { + at := n.slotOffset(i) + lo, hi := n.cellBounds(i) + fmt.Fprintf(&s, "%-6d %-9d %-12s %-9d %d\n", + i, at, fmt.Sprintf("% 02X", n.Page[at:at+SlotSize]), lo, hi-lo) + } + + fmt.Fprintf(&s, "\ncells from %d to %d, packed downward\n\n", n.FreeEnd(), page.Size-1) + fmt.Fprint(&s, Hexdump(n.Page[n.FreeEnd():], int(n.FreeEnd()))) + + fmt.Fprintf(&s, "\n%-6s %-9s %-22s %s\n", "slot", "cell at", "key bytes", "then") + fmt.Fprintln(&s, strings.Repeat("-", 68)) + for i := 0; i < count; i++ { + lo, _ := n.cellBounds(i) + key := n.KeyRef(i) + var tail string + if n.Leaf() { + r := n.Row(i) + tail = fmt.Sprintf("row id -> page %d slot %d", r.Page, r.Slot) + } else { + tail = fmt.Sprintf("child -> page %d", n.Child(i+1)) + } + fmt.Fprintf(&s, "%-6d %-9d %-22s %s\n", i, lo, fmt.Sprintf("% 02X", key), tail) + } + + if !n.Leaf() { + fmt.Fprintf(&s, "\n%d keys, %d children\n", count, n.ChildCount()) + fmt.Fprintf(&s, " child 0 = page %d (from the node header, keys below key 0)\n", n.Child(0)) + for i := 1; i <= count; i++ { + fmt.Fprintf(&s, " child %d = page %d (from slot %d, keys >= % 02X)\n", + i, n.Child(i), i-1, n.KeyRef(i-1)) + } + } + return s.String() +} + +func FanoutTable(keyLen int) string { + var s strings.Builder + fmt.Fprintf(&s, "page %d bytes, key %d bytes\n\n", page.Size, keyLen) + fmt.Fprintf(&s, " page header %4d\n", page.HeaderSize) + fmt.Fprintf(&s, " node header %4d\n", NodeHeaderSize) + fmt.Fprintf(&s, " left for entries %4d\n\n", EntryArea) + + internal := MaxKeys(page.KindBTreeInternal, keyLen) + leaf := MaxKeys(page.KindBTreeLeaf, keyLen) + fmt.Fprintf(&s, " internal entry = %d slot + %d key + %d child = %d bytes -> %d keys, %d children\n", + SlotSize, keyLen, InternalSuffix, SlotSize+keyLen+InternalSuffix, internal, internal+1) + fmt.Fprintf(&s, " leaf entry = %d slot + %d key + %d row id = %d bytes -> %d keys\n\n", + SlotSize, keyLen, LeafSuffix, SlotSize+keyLen+LeafSuffix, leaf) + + fmt.Fprintf(&s, " %-8s %-14s %s\n", "levels", "reads per get", "keys it holds") + fmt.Fprintln(&s, " "+strings.Repeat("-", 46)) + capacity := float64(leaf) + for level := 1; level <= 4; level++ { + fmt.Fprintf(&s, " %-8d %-14d %s\n", level, level, commas(capacity)) + capacity *= float64(internal + 1) + } + return s.String() +} + +func commas(v float64) string { + s := fmt.Sprintf("%.0f", v) + if len(s) > 15 { + return fmt.Sprintf("%.3g", v) + } + var out []byte + for i, c := range []byte(s) { + if i > 0 && (len(s)-i)%3 == 0 { + out = append(out, ',') + } + out = append(out, c) + } + return string(out) +} diff --git a/internal/nodepage/fuzz_test.go b/internal/nodepage/fuzz_test.go new file mode 100644 index 0000000..95694cf --- /dev/null +++ b/internal/nodepage/fuzz_test.go @@ -0,0 +1,141 @@ +package nodepage + +import ( + "bytes" + "errors" + "sort" + "testing" + + "github.com/scute-db/scutedb/internal/core" + "github.com/scute-db/scutedb/internal/page" +) + +func FuzzNodePageSurvivesArbitraryKeys(f *testing.F) { + f.Add([]byte("abcdefghijklmnop"), uint8(4), true) + f.Add([]byte{0, 0, 0, 1, 0, 0, 0, 2}, uint8(4), false) + f.Add([]byte{0xFF}, uint8(1), true) + + f.Fuzz(func(t *testing.T, raw []byte, width uint8, leaf bool) { + w := int(width)%40 + 1 + if len(raw) < w { + t.Skip() + } + + var keys [][]byte + for i := 0; i+w <= len(raw); i += w { + keys = append(keys, raw[i:i+w]) + } + sort.Slice(keys, func(a, b int) bool { return bytes.Compare(keys[a], keys[b]) < 0 }) + uniq := keys[:0] + for i, k := range keys { + if i == 0 || !bytes.Equal(k, keys[i-1]) { + uniq = append(uniq, k) + } + } + keys = uniq + if len(keys) == 0 { + t.Skip() + } + + var n Node + var err error + if leaf { + n = NewLeaf(1) + } else { + n, err = NewInternal(1, 1) + if err != nil { + t.Fatal(err) + } + n.SetFirstChild(1000) + } + + written := 0 + for i, k := range keys { + if leaf { + err = n.AppendLeaf(k, core.RowID{Page: core.PageID(i), Slot: uint16(i)}) + } else { + err = n.AppendInternal(k, core.PageID(2000+i)) + } + if err != nil { + if !errors.Is(err, ErrNoFit) { + t.Fatalf("append %d failed with %v, want ErrNoFit", i, err) + } + break + } + written++ + } + + if err := n.Validate(); err != nil { + t.Fatalf("Validate after %d appends: %v", written, err) + } + if n.KeyCount() != written { + t.Fatalf("key count %d, appended %d", n.KeyCount(), written) + } + + max := MaxKeys(n.Kind(), w) + if written > max { + t.Fatalf("fitted %d keys of %d bytes, MaxKeys says at most %d", written, w, max) + } + + for i := 0; i < written; i++ { + if !bytes.Equal(n.KeyRef(i), keys[i]) { + t.Fatalf("key %d is % 02X, want % 02X", i, n.KeyRef(i), keys[i]) + } + at, found := n.Search(keys[i]) + if !found || at != i { + t.Fatalf("Search for key %d returned (%d, %v)", i, at, found) + } + if leaf { + if want := (core.RowID{Page: core.PageID(i), Slot: uint16(i)}); n.Row(i) != want { + t.Fatalf("row %d is %+v, want %+v", i, n.Row(i), want) + } + } else if want := core.PageID(2000 + i); n.Child(i+1) != want { + t.Fatalf("child %d is %d, want %d", i+1, n.Child(i+1), want) + } + } + + if !leaf && written > 0 { + if n.Child(0) != 1000 { + t.Fatalf("first child is %d, want 1000", n.Child(0)) + } + if got := n.ChildFor(keys[0]); got != 2000 { + t.Fatalf("ChildFor(first key) = %d, want 2000", got) + } + } + }) +} + +func BenchmarkSearchFullPage(b *testing.B) { + n := NewLeaf(1) + i := 0 + for n.AppendLeaf(key(i), rid(i)) == nil { + i++ + } + b.ReportMetric(float64(n.KeyCount()), "keys/page") + probe := key(i / 2) + b.ResetTimer() + for j := 0; j < b.N; j++ { + if _, found := n.Search(probe); !found { + b.Fatal("probe key vanished") + } + } +} + +func BenchmarkDecodeAndValidate(b *testing.B) { + n := NewLeaf(1) + i := 0 + for n.AppendLeaf(key(i), rid(i)) == nil { + i++ + } + raw := page.Page(append([]byte(nil), n.Page...)) + b.ResetTimer() + for j := 0; j < b.N; j++ { + loaded, err := Load(raw) + if err != nil { + b.Fatal(err) + } + if err := loaded.Validate(); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/nodepage/nodepage.go b/internal/nodepage/nodepage.go new file mode 100644 index 0000000..e18b92a --- /dev/null +++ b/internal/nodepage/nodepage.go @@ -0,0 +1,273 @@ +package nodepage + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "sort" + + "github.com/scute-db/scutedb/internal/core" + "github.com/scute-db/scutedb/internal/page" +) + +const ( + NodeHeaderSize = 8 + SlotSize = 4 + SlotArrayStart = page.HeaderSize + NodeHeaderSize + EntryArea = page.Size - SlotArrayStart + + LeafSuffix = 6 + InternalSuffix = 4 +) + +const ( + offLink = page.HeaderSize + offLevel = page.HeaderSize + 4 + offPad = page.HeaderSize + 6 +) + +var ( + ErrNoFit = errors.New("scutedb/nodepage: entry does not fit") + ErrBadKind = errors.New("scutedb/nodepage: not a b+tree node page") + ErrBadLevel = errors.New("scutedb/nodepage: level does not match kind") + ErrInvariant = errors.New("scutedb/nodepage: invariant violated") +) + +type Node struct{ page.Page } + +func NewLeaf(id core.PageID) Node { + p := page.New(id, page.KindBTreeLeaf) + p.SetFreeStart(SlotArrayStart) + return Node{p} +} + +func NewInternal(id core.PageID, level uint16) (Node, error) { + if level == 0 { + return Node{}, fmt.Errorf("%w: internal node at level 0", ErrBadLevel) + } + p := page.New(id, page.KindBTreeInternal) + p.SetFreeStart(SlotArrayStart) + n := Node{p} + n.setLevel(level) + return n, nil +} + +func Load(p page.Page) (Node, error) { + if len(p) != page.Size { + return Node{}, core.ErrShortPage + } + k := p.Kind() + if k != page.KindBTreeLeaf && k != page.KindBTreeInternal { + return Node{}, fmt.Errorf("%w: kind is %s", ErrBadKind, k) + } + return Node{p}, nil +} + +func (n Node) Leaf() bool { return n.Kind() == page.KindBTreeLeaf } +func (n Node) KeyCount() int { return int(n.ItemCount()) } +func (n Node) Level() uint16 { return binary.BigEndian.Uint16(n.Page[offLevel:]) } +func (n Node) ChildCount() int { + if n.Leaf() { + return 0 + } + return n.KeyCount() + 1 +} + +func (n Node) setLevel(l uint16) { binary.BigEndian.PutUint16(n.Page[offLevel:], l) } + +func (n Node) link() core.PageID { return core.PageID(binary.BigEndian.Uint32(n.Page[offLink:])) } +func (n Node) setLink(id core.PageID) { binary.BigEndian.PutUint32(n.Page[offLink:], uint32(id)) } + +func (n Node) Next() core.PageID { return n.link() } +func (n Node) SetNext(id core.PageID) { n.setLink(id) } +func (n Node) FirstChild() core.PageID { return n.link() } +func (n Node) SetFirstChild(id core.PageID) { n.setLink(id) } + +func (n Node) suffixLen() int { + if n.Leaf() { + return LeafSuffix + } + return InternalSuffix +} + +func (n Node) slotOffset(i int) int { return SlotArrayStart + i*SlotSize } + +func (n Node) cellBounds(i int) (int, int) { + s := n.slotOffset(i) + off := int(binary.BigEndian.Uint16(n.Page[s:])) + length := int(binary.BigEndian.Uint16(n.Page[s+2:])) + return off, off + length +} + +func (n Node) cell(i int) []byte { + lo, hi := n.cellBounds(i) + return n.Page[lo:hi] +} + +func (n Node) KeyRef(i int) []byte { + c := n.cell(i) + return c[:len(c)-n.suffixLen()] +} + +func (n Node) Key(i int) []byte { + return append([]byte(nil), n.KeyRef(i)...) +} + +func (n Node) Row(i int) core.RowID { + c := n.cell(i) + tail := c[len(c)-LeafSuffix:] + return core.RowID{ + Page: core.PageID(binary.BigEndian.Uint32(tail)), + Slot: binary.BigEndian.Uint16(tail[4:]), + } +} + +func (n Node) Child(i int) core.PageID { + if i == 0 { + return n.FirstChild() + } + c := n.cell(i - 1) + return core.PageID(binary.BigEndian.Uint32(c[len(c)-InternalSuffix:])) +} + +func (n Node) EntrySize(keyLen int) int { return SlotSize + keyLen + n.suffixLen() } + +func (n Node) Fits(keyLen int) bool { return n.EntrySize(keyLen) <= n.FreeSpace() } + +func (n Node) appendCell(key []byte, tail []byte) error { + need := SlotSize + len(key) + len(tail) + if need > n.FreeSpace() { + return fmt.Errorf("%w: needs %d bytes, %d free", ErrNoFit, need, n.FreeSpace()) + } + start := int(n.FreeEnd()) - len(key) - len(tail) + copy(n.Page[start:], key) + copy(n.Page[start+len(key):], tail) + + s := n.slotOffset(n.KeyCount()) + binary.BigEndian.PutUint16(n.Page[s:], uint16(start)) + binary.BigEndian.PutUint16(n.Page[s+2:], uint16(len(key)+len(tail))) + + n.SetFreeEnd(uint16(start)) + n.SetFreeStart(uint16(s + SlotSize)) + n.SetItemCount(n.ItemCount() + 1) + return nil +} + +func (n Node) AppendLeaf(key []byte, rid core.RowID) error { + if !n.Leaf() { + return fmt.Errorf("%w: AppendLeaf on an internal node", ErrBadKind) + } + var tail [LeafSuffix]byte + binary.BigEndian.PutUint32(tail[:], uint32(rid.Page)) + binary.BigEndian.PutUint16(tail[4:], rid.Slot) + return n.appendCell(key, tail[:]) +} + +func (n Node) AppendInternal(key []byte, child core.PageID) error { + if n.Leaf() { + return fmt.Errorf("%w: AppendInternal on a leaf node", ErrBadKind) + } + var tail [InternalSuffix]byte + binary.BigEndian.PutUint32(tail[:], uint32(child)) + return n.appendCell(key, tail[:]) +} + +func (n Node) Search(key []byte) (int, bool) { + count := n.KeyCount() + i := sort.Search(count, func(i int) bool { return bytes.Compare(n.KeyRef(i), key) >= 0 }) + if i < count && bytes.Equal(n.KeyRef(i), key) { + return i, true + } + return i, false +} + +func (n Node) ChildFor(key []byte) core.PageID { + i := sort.Search(n.KeyCount(), func(i int) bool { return bytes.Compare(key, n.KeyRef(i)) < 0 }) + return n.Child(i) +} + +func MaxKeys(kind page.Kind, keyLen int) int { + suffix := InternalSuffix + if kind == page.KindBTreeLeaf { + suffix = LeafSuffix + } + per := SlotSize + keyLen + suffix + if per <= 0 { + return 0 + } + return EntryArea / per +} + +func Fanout(keyLen int) int { return MaxKeys(page.KindBTreeInternal, keyLen) + 1 } + +func (n Node) Validate() error { + k := n.Kind() + if k != page.KindBTreeLeaf && k != page.KindBTreeInternal { + return fmt.Errorf("%w: kind is %s", ErrBadKind, k) + } + if n.Leaf() != (n.Level() == 0) { + return fmt.Errorf("%w: kind %s at level %d", ErrBadLevel, k, n.Level()) + } + if got := binary.BigEndian.Uint16(n.Page[offPad:]); got != 0 { + return fmt.Errorf("%w: padding bytes are %d, want 0", ErrInvariant, got) + } + + count := n.KeyCount() + wantStart := SlotArrayStart + count*SlotSize + if int(n.FreeStart()) != wantStart { + return fmt.Errorf("%w: free start is %d, %d slots end at %d", + ErrInvariant, n.FreeStart(), count, wantStart) + } + if int(n.FreeEnd()) > page.Size || n.FreeEnd() < n.FreeStart() { + return fmt.Errorf("%w: free end %d is outside [%d, %d]", + ErrInvariant, n.FreeEnd(), n.FreeStart(), page.Size) + } + + if count == 0 { + if int(n.FreeEnd()) != page.Size { + return fmt.Errorf("%w: empty node has free end %d, want %d", + ErrInvariant, n.FreeEnd(), page.Size) + } + return nil + } + + suffix := n.suffixLen() + spans := make([][2]int, count) + for i := 0; i < count; i++ { + lo, hi := n.cellBounds(i) + if lo < int(n.FreeStart()) || hi > page.Size || hi <= lo { + return fmt.Errorf("%w: slot %d spans [%d, %d), outside [%d, %d)", + ErrInvariant, i, lo, hi, n.FreeStart(), page.Size) + } + if hi-lo < suffix+1 { + return fmt.Errorf("%w: slot %d holds %d bytes, too few for a %d-byte suffix and a key", + ErrInvariant, i, hi-lo, suffix) + } + spans[i] = [2]int{lo, hi} + } + + sort.Slice(spans, func(a, b int) bool { return spans[a][0] < spans[b][0] }) + if spans[0][0] != int(n.FreeEnd()) { + return fmt.Errorf("%w: lowest cell starts at %d, free end says %d", + ErrInvariant, spans[0][0], n.FreeEnd()) + } + for i := 1; i < count; i++ { + if spans[i-1][1] != spans[i][0] { + return fmt.Errorf("%w: cells [%d, %d) and [%d, %d) are not adjacent", + ErrInvariant, spans[i-1][0], spans[i-1][1], spans[i][0], spans[i][1]) + } + } + if spans[count-1][1] != page.Size { + return fmt.Errorf("%w: highest cell ends at %d, want %d", + ErrInvariant, spans[count-1][1], page.Size) + } + + for i := 1; i < count; i++ { + if bytes.Compare(n.KeyRef(i-1), n.KeyRef(i)) >= 0 { + return fmt.Errorf("%w: key %d (%X) is not less than key %d (%X)", + ErrInvariant, i-1, n.KeyRef(i-1), i, n.KeyRef(i)) + } + } + return nil +} diff --git a/internal/nodepage/nodepage_test.go b/internal/nodepage/nodepage_test.go new file mode 100644 index 0000000..a5c585f --- /dev/null +++ b/internal/nodepage/nodepage_test.go @@ -0,0 +1,456 @@ +package nodepage + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "path/filepath" + "testing" + + "github.com/scute-db/scutedb/internal/core" + "github.com/scute-db/scutedb/internal/page" +) + +func key(i int) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(i)) + return b +} + +func rid(i int) core.RowID { + return core.RowID{Page: core.PageID(i / 10), Slot: uint16(i % 10)} +} + +func TestLeafRoundTrip(t *testing.T) { + n := NewLeaf(7) + n.SetNext(9) + for i := 0; i < 50; i++ { + if err := n.AppendLeaf(key(i), rid(i)); err != nil { + t.Fatalf("AppendLeaf(%d): %v", i, err) + } + } + if err := n.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if n.ID() != 7 || n.Next() != 9 || n.Level() != 0 || !n.Leaf() { + t.Fatalf("header round trip failed: id %d next %d level %d leaf %v", + n.ID(), n.Next(), n.Level(), n.Leaf()) + } + if n.KeyCount() != 50 { + t.Fatalf("key count %d, want 50", n.KeyCount()) + } + for i := 0; i < 50; i++ { + if !bytes.Equal(n.Key(i), key(i)) { + t.Fatalf("key %d is % 02X, want % 02X", i, n.Key(i), key(i)) + } + if n.Row(i) != rid(i) { + t.Fatalf("row %d is %+v, want %+v", i, n.Row(i), rid(i)) + } + } +} + +func TestInternalRoundTrip(t *testing.T) { + n, err := NewInternal(4, 2) + if err != nil { + t.Fatalf("NewInternal: %v", err) + } + n.SetFirstChild(100) + for i := 0; i < 30; i++ { + if err := n.AppendInternal(key(i*10), core.PageID(101+i)); err != nil { + t.Fatalf("AppendInternal(%d): %v", i, err) + } + } + if err := n.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if n.Leaf() || n.Level() != 2 { + t.Fatalf("leaf %v level %d, want false and 2", n.Leaf(), n.Level()) + } + if n.ChildCount() != 31 { + t.Fatalf("child count %d, want 31", n.ChildCount()) + } + if n.Child(0) != 100 { + t.Fatalf("child 0 is %d, want 100 (it lives in the node header)", n.Child(0)) + } + for i := 1; i <= 30; i++ { + if want := core.PageID(100 + i); n.Child(i) != want { + t.Fatalf("child %d is %d, want %d", i, n.Child(i), want) + } + } +} + +func TestChildForRoutesLikeTheTreeDoes(t *testing.T) { + n, err := NewInternal(1, 1) + if err != nil { + t.Fatal(err) + } + n.SetFirstChild(10) + for i, k := range []int{20, 40, 60} { + if err := n.AppendInternal(key(k), core.PageID(11+i)); err != nil { + t.Fatal(err) + } + } + cases := []struct { + k int + want core.PageID + }{ + {0, 10}, {19, 10}, {20, 11}, {39, 11}, {40, 12}, {59, 12}, {60, 13}, {999, 13}, + } + for _, c := range cases { + if got := n.ChildFor(key(c.k)); got != c.want { + t.Errorf("ChildFor(%d) = page %d, want %d", c.k, got, c.want) + } + } +} + +func TestSearchFindsAndPlaces(t *testing.T) { + n := NewLeaf(1) + for _, k := range []int{10, 20, 30} { + if err := n.AppendLeaf(key(k), rid(k)); err != nil { + t.Fatal(err) + } + } + cases := []struct { + k int + at int + found bool + }{ + {5, 0, false}, {10, 0, true}, {15, 1, false}, {20, 1, true}, + {25, 2, false}, {30, 2, true}, {35, 3, false}, + } + for _, c := range cases { + at, found := n.Search(key(c.k)) + if at != c.at || found != c.found { + t.Errorf("Search(%d) = (%d, %v), want (%d, %v)", c.k, at, found, c.at, c.found) + } + } +} + +func TestMaxKeysMatchesWhatActuallyFits(t *testing.T) { + mkKey := func(keyLen, i int) []byte { + k := bytes.Repeat([]byte{0xAA}, keyLen) + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(i)) + n := keyLen + if n > 8 { + n = 8 + } + copy(k[keyLen-n:], b[8-n:]) + return k + } + + for keyLen := 1; keyLen <= 64; keyLen++ { + leaf := NewLeaf(1) + count := 0 + for { + if err := leaf.AppendLeaf(mkKey(keyLen, count), rid(count)); err != nil { + break + } + count++ + } + if want := MaxKeys(page.KindBTreeLeaf, keyLen); count != want { + t.Errorf("leaf with %d-byte keys held %d, MaxKeys says %d", keyLen, count, want) + } + + internal, err := NewInternal(1, 1) + if err != nil { + t.Fatal(err) + } + count = 0 + for { + if err := internal.AppendInternal(mkKey(keyLen, count), core.PageID(count)); err != nil { + break + } + count++ + } + if want := MaxKeys(page.KindBTreeInternal, keyLen); count != want { + t.Errorf("internal with %d-byte keys held %d, MaxKeys says %d", keyLen, count, want) + } + if got := Fanout(keyLen); got != count+1 { + t.Errorf("Fanout(%d) = %d, node held %d keys so it has %d children", + keyLen, got, count, count+1) + } + } +} + +func TestFullNodeRefusesAndStaysValid(t *testing.T) { + n := NewLeaf(1) + i := 0 + for { + if err := n.AppendLeaf(key(i), rid(i)); err != nil { + if !errors.Is(err, ErrNoFit) { + t.Fatalf("append %d failed with %v, want ErrNoFit", i, err) + } + break + } + i++ + } + if err := n.Validate(); err != nil { + t.Fatalf("a full node failed Validate: %v", err) + } + if n.FreeSpace() < 0 || n.FreeSpace() >= n.EntrySize(8) { + t.Fatalf("free space %d should be non-negative and smaller than one entry (%d)", + n.FreeSpace(), n.EntrySize(8)) + } + if n.Fits(8) { + t.Error("Fits says another 8-byte key fits, but Append refused one") + } +} + +func TestGoldenLeafLayout(t *testing.T) { + n := NewLeaf(7) + n.SetNext(9) + if err := n.AppendLeaf([]byte("aa"), core.RowID{Page: 3, Slot: 1}); err != nil { + t.Fatal(err) + } + if err := n.AppendLeaf([]byte("bb"), core.RowID{Page: 4, Slot: 2}); err != nil { + t.Fatal(err) + } + if err := n.Validate(); err != nil { + t.Fatal(err) + } + + want := []struct { + at int + bytes []byte + what string + }{ + {0, []byte{0, 0, 0, 7}, "page id 7"}, + {4, []byte{byte(page.KindBTreeLeaf)}, "kind btree-leaf"}, + {5, []byte{0}, "flags"}, + {6, []byte{0, 2}, "key count 2"}, + {8, []byte{0, 0x20}, "free start 32 = 24 + 2 slots"}, + {10, []byte{0x0F, 0xF0}, "free end 4080 = 4096 - 2 cells of 8"}, + {12, []byte{0, 0, 0, 0}, "reserved"}, + {16, []byte{0, 0, 0, 9}, "next leaf is page 9"}, + {20, []byte{0, 0}, "level 0"}, + {22, []byte{0, 0}, "padding"}, + {24, []byte{0x0F, 0xF8, 0, 8}, "slot 0 -> cell at 4088, 8 bytes"}, + {28, []byte{0x0F, 0xF0, 0, 8}, "slot 1 -> cell at 4080, 8 bytes"}, + {4088, []byte{'a', 'a', 0, 0, 0, 3, 0, 1}, "cell 0: key aa, row page 3 slot 1"}, + {4080, []byte{'b', 'b', 0, 0, 0, 4, 0, 2}, "cell 1: key bb, row page 4 slot 2"}, + } + for _, w := range want { + got := []byte(n.Page[w.at : w.at+len(w.bytes)]) + if !bytes.Equal(got, w.bytes) { + t.Errorf("bytes %d..%d are % 02X, want % 02X (%s)", + w.at, w.at+len(w.bytes)-1, got, w.bytes, w.what) + } + } + + for i := int(n.FreeStart()); i < int(n.FreeEnd()); i++ { + if n.Page[i] != 0 { + t.Fatalf("free space byte %d is %02X, want 00", i, n.Page[i]) + } + } +} + +func TestFirstKeysCellSitsHighestInThePage(t *testing.T) { + n := NewLeaf(1) + for i := 0; i < 5; i++ { + if err := n.AppendLeaf(key(i), rid(i)); err != nil { + t.Fatal(err) + } + } + prev := page.Size + 1 + for i := 0; i < n.KeyCount(); i++ { + lo, _ := n.cellBounds(i) + if lo >= prev { + t.Fatalf("cell %d starts at %d, not below the previous cell at %d", i, lo, prev) + } + prev = lo + } +} + +func TestKeyCopiesAndKeyRefViews(t *testing.T) { + n := NewLeaf(1) + if err := n.AppendLeaf([]byte("abcd"), rid(1)); err != nil { + t.Fatal(err) + } + cp := n.Key(0) + cp[0] = 'z' + if !bytes.Equal(n.Key(0), []byte("abcd")) { + t.Error("writing to the slice from Key changed the page; Key must copy") + } + ref := n.KeyRef(0) + ref[0] = 'z' + if !bytes.Equal(n.Key(0), []byte("zbcd")) { + t.Error("writing to the slice from KeyRef did not change the page; KeyRef must be a view") + } +} + +func TestRoundTripThroughARealFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "nodes.db") + pf, err := page.Open(path) + if err != nil { + t.Fatal(err) + } + defer pf.Close() + + leaf := NewLeaf(0) + leaf.SetNext(1) + for i := 0; i < 20; i++ { + if err := leaf.AppendLeaf(key(i), rid(i)); err != nil { + t.Fatal(err) + } + } + if err := pf.Write(leaf.Page); err != nil { + t.Fatal(err) + } + if err := pf.Sync(); err != nil { + t.Fatal(err) + } + + raw, err := pf.Read(0) + if err != nil { + t.Fatal(err) + } + back, err := Load(raw) + if err != nil { + t.Fatalf("Load: %v", err) + } + if err := back.Validate(); err != nil { + t.Fatalf("Validate after a disk round trip: %v", err) + } + if back.Next() != 1 || back.KeyCount() != 20 { + t.Fatalf("next %d count %d, want 1 and 20", back.Next(), back.KeyCount()) + } + for i := 0; i < 20; i++ { + if !bytes.Equal(back.Key(i), key(i)) || back.Row(i) != rid(i) { + t.Fatalf("entry %d did not survive the round trip", i) + } + } + if !bytes.Equal(back.Page, leaf.Page) { + t.Error("the page read back is not byte-identical to the one written") + } +} + +func TestLoadRejectsPagesThatAreNotNodes(t *testing.T) { + for _, k := range []page.Kind{page.KindFree, page.KindMeta, page.KindHeap} { + p := page.New(1, k) + if _, err := Load(p); !errors.Is(err, ErrBadKind) { + t.Errorf("Load of a %s page gave %v, want ErrBadKind", k, err) + } + } + if _, err := Load(make(page.Page, 10)); !errors.Is(err, core.ErrShortPage) { + t.Errorf("Load of a short page gave %v, want ErrShortPage", err) + } +} + +func TestNewInternalRejectsLevelZero(t *testing.T) { + if _, err := NewInternal(1, 0); !errors.Is(err, ErrBadLevel) { + t.Errorf("NewInternal at level 0 gave %v, want ErrBadLevel", err) + } +} + +func TestAppendRejectsTheWrongKind(t *testing.T) { + leaf := NewLeaf(1) + if err := leaf.AppendInternal(key(1), 2); !errors.Is(err, ErrBadKind) { + t.Errorf("AppendInternal on a leaf gave %v, want ErrBadKind", err) + } + internal, err := NewInternal(1, 1) + if err != nil { + t.Fatal(err) + } + if err := internal.AppendLeaf(key(1), rid(1)); !errors.Is(err, ErrBadKind) { + t.Errorf("AppendLeaf on an internal node gave %v, want ErrBadKind", err) + } +} + +func TestValidateCatchesCorruption(t *testing.T) { + build := func() Node { + n := NewLeaf(1) + for i := 0; i < 6; i++ { + if err := n.AppendLeaf(key(i), rid(i)); err != nil { + t.Fatal(err) + } + } + return n + } + + cases := []struct { + name string + corrupt func(Node) + }{ + {"key count no longer matches free start", func(n Node) { n.SetItemCount(5) }}, + {"free start moved", func(n Node) { n.SetFreeStart(n.FreeStart() + 4) }}, + {"free end moved", func(n Node) { n.SetFreeEnd(n.FreeEnd() - 8) }}, + {"a slot points into the slot array", func(n Node) { + binary.BigEndian.PutUint16(n.Page[n.slotOffset(2):], SlotArrayStart) + }}, + {"a slot points past the page", func(n Node) { + binary.BigEndian.PutUint16(n.Page[n.slotOffset(2):], page.Size-2) + }}, + {"a cell is too short to hold a row id", func(n Node) { + binary.BigEndian.PutUint16(n.Page[n.slotOffset(2)+2:], 4) + }}, + {"keys are out of order", func(n Node) { + a, b := n.slotOffset(2), n.slotOffset(3) + var tmp [SlotSize]byte + copy(tmp[:], n.Page[a:a+SlotSize]) + copy(n.Page[a:a+SlotSize], n.Page[b:b+SlotSize]) + copy(n.Page[b:b+SlotSize], tmp[:]) + }}, + {"two slots point at the same cell", func(n Node) { + lo, _ := n.cellBounds(3) + binary.BigEndian.PutUint16(n.Page[n.slotOffset(2):], uint16(lo)) + }}, + {"padding was used for something", func(n Node) { n.Page[offPad] = 1 }}, + {"a leaf claims a level", func(n Node) { n.setLevel(3) }}, + {"the kind byte was flipped", func(n Node) { n.SetKind(page.KindHeap) }}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + n := build() + if err := n.Validate(); err != nil { + t.Fatalf("the clean node did not validate: %v", err) + } + c.corrupt(n) + if err := n.Validate(); err == nil { + t.Errorf("Validate accepted a node where %s", c.name) + } + }) + } +} + +func TestEmptyNodeIsValid(t *testing.T) { + leaf := NewLeaf(3) + if err := leaf.Validate(); err != nil { + t.Errorf("an empty leaf failed Validate: %v", err) + } + if leaf.FreeSpace() != EntryArea { + t.Errorf("an empty leaf has %d free, want %d", leaf.FreeSpace(), EntryArea) + } + internal, err := NewInternal(3, 1) + if err != nil { + t.Fatal(err) + } + if err := internal.Validate(); err != nil { + t.Errorf("an empty internal node failed Validate: %v", err) + } +} + +func TestDescribeMentionsWhatItDecoded(t *testing.T) { + n := NewLeaf(7) + n.SetNext(9) + if err := n.AppendLeaf([]byte("aa"), core.RowID{Page: 3, Slot: 1}); err != nil { + t.Fatal(err) + } + got := n.Describe() + for _, want := range []string{ + "page 7", "btree-leaf", "next leaf = page 9", "0F F8", "row id -> page 3 slot 1", + } { + if !bytes.Contains([]byte(got), []byte(want)) { + t.Errorf("Describe output is missing %q\n%s", want, got) + } + } +} + +func TestFanoutTableIsHonestAboutOverhead(t *testing.T) { + out := FanoutTable(8) + if !bytes.Contains([]byte(out), []byte(fmt.Sprintf("%d", EntryArea))) { + t.Errorf("FanoutTable should show the %d bytes left for entries\n%s", EntryArea, out) + } +}