diff --git a/Makefile b/Makefile index a87f783..6a95e3b 100644 --- a/Makefile +++ b/Makefile @@ -70,3 +70,6 @@ demo-range: demo-delete: go run ./cmd/scutedb-demo delete + +demo-nodepage: + go run ./cmd/scutedb-demo nodepage diff --git a/README.md b/README.md index 927f955..a386a7a 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ A *scute* is one of the bony plates that make up a turtle's shell. A shell is made of plates; a database file is made of pages. -**Status:** Phases 0 and A complete. The B+Tree searches, inserts, splits, -scans ranges and deletes. All of it still lives in memory — Phase B puts it on -disk. +**Status:** Phases 0 and A complete. Phase B has started: a B+Tree node now has +a byte-for-byte on-disk format, and a tree can be walked straight out of a file +using nothing but page numbers. --- @@ -20,6 +20,7 @@ make demo # list the runnable experiments make demo-scan # watch a file-based database degrade make hexdump # write real pages and look at the bytes make demo-delete # borrow, merge and root collapse, traced live +make demo-nodepage # decode a real B+Tree page out of a real file ``` --- @@ -30,7 +31,7 @@ make demo-delete # borrow, merge and root collapse, traced live |-------|------|--------| | **0** | Foundations — interfaces, the naive database, pages | **done** | | **A** | Bytes & the B+Tree | **done** | -| B | Persistence — storage manager, buffer pool, locking | **next** | +| B | Persistence — storage manager, buffer pool, locking | **in progress** | | C | A real data store — schema, rows, indexes | | | D | Transactions — WAL, recovery, 2PL, MVCC | | | E | Beyond — LSM engine, Raft, server, query planner | | @@ -661,6 +662,146 @@ Deliberate, each one is a later step: --- +## Phase B — Persistence + +### `0x09` Nodes as pages + +Every node in `0x06` to `0x08` was a Go struct held together by pointers. A +pointer is a RAM address: it is meaningless in the next process, so the entire +tree evaporates on exit. This step gives a node a byte layout and swaps every +pointer for a **page number**. + +``` +in memory on disk +children []*node 8 bytes each child core.PageID 4 bytes each + a RAM address a page number + valid until exit valid forever +``` + +Page 12 is at byte `12 * 4096 = 49152` — in this process, in the next one, and on +another machine. That single swap is the whole step; everything below follows +from it. + +**The layout.** `internal/nodepage` is a *slotted page*, the same shape Postgres +and SQLite use: + +``` +region bytes holds +page header 0..15 id, kind, key count, free start, free end +node header 16..23 next leaf / first child, level, 2 pad +slot array 24.. 4 bytes per entry, in KEY order +free space shrinks from both ends +cells ..4095 key + payload, in INSERTION order +``` + +Slots grow up from byte 24, cells grow down from byte 4095, and free space is +whatever is left between them. The trick worth noticing: **the slot array is +sorted, the cells are not.** A slot is 4 bytes (`offset`, `length`) and inserting +a key in the middle means shifting a few 4-byte slots, never the key bytes +themselves. Binary search reads the slot array, so lookups stay `O(log n)` while +writes stay cheap. + +**Two cell shapes**, and the key length never needs storing because the suffix is +fixed: + +``` +leaf cell key bytes || row page (4) || row slot (2) key = len - 6 +internal cell key bytes || child page (4) key = len - 4 +``` + +An internal node with *n* keys has *n+1* children. The extra child has no slot to +live in, so it lives in the node header — that is what `first child` is. + +**Fanout is computed, not chosen.** With a 4096-byte page and an 8-byte key: + +``` + page header 16 + node header 8 + left for entries 4072 + + internal entry = 4 slot + 8 key + 4 child = 16 bytes -> 254 keys, 255 children + leaf entry = 4 slot + 8 key + 6 row id = 18 bytes -> 226 keys + + levels reads per get keys it holds + 1 1 226 + 2 2 57,630 + 3 3 14,695,650 + 4 4 3,747,390,750 +``` + +This corrects a number stated earlier in this file. `0x06` estimated ~340 keys +per internal node from `4080 / (8 + 4)`, which quietly ignored the node header +and the 4-byte slot every entry needs. The real figure is **254** — the +bookkeeping costs 25% of the fanout. The estimate was optimistic in the usual +direction: it counted the data and forgot the structure that makes the data +findable. + +**A real page, out of a real file.** Three pages written to disk, then read back +with `xxd` — no ScuteDB code involved in the reading: + +``` +$ xxd -s 4096 -l 32 tree.db +00001000: 0000 0001 0300 0002 0020 0fe4 0000 0000 ......... ...... +00001010: 0000 0002 0000 0000 0ff2 000e 0fe4 000e ................ +``` + +Decoding it by hand, left to right: + +| bytes | value | meaning | +|---|---|---| +| `0000 0001` | 1 | page id | +| `03` | 3 | kind, `btree-leaf` | +| `00` | 0 | flags | +| `0002` | 2 | key count | +| `0020` | 32 | free start, `24 + 2 slots x 4` | +| `0fe4` | 4068 | free end, where the lowest cell begins | +| `0000 0000` | — | reserved for the checksum in `0x13` | +| `0000 0002` | 2 | next leaf is page 2 | +| `0000` | 0 | level, 0 means leaf | +| `0000` | — | padding, keeps slots 4-byte aligned | +| `0ff2 000e` | 4082, 14 | slot 0 points at a 14-byte cell | +| `0fe4 000e` | 4068, 14 | slot 1 points at a 14-byte cell | + +Slot 0 says byte 4082 of page 1, which is file offset `4096 + 4082 = 8178`: + +``` +$ xxd -s 8178 -l 14 tree.db +00001ff2: 7fff ffff ffff ffff 0000 0064 0000 ...........d.. +``` + +`7F FF FF FF FF FF FF FF` is the key **-1**, sign bit flipped by the ordered +encoding from `0x03`. Then `0000 0064` is row page 100, and `0000` is slot 0. +Every layer built so far is visible in those fourteen bytes. + +**Walking the tree with no pointers at all**, reading pages back from the file: + +``` +looking up 30, starting at page 0 + read page 0 at byte offset 0, kind btree-internal, 1 keys + not a leaf, so follow a page id: next is page 2 + read page 2 at byte offset 8192, kind btree-leaf, 2 keys + found at slot 1 -> row id page 200 slot 30 +``` + +**How the real ones do it.** Postgres calls a page number a `BlockNumber`, a +`uint32` index into the relation file, and its B-Tree pages carry a `btpo_level` +exactly like the level field here. SQLite uses 1-based 32-bit page numbers, with +page 1 always holding the schema. InnoDB uses 32-bit page numbers inside a +tablespace with 16 KB pages. Nobody stores an address. + +**How it is verified.** `Validate` checks twelve structural facts, including that +the cells exactly tile the region from `free end` to the end of the page with no +gaps and no overlaps, and that the slot array is in strictly ascending key order. +Eleven separate corruptions are asserted to fail it. `TestGoldenLeafLayout` +pins fourteen exact byte ranges so the format cannot drift silently, and +`TestMaxKeysMatchesWhatActuallyFits` fills a page for every key length from 1 to +64 and checks the count against the arithmetic. A fuzz target built pages from +arbitrary keys for 2.38M executions. + +Run it: `make demo-nodepage`. + +--- + ## Layout ``` @@ -676,6 +817,7 @@ internal/ nullbits/ null bitmaps and SQL three-valued logic slots/ fixed-size, aligned record slots inside a page btree/ in-memory B+Tree: search, insert, split, range scans, delete + nodepage/ the on-disk byte layout of a B+Tree node ``` ## Conventions diff --git a/cmd/scutedb-demo/main.go b/cmd/scutedb-demo/main.go index 51f85a5..26af582 100644 --- a/cmd/scutedb-demo/main.go +++ b/cmd/scutedb-demo/main.go @@ -27,7 +27,7 @@ import ( func main() { if len(os.Args) < 2 { - fmt.Println("usage: scutedb-demo scan|update|race|crash|pages|header|encode|nulls|align|btree|range|delete") + fmt.Println("usage: scutedb-demo scan|update|race|crash|pages|header|encode|nulls|align|btree|range|delete|nodepage") os.Exit(2) } switch os.Args[1] { @@ -55,6 +55,8 @@ func main() { expRange() case "delete": expDelete() + case "nodepage": + expNodePage() case "crash-child": crashChild(os.Args[2]) default: diff --git a/cmd/scutedb-demo/nodepage.go b/cmd/scutedb-demo/nodepage.go new file mode 100644 index 0000000..ca8e9c7 --- /dev/null +++ b/cmd/scutedb-demo/nodepage.go @@ -0,0 +1,220 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "unsafe" + + "github.com/scute-db/scutedb/internal/codec" + "github.com/scute-db/scutedb/internal/core" + "github.com/scute-db/scutedb/internal/nodepage" + "github.com/scute-db/scutedb/internal/page" +) + +func expNodePage() { + fmt.Print("STEP 0x09 Nodes as pages\n\n") + + section1() + section2() + section3() + section4() + section5() +} + +func nodeKey(v int64) []byte { return codec.AppendKeyInt64(nil, v) } + +func section1() { + fmt.Print("1. WHY A POINTER CANNOT GO ON DISK\n\n") + + type inMemoryNode struct { + children []*int + next *int + } + var n inMemoryNode + + fmt.Printf(" an in-memory child link is a Go pointer: %d bytes\n", unsafe.Sizeof(n.next)) + fmt.Print(" it holds a RAM address, valid only while this process lives.\n") + fmt.Print(" write it to disk, restart, and it points at someone else's memory.\n\n") + fmt.Printf(" an on-disk child link is a core.PageID: %d bytes\n", unsafe.Sizeof(core.PageID(0))) + fmt.Print(" it holds a page number. page 12 is at byte 12 * 4096 = 49152,\n") + fmt.Print(" in this process, in the next one, and on another machine.\n\n") + fmt.Print(" that swap is the whole step. everything else follows from it.\n\n") +} + +func section2() { + fmt.Print("2. FANOUT IS COMPUTED, NOT CHOSEN\n\n") + fmt.Print(indent(nodepage.FanoutTable(8), " ")) + + old := page.Usable / (8 + 4) + real := nodepage.MaxKeys(page.KindBTreeInternal, 8) + fmt.Printf(" an earlier estimate in this repo said %d keys per internal node:\n", old) + fmt.Printf(" %d usable bytes / (8 key + 4 child) = %d.\n", page.Usable, old) + fmt.Printf(" the real answer is %d, because that estimate forgot the %d-byte\n", real, nodepage.NodeHeaderSize) + fmt.Printf(" node header and the %d-byte slot per entry. overhead is %d%% of fanout.\n\n", + nodepage.SlotSize, (old-real)*100/old) +} + +func section3() { + fmt.Print("3. A REAL LEAF PAGE, BYTE FOR BYTE\n\n") + + leaf := nodepage.NewLeaf(1) + leaf.SetNext(2) + for _, v := range []int64{-1, 10, 20} { + if err := leaf.AppendLeaf(nodeKey(v), core.RowID{Page: core.PageID(v + 2), Slot: uint16(v + 1)}); err != nil { + fail(err) + } + } + if err := leaf.Validate(); err != nil { + fail(err) + } + fmt.Print(indent(leaf.Describe(), " ")) + + fmt.Print(" read the first cell yourself:\n") + lo := int(leaf.FreeEnd()) + 2*(8+nodepage.LeafSuffix) + fmt.Printf(" slot 0 says the cell is at %d. bytes %d..%d are\n", lo, lo, lo+13) + fmt.Printf(" % 02X\n", []byte(leaf.Page[lo:lo+14])) + fmt.Print(" first 8 bytes = the key, big-endian with the sign bit flipped\n") + fmt.Print(" 7F FF FF FF FF FF FF FF is -1 from step 0x03\n") + fmt.Print(" next 4 bytes = row id page number\n") + fmt.Print(" last 2 bytes = row id slot number\n\n") +} + +func section4() { + fmt.Print("4. AN INTERNAL PAGE HOLDS N KEYS AND N+1 CHILDREN\n\n") + + root, err := nodepage.NewInternal(0, 1) + if err != nil { + fail(err) + } + root.SetFirstChild(1) + if err := root.AppendInternal(nodeKey(20), 2); err != nil { + fail(err) + } + if err := root.Validate(); err != nil { + fail(err) + } + fmt.Print(indent(root.Describe(), " ")) + fmt.Print(" one key, two children. the extra child has nowhere to sit in the\n") + fmt.Print(" slot array, so it lives in the node header instead.\n\n") +} + +func section5() { + fmt.Print("5. WALKING A TREE THAT IS ONLY BYTES ON DISK\n\n") + + dir, err := os.MkdirTemp("", "scutedb-nodepage") + if err != nil { + fail(err) + } + defer os.RemoveAll(dir) + path := filepath.Join(dir, "tree.db") + + pf, err := page.Open(path) + if err != nil { + fail(err) + } + + root, err := nodepage.NewInternal(0, 1) + if err != nil { + fail(err) + } + root.SetFirstChild(1) + if err := root.AppendInternal(nodeKey(20), 2); err != nil { + fail(err) + } + + left := nodepage.NewLeaf(1) + left.SetNext(2) + for _, v := range []int64{-1, 10} { + if err := left.AppendLeaf(nodeKey(v), core.RowID{Page: 100, Slot: uint16(v + 1)}); err != nil { + fail(err) + } + } + + right := nodepage.NewLeaf(2) + for _, v := range []int64{20, 30} { + if err := right.AppendLeaf(nodeKey(v), core.RowID{Page: 200, Slot: uint16(v)}); err != nil { + fail(err) + } + } + + for _, n := range []nodepage.Node{root, left, right} { + if err := n.Validate(); err != nil { + fail(err) + } + if err := pf.Write(n.Page); err != nil { + fail(err) + } + } + if err := pf.Sync(); err != nil { + fail(err) + } + if err := pf.Close(); err != nil { + fail(err) + } + + info, err := os.Stat(path) + if err != nil { + fail(err) + } + fmt.Printf(" wrote %s: %d bytes = %d pages of %d\n\n", filepath.Base(path), info.Size(), + info.Size()/page.Size, page.Size) + + reopened, err := page.Open(path) + if err != nil { + fail(err) + } + defer reopened.Close() + + for _, want := range []int64{10, 30} { + fmt.Printf(" looking up %d, starting at page 0\n", want) + id := core.PageID(0) + for { + raw, err := reopened.Read(id) + if err != nil { + fail(err) + } + n, err := nodepage.Load(raw) + if err != nil { + fail(err) + } + fmt.Printf(" read page %d at byte offset %d, kind %s, %d keys\n", + id, page.Offset(id), n.Kind(), n.KeyCount()) + if n.Leaf() { + at, found := n.Search(nodeKey(want)) + if !found { + fmt.Printf(" %d is not in this leaf\n\n", want) + break + } + r := n.Row(at) + fmt.Printf(" found at slot %d -> row id page %d slot %d\n\n", at, r.Page, r.Slot) + break + } + id = n.ChildFor(nodeKey(want)) + fmt.Printf(" not a leaf, so follow a page id: next is page %d\n", id) + } + } + + fmt.Print(" nothing above was a pointer. every hop was a page number\n") + fmt.Print(" turned into a byte offset by multiplying by 4096.\n\n") + fmt.Print(" this is how the real ones do it too:\n") + fmt.Print(" postgres BlockNumber, a uint32 page index into the relation file\n") + fmt.Print(" sqlite 32-bit page numbers, 1-based, page 1 is the schema root\n") + fmt.Print(" innodb 32-bit page numbers inside a tablespace, 16 KB pages\n\n") +} + +func indent(s string, with string) string { + out := with + for i := 0; i < len(s); i++ { + out += string(s[i]) + if s[i] == '\n' && i != len(s)-1 { + out += with + } + } + return out + "\n" +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, "demo failed:", err) + os.Exit(1) +} diff --git a/internal/nodepage/describe.go b/internal/nodepage/describe.go new file mode 100644 index 0000000..93179f4 --- /dev/null +++ b/internal/nodepage/describe.go @@ -0,0 +1,179 @@ +package nodepage + +import ( + "fmt" + "strings" + + "github.com/scute-db/scutedb/internal/page" +) + +func Hexdump(b []byte, base int) string { + var s strings.Builder + for off := 0; off < len(b); off += 16 { + end := off + 16 + if end > len(b) { + end = len(b) + } + line := b[off:end] + fmt.Fprintf(&s, "%04X ", base+off) + for i := 0; i < 16; i++ { + if i == 8 { + s.WriteByte(' ') + } + if i < len(line) { + fmt.Fprintf(&s, "%02X ", line[i]) + } else { + s.WriteString(" ") + } + } + s.WriteString(" |") + for _, c := range line { + if c >= 0x20 && c < 0x7F { + s.WriteByte(c) + } else { + s.WriteByte('.') + } + } + s.WriteString("|\n") + } + return s.String() +} + +func (n Node) Map() string { + var s strings.Builder + count := n.KeyCount() + fmt.Fprintf(&s, "%-14s %-13s %s\n", "region", "bytes", "holds") + fmt.Fprintln(&s, strings.Repeat("-", 62)) + fmt.Fprintf(&s, "%-14s %-13s %s\n", "page header", + fmt.Sprintf("0..%d", page.HeaderSize-1), "id, kind, item count, free start, free end") + fmt.Fprintf(&s, "%-14s %-13s %s\n", "node header", + fmt.Sprintf("%d..%d", page.HeaderSize, SlotArrayStart-1), n.linkMeaning()+", level, 2 pad") + fmt.Fprintf(&s, "%-14s %-13s %s\n", "slot array", + fmt.Sprintf("%d..%d", SlotArrayStart, int(n.FreeStart())-1), + fmt.Sprintf("%d slots x %d bytes, in key order", count, SlotSize)) + fmt.Fprintf(&s, "%-14s %-13s %s\n", "free space", + fmt.Sprintf("%d..%d", n.FreeStart(), int(n.FreeEnd())-1), + fmt.Sprintf("%d bytes, grows shut from both ends", n.FreeSpace())) + fmt.Fprintf(&s, "%-14s %-13s %s\n", "cells", + fmt.Sprintf("%d..%d", n.FreeEnd(), page.Size-1), + fmt.Sprintf("%d cells, packed from the end, insertion order", count)) + return s.String() +} + +func (n Node) linkMeaning() string { + if n.Leaf() { + return "next leaf" + } + return "first child" +} + +func (n Node) Describe() string { + var s strings.Builder + + fmt.Fprintf(&s, "page %d %s level %d %d keys\n\n", n.ID(), n.Kind(), n.Level(), n.KeyCount()) + s.WriteString(n.Map()) + + fmt.Fprintf(&s, "\nheaders, byte for byte\n\n") + fmt.Fprint(&s, Hexdump(n.Page[:SlotArrayStart], 0)) + fmt.Fprintf(&s, "\n%-9s %-14s %s\n", "offset", "bytes", "meaning") + fmt.Fprintln(&s, strings.Repeat("-", 62)) + row := func(off, size int, meaning string) { + fmt.Fprintf(&s, "%-9s %-14s %s\n", + fmt.Sprintf("%d..%d", off, off+size-1), + fmt.Sprintf("% 02X", n.Page[off:off+size]), meaning) + } + row(0, 4, fmt.Sprintf("page id = %d", n.ID())) + row(4, 1, fmt.Sprintf("kind = %d (%s)", n.Page[4], n.Kind())) + row(5, 1, fmt.Sprintf("flags = %08b", n.Flags())) + row(6, 2, fmt.Sprintf("key count = %d", n.KeyCount())) + row(8, 2, fmt.Sprintf("free start = %d (slot array ends here)", n.FreeStart())) + row(10, 2, fmt.Sprintf("free end = %d (lowest cell starts here)", n.FreeEnd())) + row(12, 4, "reserved for the checksum in 0x13") + row(offLink, 4, fmt.Sprintf("%s = page %d", n.linkMeaning(), n.link())) + row(offLevel, 2, fmt.Sprintf("level = %d (0 means leaf)", n.Level())) + row(offPad, 2, "padding, keeps the slot array 4-byte aligned") + + count := n.KeyCount() + if count == 0 { + fmt.Fprintf(&s, "\nno entries\n") + return s.String() + } + + fmt.Fprintf(&s, "\nslot array at %d, %d slots of %d bytes\n\n", SlotArrayStart, count, SlotSize) + fmt.Fprint(&s, Hexdump(n.Page[SlotArrayStart:n.FreeStart()], SlotArrayStart)) + fmt.Fprintf(&s, "\n%-6s %-9s %-12s %-9s %s\n", "slot", "at", "bytes", "cell at", "length") + fmt.Fprintln(&s, strings.Repeat("-", 62)) + for i := 0; i < count; i++ { + at := n.slotOffset(i) + lo, hi := n.cellBounds(i) + fmt.Fprintf(&s, "%-6d %-9d %-12s %-9d %d\n", + i, at, fmt.Sprintf("% 02X", n.Page[at:at+SlotSize]), lo, hi-lo) + } + + fmt.Fprintf(&s, "\ncells from %d to %d, packed downward\n\n", n.FreeEnd(), page.Size-1) + fmt.Fprint(&s, Hexdump(n.Page[n.FreeEnd():], int(n.FreeEnd()))) + + fmt.Fprintf(&s, "\n%-6s %-9s %-22s %s\n", "slot", "cell at", "key bytes", "then") + fmt.Fprintln(&s, strings.Repeat("-", 68)) + for i := 0; i < count; i++ { + lo, _ := n.cellBounds(i) + key := n.KeyRef(i) + var tail string + if n.Leaf() { + r := n.Row(i) + tail = fmt.Sprintf("row id -> page %d slot %d", r.Page, r.Slot) + } else { + tail = fmt.Sprintf("child -> page %d", n.Child(i+1)) + } + fmt.Fprintf(&s, "%-6d %-9d %-22s %s\n", i, lo, fmt.Sprintf("% 02X", key), tail) + } + + if !n.Leaf() { + fmt.Fprintf(&s, "\n%d keys, %d children\n", count, n.ChildCount()) + fmt.Fprintf(&s, " child 0 = page %d (from the node header, keys below key 0)\n", n.Child(0)) + for i := 1; i <= count; i++ { + fmt.Fprintf(&s, " child %d = page %d (from slot %d, keys >= % 02X)\n", + i, n.Child(i), i-1, n.KeyRef(i-1)) + } + } + return s.String() +} + +func FanoutTable(keyLen int) string { + var s strings.Builder + fmt.Fprintf(&s, "page %d bytes, key %d bytes\n\n", page.Size, keyLen) + fmt.Fprintf(&s, " page header %4d\n", page.HeaderSize) + fmt.Fprintf(&s, " node header %4d\n", NodeHeaderSize) + fmt.Fprintf(&s, " left for entries %4d\n\n", EntryArea) + + internal := MaxKeys(page.KindBTreeInternal, keyLen) + leaf := MaxKeys(page.KindBTreeLeaf, keyLen) + fmt.Fprintf(&s, " internal entry = %d slot + %d key + %d child = %d bytes -> %d keys, %d children\n", + SlotSize, keyLen, InternalSuffix, SlotSize+keyLen+InternalSuffix, internal, internal+1) + fmt.Fprintf(&s, " leaf entry = %d slot + %d key + %d row id = %d bytes -> %d keys\n\n", + SlotSize, keyLen, LeafSuffix, SlotSize+keyLen+LeafSuffix, leaf) + + fmt.Fprintf(&s, " %-8s %-14s %s\n", "levels", "reads per get", "keys it holds") + fmt.Fprintln(&s, " "+strings.Repeat("-", 46)) + capacity := float64(leaf) + for level := 1; level <= 4; level++ { + fmt.Fprintf(&s, " %-8d %-14d %s\n", level, level, commas(capacity)) + capacity *= float64(internal + 1) + } + return s.String() +} + +func commas(v float64) string { + s := fmt.Sprintf("%.0f", v) + if len(s) > 15 { + return fmt.Sprintf("%.3g", v) + } + var out []byte + for i, c := range []byte(s) { + if i > 0 && (len(s)-i)%3 == 0 { + out = append(out, ',') + } + out = append(out, c) + } + return string(out) +} diff --git a/internal/nodepage/fuzz_test.go b/internal/nodepage/fuzz_test.go new file mode 100644 index 0000000..95694cf --- /dev/null +++ b/internal/nodepage/fuzz_test.go @@ -0,0 +1,141 @@ +package nodepage + +import ( + "bytes" + "errors" + "sort" + "testing" + + "github.com/scute-db/scutedb/internal/core" + "github.com/scute-db/scutedb/internal/page" +) + +func FuzzNodePageSurvivesArbitraryKeys(f *testing.F) { + f.Add([]byte("abcdefghijklmnop"), uint8(4), true) + f.Add([]byte{0, 0, 0, 1, 0, 0, 0, 2}, uint8(4), false) + f.Add([]byte{0xFF}, uint8(1), true) + + f.Fuzz(func(t *testing.T, raw []byte, width uint8, leaf bool) { + w := int(width)%40 + 1 + if len(raw) < w { + t.Skip() + } + + var keys [][]byte + for i := 0; i+w <= len(raw); i += w { + keys = append(keys, raw[i:i+w]) + } + sort.Slice(keys, func(a, b int) bool { return bytes.Compare(keys[a], keys[b]) < 0 }) + uniq := keys[:0] + for i, k := range keys { + if i == 0 || !bytes.Equal(k, keys[i-1]) { + uniq = append(uniq, k) + } + } + keys = uniq + if len(keys) == 0 { + t.Skip() + } + + var n Node + var err error + if leaf { + n = NewLeaf(1) + } else { + n, err = NewInternal(1, 1) + if err != nil { + t.Fatal(err) + } + n.SetFirstChild(1000) + } + + written := 0 + for i, k := range keys { + if leaf { + err = n.AppendLeaf(k, core.RowID{Page: core.PageID(i), Slot: uint16(i)}) + } else { + err = n.AppendInternal(k, core.PageID(2000+i)) + } + if err != nil { + if !errors.Is(err, ErrNoFit) { + t.Fatalf("append %d failed with %v, want ErrNoFit", i, err) + } + break + } + written++ + } + + if err := n.Validate(); err != nil { + t.Fatalf("Validate after %d appends: %v", written, err) + } + if n.KeyCount() != written { + t.Fatalf("key count %d, appended %d", n.KeyCount(), written) + } + + max := MaxKeys(n.Kind(), w) + if written > max { + t.Fatalf("fitted %d keys of %d bytes, MaxKeys says at most %d", written, w, max) + } + + for i := 0; i < written; i++ { + if !bytes.Equal(n.KeyRef(i), keys[i]) { + t.Fatalf("key %d is % 02X, want % 02X", i, n.KeyRef(i), keys[i]) + } + at, found := n.Search(keys[i]) + if !found || at != i { + t.Fatalf("Search for key %d returned (%d, %v)", i, at, found) + } + if leaf { + if want := (core.RowID{Page: core.PageID(i), Slot: uint16(i)}); n.Row(i) != want { + t.Fatalf("row %d is %+v, want %+v", i, n.Row(i), want) + } + } else if want := core.PageID(2000 + i); n.Child(i+1) != want { + t.Fatalf("child %d is %d, want %d", i+1, n.Child(i+1), want) + } + } + + if !leaf && written > 0 { + if n.Child(0) != 1000 { + t.Fatalf("first child is %d, want 1000", n.Child(0)) + } + if got := n.ChildFor(keys[0]); got != 2000 { + t.Fatalf("ChildFor(first key) = %d, want 2000", got) + } + } + }) +} + +func BenchmarkSearchFullPage(b *testing.B) { + n := NewLeaf(1) + i := 0 + for n.AppendLeaf(key(i), rid(i)) == nil { + i++ + } + b.ReportMetric(float64(n.KeyCount()), "keys/page") + probe := key(i / 2) + b.ResetTimer() + for j := 0; j < b.N; j++ { + if _, found := n.Search(probe); !found { + b.Fatal("probe key vanished") + } + } +} + +func BenchmarkDecodeAndValidate(b *testing.B) { + n := NewLeaf(1) + i := 0 + for n.AppendLeaf(key(i), rid(i)) == nil { + i++ + } + raw := page.Page(append([]byte(nil), n.Page...)) + b.ResetTimer() + for j := 0; j < b.N; j++ { + loaded, err := Load(raw) + if err != nil { + b.Fatal(err) + } + if err := loaded.Validate(); err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/nodepage/nodepage.go b/internal/nodepage/nodepage.go new file mode 100644 index 0000000..e18b92a --- /dev/null +++ b/internal/nodepage/nodepage.go @@ -0,0 +1,273 @@ +package nodepage + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "sort" + + "github.com/scute-db/scutedb/internal/core" + "github.com/scute-db/scutedb/internal/page" +) + +const ( + NodeHeaderSize = 8 + SlotSize = 4 + SlotArrayStart = page.HeaderSize + NodeHeaderSize + EntryArea = page.Size - SlotArrayStart + + LeafSuffix = 6 + InternalSuffix = 4 +) + +const ( + offLink = page.HeaderSize + offLevel = page.HeaderSize + 4 + offPad = page.HeaderSize + 6 +) + +var ( + ErrNoFit = errors.New("scutedb/nodepage: entry does not fit") + ErrBadKind = errors.New("scutedb/nodepage: not a b+tree node page") + ErrBadLevel = errors.New("scutedb/nodepage: level does not match kind") + ErrInvariant = errors.New("scutedb/nodepage: invariant violated") +) + +type Node struct{ page.Page } + +func NewLeaf(id core.PageID) Node { + p := page.New(id, page.KindBTreeLeaf) + p.SetFreeStart(SlotArrayStart) + return Node{p} +} + +func NewInternal(id core.PageID, level uint16) (Node, error) { + if level == 0 { + return Node{}, fmt.Errorf("%w: internal node at level 0", ErrBadLevel) + } + p := page.New(id, page.KindBTreeInternal) + p.SetFreeStart(SlotArrayStart) + n := Node{p} + n.setLevel(level) + return n, nil +} + +func Load(p page.Page) (Node, error) { + if len(p) != page.Size { + return Node{}, core.ErrShortPage + } + k := p.Kind() + if k != page.KindBTreeLeaf && k != page.KindBTreeInternal { + return Node{}, fmt.Errorf("%w: kind is %s", ErrBadKind, k) + } + return Node{p}, nil +} + +func (n Node) Leaf() bool { return n.Kind() == page.KindBTreeLeaf } +func (n Node) KeyCount() int { return int(n.ItemCount()) } +func (n Node) Level() uint16 { return binary.BigEndian.Uint16(n.Page[offLevel:]) } +func (n Node) ChildCount() int { + if n.Leaf() { + return 0 + } + return n.KeyCount() + 1 +} + +func (n Node) setLevel(l uint16) { binary.BigEndian.PutUint16(n.Page[offLevel:], l) } + +func (n Node) link() core.PageID { return core.PageID(binary.BigEndian.Uint32(n.Page[offLink:])) } +func (n Node) setLink(id core.PageID) { binary.BigEndian.PutUint32(n.Page[offLink:], uint32(id)) } + +func (n Node) Next() core.PageID { return n.link() } +func (n Node) SetNext(id core.PageID) { n.setLink(id) } +func (n Node) FirstChild() core.PageID { return n.link() } +func (n Node) SetFirstChild(id core.PageID) { n.setLink(id) } + +func (n Node) suffixLen() int { + if n.Leaf() { + return LeafSuffix + } + return InternalSuffix +} + +func (n Node) slotOffset(i int) int { return SlotArrayStart + i*SlotSize } + +func (n Node) cellBounds(i int) (int, int) { + s := n.slotOffset(i) + off := int(binary.BigEndian.Uint16(n.Page[s:])) + length := int(binary.BigEndian.Uint16(n.Page[s+2:])) + return off, off + length +} + +func (n Node) cell(i int) []byte { + lo, hi := n.cellBounds(i) + return n.Page[lo:hi] +} + +func (n Node) KeyRef(i int) []byte { + c := n.cell(i) + return c[:len(c)-n.suffixLen()] +} + +func (n Node) Key(i int) []byte { + return append([]byte(nil), n.KeyRef(i)...) +} + +func (n Node) Row(i int) core.RowID { + c := n.cell(i) + tail := c[len(c)-LeafSuffix:] + return core.RowID{ + Page: core.PageID(binary.BigEndian.Uint32(tail)), + Slot: binary.BigEndian.Uint16(tail[4:]), + } +} + +func (n Node) Child(i int) core.PageID { + if i == 0 { + return n.FirstChild() + } + c := n.cell(i - 1) + return core.PageID(binary.BigEndian.Uint32(c[len(c)-InternalSuffix:])) +} + +func (n Node) EntrySize(keyLen int) int { return SlotSize + keyLen + n.suffixLen() } + +func (n Node) Fits(keyLen int) bool { return n.EntrySize(keyLen) <= n.FreeSpace() } + +func (n Node) appendCell(key []byte, tail []byte) error { + need := SlotSize + len(key) + len(tail) + if need > n.FreeSpace() { + return fmt.Errorf("%w: needs %d bytes, %d free", ErrNoFit, need, n.FreeSpace()) + } + start := int(n.FreeEnd()) - len(key) - len(tail) + copy(n.Page[start:], key) + copy(n.Page[start+len(key):], tail) + + s := n.slotOffset(n.KeyCount()) + binary.BigEndian.PutUint16(n.Page[s:], uint16(start)) + binary.BigEndian.PutUint16(n.Page[s+2:], uint16(len(key)+len(tail))) + + n.SetFreeEnd(uint16(start)) + n.SetFreeStart(uint16(s + SlotSize)) + n.SetItemCount(n.ItemCount() + 1) + return nil +} + +func (n Node) AppendLeaf(key []byte, rid core.RowID) error { + if !n.Leaf() { + return fmt.Errorf("%w: AppendLeaf on an internal node", ErrBadKind) + } + var tail [LeafSuffix]byte + binary.BigEndian.PutUint32(tail[:], uint32(rid.Page)) + binary.BigEndian.PutUint16(tail[4:], rid.Slot) + return n.appendCell(key, tail[:]) +} + +func (n Node) AppendInternal(key []byte, child core.PageID) error { + if n.Leaf() { + return fmt.Errorf("%w: AppendInternal on a leaf node", ErrBadKind) + } + var tail [InternalSuffix]byte + binary.BigEndian.PutUint32(tail[:], uint32(child)) + return n.appendCell(key, tail[:]) +} + +func (n Node) Search(key []byte) (int, bool) { + count := n.KeyCount() + i := sort.Search(count, func(i int) bool { return bytes.Compare(n.KeyRef(i), key) >= 0 }) + if i < count && bytes.Equal(n.KeyRef(i), key) { + return i, true + } + return i, false +} + +func (n Node) ChildFor(key []byte) core.PageID { + i := sort.Search(n.KeyCount(), func(i int) bool { return bytes.Compare(key, n.KeyRef(i)) < 0 }) + return n.Child(i) +} + +func MaxKeys(kind page.Kind, keyLen int) int { + suffix := InternalSuffix + if kind == page.KindBTreeLeaf { + suffix = LeafSuffix + } + per := SlotSize + keyLen + suffix + if per <= 0 { + return 0 + } + return EntryArea / per +} + +func Fanout(keyLen int) int { return MaxKeys(page.KindBTreeInternal, keyLen) + 1 } + +func (n Node) Validate() error { + k := n.Kind() + if k != page.KindBTreeLeaf && k != page.KindBTreeInternal { + return fmt.Errorf("%w: kind is %s", ErrBadKind, k) + } + if n.Leaf() != (n.Level() == 0) { + return fmt.Errorf("%w: kind %s at level %d", ErrBadLevel, k, n.Level()) + } + if got := binary.BigEndian.Uint16(n.Page[offPad:]); got != 0 { + return fmt.Errorf("%w: padding bytes are %d, want 0", ErrInvariant, got) + } + + count := n.KeyCount() + wantStart := SlotArrayStart + count*SlotSize + if int(n.FreeStart()) != wantStart { + return fmt.Errorf("%w: free start is %d, %d slots end at %d", + ErrInvariant, n.FreeStart(), count, wantStart) + } + if int(n.FreeEnd()) > page.Size || n.FreeEnd() < n.FreeStart() { + return fmt.Errorf("%w: free end %d is outside [%d, %d]", + ErrInvariant, n.FreeEnd(), n.FreeStart(), page.Size) + } + + if count == 0 { + if int(n.FreeEnd()) != page.Size { + return fmt.Errorf("%w: empty node has free end %d, want %d", + ErrInvariant, n.FreeEnd(), page.Size) + } + return nil + } + + suffix := n.suffixLen() + spans := make([][2]int, count) + for i := 0; i < count; i++ { + lo, hi := n.cellBounds(i) + if lo < int(n.FreeStart()) || hi > page.Size || hi <= lo { + return fmt.Errorf("%w: slot %d spans [%d, %d), outside [%d, %d)", + ErrInvariant, i, lo, hi, n.FreeStart(), page.Size) + } + if hi-lo < suffix+1 { + return fmt.Errorf("%w: slot %d holds %d bytes, too few for a %d-byte suffix and a key", + ErrInvariant, i, hi-lo, suffix) + } + spans[i] = [2]int{lo, hi} + } + + sort.Slice(spans, func(a, b int) bool { return spans[a][0] < spans[b][0] }) + if spans[0][0] != int(n.FreeEnd()) { + return fmt.Errorf("%w: lowest cell starts at %d, free end says %d", + ErrInvariant, spans[0][0], n.FreeEnd()) + } + for i := 1; i < count; i++ { + if spans[i-1][1] != spans[i][0] { + return fmt.Errorf("%w: cells [%d, %d) and [%d, %d) are not adjacent", + ErrInvariant, spans[i-1][0], spans[i-1][1], spans[i][0], spans[i][1]) + } + } + if spans[count-1][1] != page.Size { + return fmt.Errorf("%w: highest cell ends at %d, want %d", + ErrInvariant, spans[count-1][1], page.Size) + } + + for i := 1; i < count; i++ { + if bytes.Compare(n.KeyRef(i-1), n.KeyRef(i)) >= 0 { + return fmt.Errorf("%w: key %d (%X) is not less than key %d (%X)", + ErrInvariant, i-1, n.KeyRef(i-1), i, n.KeyRef(i)) + } + } + return nil +} diff --git a/internal/nodepage/nodepage_test.go b/internal/nodepage/nodepage_test.go new file mode 100644 index 0000000..a5c585f --- /dev/null +++ b/internal/nodepage/nodepage_test.go @@ -0,0 +1,456 @@ +package nodepage + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "path/filepath" + "testing" + + "github.com/scute-db/scutedb/internal/core" + "github.com/scute-db/scutedb/internal/page" +) + +func key(i int) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(i)) + return b +} + +func rid(i int) core.RowID { + return core.RowID{Page: core.PageID(i / 10), Slot: uint16(i % 10)} +} + +func TestLeafRoundTrip(t *testing.T) { + n := NewLeaf(7) + n.SetNext(9) + for i := 0; i < 50; i++ { + if err := n.AppendLeaf(key(i), rid(i)); err != nil { + t.Fatalf("AppendLeaf(%d): %v", i, err) + } + } + if err := n.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if n.ID() != 7 || n.Next() != 9 || n.Level() != 0 || !n.Leaf() { + t.Fatalf("header round trip failed: id %d next %d level %d leaf %v", + n.ID(), n.Next(), n.Level(), n.Leaf()) + } + if n.KeyCount() != 50 { + t.Fatalf("key count %d, want 50", n.KeyCount()) + } + for i := 0; i < 50; i++ { + if !bytes.Equal(n.Key(i), key(i)) { + t.Fatalf("key %d is % 02X, want % 02X", i, n.Key(i), key(i)) + } + if n.Row(i) != rid(i) { + t.Fatalf("row %d is %+v, want %+v", i, n.Row(i), rid(i)) + } + } +} + +func TestInternalRoundTrip(t *testing.T) { + n, err := NewInternal(4, 2) + if err != nil { + t.Fatalf("NewInternal: %v", err) + } + n.SetFirstChild(100) + for i := 0; i < 30; i++ { + if err := n.AppendInternal(key(i*10), core.PageID(101+i)); err != nil { + t.Fatalf("AppendInternal(%d): %v", i, err) + } + } + if err := n.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if n.Leaf() || n.Level() != 2 { + t.Fatalf("leaf %v level %d, want false and 2", n.Leaf(), n.Level()) + } + if n.ChildCount() != 31 { + t.Fatalf("child count %d, want 31", n.ChildCount()) + } + if n.Child(0) != 100 { + t.Fatalf("child 0 is %d, want 100 (it lives in the node header)", n.Child(0)) + } + for i := 1; i <= 30; i++ { + if want := core.PageID(100 + i); n.Child(i) != want { + t.Fatalf("child %d is %d, want %d", i, n.Child(i), want) + } + } +} + +func TestChildForRoutesLikeTheTreeDoes(t *testing.T) { + n, err := NewInternal(1, 1) + if err != nil { + t.Fatal(err) + } + n.SetFirstChild(10) + for i, k := range []int{20, 40, 60} { + if err := n.AppendInternal(key(k), core.PageID(11+i)); err != nil { + t.Fatal(err) + } + } + cases := []struct { + k int + want core.PageID + }{ + {0, 10}, {19, 10}, {20, 11}, {39, 11}, {40, 12}, {59, 12}, {60, 13}, {999, 13}, + } + for _, c := range cases { + if got := n.ChildFor(key(c.k)); got != c.want { + t.Errorf("ChildFor(%d) = page %d, want %d", c.k, got, c.want) + } + } +} + +func TestSearchFindsAndPlaces(t *testing.T) { + n := NewLeaf(1) + for _, k := range []int{10, 20, 30} { + if err := n.AppendLeaf(key(k), rid(k)); err != nil { + t.Fatal(err) + } + } + cases := []struct { + k int + at int + found bool + }{ + {5, 0, false}, {10, 0, true}, {15, 1, false}, {20, 1, true}, + {25, 2, false}, {30, 2, true}, {35, 3, false}, + } + for _, c := range cases { + at, found := n.Search(key(c.k)) + if at != c.at || found != c.found { + t.Errorf("Search(%d) = (%d, %v), want (%d, %v)", c.k, at, found, c.at, c.found) + } + } +} + +func TestMaxKeysMatchesWhatActuallyFits(t *testing.T) { + mkKey := func(keyLen, i int) []byte { + k := bytes.Repeat([]byte{0xAA}, keyLen) + var b [8]byte + binary.BigEndian.PutUint64(b[:], uint64(i)) + n := keyLen + if n > 8 { + n = 8 + } + copy(k[keyLen-n:], b[8-n:]) + return k + } + + for keyLen := 1; keyLen <= 64; keyLen++ { + leaf := NewLeaf(1) + count := 0 + for { + if err := leaf.AppendLeaf(mkKey(keyLen, count), rid(count)); err != nil { + break + } + count++ + } + if want := MaxKeys(page.KindBTreeLeaf, keyLen); count != want { + t.Errorf("leaf with %d-byte keys held %d, MaxKeys says %d", keyLen, count, want) + } + + internal, err := NewInternal(1, 1) + if err != nil { + t.Fatal(err) + } + count = 0 + for { + if err := internal.AppendInternal(mkKey(keyLen, count), core.PageID(count)); err != nil { + break + } + count++ + } + if want := MaxKeys(page.KindBTreeInternal, keyLen); count != want { + t.Errorf("internal with %d-byte keys held %d, MaxKeys says %d", keyLen, count, want) + } + if got := Fanout(keyLen); got != count+1 { + t.Errorf("Fanout(%d) = %d, node held %d keys so it has %d children", + keyLen, got, count, count+1) + } + } +} + +func TestFullNodeRefusesAndStaysValid(t *testing.T) { + n := NewLeaf(1) + i := 0 + for { + if err := n.AppendLeaf(key(i), rid(i)); err != nil { + if !errors.Is(err, ErrNoFit) { + t.Fatalf("append %d failed with %v, want ErrNoFit", i, err) + } + break + } + i++ + } + if err := n.Validate(); err != nil { + t.Fatalf("a full node failed Validate: %v", err) + } + if n.FreeSpace() < 0 || n.FreeSpace() >= n.EntrySize(8) { + t.Fatalf("free space %d should be non-negative and smaller than one entry (%d)", + n.FreeSpace(), n.EntrySize(8)) + } + if n.Fits(8) { + t.Error("Fits says another 8-byte key fits, but Append refused one") + } +} + +func TestGoldenLeafLayout(t *testing.T) { + n := NewLeaf(7) + n.SetNext(9) + if err := n.AppendLeaf([]byte("aa"), core.RowID{Page: 3, Slot: 1}); err != nil { + t.Fatal(err) + } + if err := n.AppendLeaf([]byte("bb"), core.RowID{Page: 4, Slot: 2}); err != nil { + t.Fatal(err) + } + if err := n.Validate(); err != nil { + t.Fatal(err) + } + + want := []struct { + at int + bytes []byte + what string + }{ + {0, []byte{0, 0, 0, 7}, "page id 7"}, + {4, []byte{byte(page.KindBTreeLeaf)}, "kind btree-leaf"}, + {5, []byte{0}, "flags"}, + {6, []byte{0, 2}, "key count 2"}, + {8, []byte{0, 0x20}, "free start 32 = 24 + 2 slots"}, + {10, []byte{0x0F, 0xF0}, "free end 4080 = 4096 - 2 cells of 8"}, + {12, []byte{0, 0, 0, 0}, "reserved"}, + {16, []byte{0, 0, 0, 9}, "next leaf is page 9"}, + {20, []byte{0, 0}, "level 0"}, + {22, []byte{0, 0}, "padding"}, + {24, []byte{0x0F, 0xF8, 0, 8}, "slot 0 -> cell at 4088, 8 bytes"}, + {28, []byte{0x0F, 0xF0, 0, 8}, "slot 1 -> cell at 4080, 8 bytes"}, + {4088, []byte{'a', 'a', 0, 0, 0, 3, 0, 1}, "cell 0: key aa, row page 3 slot 1"}, + {4080, []byte{'b', 'b', 0, 0, 0, 4, 0, 2}, "cell 1: key bb, row page 4 slot 2"}, + } + for _, w := range want { + got := []byte(n.Page[w.at : w.at+len(w.bytes)]) + if !bytes.Equal(got, w.bytes) { + t.Errorf("bytes %d..%d are % 02X, want % 02X (%s)", + w.at, w.at+len(w.bytes)-1, got, w.bytes, w.what) + } + } + + for i := int(n.FreeStart()); i < int(n.FreeEnd()); i++ { + if n.Page[i] != 0 { + t.Fatalf("free space byte %d is %02X, want 00", i, n.Page[i]) + } + } +} + +func TestFirstKeysCellSitsHighestInThePage(t *testing.T) { + n := NewLeaf(1) + for i := 0; i < 5; i++ { + if err := n.AppendLeaf(key(i), rid(i)); err != nil { + t.Fatal(err) + } + } + prev := page.Size + 1 + for i := 0; i < n.KeyCount(); i++ { + lo, _ := n.cellBounds(i) + if lo >= prev { + t.Fatalf("cell %d starts at %d, not below the previous cell at %d", i, lo, prev) + } + prev = lo + } +} + +func TestKeyCopiesAndKeyRefViews(t *testing.T) { + n := NewLeaf(1) + if err := n.AppendLeaf([]byte("abcd"), rid(1)); err != nil { + t.Fatal(err) + } + cp := n.Key(0) + cp[0] = 'z' + if !bytes.Equal(n.Key(0), []byte("abcd")) { + t.Error("writing to the slice from Key changed the page; Key must copy") + } + ref := n.KeyRef(0) + ref[0] = 'z' + if !bytes.Equal(n.Key(0), []byte("zbcd")) { + t.Error("writing to the slice from KeyRef did not change the page; KeyRef must be a view") + } +} + +func TestRoundTripThroughARealFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "nodes.db") + pf, err := page.Open(path) + if err != nil { + t.Fatal(err) + } + defer pf.Close() + + leaf := NewLeaf(0) + leaf.SetNext(1) + for i := 0; i < 20; i++ { + if err := leaf.AppendLeaf(key(i), rid(i)); err != nil { + t.Fatal(err) + } + } + if err := pf.Write(leaf.Page); err != nil { + t.Fatal(err) + } + if err := pf.Sync(); err != nil { + t.Fatal(err) + } + + raw, err := pf.Read(0) + if err != nil { + t.Fatal(err) + } + back, err := Load(raw) + if err != nil { + t.Fatalf("Load: %v", err) + } + if err := back.Validate(); err != nil { + t.Fatalf("Validate after a disk round trip: %v", err) + } + if back.Next() != 1 || back.KeyCount() != 20 { + t.Fatalf("next %d count %d, want 1 and 20", back.Next(), back.KeyCount()) + } + for i := 0; i < 20; i++ { + if !bytes.Equal(back.Key(i), key(i)) || back.Row(i) != rid(i) { + t.Fatalf("entry %d did not survive the round trip", i) + } + } + if !bytes.Equal(back.Page, leaf.Page) { + t.Error("the page read back is not byte-identical to the one written") + } +} + +func TestLoadRejectsPagesThatAreNotNodes(t *testing.T) { + for _, k := range []page.Kind{page.KindFree, page.KindMeta, page.KindHeap} { + p := page.New(1, k) + if _, err := Load(p); !errors.Is(err, ErrBadKind) { + t.Errorf("Load of a %s page gave %v, want ErrBadKind", k, err) + } + } + if _, err := Load(make(page.Page, 10)); !errors.Is(err, core.ErrShortPage) { + t.Errorf("Load of a short page gave %v, want ErrShortPage", err) + } +} + +func TestNewInternalRejectsLevelZero(t *testing.T) { + if _, err := NewInternal(1, 0); !errors.Is(err, ErrBadLevel) { + t.Errorf("NewInternal at level 0 gave %v, want ErrBadLevel", err) + } +} + +func TestAppendRejectsTheWrongKind(t *testing.T) { + leaf := NewLeaf(1) + if err := leaf.AppendInternal(key(1), 2); !errors.Is(err, ErrBadKind) { + t.Errorf("AppendInternal on a leaf gave %v, want ErrBadKind", err) + } + internal, err := NewInternal(1, 1) + if err != nil { + t.Fatal(err) + } + if err := internal.AppendLeaf(key(1), rid(1)); !errors.Is(err, ErrBadKind) { + t.Errorf("AppendLeaf on an internal node gave %v, want ErrBadKind", err) + } +} + +func TestValidateCatchesCorruption(t *testing.T) { + build := func() Node { + n := NewLeaf(1) + for i := 0; i < 6; i++ { + if err := n.AppendLeaf(key(i), rid(i)); err != nil { + t.Fatal(err) + } + } + return n + } + + cases := []struct { + name string + corrupt func(Node) + }{ + {"key count no longer matches free start", func(n Node) { n.SetItemCount(5) }}, + {"free start moved", func(n Node) { n.SetFreeStart(n.FreeStart() + 4) }}, + {"free end moved", func(n Node) { n.SetFreeEnd(n.FreeEnd() - 8) }}, + {"a slot points into the slot array", func(n Node) { + binary.BigEndian.PutUint16(n.Page[n.slotOffset(2):], SlotArrayStart) + }}, + {"a slot points past the page", func(n Node) { + binary.BigEndian.PutUint16(n.Page[n.slotOffset(2):], page.Size-2) + }}, + {"a cell is too short to hold a row id", func(n Node) { + binary.BigEndian.PutUint16(n.Page[n.slotOffset(2)+2:], 4) + }}, + {"keys are out of order", func(n Node) { + a, b := n.slotOffset(2), n.slotOffset(3) + var tmp [SlotSize]byte + copy(tmp[:], n.Page[a:a+SlotSize]) + copy(n.Page[a:a+SlotSize], n.Page[b:b+SlotSize]) + copy(n.Page[b:b+SlotSize], tmp[:]) + }}, + {"two slots point at the same cell", func(n Node) { + lo, _ := n.cellBounds(3) + binary.BigEndian.PutUint16(n.Page[n.slotOffset(2):], uint16(lo)) + }}, + {"padding was used for something", func(n Node) { n.Page[offPad] = 1 }}, + {"a leaf claims a level", func(n Node) { n.setLevel(3) }}, + {"the kind byte was flipped", func(n Node) { n.SetKind(page.KindHeap) }}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + n := build() + if err := n.Validate(); err != nil { + t.Fatalf("the clean node did not validate: %v", err) + } + c.corrupt(n) + if err := n.Validate(); err == nil { + t.Errorf("Validate accepted a node where %s", c.name) + } + }) + } +} + +func TestEmptyNodeIsValid(t *testing.T) { + leaf := NewLeaf(3) + if err := leaf.Validate(); err != nil { + t.Errorf("an empty leaf failed Validate: %v", err) + } + if leaf.FreeSpace() != EntryArea { + t.Errorf("an empty leaf has %d free, want %d", leaf.FreeSpace(), EntryArea) + } + internal, err := NewInternal(3, 1) + if err != nil { + t.Fatal(err) + } + if err := internal.Validate(); err != nil { + t.Errorf("an empty internal node failed Validate: %v", err) + } +} + +func TestDescribeMentionsWhatItDecoded(t *testing.T) { + n := NewLeaf(7) + n.SetNext(9) + if err := n.AppendLeaf([]byte("aa"), core.RowID{Page: 3, Slot: 1}); err != nil { + t.Fatal(err) + } + got := n.Describe() + for _, want := range []string{ + "page 7", "btree-leaf", "next leaf = page 9", "0F F8", "row id -> page 3 slot 1", + } { + if !bytes.Contains([]byte(got), []byte(want)) { + t.Errorf("Describe output is missing %q\n%s", want, got) + } + } +} + +func TestFanoutTableIsHonestAboutOverhead(t *testing.T) { + out := FanoutTable(8) + if !bytes.Contains([]byte(out), []byte(fmt.Sprintf("%d", EntryArea))) { + t.Errorf("FanoutTable should show the %d bytes left for entries\n%s", EntryArea, out) + } +}