diff --git a/Makefile b/Makefile index d9b76a9..a87f783 100644 --- a/Makefile +++ b/Makefile @@ -67,3 +67,6 @@ demo-btree: demo-range: go run ./cmd/scutedb-demo range + +demo-delete: + go run ./cmd/scutedb-demo delete diff --git a/README.md b/README.md index 1784f80..927f955 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,9 @@ A *scute* is one of the bony plates that make up a turtle's shell. A shell is made of plates; a database file is made of pages. -**Status:** Phase 0 complete more to go a lot of learning and doing. +**Status:** Phases 0 and A complete. The B+Tree searches, inserts, splits, +scans ranges and deletes. All of it still lives in memory — Phase B puts it on +disk. --- @@ -17,17 +19,18 @@ go test ./... # everything make demo # list the runnable experiments make demo-scan # watch a file-based database degrade make hexdump # write real pages and look at the bytes +make demo-delete # borrow, merge and root collapse, traced live ``` --- -## Roadmap +## Roadmap I am trying to follow plan first execute second | Phase | What | Status | |-------|------|--------| | **0** | Foundations — interfaces, the naive database, pages | **done** | -| A | Bytes & the B+Tree | **in progress** | -| B | Persistence — storage manager, buffer pool, locking | | +| **A** | Bytes & the B+Tree | **done** | +| B | Persistence — storage manager, buffer pool, locking | **next** | | C | A real data store — schema, rows, indexes | | | D | Transactions — WAL, recovery, 2PL, MVCC | | | E | Beyond — LSM engine, Raft, server, query planner | | @@ -538,11 +541,119 @@ fuzz target that ran 2.9M cases comparing scan output to the same brute force. Run it: `make demo-range`. -### Known gaps at the end of Phase 0 +### `0x08` B+Tree deletion + +The step most from-scratch databases stop at. Insert has one failure mode and +one answer; delete has one failure mode and **three**, any of which can cascade. + +``` +insert delete +does it fit? done still enough keys? done +too full -> split too empty -> borrow from left + or borrow from right + or merge + and the merge may cascade up + and the root may collapse +``` + +Every one of those six behaves differently for a leaf than for an internal node, +which is where the "3x harder" comes from. + +**All four repair paths in one trace** at order 4 (max 3 keys, min 1) — the demo +reports what actually happened rather than what was expected: + +``` +delete 030 -> fitted, no repair needed +delete 040 -> BORROWED from the LEFT sibling +delete 010 -> MERGED two nodes into one +delete 020 -> BORROWED from the RIGHT sibling +delete 050 -> MERGED two nodes into one +delete 060 -> BORROWED from the RIGHT sibling +delete 070 -> MERGED two nodes into one, then the ROOT COLLAPSED +``` + +**Borrowing rewrites one separator. Merging deletes one** — which is what can +make the parent underflow in turn, and so on up. The tree grew from the root and +it shrinks back to the root: + +``` +2000 keys -> height 7 +delete 1990 -> height 3 +along the way: 1487 borrows, 1485 merges, 4 root collapses +``` + +**The leaf/internal asymmetry mirrors the split asymmetry exactly.** A leaf +*split* copies its middle key up; an internal split *moves* it. A leaf *merge* +discards the separator; an internal merge *pulls it down* between the two halves, +because for an internal node that separator is the only copy of the key. + +**Ghost separators.** A separator only has to point a search the right way — it +does **not** have to name a key that exists. Deleting every key that appears as a +separator, then re-checking: + +``` +deleted all 49 keys that appeared as separators +49 of the 49 separators left now name keys that are gone +lookups that return the wrong answer: 0 of 100 +validate: +``` + +This is why deletion never has to hunt upward and repair separators — a large +amount of work the algorithm gets to skip, and the reason a stale separator is +correct rather than a bug. + +**Why some engines never merge at all.** Merging keeps the index dense: + +``` + leaves keys fill +4000 keys 500 4000 53% +after deleting half 249 2000 54% +``` + +251 leaves handed back, fill unchanged. Without merging the leaf count would +have stayed at 500 and the fill halved — the index growing while the data +shrinks. And yet **Postgres's B-Tree does not merge partially-empty pages**; it +only reclaims entirely empty ones. That is exactly why index bloat is an +operational concern there and why `REINDEX` exists. Merging costs write +amplification and page locks on a hot path, and some engines judge that trade +not worth making. + +**`index.Index` is now satisfied.** With `Delete` in place, `btree.Index` wraps +the tree to match the interface written in `0x00` — asserted by +`var _ index.Index = (*Index)(nil)`. The natural API stays natural (`Get` +returns a bool, `Put` cannot fail in memory); the adapter translates. + +**How it is verified.** A reference `map` is maintained alongside the tree +through 4,000 random insert/delete operations at five orders, with `Validate()` +after **every** operation and an exhaustive `Get` check over the whole key space +at the end. `TestMinKeysHoldsForEveryOrder` inserts 400 and deletes 400 for +every order from 3 to 40. `FuzzInsertDeleteAgainstReference` ran 344,719 +executions, each up to 3,000 operations, validating after each one. + +Run it: `make demo-delete`. + +### Phase A in one paragraph + +Six steps turned raw bytes into a working index. `codec` ships two encodings +because keys and values have different jobs — values optimise for size, keys +optimise for sorting correctly as raw bytes, and conflating them is not +recoverable later. `nullbits` makes "no value" a property of the row rather than +a magic value hidden inside it, and carries the three-valued logic that follows +from that. `slots` buys O(1) addressing inside a page for a few bytes of padding. +`btree` turns a linear scan into three or four hops, keeps the leaves chained so +a range costs one descent and then a sideways walk, and now repairs itself when +deletes empty a node out. + +What is still missing is the part that makes this a database rather than a data +structure: none of it survives a restart. Every node is a Go pointer. Phase B +replaces those pointers with page IDs, and the tree starts living on disk. + +### Known gaps at the end of Phase A Deliberate, each one is a later step: - ~~`Page.Append` writes items with no separator~~ → fixed in `0x03` +- The B+Tree is pointers in memory and nothing persists → `0x09` (nodes become pages) - No way to find item *n* without walking items 1..*n-1* → `0x0F` (slot directory) - `File.Allocate` never reuses a freed page → `0x0A` (free list) - The reserved header bytes hold no checksum → `0x13` @@ -564,7 +675,7 @@ internal/ codec/ value encoding (compact) and key encoding (ordered) nullbits/ null bitmaps and SQL three-valued logic slots/ fixed-size, aligned record slots inside a page - btree/ in-memory B+Tree: search, insert, split, range scans + btree/ in-memory B+Tree: search, insert, split, range scans, delete ``` ## Conventions diff --git a/cmd/scutedb-demo/main.go b/cmd/scutedb-demo/main.go index a324136..51f85a5 100644 --- a/cmd/scutedb-demo/main.go +++ b/cmd/scutedb-demo/main.go @@ -27,7 +27,7 @@ import ( func main() { if len(os.Args) < 2 { - fmt.Println("usage: scutedb-demo scan|update|race|crash|pages|header|encode|nulls|align|btree|range") + fmt.Println("usage: scutedb-demo scan|update|race|crash|pages|header|encode|nulls|align|btree|range|delete") os.Exit(2) } switch os.Args[1] { @@ -53,6 +53,8 @@ func main() { expBTree() case "range": expRange() + case "delete": + expDelete() case "crash-child": crashChild(os.Args[2]) default: @@ -855,3 +857,161 @@ func keysOf(it *btree.Iterator) string { } return strings.Join(parts, " ") } + +func expDelete() { + fmt.Println("STEP 0x08 B+Tree deletion") + + fmt.Print("\n1. THE PROBLEM: UNDERFLOW\n\n") + fmt.Println(" inserting can only ever make a node TOO FULL, and there is one") + fmt.Println(" answer: split it. deleting makes a node TOO EMPTY, and there are") + fmt.Println(" three answers, which is why this step is the hard one.") + fmt.Println("\n at order 4 a node may hold 3 keys and must hold at least 1.") + fmt.Println(" drop below that minimum and the node has UNDERFLOWED.") + + fmt.Print("\n2. THE THREE FIXES, AS THEY ACTUALLY HAPPEN\n\n") + tr, err := btree.New(4) + check(err) + for i := 1; i <= 8; i++ { + tr.Put([]byte(fmt.Sprintf("%03d", i*10)), core.RowID{Slot: uint16(i)}) + } + fmt.Print(" order 4: a node holds at most 3 keys and at least 1.\n\n") + dumpIndented(tr, " ") + for _, k := range []string{"030", "040", "010", "020", "050", "060", "070"} { + deleteAndReport(tr, k) + } + s := tr.Stats() + fmt.Printf("\n totals: %d borrows left, %d borrows right, %d merges, %d collapses\n", + s.BorrowsLeft, s.BorrowsRight, s.Merges, s.Collapses) + fmt.Println("\n borrowing rewrites one separator. merging deletes one, which is") + fmt.Println(" what can make the parent underflow in turn.") + + fmt.Print("\n3. THE CASCADE\n\n") + fmt.Println(" merging two children removes a key from their parent, which can") + fmt.Println(" make the PARENT underflow, which repeats all the way to the root.") + tr3, err := btree.New(4) + check(err) + for i := 0; i < 2000; i++ { + tr3.Put(key8(i), core.RowID{Slot: uint16(i % 1000)}) + } + tall := tr3.Height() + for i := 0; i < 1990; i++ { + tr3.Delete(key8(i)) + } + s3 := tr3.Stats() + fmt.Printf("\n 2000 keys -> height %d\n", tall) + fmt.Printf(" delete 1990 -> height %d\n", tr3.Height()) + fmt.Printf(" along the way: %d borrows, %d merges, %d root collapses\n", + s3.BorrowsLeft+s3.BorrowsRight, s3.Merges, s3.Collapses) + fmt.Println("\n the tree grew from the root and it shrinks back to the root.") + + fmt.Print("\n4. WHY DELETION IS THE HARD ONE\n\n") + fmt.Printf(" %-28s %s\n", "insert", "delete") + fmt.Println(" " + line(60)) + fmt.Printf(" %-28s %s\n", "does it fit? done", "still enough keys? done") + fmt.Printf(" %-28s %s\n", "too full -> split", "too empty -> borrow from left") + fmt.Printf(" %-28s %s\n", "", " or borrow from right") + fmt.Printf(" %-28s %s\n", "", " or merge") + fmt.Printf(" %-28s %s\n", "", "and the merge may cascade up") + fmt.Printf(" %-28s %s\n", "", "and the root may collapse") + fmt.Println("\n two cases against six. and every one of the six behaves") + fmt.Println(" differently for a leaf than for an internal node.") + + fmt.Print("\n5. GHOST SEPARATORS\n\n") + fmt.Println(" a separator only has to POINT the search the right way. it does") + fmt.Println(" not have to be a key that exists.") + tr4, err := btree.New(4) + check(err) + for i := 0; i < 100; i++ { + tr4.Put(key8(i), core.RowID{Slot: uint16(i)}) + } + seps := tr4.Separators() + for _, k := range seps { + tr4.Delete(k) + } + after := tr4.Separators() + ghosts := 0 + for _, k := range after { + if _, ok := tr4.Get(k); !ok { + ghosts++ + } + } + wrong := 0 + for i := 0; i < 100; i++ { + k := key8(i) + wasSep := false + for _, sp := range seps { + if bytes.Equal(sp, k) { + wasSep = true + } + } + if _, ok := tr4.Get(k); ok == wasSep { + wrong++ + } + } + fmt.Printf("\n deleted all %d keys that appeared as separators.\n", len(seps)) + fmt.Printf(" %d of the %d separators left in the tree now name keys that are gone.\n", + ghosts, len(after)) + fmt.Printf(" lookups that return the wrong answer: %d of 100\n", wrong) + fmt.Printf(" validate: %v\n", tr4.Validate()) + fmt.Println("\n so deletion never has to hunt down and repair separators above it.") + fmt.Println(" that is a large amount of work the algorithm gets to skip.") + + fmt.Print("\n6. WHY SOME ENGINES NEVER MERGE AT ALL\n\n") + tr5, err := btree.New(16) + check(err) + for i := 0; i < 4000; i++ { + tr5.Put(key8(i), core.RowID{Slot: uint16(i % 1000)}) + } + l1, k1, c1 := tr5.LeafFill() + for i := 0; i < 4000; i += 2 { + tr5.Delete(key8(i)) + } + l2, k2, c2 := tr5.LeafFill() + fmt.Printf(" %-26s %-10s %-10s %s\n", "", "leaves", "keys", "fill") + fmt.Println(" " + line(56)) + fmt.Printf(" %-26s %-10d %-10d %.0f%%\n", "4000 keys", l1, k1, 100*float64(k1)/float64(c1)) + fmt.Printf(" %-26s %-10d %-10d %.0f%%\n", "after deleting half", l2, k2, 100*float64(k2)/float64(c2)) + fmt.Printf("\n merging kept the leaves reasonably full and gave %d of them back.\n", l1-l2) + fmt.Println(" without merging, the leaf count would have stayed at", l1, "and the") + fmt.Println(" fill would have halved — the index gets bigger than the data.") + fmt.Println("\n and yet: Postgres's B-Tree does NOT merge partially-empty pages.") + fmt.Println(" it only reclaims pages that are entirely empty. that is exactly why") + fmt.Println(" index bloat is a real operational problem there, and why REINDEX") + fmt.Println(" exists. merging costs write amplification and page locks on a hot") + fmt.Println(" path, and some engines decide that trade is not worth it.") +} + +func deleteAndReport(tr *btree.Tree, k string) { + before := tr.Stats() + if !tr.Delete([]byte(k)) { + fmt.Printf("\n delete %s -> not present\n", k) + return + } + a := tr.Stats() + what := "fitted, no repair needed" + switch { + case a.Merges > before.Merges: + what = "MERGED two nodes into one" + case a.BorrowsLeft > before.BorrowsLeft: + what = "BORROWED from the LEFT sibling" + case a.BorrowsRight > before.BorrowsRight: + what = "BORROWED from the RIGHT sibling" + } + if a.Collapses > before.Collapses { + what += ", then the ROOT COLLAPSED" + } + fmt.Printf("\n delete %s -> %s\n", k, what) + dumpIndented(tr, " ") +} + +func dumpIndented(tr *btree.Tree, pad string) { + for _, ln := range strings.Split(strings.TrimRight(tr.Dump(), "\n"), "\n") { + fmt.Printf("%s%s\n", pad, ln) + } +} + +func key8(n int) []byte { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(n)) + return b +} diff --git a/internal/btree/adapter.go b/internal/btree/adapter.go new file mode 100644 index 0000000..fff7f09 --- /dev/null +++ b/internal/btree/adapter.go @@ -0,0 +1,46 @@ +package btree + +import ( + "github.com/scute-db/scutedb/internal/core" + "github.com/scute-db/scutedb/internal/index" +) + +type Index struct{ t *Tree } + +func NewIndex(order int) (*Index, error) { + t, err := New(order) + if err != nil { + return nil, err + } + return &Index{t: t}, nil +} + +func (x *Index) Tree() *Tree { return x.t } + +func (x *Index) Get(key []byte) (core.RowID, error) { + rid, ok := x.t.Get(key) + if !ok { + return core.RowID{}, core.ErrNotFound + } + return rid, nil +} + +func (x *Index) Put(key []byte, rid core.RowID) error { + x.t.Put(key, rid) + return nil +} + +func (x *Index) Delete(key []byte) error { + if !x.t.Delete(key) { + return core.ErrNotFound + } + return nil +} + +func (x *Index) Scan(from, to []byte) (index.Iterator, error) { + return x.t.Scan(from, to), nil +} + +func (x *Index) Close() error { return nil } + +var _ index.Index = (*Index)(nil) diff --git a/internal/btree/btree.go b/internal/btree/btree.go index bd02605..d70394e 100644 --- a/internal/btree/btree.go +++ b/internal/btree/btree.go @@ -35,6 +35,7 @@ type Tree struct { maxKeys int minKeys int count int + stats Stats } func New(order int) (*Tree, error) { diff --git a/internal/btree/delete.go b/internal/btree/delete.go new file mode 100644 index 0000000..06437c6 --- /dev/null +++ b/internal/btree/delete.go @@ -0,0 +1,162 @@ +package btree + +import ( + "bytes" + + "github.com/scute-db/scutedb/internal/core" +) + +type Stats struct { + BorrowsLeft int + BorrowsRight int + Merges int + Collapses int +} + +func (t *Tree) Stats() Stats { return t.stats } + +func (t *Tree) ResetStats() { t.stats = Stats{} } + +func (t *Tree) Delete(key []byte) bool { + if t.root == nil { + return false + } + if !t.delete(t.root, key) { + return false + } + t.count-- + for !t.root.leaf && len(t.root.keys) == 0 { + t.root = t.root.children[0] + t.stats.Collapses++ + } + return true +} + +func (t *Tree) delete(n *node, key []byte) bool { + if n.leaf { + i := n.leafIndex(key) + if i >= len(n.keys) || !bytes.Equal(n.keys[i], key) { + return false + } + n.keys = append(n.keys[:i], n.keys[i+1:]...) + n.rows = append(n.rows[:i], n.rows[i+1:]...) + return true + } + i := n.childIndex(key) + if !t.delete(n.children[i], key) { + return false + } + if len(n.children[i].keys) < t.minKeys { + t.repair(n, i) + } + return true +} + +func (t *Tree) repair(p *node, i int) { + if i > 0 && len(p.children[i-1].keys) > t.minKeys { + t.borrowFromLeft(p, i) + t.stats.BorrowsLeft++ + return + } + if i < len(p.children)-1 && len(p.children[i+1].keys) > t.minKeys { + t.borrowFromRight(p, i) + t.stats.BorrowsRight++ + return + } + if i > 0 { + t.merge(p, i-1) + } else { + t.merge(p, i) + } + t.stats.Merges++ +} + +func (t *Tree) borrowFromLeft(p *node, i int) { + left, child := p.children[i-1], p.children[i] + last := len(left.keys) - 1 + + if child.leaf { + child.keys = append([][]byte{left.keys[last]}, child.keys...) + child.rows = append([]core.RowID{left.rows[last]}, child.rows...) + left.keys = left.keys[:last:last] + left.rows = left.rows[:last:last] + p.keys[i-1] = cloneKey(child.keys[0]) + return + } + + lastChild := len(left.children) - 1 + child.keys = append([][]byte{p.keys[i-1]}, child.keys...) + child.children = append([]*node{left.children[lastChild]}, child.children...) + p.keys[i-1] = left.keys[last] + left.keys = left.keys[:last:last] + left.children = left.children[:lastChild:lastChild] +} + +func (t *Tree) borrowFromRight(p *node, i int) { + child, right := p.children[i], p.children[i+1] + + if child.leaf { + child.keys = append(child.keys, right.keys[0]) + child.rows = append(child.rows, right.rows[0]) + right.keys = right.keys[1:] + right.rows = right.rows[1:] + p.keys[i] = cloneKey(right.keys[0]) + return + } + + child.keys = append(child.keys, p.keys[i]) + child.children = append(child.children, right.children[0]) + p.keys[i] = right.keys[0] + right.keys = right.keys[1:] + right.children = right.children[1:] +} + +func (t *Tree) merge(p *node, i int) { + left, right := p.children[i], p.children[i+1] + + if left.leaf { + left.keys = append(left.keys, right.keys...) + left.rows = append(left.rows, right.rows...) + left.next = right.next + } else { + left.keys = append(left.keys, p.keys[i]) + left.keys = append(left.keys, right.keys...) + left.children = append(left.children, right.children...) + } + + p.keys = append(p.keys[:i], p.keys[i+1:]...) + p.children = append(p.children[:i+1], p.children[i+2:]...) +} + +func (t *Tree) LeafFill() (leaves, keys, capacity int) { + if t.root == nil { + return 0, 0, 0 + } + for n := t.firstLeaf(); n != nil; n = n.next { + leaves++ + keys += len(n.keys) + capacity += t.maxKeys + } + return leaves, keys, capacity +} + +func (t *Tree) Separators() [][]byte { + var out [][]byte + if t.root == nil { + return out + } + collectSeparators(t.root, &out) + return out +} + +func collectSeparators(n *node, out *[][]byte) { + if n.leaf { + return + } + for _, k := range n.keys { + *out = append(*out, cloneKey(k)) + } + for _, c := range n.children { + collectSeparators(c, out) + } +} diff --git a/internal/btree/delete_edge_test.go b/internal/btree/delete_edge_test.go new file mode 100644 index 0000000..a814317 --- /dev/null +++ b/internal/btree/delete_edge_test.go @@ -0,0 +1,288 @@ +package btree + +import ( + "bytes" + "errors" + "fmt" + "testing" + + "github.com/scute-db/scutedb/internal/core" +) + +func separators(n *node, out *[][]byte) { + if n.leaf { + return + } + *out = append(*out, n.keys...) + for _, c := range n.children { + separators(c, out) + } +} + +func TestDeletingASeparatorKey(t *testing.T) { + tr := mustNew(t, 4) + for i := 0; i < 200; i++ { + tr.Put(key(i), rid(i)) + } + var seps [][]byte + separators(tr.root, &seps) + if len(seps) == 0 { + t.Fatal("no separators to test") + } + target := seps[len(seps)/2] + t.Logf("deleting a key that is also a separator in an internal node") + if !tr.Delete(target) { + t.Fatal("delete of a separator key failed") + } + if err := tr.Validate(); err != nil { + t.Errorf("invariants broken after deleting a separator: %v", err) + } + if _, ok := tr.Get(target); ok { + t.Error("the deleted separator key is still findable") + } + var still [][]byte + separators(tr.root, &still) + found := false + for _, s := range still { + if string(s) == string(target) { + found = true + } + } + t.Logf("the separator itself still present in an internal node: %v", found) + for i := 0; i < 200; i++ { + want := string(key(i)) != string(target) + if _, ok := tr.Get(key(i)); ok != want { + t.Errorf("Get(%d) = %v, want %v", i, ok, want) + } + } +} + +func TestDeleteTwiceIsNotAnError(t *testing.T) { + tr := mustNew(t, 4) + for i := 0; i < 50; i++ { + tr.Put(key(i), rid(i)) + } + if !tr.Delete(key(10)) { + t.Fatal("first delete failed") + } + if tr.Delete(key(10)) { + t.Error("second delete of the same key reported success") + } + if tr.Len() != 49 { + t.Errorf("Len = %d after one real and one failed delete, want 49", tr.Len()) + } +} + +func TestDeleteDuringIterationKeepsTheTreeValid(t *testing.T) { + tr := mustNew(t, 4) + for i := 0; i < 200; i++ { + tr.Put(key(i), rid(i)) + } + defer func() { + if r := recover(); r != nil { + t.Errorf("delete during iteration panicked: %v", r) + } + }() + seen := 0 + it := tr.ScanAll() + for it.Next() { + seen++ + if seen == 5 { + for i := 100; i < 180; i++ { + tr.Delete(key(i)) + } + } + } + t.Logf("deleted 80 keys mid-scan; the iterator saw %d rows", seen) + if err := tr.Validate(); err != nil { + t.Errorf("tree broken by delete during iteration: %v", err) + } + if tr.Len() != 120 { + t.Errorf("Len = %d, want 120", tr.Len()) + } +} + +func TestAdapterSatisfiesTheInterface(t *testing.T) { + x, err := NewIndex(8) + if err != nil { + t.Fatal(err) + } + if err := x.Put(key(1), rid(1)); err != nil { + t.Fatal(err) + } + got, err := x.Get(key(1)) + if err != nil || got != rid(1) { + t.Fatalf("Get = %v, %v", got, err) + } + if _, err := x.Get(key(99)); !errors.Is(err, core.ErrNotFound) { + t.Errorf("Get of an absent key gave %v, want ErrNotFound", err) + } + if err := x.Delete(key(99)); !errors.Is(err, core.ErrNotFound) { + t.Errorf("Delete of an absent key gave %v, want ErrNotFound", err) + } + if err := x.Delete(key(1)); err != nil { + t.Errorf("Delete of a present key gave %v", err) + } + it, err := x.Scan(nil, nil) + if err != nil { + t.Fatal(err) + } + n := 0 + for it.Next() { + n++ + } + if n != 0 { + t.Errorf("scan of an emptied index returned %d rows", n) + } + if err := x.Close(); err != nil { + t.Error(err) + } + if _, err := NewIndex(2); err == nil { + t.Error("NewIndex(2) was accepted") + } +} + +func TestMinKeysHoldsForEveryOrder(t *testing.T) { + for order := 3; order <= 40; order++ { + tr := mustNew(t, order) + for i := 0; i < 400; i++ { + tr.Put(key(i), rid(i)) + } + for i := 0; i < 400; i++ { + tr.Delete(key(i)) + if err := tr.Validate(); err != nil { + t.Fatalf("order %d, after deleting %d: %v", order, i, err) + } + } + } + fmt.Println("insert 400 then delete 400, invariants held for every order from 3 to 40") +} + +func TestDeletePrefixAndEmptyKeys(t *testing.T) { + tr := mustNew(t, 3) + words := []string{"", "a", "aa", "aaa", "ab", "b", "ba", "bb"} + for i, w := range words { + tr.Put([]byte(w), rid(i)) + } + if tr.Len() != len(words) { + t.Fatalf("Len = %d, want %d", tr.Len(), len(words)) + } + for _, w := range []string{"aa", "", "b"} { + if !tr.Delete([]byte(w)) { + t.Errorf("delete of %q failed", w) + } + if err := tr.Validate(); err != nil { + t.Fatalf("after deleting %q: %v", w, err) + } + } + gone := map[string]bool{"aa": true, "": true, "b": true} + for _, w := range words { + _, ok := tr.Get([]byte(w)) + if ok == gone[w] { + t.Errorf("%q present=%v but deleted=%v", w, ok, gone[w]) + } + } + if tr.Delete(nil) { + t.Error("Delete(nil) succeeded after the empty key was already removed") + } +} + +func TestSeparatorKeysNeedNotExist(t *testing.T) { + tr := mustNew(t, 4) + for i := 0; i < 100; i++ { + tr.Put(key(i), rid(i)) + } + var seps [][]byte + separators(tr.root, &seps) + for _, s := range seps { + tr.Delete(s) + } + if err := tr.Validate(); err != nil { + t.Errorf("deleting every separator broke invariants: %v", err) + } + var remaining [][]byte + separators(tr.root, &remaining) + ghosts := 0 + for _, s := range remaining { + if _, ok := tr.Get(s); !ok { + ghosts++ + } + } + t.Logf("%d of %d separators now name keys that no longer exist", ghosts, len(remaining)) + scanned := len(collect(tr.ScanAll())) + if scanned != tr.Len() { + t.Errorf("scan found %d keys but Len says %d", scanned, tr.Len()) + } + for i := 0; i < 100; i++ { + k := key(i) + wasSep := false + for _, s := range seps { + if string(s) == string(k) { + wasSep = true + } + } + _, ok := tr.Get(k) + if ok == wasSep { + t.Errorf("key %d present=%v but was deleted=%v", i, ok, wasSep) + } + } +} + +func TestValidateCatchesABadSeparator(t *testing.T) { + tr := mustNew(t, 4) + for i := 0; i < 50; i++ { + tr.Put(key(i), rid(i)) + } + if err := tr.Validate(); err != nil { + t.Fatal(err) + } + if tr.root.leaf { + t.Fatal("expected an internal root for this test") + } + + saved := tr.root.keys[0] + tr.root.keys[0] = key(0) + if err := tr.Validate(); err == nil { + t.Error("Validate accepted a separator that puts the left subtree out of bounds") + } + tr.root.keys[0] = saved + + last := len(tr.root.keys) - 1 + saved = tr.root.keys[last] + tr.root.keys[last] = key(49) + if err := tr.Validate(); err == nil { + t.Error("Validate accepted a separator that puts the right subtree out of bounds") + } + tr.root.keys[last] = saved + + if err := tr.Validate(); err != nil { + t.Fatalf("Validate broke after restoring the separators: %v", err) + } +} + +func TestIteratorSkipsSeveralEmptyLeavesInARow(t *testing.T) { + tr := mustNew(t, 4) + for i := 0; i < 40; i++ { + tr.Put(key(i), rid(i)) + } + want := len(collect(tr.ScanAll())) + + first := tr.firstLeaf() + after := first.next + e1 := &node{leaf: true} + e2 := &node{leaf: true} + e1.next = e2 + e2.next = after + first.next = e1 + + got := collect(tr.ScanAll()) + if len(got) != want { + t.Fatalf("scan returned %d keys with two empty leaves spliced in, want %d", + len(got), want) + } + for i := 1; i < len(got); i++ { + if bytes.Compare(got[i-1], got[i]) >= 0 { + t.Fatalf("scan not sorted at %d", i) + } + } +} diff --git a/internal/btree/delete_test.go b/internal/btree/delete_test.go new file mode 100644 index 0000000..02a087c --- /dev/null +++ b/internal/btree/delete_test.go @@ -0,0 +1,271 @@ +package btree + +import ( + "bytes" + "fmt" + "math/rand" + "sort" + "testing" +) + +func TestDeleteMissingKey(t *testing.T) { + tr := mustNew(t, 4) + if tr.Delete(key(1)) { + t.Fatal("deleting from an empty tree reported success") + } + for i := 0; i < 10; i++ { + tr.Put(key(i), rid(i)) + } + if tr.Delete(key(99)) { + t.Fatal("deleting an absent key reported success") + } + if tr.Len() != 10 { + t.Fatalf("Len = %d after a failed delete, want 10", tr.Len()) + } + var zero Tree + if zero.Delete(key(1)) { + t.Fatal("delete on a zero tree reported success") + } +} + +func TestDeleteOneKey(t *testing.T) { + tr := mustNew(t, 4) + for i := 0; i < 10; i++ { + tr.Put(key(i), rid(i)) + } + if !tr.Delete(key(5)) { + t.Fatal("delete reported failure") + } + if tr.Len() != 9 { + t.Fatalf("Len = %d, want 9", tr.Len()) + } + if _, ok := tr.Get(key(5)); ok { + t.Fatal("deleted key is still findable") + } + for _, i := range []int{0, 1, 2, 3, 4, 6, 7, 8, 9} { + if _, ok := tr.Get(key(i)); !ok { + t.Fatalf("key %d disappeared", i) + } + } + if err := tr.Validate(); err != nil { + t.Fatal(err) + } +} + +func TestDeleteEverythingLeavesAnEmptyTree(t *testing.T) { + for _, order := range []int{3, 4, 7, 32} { + tr := mustNew(t, order) + for i := 0; i < 500; i++ { + tr.Put(key(i), rid(i)) + } + for i := 0; i < 500; i++ { + if !tr.Delete(key(i)) { + t.Fatalf("order %d: delete of %d failed", order, i) + } + if err := tr.Validate(); err != nil { + t.Fatalf("order %d after deleting %d: %v", order, i, err) + } + } + if tr.Len() != 0 { + t.Fatalf("order %d: Len = %d after deleting everything", order, tr.Len()) + } + if tr.Height() != 1 { + t.Fatalf("order %d: height = %d after deleting everything, want 1", order, tr.Height()) + } + if got := len(collect(tr.ScanAll())); got != 0 { + t.Fatalf("order %d: scan found %d keys in an empty tree", order, got) + } + tr.Put(key(1), rid(1)) + if _, ok := tr.Get(key(1)); !ok { + t.Fatalf("order %d: tree unusable after being emptied", order) + } + } +} + +func TestDeleteInReverseOrder(t *testing.T) { + tr := mustNew(t, 4) + for i := 0; i < 400; i++ { + tr.Put(key(i), rid(i)) + } + for i := 399; i >= 0; i-- { + if !tr.Delete(key(i)) { + t.Fatalf("delete of %d failed", i) + } + if err := tr.Validate(); err != nil { + t.Fatalf("after deleting %d: %v", i, err) + } + } + if tr.Len() != 0 { + t.Fatalf("Len = %d", tr.Len()) + } +} + +func TestHeightShrinksBackDown(t *testing.T) { + tr := mustNew(t, 4) + for i := 0; i < 2000; i++ { + tr.Put(key(i), rid(i)) + } + tall := tr.Height() + for i := 0; i < 1990; i++ { + tr.Delete(key(i)) + } + short := tr.Height() + fmt.Printf("order 4: 2000 keys -> height %d, then 10 keys -> height %d (%d collapses)\n", + tall, short, tr.Stats().Collapses) + if short >= tall { + t.Fatalf("height did not shrink: %d then %d", tall, short) + } + if err := tr.Validate(); err != nil { + t.Fatal(err) + } +} + +func TestEveryRepairPathGetsExercised(t *testing.T) { + tr := mustNew(t, 4) + r := rand.New(rand.NewSource(7)) + live := map[int]bool{} + for i := 0; i < 3000; i++ { + k := r.Intn(600) + if live[k] && r.Intn(2) == 0 { + tr.Delete(key(k)) + delete(live, k) + } else { + tr.Put(key(k), rid(k)) + live[k] = true + } + if err := tr.Validate(); err != nil { + t.Fatalf("after operation %d: %v", i, err) + } + } + for k := 0; k < 600; k++ { + tr.Delete(key(k)) + if err := tr.Validate(); err != nil { + t.Fatalf("draining, after deleting %d: %v", k, err) + } + } + s := tr.Stats() + fmt.Printf("3000 mixed operations then a full drain: %d borrows left, %d borrows right, %d merges, %d collapses\n", + s.BorrowsLeft, s.BorrowsRight, s.Merges, s.Collapses) + if s.BorrowsLeft == 0 { + t.Error("borrow-from-left never happened; that path is untested") + } + if s.BorrowsRight == 0 { + t.Error("borrow-from-right never happened; that path is untested") + } + if s.Merges == 0 { + t.Error("merge never happened; that path is untested") + } + if s.Collapses == 0 { + t.Error("root collapse never happened; that path is untested") + } +} + +func TestTreeMatchesAReferenceMapThroughout(t *testing.T) { + for _, order := range []int{3, 4, 5, 8, 16} { + tr := mustNew(t, order) + want := map[int]bool{} + r := rand.New(rand.NewSource(int64(order) * 31)) + + for step := 0; step < 4000; step++ { + k := r.Intn(800) + if r.Intn(100) < 45 { + gotOK := tr.Delete(key(k)) + wantOK := want[k] + if gotOK != wantOK { + t.Fatalf("order %d step %d: Delete(%d) = %v, want %v", + order, step, k, gotOK, wantOK) + } + delete(want, k) + } else { + tr.Put(key(k), rid(k)) + want[k] = true + } + + if err := tr.Validate(); err != nil { + t.Fatalf("order %d step %d: %v", order, step, err) + } + if tr.Len() != len(want) { + t.Fatalf("order %d step %d: Len = %d, reference has %d", + order, step, tr.Len(), len(want)) + } + } + + var expect []int + for k := range want { + expect = append(expect, k) + } + sort.Ints(expect) + + got := collect(tr.ScanAll()) + if len(got) != len(expect) { + t.Fatalf("order %d: scan returned %d keys, reference has %d", + order, len(got), len(expect)) + } + for i := range got { + if !bytes.Equal(got[i], key(expect[i])) { + t.Fatalf("order %d: scan position %d differs from the reference", order, i) + } + } + for _, k := range expect { + if _, ok := tr.Get(key(k)); !ok { + t.Fatalf("order %d: key %d in reference but missing from tree", order, k) + } + } + for k := 0; k < 800; k++ { + if _, ok := tr.Get(key(k)); ok != want[k] { + t.Fatalf("order %d: Get(%d) = %v, reference says %v", order, k, ok, want[k]) + } + } + } +} + +func TestScanStillWorksAfterDeletes(t *testing.T) { + tr := mustNew(t, 4) + for i := 0; i < 300; i++ { + tr.Put(key(i), rid(i)) + } + for i := 0; i < 300; i += 3 { + tr.Delete(key(i)) + } + got := collect(tr.Scan(key(50), key(100))) + var want [][]byte + for i := 50; i < 100; i++ { + if i%3 != 0 { + want = append(want, key(i)) + } + } + if len(got) != len(want) { + t.Fatalf("scan after deletes returned %d keys, want %d", len(got), len(want)) + } + for i := range got { + if !bytes.Equal(got[i], want[i]) { + t.Fatalf("position %d differs", i) + } + } + if err := tr.Validate(); err != nil { + t.Fatal(err) + } +} + +func TestLeafChainSurvivesMerges(t *testing.T) { + tr := mustNew(t, 3) + for i := 0; i < 600; i++ { + tr.Put(key(i), rid(i)) + } + for i := 0; i < 600; i += 2 { + tr.Delete(key(i)) + if err := tr.Validate(); err != nil { + t.Fatalf("after deleting %d: %v", i, err) + } + } + keys := 0 + for n := tr.firstLeaf(); n != nil; n = n.next { + keys += len(n.keys) + } + if keys != tr.Len() { + t.Fatalf("chain holds %d keys, tree has %d", keys, tr.Len()) + } + if tr.Stats().Merges == 0 { + t.Fatal("expected merges during this pattern") + } +} diff --git a/internal/btree/fuzz_test.go b/internal/btree/fuzz_test.go index 11d7012..a661db3 100644 --- a/internal/btree/fuzz_test.go +++ b/internal/btree/fuzz_test.go @@ -117,3 +117,64 @@ func FuzzScanMatchesBruteForce(f *testing.F) { } }) } + +func FuzzInsertDeleteAgainstReference(f *testing.F) { + f.Add(4, []byte{1, 2, 3, 0x81, 0x82}) + f.Add(3, []byte{5, 5, 0x85, 0x85}) + f.Add(5, []byte{0x80, 0x81, 0x82}) + f.Add(64, []byte{}) + + f.Fuzz(func(t *testing.T, order int, ops []byte) { + if order < MinOrder || order > 128 || len(ops) > 3000 { + return + } + tr, err := New(order) + if err != nil { + return + } + want := map[byte]bool{} + + for _, op := range ops { + k := op & 0x7F + if op&0x80 != 0 { + gotOK := tr.Delete([]byte{k}) + if gotOK != want[k] { + t.Fatalf("Delete(%d) = %v, reference says %v", k, gotOK, want[k]) + } + delete(want, k) + } else { + tr.Put([]byte{k}, core.RowID{Slot: uint16(k)}) + want[k] = true + } + if err := tr.Validate(); err != nil { + t.Fatalf("after op %02X: %v", op, err) + } + if tr.Len() != len(want) { + t.Fatalf("after op %02X: Len = %d, reference has %d", op, tr.Len(), len(want)) + } + } + + var seen []byte + for it := tr.ScanAll(); it.Next(); { + seen = append(seen, it.Key()[0]) + } + if len(seen) != len(want) { + t.Fatalf("scan returned %d keys, reference has %d", len(seen), len(want)) + } + for i := 1; i < len(seen); i++ { + if seen[i-1] >= seen[i] { + t.Fatalf("scan not sorted at %d", i) + } + } + for _, k := range seen { + if !want[k] { + t.Fatalf("scan returned %d which is not in the reference", k) + } + } + for k := byte(0); k < 128; k++ { + if _, ok := tr.Get([]byte{k}); ok != want[k] { + t.Fatalf("Get(%d) = %v, reference says %v", k, ok, want[k]) + } + } + }) +} diff --git a/internal/slots/regression_test.go b/internal/slots/regression_test.go index 3a27a28..ff610e3 100644 --- a/internal/slots/regression_test.go +++ b/internal/slots/regression_test.go @@ -2,6 +2,7 @@ package slots import ( "errors" + "strings" "testing" "github.com/scute-db/scutedb/internal/page" @@ -9,9 +10,13 @@ import ( func TestNegativeOrZeroRecordSizeIsRejected(t *testing.T) { for _, size := range []int{-5, -1, 0, page.Size + 1} { - if _, err := NewLayout(size, 8); !errors.Is(err, ErrNoFit) { + _, err := NewLayout(size, 8) + if !errors.Is(err, ErrNoFit) { t.Fatalf("NewLayout(%d, 8) gave %v, want ErrNoFit", size, err) } + if !strings.Contains(err.Error(), "record size") { + t.Fatalf("NewLayout(%d, 8) reported %q; the explicit size guard should name the record size, not derive a nonsense slot size", size, err) + } } }