Add B+Tree deletion with borrowing, merging and root collapse

Makefile

@@ -67,3 +67,6 @@ demo-btree:
 
 demo-range:
 	go run ./cmd/scutedb-demo range
+
+demo-delete:
+	go run ./cmd/scutedb-demo delete

README.md

@@ -6,7 +6,9 @@ A *scute* is one of the bony plates that make up a turtle's shell. A shell is
 made of plates; a database file is made of pages. 
 
 
-**Status:** Phase 0 complete more to go a lot of learning and doing.
+**Status:** Phases 0 and A complete. The B+Tree searches, inserts, splits,
+scans ranges and deletes. All of it still lives in memory — Phase B puts it on
+disk.
 
 ---
 
@@ -17,17 +19,18 @@ go test ./...              # everything
 make demo                  # list the runnable experiments
 make demo-scan             # watch a file-based database degrade
 make hexdump               # write real pages and look at the bytes
+make demo-delete           # borrow, merge and root collapse, traced live
 ```
 
 ---
 
-## Roadmap
+## Roadmap I am trying to follow plan first execute second
 
 | Phase | What | Status |
 |-------|------|--------|
 | **0** | Foundations — interfaces, the naive database, pages | **done** |
-| A | Bytes & the B+Tree | **in progress** |
-| B | Persistence — storage manager, buffer pool, locking | |
+| **A** | Bytes & the B+Tree | **done** |
+| B | Persistence — storage manager, buffer pool, locking | **next** |
 | C | A real data store — schema, rows, indexes | |
 | D | Transactions — WAL, recovery, 2PL, MVCC | |
 | E | Beyond — LSM engine, Raft, server, query planner | |
@@ -538,11 +541,119 @@ fuzz target that ran 2.9M cases comparing scan output to the same brute force.
 
 Run it: `make demo-range`.
 
-### Known gaps at the end of Phase 0
+### `0x08` B+Tree deletion
+
+The step most from-scratch databases stop at. Insert has one failure mode and
+one answer; delete has one failure mode and **three**, any of which can cascade.
+
+```
+insert                     delete
+does it fit? done          still enough keys? done
+too full -> split          too empty -> borrow from left
+                                     or borrow from right
+                                     or merge
+                           and the merge may cascade up
+                           and the root may collapse
+```
+
+Every one of those six behaves differently for a leaf than for an internal node,
+which is where the "3x harder" comes from.
+
+**All four repair paths in one trace** at order 4 (max 3 keys, min 1) — the demo
+reports what actually happened rather than what was expected:
+
+```
+delete 030 -> fitted, no repair needed
+delete 040 -> BORROWED from the LEFT sibling
+delete 010 -> MERGED two nodes into one
+delete 020 -> BORROWED from the RIGHT sibling
+delete 050 -> MERGED two nodes into one
+delete 060 -> BORROWED from the RIGHT sibling
+delete 070 -> MERGED two nodes into one, then the ROOT COLLAPSED
+```
+
+**Borrowing rewrites one separator. Merging deletes one** — which is what can
+make the parent underflow in turn, and so on up. The tree grew from the root and
+it shrinks back to the root:
+
+```
+2000 keys      -> height 7
+delete 1990    -> height 3
+along the way: 1487 borrows, 1485 merges, 4 root collapses
+```
+
+**The leaf/internal asymmetry mirrors the split asymmetry exactly.** A leaf
+*split* copies its middle key up; an internal split *moves* it. A leaf *merge*
+discards the separator; an internal merge *pulls it down* between the two halves,
+because for an internal node that separator is the only copy of the key.
+
+**Ghost separators.** A separator only has to point a search the right way — it
+does **not** have to name a key that exists. Deleting every key that appears as a
+separator, then re-checking:
+
+```
+deleted all 49 keys that appeared as separators
+49 of the 49 separators left now name keys that are gone
+lookups that return the wrong answer: 0 of 100
+validate: <nil>
+```
+
+This is why deletion never has to hunt upward and repair separators — a large
+amount of work the algorithm gets to skip, and the reason a stale separator is
+correct rather than a bug.
+
+**Why some engines never merge at all.** Merging keeps the index dense:
+
+```
+                     leaves   keys   fill
+4000 keys            500      4000   53%
+after deleting half  249      2000   54%
+```
+
+251 leaves handed back, fill unchanged. Without merging the leaf count would
+have stayed at 500 and the fill halved — the index growing while the data
+shrinks. And yet **Postgres's B-Tree does not merge partially-empty pages**; it
+only reclaims entirely empty ones. That is exactly why index bloat is an
+operational concern there and why `REINDEX` exists. Merging costs write
+amplification and page locks on a hot path, and some engines judge that trade
+not worth making.
+
+**`index.Index` is now satisfied.** With `Delete` in place, `btree.Index` wraps
+the tree to match the interface written in `0x00` — asserted by
+`var _ index.Index = (*Index)(nil)`. The natural API stays natural (`Get`
+returns a bool, `Put` cannot fail in memory); the adapter translates.
+
+**How it is verified.** A reference `map` is maintained alongside the tree
+through 4,000 random insert/delete operations at five orders, with `Validate()`
+after **every** operation and an exhaustive `Get` check over the whole key space
+at the end. `TestMinKeysHoldsForEveryOrder` inserts 400 and deletes 400 for
+every order from 3 to 40. `FuzzInsertDeleteAgainstReference` ran 344,719
+executions, each up to 3,000 operations, validating after each one.
+
+Run it: `make demo-delete`.
+
+### Phase A in one paragraph
+
+Six steps turned raw bytes into a working index. `codec` ships two encodings
+because keys and values have different jobs — values optimise for size, keys
+optimise for sorting correctly as raw bytes, and conflating them is not
+recoverable later. `nullbits` makes "no value" a property of the row rather than
+a magic value hidden inside it, and carries the three-valued logic that follows
+from that. `slots` buys O(1) addressing inside a page for a few bytes of padding.
+`btree` turns a linear scan into three or four hops, keeps the leaves chained so
+a range costs one descent and then a sideways walk, and now repairs itself when
+deletes empty a node out.
+
+What is still missing is the part that makes this a database rather than a data
+structure: none of it survives a restart. Every node is a Go pointer. Phase B
+replaces those pointers with page IDs, and the tree starts living on disk.
+
+### Known gaps at the end of Phase A
 
 Deliberate, each one is a later step:
 
 - ~~`Page.Append` writes items with no separator~~ → fixed in `0x03`
+- The B+Tree is pointers in memory and nothing persists → `0x09` (nodes become pages)
 - No way to find item *n* without walking items 1..*n-1* → `0x0F` (slot directory)
 - `File.Allocate` never reuses a freed page → `0x0A` (free list)
 - The reserved header bytes hold no checksum → `0x13`
@@ -564,7 +675,7 @@ internal/
   codec/            value encoding (compact) and key encoding (ordered)
   nullbits/         null bitmaps and SQL three-valued logic
   slots/            fixed-size, aligned record slots inside a page
-  btree/            in-memory B+Tree: search, insert, split, range scans
+  btree/            in-memory B+Tree: search, insert, split, range scans, delete
 ```
 
 ## Conventions

cmd/scutedb-demo/main.go

@@ -27,7 +27,7 @@ import (
 
 func main() {
 	if len(os.Args) < 2 {
-		fmt.Println("usage: scutedb-demo scan|update|race|crash|pages|header|encode|nulls|align|btree|range")
+		fmt.Println("usage: scutedb-demo scan|update|race|crash|pages|header|encode|nulls|align|btree|range|delete")
 		os.Exit(2)
 	}
 	switch os.Args[1] {
@@ -53,6 +53,8 @@ func main() {
 		expBTree()
 	case "range":
 		expRange()
+	case "delete":
+		expDelete()
 	case "crash-child":
 		crashChild(os.Args[2])
 	default:
@@ -855,3 +857,161 @@ func keysOf(it *btree.Iterator) string {
 	}
 	return strings.Join(parts, " ")
 }
+
+func expDelete() {
+	fmt.Println("STEP 0x08  B+Tree deletion")
+
+	fmt.Print("\n1. THE PROBLEM: UNDERFLOW\n\n")
+	fmt.Println("   inserting can only ever make a node TOO FULL, and there is one")
+	fmt.Println("   answer: split it. deleting makes a node TOO EMPTY, and there are")
+	fmt.Println("   three answers, which is why this step is the hard one.")
+	fmt.Println("\n   at order 4 a node may hold 3 keys and must hold at least 1.")
+	fmt.Println("   drop below that minimum and the node has UNDERFLOWED.")
+
+	fmt.Print("\n2. THE THREE FIXES, AS THEY ACTUALLY HAPPEN\n\n")
+	tr, err := btree.New(4)
+	check(err)
+	for i := 1; i <= 8; i++ {
+		tr.Put([]byte(fmt.Sprintf("%03d", i*10)), core.RowID{Slot: uint16(i)})
+	}
+	fmt.Print("   order 4: a node holds at most 3 keys and at least 1.\n\n")
+	dumpIndented(tr, "     ")
+	for _, k := range []string{"030", "040", "010", "020", "050", "060", "070"} {
+		deleteAndReport(tr, k)
+	}
+	s := tr.Stats()
+	fmt.Printf("\n   totals: %d borrows left, %d borrows right, %d merges, %d collapses\n",
+		s.BorrowsLeft, s.BorrowsRight, s.Merges, s.Collapses)
+	fmt.Println("\n   borrowing rewrites one separator. merging deletes one, which is")
+	fmt.Println("   what can make the parent underflow in turn.")
+
+	fmt.Print("\n3. THE CASCADE\n\n")
+	fmt.Println("   merging two children removes a key from their parent, which can")
+	fmt.Println("   make the PARENT underflow, which repeats all the way to the root.")
+	tr3, err := btree.New(4)
+	check(err)
+	for i := 0; i < 2000; i++ {
+		tr3.Put(key8(i), core.RowID{Slot: uint16(i % 1000)})
+	}
+	tall := tr3.Height()
+	for i := 0; i < 1990; i++ {
+		tr3.Delete(key8(i))
+	}
+	s3 := tr3.Stats()
+	fmt.Printf("\n   2000 keys      -> height %d\n", tall)
+	fmt.Printf("   delete 1990    -> height %d\n", tr3.Height())
+	fmt.Printf("   along the way: %d borrows, %d merges, %d root collapses\n",
+		s3.BorrowsLeft+s3.BorrowsRight, s3.Merges, s3.Collapses)
+	fmt.Println("\n   the tree grew from the root and it shrinks back to the root.")
+
+	fmt.Print("\n4. WHY DELETION IS THE HARD ONE\n\n")
+	fmt.Printf("   %-28s %s\n", "insert", "delete")
+	fmt.Println("   " + line(60))
+	fmt.Printf("   %-28s %s\n", "does it fit? done", "still enough keys? done")
+	fmt.Printf("   %-28s %s\n", "too full -> split", "too empty -> borrow from left")
+	fmt.Printf("   %-28s %s\n", "", "           or borrow from right")
+	fmt.Printf("   %-28s %s\n", "", "           or merge")
+	fmt.Printf("   %-28s %s\n", "", "and the merge may cascade up")
+	fmt.Printf("   %-28s %s\n", "", "and the root may collapse")
+	fmt.Println("\n   two cases against six. and every one of the six behaves")
+	fmt.Println("   differently for a leaf than for an internal node.")
+
+	fmt.Print("\n5. GHOST SEPARATORS\n\n")
+	fmt.Println("   a separator only has to POINT the search the right way. it does")
+	fmt.Println("   not have to be a key that exists.")
+	tr4, err := btree.New(4)
+	check(err)
+	for i := 0; i < 100; i++ {
+		tr4.Put(key8(i), core.RowID{Slot: uint16(i)})
+	}
+	seps := tr4.Separators()
+	for _, k := range seps {
+		tr4.Delete(k)
+	}
+	after := tr4.Separators()
+	ghosts := 0
+	for _, k := range after {
+		if _, ok := tr4.Get(k); !ok {
+			ghosts++
+		}
+	}
+	wrong := 0
+	for i := 0; i < 100; i++ {
+		k := key8(i)
+		wasSep := false
+		for _, sp := range seps {
+			if bytes.Equal(sp, k) {
+				wasSep = true
+			}
+		}
+		if _, ok := tr4.Get(k); ok == wasSep {
+			wrong++
+		}
+	}
+	fmt.Printf("\n   deleted all %d keys that appeared as separators.\n", len(seps))
+	fmt.Printf("   %d of the %d separators left in the tree now name keys that are gone.\n",
+		ghosts, len(after))
+	fmt.Printf("   lookups that return the wrong answer: %d of 100\n", wrong)
+	fmt.Printf("   validate: %v\n", tr4.Validate())
+	fmt.Println("\n   so deletion never has to hunt down and repair separators above it.")
+	fmt.Println("   that is a large amount of work the algorithm gets to skip.")
+
+	fmt.Print("\n6. WHY SOME ENGINES NEVER MERGE AT ALL\n\n")
+	tr5, err := btree.New(16)
+	check(err)
+	for i := 0; i < 4000; i++ {
+		tr5.Put(key8(i), core.RowID{Slot: uint16(i % 1000)})
+	}
+	l1, k1, c1 := tr5.LeafFill()
+	for i := 0; i < 4000; i += 2 {
+		tr5.Delete(key8(i))
+	}
+	l2, k2, c2 := tr5.LeafFill()
+	fmt.Printf("   %-26s %-10s %-10s %s\n", "", "leaves", "keys", "fill")
+	fmt.Println("   " + line(56))
+	fmt.Printf("   %-26s %-10d %-10d %.0f%%\n", "4000 keys", l1, k1, 100*float64(k1)/float64(c1))
+	fmt.Printf("   %-26s %-10d %-10d %.0f%%\n", "after deleting half", l2, k2, 100*float64(k2)/float64(c2))
+	fmt.Printf("\n   merging kept the leaves reasonably full and gave %d of them back.\n", l1-l2)
+	fmt.Println("   without merging, the leaf count would have stayed at", l1, "and the")
+	fmt.Println("   fill would have halved — the index gets bigger than the data.")
+	fmt.Println("\n   and yet: Postgres's B-Tree does NOT merge partially-empty pages.")
+	fmt.Println("   it only reclaims pages that are entirely empty. that is exactly why")
+	fmt.Println("   index bloat is a real operational problem there, and why REINDEX")
+	fmt.Println("   exists. merging costs write amplification and page locks on a hot")
+	fmt.Println("   path, and some engines decide that trade is not worth it.")
+}
+
+func deleteAndReport(tr *btree.Tree, k string) {
+	before := tr.Stats()
+	if !tr.Delete([]byte(k)) {
+		fmt.Printf("\n   delete %s -> not present\n", k)
+		return
+	}
+	a := tr.Stats()
+	what := "fitted, no repair needed"
+	switch {
+	case a.Merges > before.Merges:
+		what = "MERGED two nodes into one"
+	case a.BorrowsLeft > before.BorrowsLeft:
+		what = "BORROWED from the LEFT sibling"
+	case a.BorrowsRight > before.BorrowsRight:
+		what = "BORROWED from the RIGHT sibling"
+	}
+	if a.Collapses > before.Collapses {
+		what += ", then the ROOT COLLAPSED"
+	}
+	fmt.Printf("\n   delete %s -> %s\n", k, what)
+	dumpIndented(tr, "     ")
+}
+
+func dumpIndented(tr *btree.Tree, pad string) {
+	for _, ln := range strings.Split(strings.TrimRight(tr.Dump(), "\n"), "\n") {
+		fmt.Printf("%s%s\n", pad, ln)
+	}
+}
+
+func key8(n int) []byte {
+	b := make([]byte, 8)
+	binary.BigEndian.PutUint64(b, uint64(n))
+	return b
+}

internal/btree/adapter.go

@@ -0,0 +1,46 @@
+package btree
+
+import (
+	"github.com/scute-db/scutedb/internal/core"
+	"github.com/scute-db/scutedb/internal/index"
+)
+
+type Index struct{ t *Tree }
+
+func NewIndex(order int) (*Index, error) {
+	t, err := New(order)
+	if err != nil {
+		return nil, err
+	}
+	return &Index{t: t}, nil
+}
+
+func (x *Index) Tree() *Tree { return x.t }
+
+func (x *Index) Get(key []byte) (core.RowID, error) {
+	rid, ok := x.t.Get(key)
+	if !ok {
+		return core.RowID{}, core.ErrNotFound
+	}
+	return rid, nil
+}
+
+func (x *Index) Put(key []byte, rid core.RowID) error {
+	x.t.Put(key, rid)
+	return nil
+}
+
+func (x *Index) Delete(key []byte) error {
+	if !x.t.Delete(key) {
+		return core.ErrNotFound
+	}
+	return nil
+}
+
+func (x *Index) Scan(from, to []byte) (index.Iterator, error) {
+	return x.t.Scan(from, to), nil
+}
+
+func (x *Index) Close() error { return nil }
+
+var _ index.Index = (*Index)(nil)