From bb7053c8b9497a07ba9d11149958ffc10e70a5d7 Mon Sep 17 00:00:00 2001 From: Syed Suhail Ahmed <114918598+suhailopensource@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:18:29 +0530 Subject: [PATCH] 0x03: binary serialization Splits encoding into two jobs with different goals. Values (varint, zigzag, length-prefixed) optimise for size; keys (fixed-width big-endian, sign-flipped) optimise for byte-comparable ordering, because the B+Tree in 0x06 compares keys with bytes.Compare and nothing else. Length prefixing fixes the item-boundary problem left by 0x02. Three fuzz targets cover round-tripping, key order preservation, and decoder robustness against arbitrary input; the varint implementation is verified byte-identical to encoding/binary. Also corrects the README's page-size justification: 4096 matches the filesystem block size, not the VM page size, which is 16 KB on Apple Silicon. --- Makefile | 10 +- README.md | 79 +++++++++++++-- cmd/scutedb-demo/main.go | 103 +++++++++++++++++--- internal/codec/codec_test.go | 180 +++++++++++++++++++++++++++++++++++ internal/codec/fuzz_test.go | 92 ++++++++++++++++++ internal/codec/key.go | 55 +++++++++++ internal/codec/value.go | 69 ++++++++++++++ internal/codec/varint.go | 58 +++++++++++ 8 files changed, 628 insertions(+), 18 deletions(-) create mode 100644 internal/codec/codec_test.go create mode 100644 internal/codec/fuzz_test.go create mode 100644 internal/codec/key.go create mode 100644 internal/codec/value.go create mode 100644 internal/codec/varint.go diff --git a/Makefile b/Makefile index a5163c0..5b1861d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all test race vet fmt lint bench demo clean +.PHONY: all test race vet fmt lint bench demo fuzz clean all: fmt vet test @@ -47,3 +47,11 @@ hexdump: demo-pages clean: go clean ./... rm -f /tmp/scutedb-pages.db + +demo-encode: + go run ./cmd/scutedb-demo encode + +fuzz: + go test ./internal/codec/ -run=XXX -fuzz=FuzzValueRoundTrip -fuzztime=30s + go test ./internal/codec/ -run=XXX -fuzz=FuzzKeyOrderingInt64 -fuzztime=30s + go test ./internal/codec/ -run=XXX -fuzz=FuzzDecoderNeverPanics -fuzztime=30s diff --git a/README.md b/README.md index 9c5aeae..26c1353 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A *scute* is one of the bony plates that make up a turtle's shell. A shell is made of plates; a database file is made of pages. -**Status:** Phase 0 complete (3 of 35 steps). +**Status:** Phase 0 complete, Phase A started (4 of 35 steps). --- @@ -26,7 +26,7 @@ make hexdump # write real pages and look at the bytes | Phase | What | Steps | Status | |-------|------|-------|--------| | **0** | Foundations — interfaces, the naive database, pages | `0x00`–`0x02` | **done** | -| A | Bytes & the B+Tree | `0x03`–`0x08` | next | +| A | Bytes & the B+Tree | `0x03`–`0x08` | **in progress** | | B | Persistence — storage manager, buffer pool, locking | `0x09`–`0x0D` | | | C | A real data store — schema, rows, indexes | `0x0E`–`0x12` | | | D | Transactions — WAL, recovery, 2PL, MVCC | `0x13`–`0x1A` | | @@ -118,8 +118,11 @@ Every failure above has the same root cause: no fixed unit. Records of unpredictable length at unpredictable offsets cannot be found, updated, or handed to a writer safely. -`internal/page` fixes the unit at **4096 bytes** — the machine's virtual-memory -page size, and a whole number of 512-byte disk sectors. Postgres uses 8 KB, +`internal/page` fixes the unit at **4096 bytes** — matching the APFS/ext4 +filesystem block size, and a whole number of 512-byte disk sectors. (It does +*not* match the virtual-memory page size everywhere: x86-64 uses 4 KB but Apple +Silicon uses 16 KB. The filesystem block is the alignment that matters for +I/O.) Postgres uses 8 KB, SQLite defaults to 4 KB, InnoDB uses 16 KB. **Page header — 16 bytes, big-endian:** @@ -157,12 +160,75 @@ Reading page 900,000 costs exactly what reading page 0 costs. describes itself truthfully. `make demo-header` decodes every field with the arithmetic shown. +--- + +## Phase A — Bytes & the B+Tree + +### `0x03` Binary serialization + +`internal/page` could store bytes but not tell two items apart: appending +`"scute-db"` then `"hello"` produced `"scute-dbhello"` with no recoverable +boundary. `internal/codec` gives bytes structure. + +The package is deliberately split in two, because **keys and values have +different jobs**: + +| | goal | encoding | +|---|---|---| +| **values** (`value.go`) | be small | varint, zigzag for signed, length-prefixed bytes | +| **keys** (`key.go`) | sort correctly as raw bytes | fixed-width big-endian, sign-flipped | + +Conflating these is a common mistake, and it is not recoverable later: an index +built on a non-order-preserving key encoding returns wrong answers for every +range query. + +**Length prefixing** solves the boundary problem — write the length, then the +bytes: + +``` +08 73 63 75 74 65 2D 64 62 05 68 65 6C 6C 6F +^^ 8 bytes follow ^^ 5 bytes follow +``` + +**Varints** cost 1 byte for values under 128 and grow to 10 for the largest +`uint64`, versus a flat 8 for fixed-width. Signed values use zigzag first, so +`-1` costs 1 byte rather than 10. Verified byte-identical to `encoding/binary`. + +**Keys use big-endian** because the B+Tree will compare them with +`bytes.Compare` and nothing else. Sorting `[1 2 255 256 300]` by raw bytes: + +``` +big-endian [1 2 255 256 300] correct +little-endian [256 1 2 300 255] wrong +varint [1 2 256 300 255] wrong +``` + +**Signed keys flip the top bit.** In two's complement `-1` is all `FF` bytes and +sorts above `1`. XOR-ing the sign bit shifts the signed range onto the unsigned +range, preserving order: + +``` +value raw two's complement key encoding +-1 FF FF FF FF FF FF FF FF 7F FF FF FF FF FF FF FF + 0 00 00 00 00 00 00 00 00 80 00 00 00 00 00 00 00 + 1 00 00 00 00 00 00 00 01 80 00 00 00 00 00 00 01 +``` + +Floats use the same idea: flip the sign bit if positive, flip every bit if +negative. + +**Round-trip property tests.** Table tests cover the specific cases; three Go +fuzz targets cover the rest — value round-tripping, key order preservation +(`a < b` must imply `bytes.Compare < 0`, for every pair the fuzzer can find), +and that no decoder panics on arbitrary input. Roughly 480k executions/sec. + +Run it: `make demo-encode`. + ### Known gaps at the end of Phase 0 Deliberate, each one is a later step: -- `Page.Append` writes items with no separator, so `"scute-dbhello"` cannot be - split back apart → `0x03` +- ~~`Page.Append` writes items with no separator~~ → fixed in `0x03` - No way to find item *n* without walking items 1..*n-1* → `0x0F` (slot directory) - `File.Allocate` never reuses a freed page → `0x0A` (free list) - The reserved header bytes hold no checksum → `0x13` @@ -181,6 +247,7 @@ internal/ storage/ Engine interface naive/ the throwaway from 0x01 page/ fixed-size pages, header codec, header decoder + codec/ value encoding (compact) and key encoding (ordered) ``` ## Conventions diff --git a/cmd/scutedb-demo/main.go b/cmd/scutedb-demo/main.go index d6b3724..3e576fe 100644 --- a/cmd/scutedb-demo/main.go +++ b/cmd/scutedb-demo/main.go @@ -2,14 +2,19 @@ package main import ( "bufio" + "bytes" + "encoding/binary" "fmt" + "math" "os" "os/exec" "path/filepath" + "sort" "strconv" "sync" "time" + "github.com/suhailopensource/ScuteDB/internal/codec" "github.com/suhailopensource/ScuteDB/internal/core" "github.com/suhailopensource/ScuteDB/internal/naive" "github.com/suhailopensource/ScuteDB/internal/page" @@ -17,7 +22,7 @@ import ( func main() { if len(os.Args) < 2 { - fmt.Println("usage: scutedb-demo scan|update|race|crash|pages|header") + fmt.Println("usage: scutedb-demo scan|update|race|crash|pages|header|encode") os.Exit(2) } switch os.Args[1] { @@ -33,6 +38,8 @@ func main() { expPages() case "header": expHeader() + case "encode": + expEncode() case "crash-child": crashChild(os.Args[2]) default: @@ -70,7 +77,7 @@ func expScan() { _, _, _, size, err := db.Stats() check(err) fmt.Printf("%12s %12s %14s %14s\n", - commas(int64(n)), bytes(size), el.Round(time.Microsecond), + commas(int64(n)), humanBytes(size), el.Round(time.Microsecond), (el / time.Duration(n)).Round(time.Nanosecond)) db.Close() } @@ -100,9 +107,9 @@ func expUpdate() { lines, live, _, after, err := db.Stats() check(err) - fmt.Printf(" file before update %s\n", bytes(before)) - fmt.Printf(" bytes that changed %s (the new record)\n", bytes(int64(len(rec)))) - fmt.Printf(" file after update %s (it GREW)\n", bytes(after)) + fmt.Printf(" file before update %s\n", humanBytes(before)) + fmt.Printf(" bytes that changed %s (the new record)\n", humanBytes(int64(len(rec)))) + fmt.Printf(" file after update %s (it GREW)\n", humanBytes(after)) fmt.Printf(" lines on disk %s\n", commas(int64(lines))) fmt.Printf(" distinct live keys %s <- %s lines are garbage\n\n", commas(int64(live)), commas(int64(lines-live))) @@ -113,9 +120,9 @@ func expUpdate() { el := time.Since(start) fmt.Printf(" To reclaim that space we must rewrite the whole file:\n") - fmt.Printf(" compaction wrote %s in %s\n", bytes(written), el.Round(time.Millisecond)) + fmt.Printf(" compaction wrote %s in %s\n", humanBytes(written), el.Round(time.Millisecond)) fmt.Printf(" write amplification %.0fx (%s written to change %s)\n\n", - float64(written)/float64(len(rec)), bytes(written), bytes(int64(len(rec)))) + float64(written)/float64(len(rec)), humanBytes(written), humanBytes(int64(len(rec)))) fmt.Println(" Steps 0x02 and 0x0F fix this: change a 4 KB page, write a 4 KB page.") db.Close() } @@ -173,7 +180,7 @@ func expRace() { fmt.Printf(" damaged records %s\n", commas(int64(torn))) fmt.Printf(" records LOST %s\n", commas(int64(1000-good))) fmt.Printf(" file size %s (expected %s)\n\n", - bytes(st.Size()), bytes(int64(1000*len("writer0:record0000\tvalue\n")))) + humanBytes(st.Size()), humanBytes(int64(1000*len("writer0:record0000\tvalue\n")))) fmt.Println(" Both goroutines read the same offset and wrote on top of") fmt.Println(" each other. Run this with -race and Go names the exact lines.") fmt.Println(" Step 0x0C is where we make this impossible.") @@ -219,7 +226,7 @@ func expCrash() { st, _ := os.Stat(path) fmt.Printf(" child reported writing %s records\n", commas(int64(claimed))) - fmt.Printf(" actually on disk %s records (%s)\n", commas(int64(survived)), bytes(st.Size())) + fmt.Printf(" actually on disk %s records (%s)\n", commas(int64(survived)), humanBytes(st.Size())) fmt.Printf(" VANISHED %s records\n\n", commas(int64(claimed-survived))) fmt.Println(" Put() returned nil for every one of those. The records were") fmt.Println(" sitting in a userspace buffer that died with the process -") @@ -272,7 +279,7 @@ func expPages() { st, err := os.Stat(path) check(err) fmt.Printf("\n file size %s (exactly %d x %d, no more, no less)\n", - bytes(st.Size()), st.Size()/page.Size, page.Size) + humanBytes(st.Size()), st.Size()/page.Size, page.Size) pf2, err := page.Open(path) check(err) @@ -330,7 +337,7 @@ func commas(n int64) string { return string(out) } -func bytes(n int64) string { +func humanBytes(n int64) string { switch { case n < 1024: return fmt.Sprintf("%d B", n) @@ -340,3 +347,77 @@ func bytes(n int64) string { return fmt.Sprintf("%.1f MB", float64(n)/(1024*1024)) } } + +func expEncode() { + fmt.Println("STEP 0x03 binary serialization") + + fmt.Print("\n1. THE PROBLEM LEFT OVER FROM 0x02\n\n") + p := page.New(1, page.KindHeap) + p.Append([]byte("scute-db")) + p.Append([]byte("hello")) + fmt.Printf(" raw appends: %q\n", string(p[page.HeaderSize:p.FreeStart()])) + fmt.Println(" two items, nothing marks the boundary. unrecoverable.") + + fmt.Print("\n2. THE FIX: LENGTH PREFIXING\n\n") + q := page.New(1, page.KindHeap) + q.Append(codec.AppendString(nil, "scute-db")) + q.Append(codec.AppendString(nil, "hello")) + body := q[page.HeaderSize:q.FreeStart()] + fmt.Printf(" bytes: % 02X\n", body) + fmt.Printf(" as text: %q\n", string(body)) + off := 0 + for i := 1; off < len(body); i++ { + s, n, err := codec.String(body[off:]) + if err != nil { + fmt.Println(" decode error:", err) + break + } + fmt.Printf(" item %d: len=%-3d %-12q bytes %d..%d\n", i, len(s), s, off, off+n-1) + off += n + } + + fmt.Print("\n3. FIXED WIDTH vs VARINT\n\n") + fmt.Printf(" %-22s %-10s %-10s %s\n", "value", "fixed 8B", "varint", "saved") + fmt.Println(" " + line(56)) + for _, v := range []uint64{0, 1, 127, 128, 1000, 1 << 20, 1 << 40, math.MaxUint64} { + n := len(codec.AppendUvarint(nil, v)) + fmt.Printf(" %-22d %-10d %-10d %+d bytes\n", v, 8, n, 8-n) + } + + fmt.Print("\n4. WHY KEYS MUST BE BIG-ENDIAN\n\n") + nums := []uint64{1, 2, 255, 256, 300} + fmt.Printf(" %-8s %-27s %s\n", "value", "big-endian (low 4 bytes)", "varint") + fmt.Println(" " + line(52)) + for _, v := range nums { + be := codec.AppendKeyUint64(nil, v) + fmt.Printf(" %-8d %-27s % 02X\n", v, fmt.Sprintf("% 02X", be[4:]), codec.AppendUvarint(nil, v)) + } + fmt.Print("\n the same five values, sorted purely by their bytes:\n") + fmt.Printf(" big-endian %v correct\n", + sortedBy(nums, func(v uint64) []byte { return codec.AppendKeyUint64(nil, v) })) + fmt.Printf(" little-endian %v wrong\n", + sortedBy(nums, func(v uint64) []byte { return binary.LittleEndian.AppendUint64(nil, v) })) + fmt.Printf(" varint %v wrong\n", + sortedBy(nums, func(v uint64) []byte { return codec.AppendUvarint(nil, v) })) + fmt.Println("\n The B+Tree will compare keys with bytes.Compare and nothing else.") + fmt.Println(" Only the first encoding survives that.") + + fmt.Print("\n5. SIGNED KEYS NEED A SIGN FLIP\n\n") + fmt.Printf(" %-8s %-26s %s\n", "value", "raw two's complement", "key encoding") + fmt.Println(" " + line(60)) + for _, v := range []int64{-2, -1, 0, 1, 2} { + raw := binary.BigEndian.AppendUint64(nil, uint64(v)) + fmt.Printf(" %-8d %-26s % 02X\n", v, + fmt.Sprintf("% 02X", raw), codec.AppendKeyInt64(nil, v)) + } + fmt.Println("\n Raw, -1 is all FF bytes and sorts ABOVE 1. Flipping the top bit") + fmt.Println(" shifts the signed range so bytewise order matches numeric order.") +} + +func sortedBy(nums []uint64, enc func(uint64) []byte) []uint64 { + out := append([]uint64(nil), nums...) + sort.Slice(out, func(i, j int) bool { + return bytes.Compare(enc(out[i]), enc(out[j])) < 0 + }) + return out +} diff --git a/internal/codec/codec_test.go b/internal/codec/codec_test.go new file mode 100644 index 0000000..53c72e6 --- /dev/null +++ b/internal/codec/codec_test.go @@ -0,0 +1,180 @@ +package codec + +import ( + "bytes" + "encoding/binary" + "math" + "math/rand" + "sort" + "testing" +) + +func TestVarintMatchesStdlib(t *testing.T) { + values := []uint64{0, 1, 127, 128, 300, 16383, 16384, 1 << 32, math.MaxUint64} + for _, v := range values { + ours := AppendUvarint(nil, v) + theirs := binary.AppendUvarint(nil, v) + if !bytes.Equal(ours, theirs) { + t.Fatalf("uvarint(%d): ours % 02X, stdlib % 02X", v, ours, theirs) + } + if got := UvarintLen(v); got != len(ours) { + t.Fatalf("UvarintLen(%d) = %d, encoded %d bytes", v, got, len(ours)) + } + } + for _, v := range []int64{0, 1, -1, 63, -64, math.MaxInt64, math.MinInt64} { + ours := AppendVarint(nil, v) + theirs := binary.AppendVarint(nil, v) + if !bytes.Equal(ours, theirs) { + t.Fatalf("varint(%d): ours % 02X, stdlib % 02X", v, ours, theirs) + } + } +} + +func TestValueRoundTrip(t *testing.T) { + buf := []byte{} + buf = AppendUint64(buf, 4096) + buf = AppendInt64(buf, -42) + buf = AppendBool(buf, true) + buf = AppendFloat64(buf, 3.5) + buf = AppendString(buf, "scute-db") + buf = AppendBytes(buf, []byte{0xDE, 0xAD}) + + off := 0 + u, n, err := Uint64(buf[off:]) + if err != nil || u != 4096 { + t.Fatalf("uint64 = %d, %v", u, err) + } + off += n + i, n, err := Int64(buf[off:]) + if err != nil || i != -42 { + t.Fatalf("int64 = %d, %v", i, err) + } + off += n + b, n, err := Bool(buf[off:]) + if err != nil || !b { + t.Fatalf("bool = %v, %v", b, err) + } + off += n + f, n, err := Float64(buf[off:]) + if err != nil || f != 3.5 { + t.Fatalf("float64 = %v, %v", f, err) + } + off += n + s, n, err := String(buf[off:]) + if err != nil || s != "scute-db" { + t.Fatalf("string = %q, %v", s, err) + } + off += n + raw, n, err := Bytes(buf[off:]) + if err != nil || !bytes.Equal(raw, []byte{0xDE, 0xAD}) { + t.Fatalf("bytes = % 02X, %v", raw, err) + } + off += n + if off != len(buf) { + t.Fatalf("consumed %d of %d bytes", off, len(buf)) + } +} + +func TestTruncatedInputIsAnError(t *testing.T) { + full := AppendString(nil, "hello") + for cut := 0; cut < len(full); cut++ { + if _, _, err := String(full[:cut]); err == nil { + t.Fatalf("String(%d of %d bytes) returned no error", cut, len(full)) + } + } + if _, _, err := Uvarint([]byte{0x80, 0x80, 0x80}); err != ErrTruncated { + t.Fatalf("unterminated varint gave %v, want ErrTruncated", err) + } + overflow := []byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x02} + if _, _, err := Uvarint(overflow); err != ErrOverflow { + t.Fatalf("11-byte varint gave %v, want ErrOverflow", err) + } +} + +func TestKeyOrderingUint64(t *testing.T) { + r := rand.New(rand.NewSource(1)) + nums := make([]uint64, 500) + for i := range nums { + nums[i] = r.Uint64() + } + sort.Slice(nums, func(i, j int) bool { return nums[i] < nums[j] }) + for i := 1; i < len(nums); i++ { + a := AppendKeyUint64(nil, nums[i-1]) + b := AppendKeyUint64(nil, nums[i]) + if bytes.Compare(a, b) > 0 { + t.Fatalf("%d encodes above %d", nums[i-1], nums[i]) + } + } +} + +func TestKeyOrderingInt64(t *testing.T) { + nums := []int64{math.MinInt64, -1 << 40, -300, -2, -1, 0, 1, 2, 300, 1 << 40, math.MaxInt64} + for i := 1; i < len(nums); i++ { + a := AppendKeyInt64(nil, nums[i-1]) + b := AppendKeyInt64(nil, nums[i]) + if bytes.Compare(a, b) >= 0 { + t.Fatalf("%d encodes >= %d\n %d -> % 02X\n %d -> % 02X", + nums[i-1], nums[i], nums[i-1], a, nums[i], b) + } + } +} + +func TestKeyOrderingFloat64(t *testing.T) { + nums := []float64{math.Inf(-1), -1e10, -1.5, -0.0, 0.0, 1.5, 1e10, math.Inf(1)} + for i := 1; i < len(nums); i++ { + a := AppendKeyFloat64(nil, nums[i-1]) + b := AppendKeyFloat64(nil, nums[i]) + if bytes.Compare(a, b) > 0 { + t.Fatalf("%v encodes above %v", nums[i-1], nums[i]) + } + } +} + +func TestKeyRoundTrip(t *testing.T) { + for _, v := range []int64{math.MinInt64, -1, 0, 1, math.MaxInt64} { + got, _, err := KeyInt64(AppendKeyInt64(nil, v)) + if err != nil || got != v { + t.Fatalf("int64 key %d round-tripped to %d (%v)", v, got, err) + } + } + for _, v := range []float64{math.Inf(-1), -1.5, -0.0, 0.0, 1.5, math.Inf(1), math.NaN()} { + got, _, err := KeyFloat64(AppendKeyFloat64(nil, v)) + if err != nil || math.Float64bits(got) != math.Float64bits(v) { + t.Fatalf("float64 key %v round-tripped to %v (%v)", v, got, err) + } + } +} + +func TestVarintIsNotOrderPreserving(t *testing.T) { + a := AppendUvarint(nil, 255) + b := AppendUvarint(nil, 256) + if bytes.Compare(a, b) <= 0 { + t.Fatal("varint preserved order for 255 vs 256; the test intends to show it does not") + } + t.Logf("255 -> % 02X", a) + t.Logf("256 -> % 02X", b) + t.Logf("bytewise, 255 sorts ABOVE 256, because a varint puts the LOW bits first") + + big := AppendKeyUint64(nil, 255) + bigger := AppendKeyUint64(nil, 256) + if bytes.Compare(big, bigger) >= 0 { + t.Fatal("big-endian key encoding failed to preserve order") + } + t.Logf("as keys: 255 -> % 02X", big) + t.Logf(" 256 -> % 02X correct order", bigger) +} + +func TestLittleEndianBreaksKeyOrdering(t *testing.T) { + nums := []uint64{1, 2, 256, 300} + le := make([][]byte, len(nums)) + for i, n := range nums { + le[i] = binary.LittleEndian.AppendUint64(nil, n) + } + sorted := append([][]byte(nil), le...) + sort.Slice(sorted, func(i, j int) bool { return bytes.Compare(sorted[i], sorted[j]) < 0 }) + if bytes.Equal(sorted[0], le[0]) { + t.Fatal("little-endian preserved order; expected it not to") + } + t.Logf("input order: %v", nums) + t.Logf("little-endian byte order puts %d first", binary.LittleEndian.Uint64(sorted[0])) +} diff --git a/internal/codec/fuzz_test.go b/internal/codec/fuzz_test.go new file mode 100644 index 0000000..9b7311f --- /dev/null +++ b/internal/codec/fuzz_test.go @@ -0,0 +1,92 @@ +package codec + +import ( + "bytes" + "math" + "testing" +) + +func FuzzValueRoundTrip(f *testing.F) { + f.Add(uint64(0), int64(0), false, 0.0, "") + f.Add(uint64(4096), int64(-42), true, 3.5, "scute-db") + f.Add(uint64(math.MaxUint64), int64(math.MinInt64), true, math.Inf(-1), "\x00\xff mixed") + + f.Fuzz(func(t *testing.T, u uint64, i int64, b bool, fl float64, s string) { + buf := AppendUint64(nil, u) + buf = AppendInt64(buf, i) + buf = AppendBool(buf, b) + buf = AppendFloat64(buf, fl) + buf = AppendString(buf, s) + + off := 0 + gu, n, err := Uint64(buf[off:]) + if err != nil || gu != u { + t.Fatalf("uint64 %d -> %d (%v)", u, gu, err) + } + off += n + gi, n, err := Int64(buf[off:]) + if err != nil || gi != i { + t.Fatalf("int64 %d -> %d (%v)", i, gi, err) + } + off += n + gb, n, err := Bool(buf[off:]) + if err != nil || gb != b { + t.Fatalf("bool %v -> %v (%v)", b, gb, err) + } + off += n + gf, n, err := Float64(buf[off:]) + if err != nil || math.Float64bits(gf) != math.Float64bits(fl) { + t.Fatalf("float64 %v -> %v (%v)", fl, gf, err) + } + off += n + gs, n, err := String(buf[off:]) + if err != nil || gs != s { + t.Fatalf("string %q -> %q (%v)", s, gs, err) + } + off += n + if off != len(buf) { + t.Fatalf("consumed %d of %d bytes", off, len(buf)) + } + }) +} + +func FuzzKeyOrderingInt64(f *testing.F) { + f.Add(int64(0), int64(1)) + f.Add(int64(-1), int64(1)) + f.Add(int64(math.MinInt64), int64(math.MaxInt64)) + + f.Fuzz(func(t *testing.T, a, b int64) { + ea := AppendKeyInt64(nil, a) + eb := AppendKeyInt64(nil, b) + cmpNum := 0 + switch { + case a < b: + cmpNum = -1 + case a > b: + cmpNum = 1 + } + if got := bytes.Compare(ea, eb); got != cmpNum { + t.Fatalf("%d vs %d: numeric %d, bytewise %d\n % 02X\n % 02X", + a, b, cmpNum, got, ea, eb) + } + }) +} + +func FuzzDecoderNeverPanics(f *testing.F) { + f.Add([]byte{}) + f.Add([]byte{0xFF}) + f.Add([]byte{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}) + f.Add([]byte{0x05, 'h', 'e'}) + + f.Fuzz(func(t *testing.T, data []byte) { + Uvarint(data) + Varint(data) + Bool(data) + Float64(data) + Bytes(data) + String(data) + KeyUint64(data) + KeyInt64(data) + KeyFloat64(data) + }) +} diff --git a/internal/codec/key.go b/internal/codec/key.go new file mode 100644 index 0000000..1eef348 --- /dev/null +++ b/internal/codec/key.go @@ -0,0 +1,55 @@ +package codec + +import ( + "encoding/binary" + "math" +) + +const signBit = uint64(1) << 63 + +func AppendKeyUint64(dst []byte, v uint64) []byte { + return binary.BigEndian.AppendUint64(dst, v) +} + +func KeyUint64(src []byte) (uint64, int, error) { + if len(src) < 8 { + return 0, 0, ErrTruncated + } + return binary.BigEndian.Uint64(src), 8, nil +} + +func AppendKeyInt64(dst []byte, v int64) []byte { + return binary.BigEndian.AppendUint64(dst, uint64(v)^signBit) +} + +func KeyInt64(src []byte) (int64, int, error) { + if len(src) < 8 { + return 0, 0, ErrTruncated + } + return int64(binary.BigEndian.Uint64(src) ^ signBit), 8, nil +} + +func AppendKeyFloat64(dst []byte, f float64) []byte { + b := math.Float64bits(f) + if b&signBit != 0 { + b = ^b + } else { + b |= signBit + } + return binary.BigEndian.AppendUint64(dst, b) +} + +func KeyFloat64(src []byte) (float64, int, error) { + if len(src) < 8 { + return 0, 0, ErrTruncated + } + b := binary.BigEndian.Uint64(src) + if b&signBit != 0 { + b &^= signBit + } else { + b = ^b + } + return math.Float64frombits(b), 8, nil +} + +func AppendKeyString(dst []byte, s string) []byte { return append(dst, s...) } diff --git a/internal/codec/value.go b/internal/codec/value.go new file mode 100644 index 0000000..dc613f2 --- /dev/null +++ b/internal/codec/value.go @@ -0,0 +1,69 @@ +package codec + +import ( + "encoding/binary" + "math" +) + +func AppendUint64(dst []byte, v uint64) []byte { return AppendUvarint(dst, v) } + +func Uint64(src []byte) (uint64, int, error) { return Uvarint(src) } + +func AppendInt64(dst []byte, v int64) []byte { return AppendVarint(dst, v) } + +func Int64(src []byte) (int64, int, error) { return Varint(src) } + +func AppendBool(dst []byte, b bool) []byte { + if b { + return append(dst, 1) + } + return append(dst, 0) +} + +func Bool(src []byte) (bool, int, error) { + if len(src) < 1 { + return false, 0, ErrTruncated + } + return src[0] != 0, 1, nil +} + +func AppendFloat64(dst []byte, f float64) []byte { + return binary.BigEndian.AppendUint64(dst, math.Float64bits(f)) +} + +func Float64(src []byte) (float64, int, error) { + if len(src) < 8 { + return 0, 0, ErrTruncated + } + return math.Float64frombits(binary.BigEndian.Uint64(src)), 8, nil +} + +func AppendBytes(dst []byte, b []byte) []byte { + dst = AppendUvarint(dst, uint64(len(b))) + return append(dst, b...) +} + +func Bytes(src []byte) ([]byte, int, error) { + n, hdr, err := Uvarint(src) + if err != nil { + return nil, 0, err + } + end := hdr + int(n) + if n > uint64(len(src)) || end > len(src) || end < hdr { + return nil, 0, ErrTruncated + } + return src[hdr:end], end, nil +} + +func AppendString(dst []byte, s string) []byte { + dst = AppendUvarint(dst, uint64(len(s))) + return append(dst, s...) +} + +func String(src []byte) (string, int, error) { + b, n, err := Bytes(src) + if err != nil { + return "", 0, err + } + return string(b), n, nil +} diff --git a/internal/codec/varint.go b/internal/codec/varint.go new file mode 100644 index 0000000..5943edf --- /dev/null +++ b/internal/codec/varint.go @@ -0,0 +1,58 @@ +package codec + +import "errors" + +const MaxVarintLen64 = 10 + +var ( + ErrTruncated = errors.New("scutedb/codec: buffer ended mid-value") + ErrOverflow = errors.New("scutedb/codec: varint overflows 64 bits") +) + +func AppendUvarint(dst []byte, v uint64) []byte { + for v >= 0x80 { + dst = append(dst, byte(v)|0x80) + v >>= 7 + } + return append(dst, byte(v)) +} + +func Uvarint(src []byte) (uint64, int, error) { + var x uint64 + var s uint + for i := 0; i < len(src); i++ { + b := src[i] + if i == MaxVarintLen64-1 && b > 1 { + return 0, 0, ErrOverflow + } + if b < 0x80 { + return x | uint64(b)<= 0x80 { + v >>= 7 + n++ + } + return n +} + +func zigzag(v int64) uint64 { return uint64(v<<1) ^ uint64(v>>63) } + +func unzigzag(u uint64) int64 { return int64(u>>1) ^ -int64(u&1) } + +func AppendVarint(dst []byte, v int64) []byte { return AppendUvarint(dst, zigzag(v)) } + +func Varint(src []byte) (int64, int, error) { + u, n, err := Uvarint(src) + if err != nil { + return 0, 0, err + } + return unzigzag(u), n, nil +}