Add internal/nullbits, and fix three key-encoding defects in codec

Makefile

@@ -55,3 +55,6 @@ fuzz:
 	go test ./internal/codec/ -run=XXX -fuzz=FuzzValueRoundTrip -fuzztime=30s
 	go test ./internal/codec/ -run=XXX -fuzz=FuzzKeyOrderingInt64 -fuzztime=30s
 	go test ./internal/codec/ -run=XXX -fuzz=FuzzDecoderNeverPanics -fuzztime=30s
+
+demo-nulls:
+	go run ./cmd/scutedb-demo nulls

README.md

@@ -217,13 +217,101 @@ value   raw two's complement      key encoding
 Floats use the same idea: flip the sign bit if positive, flip every bit if
 negative.
 
-**Round-trip property tests.** Table tests cover the specific cases; three Go
-fuzz targets cover the rest — value round-tripping, key order preservation
-(`a < b` must imply `bytes.Compare < 0`, for every pair the fuzzer can find),
-and that no decoder panics on arbitrary input. Roughly 480k executions/sec.
+**Round-trip property tests.** Table tests cover the specific cases; four Go
+fuzz targets cover the rest — value round-tripping, integer and float key order
+preservation (`a < b` must imply `bytes.Compare < 0`, for every pair the fuzzer
+can find), and that no decoder panics on arbitrary input. Roughly 430–480k
+executions/sec.
+
+Two correctness details the float fuzzer exists to protect:
+
+- **`-0.0` is normalised to `+0.0` before encoding.** IEEE-754 says they are
+  equal, but their bit patterns differ maximally, so without normalisation an
+  index would place them at opposite ends and `WHERE x = 0.0` would miss rows
+  stored as `-0.0`.
+- **All NaN payloads canonicalise to one.** NaN has many bit patterns; without
+  this, two NaNs would be different index keys. They sort above every real
+  number, matching Postgres.
+
+**`Bytes` copies, `BytesRef` does not.** `BytesRef` returns a view into the
+caller's buffer for hot paths; `Bytes` copies. The distinction matters because
+page buffers are recycled by the buffer pool, so a view outlives its backing
+bytes. The name is the only warning, so the split is deliberate rather than a
+single ambiguous function.
 
 Run it: `make demo-encode`.
 
+### `0x04` Nulls
+
+A byte of zeros in a page is ambiguous: it may be the number `0`, or a field
+that was never given a value. Those mean different things and must be
+distinguishable.
+
+Sentinel values (`-1`, `0`, `MinInt64`, `""`) do not work, because every
+sentinel removes a legal value from the type's range. MySQL's `0000-00-00` date
+and the `-1` "no sensor reading" convention are the same mistake in production.
+
+`internal/nullbits` puts the answer outside the data instead: a **bitmap in the
+record header**, one bit per field.
+
+```
+field    value      stored?
+id       42         yes
+name     suhail     yes
+age      NULL       NO - 0 bytes
+email    NULL       NO - 0 bytes
+score    9.1        yes
+
+bitmap:   ..NN.---   (N null, . present, - padding)
+record:   0C 54 06 73 75 68 61 69 6C 40 22 33 33 33 33 33 33
+          ^^ header, then only the three present values
+```
+
+Two consequences worth stating explicitly:
+
+- **A null field occupies zero bytes.** The bitmap is not a description of the
+  data, it is the only record that the field exists at all.
+- **The bitmap must precede the values.** A reader cannot parse the value area
+  without first knowing which fields were skipped. This is why it lives in the
+  header, and it is why Postgres puts `t_bits` in its tuple header.
+
+One bit per field rather than one byte: a 32-column row spends 4 bytes instead
+of 32.
+
+The bitmap is sized in whole bytes, so a 5-field record has 3 unused trailing
+bits. The bitmap does not know the field count — the schema does. `Describe`
+renders padding as `-` so this is visible rather than misleading.
+
+**Three-valued logic.** SQL's `NULL` means *unknown*, not *empty*, which makes
+comparison return `UNKNOWN` rather than true or false. `internal/nullbits`
+implements `Bool3` with the SQL truth tables. The consequence:
+
+```
+age is NULL
+
+age = 30                  -> UNKNOWN
+age != 30                 -> UNKNOWN
+age = 30 OR age != 30     -> UNKNOWN
+WHERE keeps the row?      -> false
+```
+
+A condition that holds for every number that exists still excludes the row,
+because `WHERE` keeps `TRUE` only. That is why SQL needs `IS NULL` as separate
+syntax: `= NULL` can never be true.
+
+Unknown does not always propagate — `FALSE AND UNKNOWN` is `FALSE`, because the
+answer is already decided. The truth tables are pinned by test.
+
+Two API details that fall out of getting this right:
+
+- `Count(fields)` and `Any(fields)` take the field count rather than reading the
+  whole bitmap. A bitmap is sized in whole bytes, so a corrupt or stray padding
+  bit would otherwise be reported as a phantom null field.
+- `Bool3` normalises out-of-range values to `UNKNOWN`, so a garbage value cannot
+  make `AND` return `TRUE`.
+
+Run it: `make demo-nulls`.
+
 ### Known gaps at the end of Phase 0
 
 Deliberate, each one is a later step:
@@ -248,6 +336,7 @@ internal/
   naive/            the throwaway from 0x01
   page/             fixed-size pages, header codec, header decoder
   codec/            value encoding (compact) and key encoding (ordered)
+  nullbits/         null bitmaps and SQL three-valued logic
 ```
 
 ## Conventions

cmd/scutedb-demo/main.go

@@ -17,12 +17,13 @@ import (
 	"github.com/scute-db/scutedb/internal/codec"
 	"github.com/scute-db/scutedb/internal/core"
 	"github.com/scute-db/scutedb/internal/naive"
+	"github.com/scute-db/scutedb/internal/nullbits"
 	"github.com/scute-db/scutedb/internal/page"
 )
 
 func main() {
 	if len(os.Args) < 2 {
-		fmt.Println("usage: scutedb-demo scan|update|race|crash|pages|header|encode")
+		fmt.Println("usage: scutedb-demo scan|update|race|crash|pages|header|encode|nulls")
 		os.Exit(2)
 	}
 	switch os.Args[1] {
@@ -40,6 +41,8 @@ func main() {
 		expHeader()
 	case "encode":
 		expEncode()
+	case "nulls":
+		expNulls()
 	case "crash-child":
 		crashChild(os.Args[2])
 	default:
@@ -421,3 +424,122 @@ func sortedBy(nums []uint64, enc func(uint64) []byte) []uint64 {
 	})
 	return out
 }
+
+func expNulls() {
+	fmt.Println("STEP 0x04  nulls and the zero problem")
+
+	fmt.Print("\n1. THE PROBLEM\n\n")
+	p := page.New(1, page.KindHeap)
+	p.Append(codec.AppendInt64(nil, 0))
+	body := p[page.HeaderSize : page.HeaderSize+8]
+	fmt.Printf("   a slot holding the number 0:   % 02X\n", body[:1])
+	fmt.Printf("   an untouched slot:             % 02X\n", p[100:101])
+	fmt.Println("   identical bytes. 'age is 0' and 'age was never given' are")
+	fmt.Println("   indistinguishable, and they mean completely different things.")
+
+	fmt.Print("\n2. THE TEMPTING FIX: A SENTINEL VALUE\n\n")
+	fmt.Println("   pick a magic value to mean 'missing':")
+	fmt.Printf("   %-22s %s\n", "sentinel", "what you can no longer store")
+	fmt.Println("   " + line(56))
+	for _, c := range [][2]string{
+		{"-1", "any legitimately negative value"},
+		{"0", "a real zero: a balance, a count, a temperature"},
+		{"math.MinInt64", "the smallest valid integer"},
+		{"\"\" (empty string)", "an intentionally blank field"},
+	} {
+		fmt.Printf("   %-22s %s\n", c[0], c[1])
+	}
+	fmt.Println("\n   every sentinel steals a legal value out of the type's range.")
+	fmt.Println("   MySQL's 0000-00-00 date and the -1 'no reading' sensor convention")
+	fmt.Println("   are the same mistake, shipped.")
+
+	fmt.Print("\n3. THE FIX: A NULL BITMAP IN THE RECORD HEADER\n\n")
+	names := []string{"id", "name", "age", "email", "score"}
+	values := []any{int64(42), "suhail", nil, nil, 9.1}
+
+	bm := nullbits.New(len(names))
+	var data []byte
+	for i, v := range values {
+		if v == nil {
+			bm.SetNull(i)
+			continue
+		}
+		switch x := v.(type) {
+		case int64:
+			data = codec.AppendInt64(data, x)
+		case string:
+			data = codec.AppendString(data, x)
+		case float64:
+			data = codec.AppendFloat64(data, x)
+		}
+	}
+	record := append(append([]byte(nil), bm...), data...)
+
+	fmt.Printf("   %-8s %-10s %s\n", "field", "value", "stored?")
+	fmt.Println("   " + line(40))
+	for i, n := range names {
+		mark, v := "yes", fmt.Sprint(values[i])
+		if bm.IsNull(i) {
+			mark, v = "NO - 0 bytes", "NULL"
+		}
+		fmt.Printf("   %-8s %-10s %s\n", n, v, mark)
+	}
+	fmt.Printf("\n   bitmap:   %s   (%d byte; N null, . present, - padding)\n", bm.Describe(len(names)), len(bm))
+	fmt.Printf("   raw:      % 02X\n", []byte(bm))
+	fmt.Printf("   record:   % 02X\n", record)
+	fmt.Printf("             ^^ header, then only the %d present values\n", len(names)-bm.Count(len(names)))
+
+	off := len(bm)
+	fmt.Println("\n   decoding: the bitmap tells the reader which fields to expect")
+	for i, n := range names {
+		if bm.IsNull(i) {
+			fmt.Printf("     %-8s NULL          (skip, nothing was written)\n", n)
+			continue
+		}
+		switch i {
+		case 0:
+			v, adv, _ := codec.Int64(record[off:])
+			fmt.Printf("     %-8s %-13d bytes %d..%d\n", n, v, off, off+adv-1)
+			off += adv
+		case 1:
+			v, adv, _ := codec.String(record[off:])
+			fmt.Printf("     %-8s %-13q bytes %d..%d\n", n, v, off, off+adv-1)
+			off += adv
+		case 4:
+			v, adv, _ := codec.Float64(record[off:])
+			fmt.Printf("     %-8s %-13v bytes %d..%d\n", n, v, off, off+adv-1)
+			off += adv
+		}
+	}
+
+	fmt.Print("\n4. WHY A BITMAP AND NOT A FLAG BYTE PER FIELD\n\n")
+	fmt.Printf("   %-10s %-16s %-16s %s\n", "fields", "1 flag byte each", "bitmap", "saved per row")
+	fmt.Println("   " + line(60))
+	for _, n := range []int{5, 8, 32, 100} {
+		fmt.Printf("   %-10d %-16d %-16d %d bytes\n", n, n, nullbits.Size(n), n-nullbits.Size(n))
+	}
+
+	fmt.Print("\n5. NULL IS NOT A VALUE. IT IS 'UNKNOWN'.\n\n")
+	all := []nullbits.Bool3{nullbits.True, nullbits.False, nullbits.Unknown}
+	fmt.Printf("   %-24s %-10s %-10s %s\n", "", "AND", "OR", "")
+	for _, a := range all {
+		for _, b := range all {
+			fmt.Printf("   %-9v %-9v -> %-9v %-9v\n", a, b, a.And(b), a.Or(b))
+		}
+	}
+	fmt.Println("\n   note FALSE AND UNKNOWN = FALSE, and TRUE OR UNKNOWN = TRUE.")
+	fmt.Println("   unknown does not always spread; sometimes the answer is already decided.")
+
+	fmt.Print("\n6. THE CONSEQUENCE\n\n")
+	var age *int64
+	thirty := int64(30)
+	eq := nullbits.EqualInt64(age, &thirty)
+	fmt.Printf("   age is NULL\n\n")
+	fmt.Printf("   age = 30                  -> %v\n", eq)
+	fmt.Printf("   age != 30                 -> %v\n", eq.Not())
+	fmt.Printf("   age = 30 OR age != 30     -> %v\n", eq.Or(eq.Not()))
+	fmt.Printf("   WHERE keeps the row?      -> %v\n", eq.Or(eq.Not()).IsTrue())
+	fmt.Println("\n   a condition that is true for every number alive excludes this row.")
+	fmt.Println("   WHERE keeps TRUE only, never UNKNOWN. this is why SQL needs IS NULL:")
+	fmt.Printf("   age IS NULL               -> %v\n", nullbits.IsNull(age))
+}

internal/codec/fuzz_test.go

@@ -90,3 +90,44 @@ func FuzzDecoderNeverPanics(f *testing.F) {
 		KeyFloat64(data)
 	})
 }
+
+func FuzzKeyOrderingFloat64(f *testing.F) {
+	f.Add(0.0, 1.0)
+	f.Add(math.Copysign(0, -1), 0.0)
+	f.Add(math.Inf(-1), math.Inf(1))
+	f.Add(math.NaN(), 1.0)
+
+	f.Fuzz(func(t *testing.T, a, b float64) {
+		ea := AppendKeyFloat64(nil, a)
+		eb := AppendKeyFloat64(nil, b)
+		got := bytes.Compare(ea, eb)
+
+		aNaN, bNaN := math.IsNaN(a), math.IsNaN(b)
+		switch {
+		case aNaN && bNaN:
+			if got != 0 {
+				t.Fatalf("two NaNs compared %d, want 0", got)
+			}
+		case aNaN:
+			if got <= 0 {
+				t.Fatalf("NaN vs %v compared %d, want > 0", b, got)
+			}
+		case bNaN:
+			if got >= 0 {
+				t.Fatalf("%v vs NaN compared %d, want < 0", a, got)
+			}
+		case a < b:
+			if got >= 0 {
+				t.Fatalf("%v < %v but bytes compared %d", a, b, got)
+			}
+		case a > b:
+			if got <= 0 {
+				t.Fatalf("%v > %v but bytes compared %d", a, b, got)
+			}
+		default:
+			if got != 0 {
+				t.Fatalf("%v == %v but bytes compared %d\n  % 02X\n  % 02X", a, b, got, ea, eb)
+			}
+		}
+	})
+}

internal/codec/key.go

@@ -30,6 +30,12 @@ func KeyInt64(src []byte) (int64, int, error) {
 }
 
 func AppendKeyFloat64(dst []byte, f float64) []byte {
+	if f == 0 {
+		f = 0
+	}
+	if math.IsNaN(f) {
+		f = math.NaN()
+	}
 	b := math.Float64bits(f)
 	if b&signBit != 0 {
 		b = ^b

internal/codec/regression_test.go

@@ -0,0 +1,68 @@
+package codec
+
+import (
+	"bytes"
+	"math"
+	"testing"
+)
+
+func TestNegativeZeroEncodesLikePositiveZero(t *testing.T) {
+	pos := AppendKeyFloat64(nil, 0.0)
+	neg := AppendKeyFloat64(nil, math.Copysign(0, -1))
+	if !bytes.Equal(pos, neg) {
+		t.Fatalf("-0.0 encodes to % 02X but +0.0 encodes to % 02X; they are numerically equal", neg, pos)
+	}
+}
+
+func TestAllNaNsEncodeIdentically(t *testing.T) {
+	a := AppendKeyFloat64(nil, math.NaN())
+	b := AppendKeyFloat64(nil, math.Float64frombits(0x7FF8000000000042))
+	if !bytes.Equal(a, b) {
+		t.Fatalf("two NaN payloads encode differently: % 02X vs % 02X", a, b)
+	}
+}
+
+func TestNaNSortsAboveEverything(t *testing.T) {
+	nan := AppendKeyFloat64(nil, math.NaN())
+	for _, v := range []float64{math.Inf(-1), -1e300, 0, 1e300, math.Inf(1)} {
+		if bytes.Compare(nan, AppendKeyFloat64(nil, v)) <= 0 {
+			t.Fatalf("NaN does not sort above %v", v)
+		}
+	}
+}
+
+func TestBytesDoesNotAliasSource(t *testing.T) {
+	buf := AppendBytes(nil, []byte{1, 2, 3})
+	out, _, err := Bytes(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	buf[1] = 99
+	if out[0] != 1 {
+		t.Fatal("Bytes() returned a slice aliasing the caller's buffer")
+	}
+}
+
+func TestBytesRefDoesAliasSource(t *testing.T) {
+	buf := AppendBytes(nil, []byte{1, 2, 3})
+	out, _, err := BytesRef(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	buf[1] = 99
+	if out[0] != 99 {
+		t.Fatal("BytesRef() copied; it is documented as returning a view")
+	}
+}
+
+func TestStringDoesNotAliasSource(t *testing.T) {
+	buf := AppendString(nil, "abc")
+	out, _, err := String(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	buf[1] = 'z'
+	if out != "abc" {
+		t.Fatalf("String() = %q after mutating the source; it must copy", out)
+	}
+}