diff --git a/TESTING.md b/TESTING.md index 89351a3..ff6b11c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -35,6 +35,7 @@ go test -run xxx -fuzz FuzzSelectF128 -fuzztime 10m | Exhaustive boundary grids | Enumerates small parameter domains and checks digest integers below, at, and above every exact CDF boundary, with anti-vacuity counters | `cdf_exhaustive_test.go`, `oracle_hardening_test.go` | | Higher-precision convergence | Requires the 512- and 1024-bit CDF walks to converge and compares them with exact small-domain results and certified large-money results | `selectAtPrecision` in `f128_exact_test.go`, tests in `oracle_hardening_test.go` | | Internal trajectory and liveness | Checks PMF/CDF error envelopes and monotonicity, audits the maximum-domain exponent path, proves bounded freeze permanence, and pins CDF-evaluation limits | `f128_trajectory_test.go` | +| Frozen-tail output pinning | Compares promoted indexes with high-precision and Boost tail counts, and bounds maximum-digest outputs across current committee sizes, stake fractions, and supply scales | `TestSelectF128CurrentConsensusFrozenTail` and `TestSelectF128CurrentCommitteeOutputCeiling` in `f128_test.go` | | Metamorphic properties | Checks digest monotonicity, power-of-two and arbitrary common-factor probability scaling, primitive identities, and arithmetic order properties without a numeric oracle | `f128_rapid_test.go` | | Distribution sanity | Checks aggregate selection weight against the expected binomial mean without reusing the CDF formula | `TestSelectF128Distribution` in `f128_exact_test.go` | | Arb-certified quantiles | Uses Arb's regularized incomplete beta implementation to certify large-money quantile inequalities with rigorous dyadic endpoints | `tools/generate_arb_oracle.py`, `testdata/f128_arb_certificates.json`, `f128_arb_certificate_test.go` | @@ -66,9 +67,11 @@ those endpoints enclose the true incomplete-beta CDF. ## Important test semantics -- The near-one frozen-tail sliver is defined to return `money`. Exact-math - tolerance tests exclude that interval deliberately; dedicated tests pin its - behavior and liveness at current-scale values. +- The near-one frozen-tail sliver promotes the first permanently frozen CDF + boundary to 1 and returns its finite index. Exact-math tolerance tests + exclude that interval deliberately because this is a precision policy, not + the exact binomial quantile; dedicated tests pin its behavior and liveness + at current-scale values. - `money` must remain below `SelectF128MaxMoney`. The maximum-domain exponent audit checks the worst representable `1-p` combination without performing a supply-sized walk. diff --git a/f128.go b/f128.go index 8d87a1e..2866ec5 100644 --- a/f128.go +++ b/f128.go @@ -37,8 +37,9 @@ import ( // It also shapes the ratio == 1.0 edge (all-0xff digest, or any digest with // >= 129 leading one bits): for some distributions the accumulated cdf rounds // up to exactly 1.0 at an early j and the walk stops there; in others it stays -// below 1.0 for all j < money and the walk runs to money (see -// TestSelectF128RatioExactlyOne and the SelectF128 doc comment). +// below 1.0 and either freezes, causing the walk to return the promoted freeze +// index, or remains live through every j < money and legitimately falls through +// to money (see TestSelectF128RatioExactlyOne and the SelectF128 doc comment). type f128 struct { hi, lo uint64 // exp is explicitly 64-bit: int is 32 bits on 386/arm, and a @@ -525,16 +526,21 @@ func (b *binomialF128) cdf(j uint64) f128 { // double boundary = cdf(dist, j); boundary := dist.cdf(j) // if (ratio <= boundary) { if ratio.cmp(boundary) <= 0 { // return j; return j -// } } -// } } -// return money; return money -// } } +// } } +// if dist.frozen { +// return j +// } +// } } +// return money; return money +// } } // // Boost computes cdf(dist, j) = ibetac(j+1, n-j, p) afresh each step in hardware // double, whereas dist.cdf(j) returns the same mathematical value as a running // PMF sum in software f128 (see binomialF128). The f128 path also receives the // digest ratio directly at f128 precision, and the success probability as its // exact integer numerator and denominator rather than a float64 quotient. +// The frozen branch has no C++ counterpart: it is the documented SelectF128 +// policy for an f128 running sum that can no longer represent later CDF mass. // // Precondition: money < SelectF128MaxMoney (2^56). Below that bound no int64 // exponent arithmetic in the walk can wrap, even at the most extreme @@ -544,10 +550,18 @@ func (b *binomialF128) cdf(j uint64) f128 { // bound is undefined (Boost's Select cannot evaluate such money either). func binomialCDFWalkF128(expectedSize, totalMoney uint64, ratio f128, money uint64) uint64 { dist := newBinomialF128(expectedSize, totalMoney, money) - if dist == nil { // p >= 1: cdf(j)==0 for j= totalMoney. + // For nonzero totalMoney this is p >= 1: cdf(j)==0 for j= in { + t.Fatalf("%s: monotonicity across the frozen transition: crossing %d >= freeze index %d", c.name, below, in) + } + if got := SelectF128(c.money, c.total, c.expected, dBelow); got != below { + t.Fatalf("%s: SelectF128(below)=%d != instrumented walk %d", c.name, got, below) + } + if got := SelectF128(c.money, c.total, c.expected, dIn); got != in { + t.Fatalf("%s: SelectF128(in)=%d != instrumented walk %d", c.name, got, in) + } + } +} diff --git a/f128_test.go b/f128_test.go index 2b00ee2..5d7b2d0 100644 --- a/f128_test.go +++ b/f128_test.go @@ -44,7 +44,9 @@ const testOracleMaxCDFSteps = uint64(20_000) // money (up to ~2^51) where it is likewise exact. (A truly unbounded oracle would // need big.Rat, which is infeasible at large money -- (1-p)^money has a // total^money denominator -- so this range, covering all reachable inputs, is the -// practical maximum.) +// practical maximum.) The oracle also mirrors the production frozen-tail +// policy: once a shrinking PMF addition no longer moves the CDF, it promotes +// that first frozen boundary and returns its index. func selectBigOracle(money uint64, totalMoney uint64, expectedSize uint64, vrfOutput Digest) uint64 { const prec = f128MantBits ratio := digestRatioBig(vrfOutput, prec) @@ -66,6 +68,8 @@ func selectBigOracle(money uint64, totalMoney uint64, expectedSize uint64, vrfOu return 0 } for j := uint64(1); j < money && j <= testOracleMaxCDFSteps; j++ { + pmfPrev := pmf + cdfPrev := cdf factor := new(big.Float).SetPrec(prec).Quo( new(big.Float).SetPrec(prec).SetUint64(money-j+1), new(big.Float).SetPrec(prec).SetUint64(j)) @@ -75,6 +79,9 @@ func selectBigOracle(money uint64, totalMoney uint64, expectedSize uint64, vrfOu if cdf.Cmp(ratio) >= 0 { return j } + if cdf.Cmp(cdfPrev) == 0 && pmf.Cmp(pmfPrev) < 0 { + return j + } } if money > testOracleMaxCDFSteps { panic("selectBigOracle exceeded the test oracle step budget") @@ -118,10 +125,9 @@ func FuzzSelectF128(f *testing.F) { f.Add(uint64(100), uint64(1000), uint64(2000), make([]byte, 32)) // expectedSize > totalMoney // all-0xff digest: ratio is exactly 1.0, the cdf-reaches-1.0 regime f.Add(uint64(1954), uint64(1_999_999_999_999_964), uint64(1500), bytes.Repeat([]byte{0xff}, 32)) - // fall-through to money by full walk (p=1/2: the pmf never drops below - // cum's half-ulp, so the CDF never freezes) and by the freeze short-circuit - // (tiny p: pmf underflows within a few steps); the second must equal the - // oracle's unshortened walk + // A legitimate fall-through to money (p=1/2: the pmf never drops below + // cum's half-ulp, so the CDF never freezes) and the promoted-freeze path + // (tiny p: the CDF stops moving within a few steps). f.Add(uint64(100), uint64(200), uint64(100), bytes.Repeat([]byte{0xff}, 32)) f.Add(uint64(1954), uint64(1_999_999_999_999_960), uint64(1500), bytes.Repeat([]byte{0xff}, 32)) // expectedSize > totalMoney with a nonzero digest: the degenerate path's money return @@ -320,15 +326,15 @@ func TestSelectF128NearMaximumDigest(t *testing.T) { // TestSelectF128RatioExactlyOne pins the walk when the f128 ratio is exactly // 1.0: mathematically for the all-0xff digest, and by 128-bit rounding for any // digest with at least 129 leading one bits. With the f128-rounded threshold -// fixed at 1.0, money is the exact-CDF count; the walk returns an earlier j -// only when the accumulated f128 CDF happens to round up to exactly 1.0 (see -// the SelectF128 doc comment). Each case pins one branch: +// fixed at 1.0, money is the exact-CDF count. SelectF128 deliberately returns +// an earlier finite index when the accumulated f128 CDF either rounds up to +// exactly 1.0 or freezes below it (see the SelectF128 doc comment). Each case +// pins one branch: // // - money=1954 with total=1_999_999_999_999_964: stops early at j=3, while -// the same distribution with total=2_000_000_000_000_000 (36 more) falls -// through to money. A hair-trigger pair pinned together: if a rounding -// change flips either, the cdf trajectory moved by an ulp -- the walk did -// not break. +// the same distribution with total=2_000_000_000_000_000 (36 more) +// freezes at j=5. A hair-trigger pair pinned together: if a rounding +// change flips either, the CDF trajectory moved by an ulp. // - money=100, p=1/2: provably falls through to money -- cdf(99) is // 1 - 2^-100, which sits 2^28 ulps below 1.0, a gap no rounding can // bridge. @@ -354,7 +360,7 @@ func TestSelectF128RatioExactlyOne(t *testing.T) { want uint64 }{ {1954, 1_999_999_999_999_964, 1500, 3}, - {1954, 2_000_000_000_000_000, 1500, 1954}, + {1954, 2_000_000_000_000_000, 1500, 5}, {100, 200, 100, 100}, {129, 258, 129, 128}, } @@ -538,7 +544,7 @@ func TestDivUVsBig(t *testing.T) { } } -// TestSelectF128CurrentConsensusFrozenTail pins the accepted frozen-tail +// TestSelectF128CurrentConsensusFrozenTail pins the promoted frozen-tail // behavior at values admitted by current go-algorand consensus parameters. // Consensus v41 inherits NumProposers=20, NextCommitteeSize=5000, and // MinBalance=100,000 microalgos. Its payout-eligibility interval is 30,000 @@ -549,11 +555,14 @@ func TestDivUVsBig(t *testing.T) { // // In every case q=(1-p) rounds downward. Raising q to money scales every PMF // term down enough that the accumulated f128 CDF freezes below the chosen -// digest ratio. SelectF128 defines this interval to return money; completion -// is also the liveness assertion, since the unshortened walk would perform up -// to money no-op iterations. The deployed Boost walk does not share the -// plateau: the digest rounds to binary64 1.0, and its independently evaluated -// CDF reaches 1.0 at the finite values pinned in boostWant. +// digest ratio. SelectF128 promotes the first frozen boundary to 1 and returns +// its index, so every result remains committee-scale instead of falling +// through to money after up to money no-op iterations. The deployed Boost walk +// does not share the plateau: the digest rounds to binary64 1.0, and its +// independently evaluated CDF reaches 1.0 at the finite values in boostWant. +// highWant records the 512-bit recurrence's finite tail quantile to make the +// deliberate approximation visible: the promoted index is not exact, but it +// remains in the same committee-scale neighborhood instead of returning stake. func TestSelectF128CurrentConsensusFrozenTail(t *testing.T) { const ( mainnetSupply = uint64(10_000_000_000_000_000) @@ -564,19 +573,28 @@ func TestSelectF128CurrentConsensusFrozenTail(t *testing.T) { total uint64 expected uint64 clearBit uint + f128Want uint64 + highWant uint64 boostWant uint64 }{ - {"proposer committee", 1_999_999_999_999_999, 1_999_999_999_999_999, 20, 175, 67}, - {"base minimum balance", 100_000, mainnetSupply, 5_000, 141, 2}, - {"payout minimum balance", 30_000_000_000, mainnetSupply, 5_000, 159, 6}, - {"payout maximum balance", 70_000_000_000_000, mainnetSupply, 5_000, 170, 94}, - {"mainnet supply ceiling", mainnetSupply, mainnetSupply, 5_000, 178, 5_598}, + {"proposer committee", 1_999_999_999_999_999, 1_999_999_999_999_999, 20, 175, 104, 81, 67}, + {"base minimum balance", 100_000, mainnetSupply, 5_000, 141, 6, 4, 2}, + {"payout minimum balance", 30_000_000_000, mainnetSupply, 5_000, 159, 15, 11, 6}, + {"payout maximum balance", 70_000_000_000_000, mainnetSupply, 5_000, 170, 138, 114, 94}, + {"mainnet supply ceiling", mainnetSupply, mainnetSupply, 5_000, 178, 5_945, 5_730, 5_598}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { d := maxDigestMinusPowerOfTwo(test.clearBit) - if got := SelectF128(test.money, test.total, test.expected, d); got != test.money { - t.Fatalf("SelectF128=%d, want money=%d for a ratio above the CDF plateau", got, test.money) + got := SelectF128(test.money, test.total, test.expected, d) + if got != test.f128Want { + t.Fatalf("SelectF128=%d, want promoted freeze index %d", got, test.f128Want) + } + if oracle := selectBigOracle(test.money, test.total, test.expected, d); oracle != got { + t.Fatalf("big.Float oracle=%d disagrees with SelectF128=%d", oracle, got) + } + if got := selectHighPrec(test.money, test.total, test.expected, d); got != test.highWant { + t.Fatalf("high-precision selector=%d, want finite tail count %d", got, test.highWant) } if got := Select(test.money, test.total, float64(test.expected), d); got != test.boostWant { t.Fatalf("Boost Select=%d, want finite tail count %d", got, test.boostWant) @@ -585,16 +603,118 @@ func TestSelectF128CurrentConsensusFrozenTail(t *testing.T) { } } +// TestSelectF128CurrentCommitteeOutputCeiling checks the maximum digest, and +// therefore the maximum SelectF128 result by digest monotonicity, across every +// current go-algorand committee size at representative supply scales. It also +// advances the sole-staker recurrence directly to confirm that the freeze +// itself fires below the ceiling; an API result can instead cross earlier if +// its CDF rounds to 1. With the protocol invariant money<=totalMoney, +// money==totalMoney maximizes the mean at expectedSize. The factor is a +// current-parameter regression ceiling, not a proof over every valid input and +// not part of the API: changing precision or committee parameters requires +// re-deriving it rather than silently turning it into a consensus cap. +func TestSelectF128CurrentCommitteeOutputCeiling(t *testing.T) { + committees := []uint64{20, 500, 1500, 2400, 2990, 5000, 6000} + totals := []uint64{ + 2_000_000_000_000_000, + 10_000_000_000_000_000, + SelectF128MaxMoney - 1, + } + const maxCurrentFreezeMultiple = uint64(6) + + var maximum Digest + for i := range maximum { + maximum[i] = 0xff + } + for _, expected := range committees { + for _, total := range totals { + ceiling := maxCurrentFreezeMultiple * expected + if total <= ceiling { + t.Fatalf("expected=%d total=%d: test cannot distinguish stake from ceiling %d", + expected, total, ceiling) + } + + // A sole online account has the largest mean admitted by the + // money<=total protocol invariant. Inspect that recurrence directly: + // the maximum-digest API path may cross a CDF rounded to 1 before it + // reaches the first permanently frozen boundary. + b := newBinomialF128(expected, total, total) + if b == nil { + t.Fatalf("expected=%d total=%d: degenerate sole-staker distribution", expected, total) + } + limit := 10 * ceiling + b.cdf(limit) + if !b.frozen { + t.Fatalf("expected=%d total=%d: CDF did not freeze within %d steps", expected, total, limit) + } + if b.at > ceiling { + t.Fatalf("expected=%d total=%d: freeze index %d above current ceiling %d", + expected, total, b.at, ceiling) + } + + stakes := map[uint64]struct{}{ + 100_000: {}, + total / 1_000_000: {}, + total / 1_000: {}, + total / 10: {}, + total / 2: {}, + total - 1: {}, + total: {}, + } + var observedMax uint64 + for money := range stakes { + if money == 0 || money > total { + continue + } + got := SelectF128(money, total, expected, maximum) + if got > ceiling { + t.Fatalf("expected=%d money=%d total=%d: maximum digest selected %d, above current ceiling %d", + expected, money, total, got, ceiling) + } + if got > observedMax { + observedMax = got + } + } + t.Logf("expected=%d total=%d: freeze index %d, maximum observed output %d (ceiling %d)", + expected, total, b.at, observedMax, ceiling) + } + } +} + +// TestSelectF128OutputCeilingRequiresStakeInvariant prevents the +// current-committee regression ceiling above from being mistaken for a +// library-wide result cap. If money exceeds totalMoney, the mean +// money*expectedSize/totalMoney can exceed the committee size by an arbitrary +// factor; promotion only handles the unresolved frozen tail and must not clip +// an ordinary crossing. +func TestSelectF128OutputCeilingRequiresStakeInvariant(t *testing.T) { + var maximum Digest + for i := range maximum { + maximum[i] = 0xff + } + if got := SelectF128(1_000_000, 100, 20, maximum); got != 204_858 { + t.Fatalf("SelectF128=%d, want ordinary high-mean crossing 204858", got) + } +} + // TestSelectF128FrozenTailReportedCase retains the original supply-sized // reproducer: pmf(0)'s trial-count-amplified rounding leaves the accumulated // CDF around 2^-78 below 1 when money == totalMoney == 2e15 and committee size // is the current certification size 1500. The ratio 1-2^-80 is above that -// plateau, while Boost terminates at its binary64 tail boundary. +// plateau, so SelectF128 returns the promoted index 2032; the 512-bit +// recurrence and Boost terminate at finite tail counts 1913 and 1832. func TestSelectF128FrozenTailReportedCase(t *testing.T) { const onlineStake = uint64(2_000_000_000_000_000) d := maxDigestMinusPowerOfTwo(176) // ratio ~= 1 - 2^-80 - if got := SelectF128(onlineStake, onlineStake, 1500, d); got != onlineStake { - t.Fatalf("SelectF128=%d, want money=%d for a ratio above the CDF plateau", got, onlineStake) + got := SelectF128(onlineStake, onlineStake, 1500, d) + if got != 2032 { + t.Fatalf("SelectF128=%d, want promoted freeze index 2032", got) + } + if oracle := selectBigOracle(onlineStake, onlineStake, 1500, d); oracle != got { + t.Fatalf("big.Float oracle=%d disagrees with SelectF128=%d", oracle, got) + } + if got := selectHighPrec(onlineStake, onlineStake, 1500, d); got != 1913 { + t.Fatalf("high-precision selector=%d, want finite tail count 1913", got) } if got := Select(onlineStake, onlineStake, 1500, d); got != 1832 { t.Fatalf("Boost Select=%d, want finite tail count 1832", got) @@ -605,7 +725,7 @@ func TestSelectF128FrozenTailReportedCase(t *testing.T) { for i := range d { d[i] = 0xff } - if got := SelectF128(onlineStake, onlineStake, 1500, d); got != onlineStake { - t.Fatalf("SelectF128=%d, want money=%d for ratio exactly 1.0", got, onlineStake) + if got := SelectF128(onlineStake, onlineStake, 1500, d); got != 2032 { + t.Fatalf("SelectF128=%d, want promoted freeze index 2032 for ratio exactly 1.0", got) } } diff --git a/f128_trajectory_test.go b/f128_trajectory_test.go index 019bf91..cdd57b2 100644 --- a/f128_trajectory_test.go +++ b/f128_trajectory_test.go @@ -248,11 +248,13 @@ func TestSelectF128TrajectoryErrorBudget(t *testing.T) { } } -// TestSelectF128FreezePermanence ignores the production short-circuit after a +// TestSelectF128FreezePermanence ignores the production frozen branch after a // bounded walk freezes and explicitly advances the recurrence to money-1. // Every later PMF must remain non-increasing and every rounded CDF add must be -// a no-op, proving on the exercised grid that the optimization returns the -// same answer as the otherwise impractical unshortened walk. +// a no-op, proving on the exercised grid that no later boundary can move and +// the early exit forfeits no crossing. It does not claim to match the +// unshortened walk's result: that walk would fall through to money, which the +// frozen-tail policy deliberately replaces with the promoted index. func TestSelectF128FreezePermanence(t *testing.T) { tests := []struct { money, total, expected uint64 @@ -307,7 +309,7 @@ func selectF128WithStepCount(money, total, expected uint64, d Digest) (selected, return j, evaluations, false } if dist.frozen { - return money, evaluations, true + return j, evaluations, true } } return money, evaluations, false @@ -315,7 +317,7 @@ func selectF128WithStepCount(money, total, expected uint64, d Digest) (selected, // TestSelectF128ConsensusStepBounds makes liveness deterministic by counting // CDF evaluations rather than timing them. The cases cover certified tails -// outside the frozen sliver and defined money-returning cases inside it. +// outside the frozen sliver and promoted finite boundaries inside it. func TestSelectF128ConsensusStepBounds(t *testing.T) { const ( online = uint64(2_000_000_000_000_000) @@ -331,9 +333,9 @@ func TestSelectF128ConsensusStepBounds(t *testing.T) { {"online 1500 certified tail", online, online, 1500, maxDigestMinusPowerOfTwo(196), 1852, false}, {"online 6000 certified tail", online, online, 6000, maxDigestMinusPowerOfTwo(200), 6667, false}, {"supply 5000 certified tail", supply, supply, 5000, maxDigestMinusPowerOfTwo(190), 5667, false}, - {"proposer frozen tail", online - 1, online - 1, 20, maxDigestMinusPowerOfTwo(175), online - 1, true}, - {"base minimum frozen tail", 100_000, supply, 5000, maxDigestMinusPowerOfTwo(141), 100_000, true}, - {"supply frozen tail", supply, supply, 5000, maxDigestMinusPowerOfTwo(178), supply, true}, + {"proposer frozen tail", online - 1, online - 1, 20, maxDigestMinusPowerOfTwo(175), 104, true}, + {"base minimum frozen tail", 100_000, supply, 5000, maxDigestMinusPowerOfTwo(141), 6, true}, + {"supply frozen tail", supply, supply, 5000, maxDigestMinusPowerOfTwo(178), 5945, true}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { diff --git a/mutation_check.go b/mutation_check.go index 4c9f22c..ff49daa 100644 --- a/mutation_check.go +++ b/mutation_check.go @@ -94,10 +94,20 @@ var mutants = []mutant{ new: "for j := uint64(1); j < money; j++ {\n\t\tboundary := dist.cdf(j)", }, {name: "walk-strict-compare", old: "P(X <= j)\n\t\tif ratio.cmp(boundary) <= 0 {", new: "P(X <= j)\n\t\tif ratio.cmp(boundary) < 0 {"}, + { + name: "frozen-branch-return-money", + old: "monotonicity in the digest.\n\t\t\treturn j", + new: "monotonicity in the digest.\n\t\t\treturn money", + }, + { + name: "frozen-branch-off-by-one", + old: "monotonicity in the digest.\n\t\t\treturn j", + new: "monotonicity in the digest.\n\t\t\treturn j + 1", + }, {name: "freeze-fire-on-change", old: "if b.cum.cmp(cumPrev) == 0 && b.pmf.cmp(pmfPrev) < 0 {", new: "if b.cum.cmp(cumPrev) != 0 && b.pmf.cmp(pmfPrev) < 0 {"}, { name: "freeze-nonstrict-pmf", old: "b.pmf.cmp(pmfPrev) < 0 {", new: "b.pmf.cmp(pmfPrev) <= 0 {", - equivalent: "an equal pmf whose add was a no-op still freezes cum forever: the step factor is non-increasing, so later pmfs stay <= this one and later adds stay no-ops", + equivalent: "the mutant widens the trigger to equal-pmf no-ops, but no such step exists, so the observable freeze index is unchanged: pmf(k) == pmf(k-1) needs the rounded step multiplier within ~an ulp of 1, which the strictly decreasing factor satisfies only adjacent to the PMF mode, where the term is within ulps of the distribution's maximum and cum <= (k+1)*pmf, giving pmf/cum >= 1/(k+1) > 2^-57 for money < 2^56 -- far above the ~2^-129 no-op threshold, so the add always moves cum there", }, {name: "divstep-cap-boundary", old: "if uHi >= v1 {", new: "if uHi > v1 {"}, { @@ -131,6 +141,12 @@ var mutants = []mutant{ old: "new(big.Float).SetPrec(prec).SetUint64(money-j+1),", new: "new(big.Float).SetPrec(prec).SetUint64(money-j),", }, + { + name: "big-oracle-ignore-freeze", + target: "f128_test.go", + old: "if cdf.Cmp(cdfPrev) == 0 && pmf.Cmp(pmfPrev) < 0 {\n\t\t\treturn j\n\t\t}", + new: "if cdf.Cmp(cdfPrev) == 0 && pmf.Cmp(pmfPrev) < 0 {\n\t\t\treturn money\n\t\t}", + }, { name: "digest-oracle-denominator", target: "f128_test.go", diff --git a/sortition.go b/sortition.go index 77dd5cd..431f009 100644 --- a/sortition.go +++ b/sortition.go @@ -100,49 +100,42 @@ const SelectF128MaxMoney = uint64(1) << 56 // one, so replacing Select with SelectF128 remains a protocol-gated, // network-coordinated consensus change. // -// One tail edge: the top ~2^-129 of digest space rounds to an f128 ratio of -// exactly 1.0 (only the all-0xff digest IS exactly 1.0; the rest of the -// interval rounds up to it). With the threshold fixed at 1.0, the exact CDF is -// < 1 for every j < money and the walk never evaluates cdf(money) == 1, so the -// exact-CDF count is money -- and the walk returns money unless the accumulated -// f128 CDF happens to round up to exactly 1.0 at an earlier j (a rounding -// artifact), in which case it returns that j. Both outcomes occur, decided -// per-distribution at ulp granularity: the money=1954 case in -// TestSelectF128RatioExactlyOne stops at j=3, while the same distribution with -// total=2_000_000_000_000_000 falls through to money. Both match the 128-bit -// big.Float oracle. Boost's double CDF -- evaluated independently per j via -// ibetac rather than accumulated -- can also saturate to 1.0 on its far coarser -// grid, potentially at a different (typically earlier) j. At these near-maximum -// digests the two implementations can therefore return wildly different counts: -// one may stop within a few steps of the binomial tail while the other returns -// the full trial count money. A uniform VRF output lands in this interval with -// probability about 2^-129 per credential. The event is possible, but the -// consensus threat model treats it as negligible and assumes the registered -// key and unpredictable seed prevent an adversary from targeting it. This is -// therefore a documented statistical edge rather than a case special-cased by -// the implementation. +// FROZEN-TAIL POLICY. Rounding q=1-p once and then raising it to money can +// scale every PMF term by a common error of about money*2^-129. When that +// error is downward, the accumulated f128 CDF can settle at a plateau below +// 1. A digest ratio above the plateau would never cross another represented +// boundary, and a literal walk would eventually fall through and return +// money after as many as money no-op iterations. // -// The same holds in a wider sliver just below 1.0. pmf(0) = (1-p)^money -// amplifies the 2^-129 rounding of 1-p by up to the trial count, and pmf(0) -// scales every PMF term, so the accumulated CDF settles at a plateau that can -// sit as much as ~money*2^-129 below 1 (~2^-78 at 2e15 microalgos of stake, -// and ~2^-76 at the 10^16-microalgo mainnet supply ceiling). A digest ratio -// between that plateau and 1.0 sits above every boundary without rounding to -// 1.0; the walk detects the frozen CDF and immediately returns money, the same -// result the plain walk would reach after up to money no-op iterations. +// Once an addition leaves the CDF unchanged while the PMF is strictly +// shrinking, every later term is no larger and every later CDF addition is +// also a no-op. SelectF128 therefore promotes that first frozen boundary to 1 +// and returns its index. This assigns the unresolved tail to one finite result +// instead of treating the account's entire stake as its selection weight. It +// is a deliberate approximation: affected digests no longer distinguish the +// true binomial quantiles beyond the precision horizon, and the result differs +// from the literal fall-through in the C++ reference loop. // -// These outputs are possible under current go-algorand committee and balance -// bounds; TestSelectF128CurrentConsensusFrozenTail pins examples from the base -// account minimum through the mainnet supply ceiling. For an account with -// stake m, the affected interval is approximately m*2^-129, and summing that -// first-order bound over all online accounts gives approximately -// totalMoney*2^-129 per committee selection, independent of how stake is -// split. The consensus rationale for accepting the edge is probabilistic, not -// impossibility: registered keys and an unpredictable seed are assumed to -// prevent targeting the interval. Within it the count is DEFINED as money -// rather than the exact binomial-tail crossing. Computing pmf(0) with guard -// bits could narrow the interval, at the cost of additional consensus-critical -// arithmetic and audit surface. +// The rule also applies when the digest ratio rounded to exactly 1. The top +// ~2^-129 of digest space does so at f128 precision; only the all-0xff digest +// is exactly 1 under the digest/(2^256-1) mapping. If the accumulated CDF +// rounds to 1 before freezing, the ordinary inclusive boundary wins. If it +// freezes below 1, the promoted freeze index wins. At small money the CDF can +// instead remain live and below 1 through every j < money, in which case the +// ordinary loop legitimately falls through to money, the exact inverse-CDF +// count for ratio 1. TestSelectF128RatioExactlyOne pins all three trajectories. +// +// The frozen sliver is approximately money*2^-129 wide. Summed over online +// accounts, its first-order rate is approximately totalMoney*2^-129 per +// committee selection, independent of how stake is split. Under the protocol +// invariant money <= totalMoney, the binomial mean is at most expectedSize; +// current-parameter regression tests pin the promoted results at +// committee-scale indexes from the base account minimum through the mainnet +// supply ceiling. Callers that violate money <= totalMoney can have a mean and +// selection result larger than expectedSize; this tail policy is not a general +// output cap. Computing pmf(0) with guard bits would narrow the frozen sliver +// and move the promoted indexes; changing that precision policy is therefore +// also a consensus change. func SelectF128(money uint64, totalMoney uint64, expectedSize uint64, vrfOutput Digest) uint64 { ratio := f128FromDigestRatio(vrfOutput) return binomialCDFWalkF128(expectedSize, totalMoney, ratio, money)