From 39d8084c7f6834f15dd045305fa9d8d7580977f3 Mon Sep 17 00:00:00 2001 From: lmtlssss <261705307+lmtlssss@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:21:38 -0800 Subject: [PATCH 01/12] proofs/exact-llm: route t3, every product of exact-16 by adds and subtracts in a C kernel (t3.c) --- proofs/exact-llm/exact_llm.py | 209 ++++++++++++++++++++++++++-- proofs/exact-llm/routes_test.py | 32 ++++- proofs/exact-llm/t3.c | 238 ++++++++++++++++++++++++++++++++ 3 files changed, 465 insertions(+), 14 deletions(-) create mode 100644 proofs/exact-llm/t3.c diff --git a/proofs/exact-llm/exact_llm.py b/proofs/exact-llm/exact_llm.py index 80feb70..4441623 100644 --- a/proofs/exact-llm/exact_llm.py +++ b/proofs/exact-llm/exact_llm.py @@ -14,9 +14,10 @@ float32 baseline and onnxruntime where an export is pinned. usage: - python exact_llm.py check [--model M] [--routes f64,f32,i8] [--no-float] [--no-onnx] [--peaks FILE] + python exact_llm.py check [--model M] [--routes f64,f32,i8,t3] [--no-float] [--no-onnx] [--peaks FILE] digests, the hashes against expected*.sha256, baselines, distance - python exact_llm.py hash --engine exact|exact16-f64|exact16-f32|exact16-i8|float32|onnx [--batch N] [--threads N] + python exact_llm.py hash --engine exact|exact16-f64|exact16-f32|exact16-i8|exact16-t3|float32|onnx [--batch N] + [--threads N] [--model M] """ import gzip @@ -515,6 +516,10 @@ def __init__(self, W=None, keep_f64=False, clamps=None): self.f64 = {} self.eps = [int(round(EPS * 2 ** (2 * A - 2 * s))) for s in range(A + 1)] # eps at 2^-(32 - 2s) + def mul(self, a, b, kind="element-wise"): + """The element-wise integer product a * b (int64): NumPy's here; the t3 route's by adds.""" + return a * b + def clamp(self, x, site, layer=None): c = self.C[site] a = np.abs(x) @@ -554,14 +559,14 @@ def rms(self, x, wname): top = np.abs(x).max(-1, keepdims=True) sh = np.maximum(0, np.frexp(top.astype(np.float64))[1] - NORM_BITS).astype(np.int64) # bit length - NORM_BITS xs = x >> sh - ss = (xs * xs).sum(-1, keepdims=True) # at 2^-(32 - 2 sh), below 2^62 + ss = self.mul(xs, xs).sum(-1, keepdims=True) # at 2^-(32 - 2 sh), below 2^62 ms = fdiv(ss, HIDDEN) assert (sh <= A).all() eps = np.array(self.eps, dtype=np.int64)[sh] r = isqrt(ms + np.maximum(eps, 1)) # sqrt(mean x^2 + eps) at 2^-(16 - sh) y = fdiv(x << (A - sh), r) # x / r at 2^-16 assert (np.abs(y) <= RMS_Y << A).all() # |x_i| <= sqrt(HIDDEN) rms: bounds.py - return rshift_round(y * np.asarray(self.W[wname]), A) + return rshift_round(self.mul(y, np.asarray(self.W[wname])), A) def rope_tables(self, pos): """cos and sin of each position, read once per forward pass: [B, 1, L, 32] at 2^-20.""" @@ -570,19 +575,20 @@ def rope_tables(self, pos): def rope(self, x, cs): c, s = cs x1, x2 = x[..., :HEAD // 2], x[..., HEAD // 2:] - return np.concatenate([rshift_round(x1 * c - x2 * s, ROPE), rshift_round(x2 * c + x1 * s, ROPE)], -1) + m = self.mul + return np.concatenate([rshift_round(m(x1, c) - m(x2, s), ROPE), rshift_round(m(x2, c) + m(x1, s), ROPE)], -1) def exp(self, t): k = np.minimum(t >> EXP_STEP, EXP_N) r = t & ((1 << EXP_STEP) - 1) - return (self.Et[k] * ((1 << EXP_STEP) - r) + self.Et[np.minimum(k + 1, EXP_N)] * r) >> EXP_STEP + return (self.mul(self.Et[k], (1 << EXP_STEP) - r) + self.mul(self.Et[np.minimum(k + 1, EXP_N)], r)) >> EXP_STEP def silu(self, x): top = SILU_N << SILU_STEP u = x + (SILU_LO << A) uc = np.clip(u, 0, top - 1) k, r = uc >> SILU_STEP, uc & ((1 << SILU_STEP) - 1) - g = (self.St[k] * ((1 << SILU_STEP) - r) + self.St[k + 1] * r) >> SILU_STEP + g = (self.mul(self.St[k], (1 << SILU_STEP) - r) + self.mul(self.St[k + 1], r)) >> SILU_STEP return np.where((u >= 0) & (u < top), g, np.where(u < 0, 0, x)) def cache_entry(self, k, v): @@ -631,7 +637,7 @@ def forward(self, ids, pos, cache, kvalid): h = self.prepare(self.rms(x, p + "post_attention_layernorm.weight").reshape(B * L, HIDDEN), "mlp input") g = self.clamp(self.linear(h, p + "mlp.gate_proj.weight", "mlp input"), "gate") u = self.clamp(self.linear(h, p + "mlp.up_proj.weight", "mlp input"), "up") - a = self.prepare(rshift_round(self.silu(g) * u, A), "down input") + a = self.prepare(rshift_round(self.mul(self.silu(g), u), A), "down input") x = x + self.linear(a, p + "mlp.down_proj.weight", "down input").reshape(B, L, HIDDEN) x = self.clamp(x, "residual", LAYERS) h = self.rms(x, "model.norm.weight").reshape(B * L, HIDDEN) @@ -661,7 +667,9 @@ def logits(self, h): PEEL16 = 2 CHUNK32 = 1024 # f32 route: inputs per chunk, 1,024 x 2^14 = 2^24 N_I8 = 33025 # i8 route: n 255^2 < 2^31 for n <= 33,025 -ROUTES = ("f64", "f32", "i8") +ROUTES = ("f64", "f32", "i8") # check's default +ROUTES_T3 = ("t3",) # by adds and subtracts (t3.c), where a C compiler is installed: --routes t3 +TOP16 = 15 # t3: highest signed-digit position of a 16-bit value; 16 STEP words per output row def shift_round(v, k): @@ -756,6 +764,166 @@ def __call__(self, a, b): return self.s.run(None, {"A": a, "B": b})[0].astype(np.int64) +T3_DIR = os.environ.get("EXACT_LLM_T3", os.path.join(CACHE, "t3")) # the compiled kernel and the weight streams +T3_THREADS = int(os.environ.get("EXACT_LLM_T3_THREADS", "0")) or os.cpu_count() or 1 + + +class T3: + """The t3 route's kernel, t3.c, compiled once with the C compiler `cc` (or $CC) into T3_DIR and opened with + ctypes. Matrix products (mm) and element-wise products (mul) by add, subtract, shift by a constant and bitwise + logic. A registered weight matrix is compiled to its stream once and kept in T3_DIR (2 bytes per nonzero digit + plus 16 words per output row); any other right operand is recoded at each product. Counters per kind: digit + adds and subtracts, Horner adds, values recoded, recoding operations.""" + KINDS = ("weights", "peeled", "attention", "element-wise", "weight streams, built once") + + def __init__(self): + import ctypes + import subprocess + src = os.path.join(HERE, "t3.c") + tag = hashlib.sha256(open(src, "rb").read()).hexdigest()[:12] + os.makedirs(T3_DIR, exist_ok=True) + self.path = os.path.join(T3_DIR, "t3-%s.%s" % (tag, "dylib" if sys.platform == "darwin" else "so")) + if not os.path.exists(self.path): + cc = os.environ.get("CC", "cc") + subprocess.run([cc, "-O2", "-shared", "-fPIC", "-pthread", src, "-o", self.path + ".part"], check=True) + os.replace(self.path + ".part", self.path) + L = self.lib = ctypes.CDLL(self.path) + i64, p = ctypes.c_int64, ctypes.c_void_p + L.t3_compile_masks.argtypes = [i64, i64, p, i64, p, p, p, p] + L.t3_compile_masks.restype = i64 + L.t3_compile_fill.argtypes = [i64, i64, p, p, p] + L.t3_run.argtypes = [p, p, i64, i64, i64, p, p, i64, p, p] + L.t3_mul.argtypes = [i64, p, p, p, ctypes.c_uint64, p] + self.cnt = {k: np.zeros(4, dtype=np.uint64) for k in self.KINDS} + self.mats = [] # registered weights: (first byte, last byte, name, row stride) + self.rowoffs = {} + self.buf = np.empty(0, np.uint16) + self.log = None # a list: one entry per product, when tracing + + @staticmethod + def ptr(a): + return a.ctypes.data + + def register(self, W): + """The int16 weight matrices [in, out] (memory-mapped) whose streams are compiled once and kept.""" + for name, w in W.items(): + if w.dtype == np.int16 and w.ndim == 2: + self.mats.append((self.ptr(w), self.ptr(w) + w.nbytes, name, w.shape[1])) + + def weight_key(self, b): + if b.dtype != np.int16 or b.ndim != 2 or b.strides[1] != 2: + return None + at = self.ptr(b) + for lo, hi, name, ld in self.mats: + if lo <= at < hi and b.strides[0] == 2 * ld and (at - lo) // 2 < ld: + return name, (at - lo) // 2, b.shape[1] + return None + + def compile(self, b, kind): + """b [n_in, M] (|entries| <= 2^15 - 1) -> the stream (uint16) and the row offsets (int64 [M + 1]).""" + w = np.ascontiguousarray(b, dtype=np.int32) + n_in, M = w.shape + assert n_in < (1 << 14) + pm, mm = np.empty((M, n_in), np.uint16), np.empty((M, n_in), np.uint16) + rowoff = np.empty(M + 1, np.int64) + total = self.lib.t3_compile_masks(n_in, M, self.ptr(w), M, self.ptr(pm), self.ptr(mm), self.ptr(rowoff), + self.ptr(self.cnt[kind])) + stream = np.empty(total, np.uint16) + self.lib.t3_compile_fill(n_in, M, self.ptr(pm), self.ptr(mm), self.ptr(stream)) + return stream, rowoff + + def weight_stream(self, key, b): + name, c0, M = key + base = os.path.join(T3_DIR, "streams", "%s.c%d.m%d" % (name, c0, M)) + if key not in self.rowoffs: + if not (os.path.exists(base + ".words") and os.path.exists(base + ".rowoff.npy")): + os.makedirs(os.path.dirname(base), exist_ok=True) + stream, rowoff = self.compile(b, "weight streams, built once") + with open(base + ".words.part", "wb") as f: + f.write(stream.astype(" 1: # rows split at equal shares of the stream's words + part = np.searchsorted(rowoff, np.linspace(0, rowoff[-1], threads + 1)).astype(np.int64) + part[0], part[-1] = 0, M + else: + part = np.array([0, M], np.int64) + before = self.cnt[kind].copy() + self.lib.t3_run(self.ptr(stream), self.ptr(rowoff), x.shape[0], Np, N, self.ptr(x), self.ptr(y), + part.size - 1, self.ptr(part), self.ptr(self.cnt[kind])) + if self.log is not None: + d = self.cnt[kind] - before + self.log.append((kind, x.shape[0], M, N, int(d[0]), int(d[1]))) + return y[:, :N] + + def mm2(self, a, b): + N, n_in = a.shape + M = b.shape[1] + if N == 0 or M == 0: + return np.zeros((N, M), np.int64) + x = np.ascontiguousarray(a.T, dtype=np.int32) + key = self.weight_key(b) + if key is not None: + stream, rowoff = self.weight_stream(key, b) + return self.run(stream, rowoff, x, "weights", T3_THREADS).T + stream, rowoff = self.compile(b, "attention") + return self.run(stream, rowoff, x, "attention", 1).T + + def mm(self, a, b): + """a @ b exactly (int64) for integer arrays with |entries| <= 2^15 - 1, a [..., N, n], b [..., n, M].""" + a, b = np.asarray(a), np.asarray(b) + if a.ndim == 2 and b.ndim == 2: + return self.mm2(a, b) + lead = np.broadcast_shapes(a.shape[:-2], b.shape[:-2]) + a = np.broadcast_to(a, lead + a.shape[-2:]) + b = np.broadcast_to(b, lead + b.shape[-2:]) + out = np.empty(lead + (a.shape[-2], b.shape[-1]), np.int64) + for i in np.ndindex(*lead): + out[i] = self.mm2(a[i], b[i]) + return out + + def mul(self, a, b, kind="element-wise"): + """a * b element-wise (int64, broadcast), the operand of smaller magnitude recoded into signed digits.""" + a, b = np.broadcast_arrays(np.asarray(a, dtype=np.int64), np.asarray(b, dtype=np.int64)) + out = np.empty(a.shape, np.int64) + if a.size == 0: + return out + ma, mb = int(np.abs(a).max()), int(np.abs(b).max()) + if ma < mb: + a, b, mb = b, a, ma + assert mb < (1 << 62) + a, b = np.ascontiguousarray(a), np.ascontiguousarray(b) + self.lib.t3_mul(a.size, self.ptr(a), self.ptr(b), self.ptr(out), 1 << mb.bit_length(), self.ptr(self.cnt[kind])) + return out + + +_T3 = [] + + +def t3_kernel(): + if not _T3: + _T3.append(T3()) + return _T3[0] + + def mm16(a, b, route, mmi=None): """a @ b exactly, as int64, for integer arrays with every |entry| <= 2^15 - 1 (a [..., N, n], b [..., n, M]). f64: one float64 product, |partial sums| <= n 2^30 < 2^53. @@ -764,10 +932,14 @@ def mm16(a, b, route, mmi=None): i8: unsigned: u = a + 2^15 in [1, 2^16 - 1], u = 256 u1 + u0 with u1, u0 in [0, 255] (uint8), the same for b; four int32 MatMulInteger products of uint8 by uint8, partial sums <= n 255^2 < 2^31; then, in int64, a b = u_a u_b - 2^15 (row sums of a) - 2^15 (column sums of b) - n 2^30. - Then a b (u_a u_b on the i8 route) = 2^16 G11 + 2^8 (G10 + G01) + G00, G_ij = a_i b_j, in int64.""" + Then a b (u_a u_b on the i8 route) = 2^16 G11 + 2^8 (G10 + G01) + G00, G_ij = a_i b_j, in int64. + t3: b in signed digits (non-adjacent form, positions 0 to 15), each column of b a stream of ADD i, SUB i and + STEP words; t3.c runs the stream on the columns of a with adds and subtracts only (class T3).""" n = a.shape[-1] if route == "f64": return (a.astype(np.float64) @ b.astype(np.float64)).astype(np.int64) + if route == "t3": + return t3_kernel().mm(a, b) # the digits in the operand's own type when it is int16 (the weights: no widening; every step below stays within # [-2^15, 2^15)), else in int64 a = np.asarray(a, dtype=np.int64) @@ -914,6 +1086,12 @@ def __init__(self, route="f64"): self.mmi = MatMulInteger() if route == "i8" else None self.extra = 0 # vectors whose rounding needed one more shift self.p_held = 0 # p values held at 2^15 - 1 + if route == "t3": + self.t3 = t3_kernel() + self.t3.register(self.W16) + + def mul(self, a, b, kind="element-wise"): + return self.t3.mul(a, b, kind) if self.route == "t3" else a * b def prepare(self, x, site): """A product input as every product that reads it needs it, made once: clamped at its site, its PEEL16 @@ -937,7 +1115,7 @@ def product(self, prep, name, dtype=np.int64): wc = w[:, c:c + 8192] # int16, memory-mapped T = np.left_shift(mm16(xb, wc, self.route, self.mmi), sh) for j in range(PEEL16): - T += xp[:, j:j + 1] * wc[idx[:, j]] # int64 times int16: int64 + T += self.mul(xp[:, j:j + 1], wc[idx[:, j]], "peeled") # int64 times int16: int64 blk = shift_round(T, g[None, c:c + 8192]) assert dtype == np.int64 or np.abs(blk).max() < (1 << 31) y[:, c:c + 8192] = blk @@ -1035,7 +1213,7 @@ def run_window(model, ids): def engine(name, threads=0): if name == "exact": return Exact() - if name in ("exact16-" + r for r in ROUTES): + if name in ("exact16-" + r for r in ROUTES + ROUTES_T3): return Exact16(name.split("-")[1]) if name == "float32": return Float32() @@ -1123,6 +1301,9 @@ def main(argv): if name.startswith("exact16"): print("route %s; vectors with one more shift %d; p held at 2^15 - 1: %d" % ( eng.mmi.name if eng.mmi else eng.route, eng.extra, eng.p_held)) + if eng.route == "t3": + print("t3 counters (digit adds, Horner adds, values recoded, recoding operations):", + {k: [int(v) for v in c] for k, c in eng.t3.cnt.items()}) return 0 if len(argv) > 1 and argv[1] == "text": name, save = arg("--engine", "exact"), arg("--save", None) @@ -1171,6 +1352,10 @@ def main(argv): print("exact16 logits %s continuations %s %s route %s, %.0f s, clamped %d" % ( h16l, h16c, "MATCH" if ok16 else "DIFFERENT", m16.mmi.name if m16.mmi else r, time.time() - t0, sum(m16.clamped.values()))) + if r == "t3": + print(" t3 kernel %s; digit adds, Horner adds, values recoded, recoding operations: %s" % ( + os.path.basename(m16.t3.path), "; ".join("%s %s" % (k, ", ".join("{:,}".format(int(v)) for v in c)) + for k, c in m16.t3.cnt.items()))) if not kept16: print(" peaks (real) / clamps: %s" % peaks_line(m16)) if arg("--peaks", None): diff --git a/proofs/exact-llm/routes_test.py b/proofs/exact-llm/routes_test.py index 443b04e..4002048 100644 --- a/proofs/exact-llm/routes_test.py +++ b/proofs/exact-llm/routes_test.py @@ -7,6 +7,7 @@ # with the products negative. "digit extremes" mixes, position by position, the eight values whose digits sit at the # ends of both routes' digit ranges. A kernel that saturates or rounds anywhere fails one of these. import os +import shutil import sys import numpy as np @@ -17,6 +18,10 @@ L = X.LIM16 mmi = X.MatMulInteger() print("i8 product engine:", mmi.name) +# t3 (t3.c, by adds and subtracts) where a C compiler is installed +T3 = shutil.which(os.environ.get("CC", "cc")) is not None +ROUTES = X.ROUTES + (X.ROUTES_T3 if T3 else ()) +print("t3 kernel:", X.t3_kernel().path if T3 else "no C compiler: not tested") shapes = [((5, 576), (576, 300)), ((3, 1023), (1023, 7)), ((3, 1024), (1024, 7)), ((3, 1025), (1025, 7)), ((4, 1536), (1536, 64)), ((2, 3, 27, 64), (2, 3, 64, 40)), ((2, 3, 27, 600), (2, 3, 600, 64))] # digits (i8: of a + 2^15, high and low; f32: balanced high and low): 32767 (255, 255; 128, -1), -32767 (0, 1; @@ -41,14 +46,14 @@ a, b = rng.integers(0, L + 1, sa), rng.integers(-L, L + 1, sb) a, b = a.astype(np.int64), b.astype(np.int64) ref = np.matmul(a, b) # int64, exact: |sum| < 1536 * 2^30 - for r in X.ROUTES: + for r in ROUTES: for bb in (b, b.astype(np.int16)): # the weights come as int16 y = X.mm16(a, bb, r, mmi) if y.dtype != np.int64 or y.shape != ref.shape or not (y == ref).all(): bad += 1 print("DIFFERENT", r, sa, sb, kind, bb.dtype) print("cases %d (%d shapes x %d kinds, int64 and int16 right operands), routes %d, different %d" % ( - len(shapes) * len(KINDS), len(shapes), len(KINDS), len(X.ROUTES), bad)) + len(shapes) * len(KINDS), len(shapes), len(KINDS), len(ROUTES), bad)) # the extremes have the digits stated above u = EXTREMES + (1 << 15) h = (EXTREMES + 128) >> 8 @@ -72,4 +77,27 @@ # peel_index: ties go to the lower index x = np.array([[5, -9, 9, 3, -9, 1]], dtype=np.int64) print("peel_index of", x.tolist()[0], "k=2:", sorted(X.peel_index(x, 2)[0].tolist()), "(expect [1, 2])") +if T3: + k3 = X.t3_kernel() + # every 16-bit value through the stream: y = 1 . b for b = -32767 .. 32767, one input, 65,535 output rows; then + # the digits as the kernel forms them: nonzero digits per value against the census formula, popcount(h ^ (m + h)) + v = np.arange(-L, L + 1, dtype=np.int64) + y = k3.mm(np.ones((1, 1), np.int64), v[None]) + m = np.abs(v) + naf = np.bitwise_count(((m >> 1) ^ (m + (m >> 1))).astype(np.uint64)).astype(np.int64) + before = k3.cnt["attention"].copy() + k3.mm(np.ones((1, 1), np.int64), v[None]) + adds = int(k3.cnt["attention"][0] - before[0]) + ok3 = (y[0] == v).all() and adds == int(naf.sum()) + bad += 0 if ok3 else 1 + print("t3 stream, every value -32767..32767: %s; digit adds %d, census formula %d, at most %d per value" % ( + "exact" if (y[0] == v).all() else "DIFFERENT", adds, int(naf.sum()), int(naf.max()))) + # element-wise: random int64 pairs whose product fits, the extremes, and signs + a = rng.integers(-(1 << 31), 1 << 31, 200000) + b = rng.integers(-(1 << 31), 1 << 31, 200000) + e = np.array([0, 1, -1, L, -L, (1 << 31) - 1, -(1 << 31), (1 << 40) + 3, -(1 << 41) - 5]) + a, b = np.concatenate([a, np.repeat(e, e.size)]), np.concatenate([b, np.tile(e[::-1], e.size)]) + okm = (k3.mul(a, b) == a * b).all() and (k3.mul(b[:, None], a[None, :50]) == b[:, None] * a[None, :50]).all() + bad += 0 if okm else 1 + print("t3 element-wise products, %d pairs and a broadcast: %s" % (a.size, "exact" if okm else "DIFFERENT")) sys.exit(1 if bad else 0) diff --git a/proofs/exact-llm/t3.c b/proofs/exact-llm/t3.c new file mode 100644 index 0000000..63611c0 --- /dev/null +++ b/proofs/exact-llm/t3.c @@ -0,0 +1,238 @@ +/* t3.c: the t3 route of exact-16. Matrix products and element-wise products by add, subtract, shift by a constant + * and bitwise logic; no multiply instruction (t3_nomul.py reads the compiled object and checks). + * + * Signed digits. For m >= 0: h = m >> 1, s = m + h, c = h ^ s, plus = s & c, minus = h & c; then m = plus - minus, + * plus & minus = 0, and no two neighbouring positions are nonzero (the non-adjacent form, the fewest nonzero signed + * digits). A negative value is recoded as its magnitude 0 - v with plus and minus exchanged. |v| <= 2^15 - 1 has its + * digits at positions 0 to 15. + * + * The stream. A matrix W [n_in][M] (inputs by outputs) is compiled to 16-bit words, output row by output row; per + * row, for digit position k = 15 down to 0: ADD i for each input whose weight has digit +1 at k, then SUB i for each + * with -1 at k (i ascending), then STEP (k > 0) or EMIT (k = 0). + * word = op << 14 | i op 0 ADD, 1 SUB, 2 STEP, 3 EMIT; i < 2^14 + * The machine. State: N lanes (one per input vector), each with s (int32) and acc (int64); x[i][lane] the inputs. + * ADD i s = s + x[i] + * SUB i s = s - x[i] + * STEP acc = acc + acc; acc = acc + s; s = 0 (Horner over the digit positions: 2 adds) + * EMIT STEP, then y[row] = acc; acc = 0; row = row + 1 + * So y[row] = sum_k 2^k (sum of x over digit +1 at k - sum over digit -1 at k) = sum_i W[i][row] x[i], exactly: + * |s| <= n_in (2^15 - 1) < 2^31 for n_in < 2^16; after the STEP of position k, acc = sum_i x[i] (the digits of + * W[i][row] at positions k..15, as a number), each such number below 2^16 in magnitude: |acc| < n_in 2^31 < 2^47. + * The lanes run 8 at a time (N a multiple of 8, padded with zero inputs); the counters count the nreal lanes. + * + * Counters (uint64, added to, never reset here): cnt[0] digit adds and subtracts, lane by lane; cnt[1] Horner adds; + * cnt[2] values recoded; cnt[3] recoding operations (shift, add, xor, and, and, plus a subtract for a negative). + */ +#include +#include +#include + +#define OP_ADD 0u +#define OP_SUB 1u +#define OP_STEP 2u +#define OP_EMIT 3u +#define TOP 15 +/* An empty barrier on a register: the compiler may not fold a loop of adds around it into a multiply. */ +#define KEEP(v) __asm__ volatile("" : "+r"(v)) + +/* The signed digits of v (|v| < 2^62): *pl and *mi with v = *pl - *mi. Returns the operations used. */ +static inline uint64_t recode(int64_t v, uint64_t *pl, uint64_t *mi) { + uint64_t neg = v < 0; + uint64_t m = neg ? (uint64_t)0 - (uint64_t)v : (uint64_t)v; + uint64_t h = m >> 1, s = m + h, c = h ^ s; + uint64_t p = s & c, q = h & c; + *pl = neg ? q : p; + *mi = neg ? p : q; + return neg ? 6 : 5; +} + +/* Pass 1. W int32 [n_in][M], row stride ld (elements). Writes the masks Pm, Mm [M][n_in] (16 bits each) and + * rowoff[0..M] (word offsets of each row in the stream). Returns the stream length in words. */ +int64_t t3_compile_masks(int64_t n_in, int64_t M, const int32_t *W, int64_t ld, uint16_t *Pm, uint16_t *Mm, + int64_t *rowoff, uint64_t *cnt) { + const int32_t *wr = W; + for (int64_t i = 0; i < n_in; i++) { + uint16_t *pp = Pm + i, *mm = Mm + i; + for (int64_t j = 0; j < M; j++) { + uint64_t p, q; + cnt[3] += recode(wr[j], &p, &q); + cnt[2] += 1; + *pp = (uint16_t)p; + *mm = (uint16_t)q; + pp += n_in; + mm += n_in; + } + wr += ld; + } + int64_t off = 0; + const uint16_t *pp = Pm, *mm = Mm; + for (int64_t j = 0; j < M; j++) { + rowoff[j] = off; + off += TOP + 1; /* the STEP and EMIT words */ + for (int64_t i = 0; i < n_in; i++) { + uint32_t d = (uint32_t)pp[i] | (uint32_t)mm[i]; + while (d) { /* one per nonzero digit */ + off += d & 1u; + d >>= 1; + } + } + pp += n_in; + mm += n_in; + } + rowoff[M] = off; + return off; +} + +/* Pass 2: the words, from the masks. */ +void t3_compile_fill(int64_t n_in, int64_t M, const uint16_t *Pm, const uint16_t *Mm, uint16_t *stream) { + uint16_t *o = stream; + const uint16_t *pp = Pm, *mm = Mm; + for (int64_t j = 0; j < M; j++) { + uint32_t bit = 1u << TOP; + while (bit) { + for (int64_t i = 0; i < n_in; i++) + if (pp[i] & bit) + *o++ = (uint16_t)((OP_ADD << 14) | (uint32_t)i); + for (int64_t i = 0; i < n_in; i++) + if (mm[i] & bit) + *o++ = (uint16_t)((OP_SUB << 14) | (uint32_t)i); + bit >>= 1; + *o++ = (uint16_t)((bit ? OP_STEP : OP_EMIT) << 14); + } + pp += n_in; + mm += n_in; + } +} + +/* The machine on rows [r0, r1) of a stream. x int32 [n_in][N]; y int64 [M][N]; N a multiple of 8. */ +typedef struct { + const uint16_t *stream; + const int64_t *rowoff; + int64_t r0, r1, n_in, N, nreal; + const int32_t *x; + int64_t *y; + uint64_t cnt[4]; +} job_t; + +typedef int32_t v8 __attribute__((vector_size(32))); /* 8 lanes of s */ +typedef int64_t v8l __attribute__((vector_size(64))); /* 8 lanes of acc */ + +static void *run_rows(void *arg) { + job_t *jb = (job_t *)arg; + int64_t N = jb->N; + const int32_t **xrow = (const int32_t **)malloc(((size_t)jb->n_in + 1) * sizeof(*xrow)); + const int32_t *xp = jb->x; + for (int64_t i = 0; i < jb->n_in; i++) { /* row addresses by repeated addition */ + xrow[i] = xp; + xp += N; + KEEP(xp); + } + int64_t *yrow = jb->y; + for (int64_t r = 0; r < jb->r0; r++) { + yrow += N; + KEEP(yrow); /* an add per row, not one multiply */ + } + uint64_t adds = 0, horner = 0; + for (int64_t r = jb->r0; r < jb->r1; r++) { + const uint16_t *start = jb->stream + jb->rowoff[r]; + int64_t left = jb->nreal; /* real lanes not yet run */ + for (int64_t c = 0; c < N; c += 8) { /* the row's words once per 8 lanes */ + uint64_t nl = left < 8 ? (uint64_t)left : 8u; + left -= 8; + const uint16_t *p = start; + v8 s = {0, 0, 0, 0, 0, 0, 0, 0}; + v8l acc = {0, 0, 0, 0, 0, 0, 0, 0}; + for (;;) { + uint32_t w = *p++; + uint32_t op = w >> 14; + if (op == OP_ADD) { + v8 xv; + __builtin_memcpy(&xv, xrow[w & 0x3FFFu] + c, sizeof xv); + s += xv; + adds += nl; + } else if (op == OP_SUB) { + v8 xv; + __builtin_memcpy(&xv, xrow[w & 0x3FFFu] + c, sizeof xv); + s -= xv; + adds += nl; + } else { + acc = acc + acc; + acc = acc + __builtin_convertvector(s, v8l); + s = (v8){0, 0, 0, 0, 0, 0, 0, 0}; + horner += nl; + horner += nl; + if (op == OP_EMIT) + break; + } + } + __builtin_memcpy(yrow + c, &acc, sizeof acc); + } + yrow += N; + } + jb->cnt[0] = adds; + jb->cnt[1] = horner; + free(xrow); + return NULL; +} + +/* y [M][N] = W^T x for the compiled stream, rows split at part[0..nthreads] (part[0] = 0, part[nthreads] = M). */ +void t3_run(const uint16_t *stream, const int64_t *rowoff, int64_t n_in, int64_t N, int64_t nreal, const int32_t *x, int64_t *y, + int64_t nthreads, const int64_t *part, uint64_t *cnt) { + job_t jobs[64]; + pthread_t th[64]; + if (nthreads > 64) + nthreads = 64; + for (int64_t t = 0; t < nthreads; t++) { + job_t *jb = &jobs[t]; + jb->stream = stream; + jb->rowoff = rowoff; + jb->r0 = part[t]; + jb->r1 = part[t + 1]; + jb->n_in = n_in; + jb->N = N; + jb->nreal = nreal; + jb->x = x; + jb->y = y; + if (nthreads == 1) + run_rows(jb); + else + pthread_create(&th[t], NULL, run_rows, jb); + } + for (int64_t t = 0; t < nthreads; t++) { + if (nthreads > 1) + pthread_join(th[t], NULL); + cnt[0] += jobs[t].cnt[0]; + cnt[1] += jobs[t].cnt[1]; + } +} + +/* out[e] = a[e] b[e] for n elements, b recoded at run time (|b| < 2^62, its digits at positions 0..K, top = 2^K): + * Horner from position K down: acc = acc + acc, then + a at a digit +1, - a at a digit -1. Arithmetic mod 2^64, as + * NumPy's int64 product. */ +void t3_mul(int64_t n, const int64_t *a, const int64_t *b, int64_t *out, uint64_t top, uint64_t *cnt) { + uint64_t adds = 0, horner = 0, ops = 0; + for (int64_t e = 0; e < n; e++) { + uint64_t pl, mi; + ops += recode(b[e], &pl, &mi); + uint64_t x = (uint64_t)a[e], acc = 0, bit = top; + while (bit) { + acc = acc + acc; + horner += 1; + KEEP(horner); + if (pl & bit) { + acc = acc + x; + adds += 1; + } + if (mi & bit) { + acc = acc - x; + adds += 1; + } + bit >>= 1; + } + out[e] = (int64_t)acc; + } + cnt[0] += adds; + cnt[1] += horner; + cnt[2] += (uint64_t)n; + cnt[3] += ops; +} From 5956ea047eb96b4c8dbcf8bcdf70d89da7c09ed7 Mon Sep 17 00:00:00 2001 From: lmtlssss <261705307+lmtlssss@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:21:39 -0800 Subject: [PATCH 02/12] proofs/exact-llm: t3_nomul.py, no multiply instruction in the t3 kernel; the stream machine for rv32i --- proofs/exact-llm/t3_nomul.py | 87 ++++++++++++++++++++++++++++++++++++ proofs/exact-llm/t3_rv32.c | 39 ++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 proofs/exact-llm/t3_nomul.py create mode 100644 proofs/exact-llm/t3_rv32.c diff --git a/proofs/exact-llm/t3_nomul.py b/proofs/exact-llm/t3_nomul.py new file mode 100644 index 0000000..7bfed84 --- /dev/null +++ b/proofs/exact-llm/t3_nomul.py @@ -0,0 +1,87 @@ +"""The t3 kernel has no multiplier: the compiled object's disassembly, function by function, holds no multiply, +multiply-add or divide instruction, scalar or vector, integer or floating point. + +usage: python t3_nomul.py [OBJECT] (default: the kernel exact_llm.py builds and loads, t3-.dylib or .so) + +Reads `objdump -d` (LLVM's or GNU's). Prints each function's instruction histogram and the whole object's; exit 1 +if any function holds an instruction of the lists below. arm64 and x86-64 mnemonics; Apple's objdump writes a +vector arrangement into the mnemonic (mul.4s), GNU's into the operands, and both are read by the stem. +""" +import collections +import os +import re +import subprocess +import sys + +ARM64_MUL = set("""mul madd msub mneg smull umull smulh umulh smaddl smsubl umaddl umsubl smnegl umnegl + mla mls smull2 umull2 smlal smlal2 smlsl smlsl2 umlal umlal2 umlsl umlsl2 sqdmulh sqrdmulh sqdmull sqdmull2 + sqdmlal sqdmlal2 sqdmlsl sqdmlsl2 sqrdmlah sqrdmlsh pmul pmull pmull2 sdot udot usdot sudot smmla ummla usmmla + fmul fmulx fmla fmls fnmul fmadd fmsub fnmadd fnmsub bfdot bfmmla bfmlalb bfmlalt fmmla""".split()) +ARM64_DIV = {"sdiv", "udiv", "fdiv", "fsqrt", "frecpe", "frecps", "frsqrte", "frsqrts"} +X86_MUL = ("mul", "imul", "mulx", "pmul", "vpmul", "pmadd", "vpmadd", "vfmadd", "vfmsub", "vfnmadd", "vfnmsub", + "vpdp", "mulss", "mulsd", "mulps", "mulpd", "vmul") +X86_DIV = ("div", "idiv", "divss", "divsd", "divps", "divpd", "vdiv", "sqrt", "vsqrt") +# RISC-V M extension (and its 64-bit word forms); on rv32i a C multiply becomes a call to a library routine instead +RISCV_MUL = {"mulh", "mulhsu", "mulhu", "mulw", "divu", "rem", "remu", "divw", "divuw", "remw", "remuw"} +LIBCALLS = ("__mul", "__div", "__udiv", "__mod", "__umod") + + +def stem(mnemonic): + return mnemonic.split(".")[0].lower() + + +def flagged(m): + s = stem(m) + if s in ARM64_MUL or s in ARM64_DIV or s in RISCV_MUL: + return True + return s.startswith(X86_MUL) or s.startswith(X86_DIV) + + +def disassemble(path): + tool = os.environ.get("OBJDUMP", "objdump") + out = subprocess.run([tool, "-d", "-r", "--no-show-raw-insn", path], capture_output=True, text=True, + check=True).stdout + calls = sorted(set(re.findall(r"\b(__(?:mul|div|udiv|mod|umod)\w*)", out))) + if calls: + raise SystemExit("library multiply or divide referenced: %s\nMULTIPLIER FOUND" % ", ".join(calls)) + funcs, name = collections.OrderedDict(), None + for line in out.splitlines(): + m = re.match(r"^[0-9a-fA-F]+ <(.+)>:$", line.strip()) + if m: + name = m.group(1) + funcs.setdefault(name, collections.Counter()) + continue + m = re.match(r"^\s*[0-9a-fA-F]+:\s+([a-z][a-z0-9.]*)", line) # address, then the mnemonic + if m and name is not None: + funcs[name][m.group(1)] += 1 + return funcs + + +def main(argv): + if len(argv) > 1: + path = argv[1] + else: + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import exact_llm as X + path = X.t3_kernel().path + funcs = disassemble(path) + total, bad = collections.Counter(), [] + print("object %s" % path) + for name, h in funcs.items(): + if not h: + continue + total.update(h) + hits = sorted(m for m in h if flagged(m)) + bad += [(name, m, h[m]) for m in hits] + print("%-22s %5d instructions; multiply or divide: %s" % ( + name, sum(h.values()), ", ".join("%s %d" % (m, h[m]) for m in hits) or "none")) + print("histogram, all functions (%d instructions): %s" % ( + sum(total.values()), ", ".join("%s %d" % kv for kv in sorted(total.items(), key=lambda kv: (-kv[1], kv[0]))))) + print("multiply, multiply-add or divide instructions: %d%s" % ( + sum(n for _, _, n in bad), "" if not bad else " (" + "; ".join("%s in %s x%d" % (m, f, n) for f, m, n in bad) + ")")) + print("NO MULTIPLIER" if not bad else "MULTIPLIER FOUND") + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/proofs/exact-llm/t3_rv32.c b/proofs/exact-llm/t3_rv32.c new file mode 100644 index 0000000..7605ad0 --- /dev/null +++ b/proofs/exact-llm/t3_rv32.c @@ -0,0 +1,39 @@ +/* t3_rv32.c: the t3 stream machine of t3.c, one lane, freestanding, for a core without a multiplier (rv32i). + * No libc, no threads, no vector types. The same words: op << 14 | i, op 0 ADD, 1 SUB, 2 STEP, 3 EMIT. + * ADD i s = s + x[i] + * SUB i s = s - x[i] + * STEP acc = acc + acc; acc = acc + s; s = 0 + * EMIT STEP, then y[row] = acc; acc = 0; row = row + 1 + * s is 32 bits, acc 64 bits (on rv32i two registers: add, sltu for the carry, add). Returns the words run. + * + * Build (Homebrew LLVM 23, no emulator here): + * clang --target=riscv32-unknown-elf -march=rv32i -mabi=ilp32 -O2 -ffreestanding -nostdlib -c t3_rv32.c + * python t3_nomul.py t3_rv32.o + */ +#include + +uint32_t t3_rv32_run(const uint16_t *p, int32_t rows, const int32_t *x, int64_t *y) { + int32_t s = 0; + int64_t acc = 0; + uint32_t words = 0; + while (rows > 0) { + uint32_t w = *p++; + uint32_t op = w >> 14; + words += 1; + if (op == 0u) { + s += x[w & 0x3FFFu]; + } else if (op == 1u) { + s -= x[w & 0x3FFFu]; + } else { + acc = acc + acc; + acc = acc + (int64_t)s; + s = 0; + if (op == 3u) { + *y++ = acc; + acc = 0; + rows -= 1; + } + } + } + return words; +} From 136d4c71b4fd279002fd26982c95851ca9bf3431 Mon Sep 17 00:00:00 2001 From: lmtlssss <261705307+lmtlssss@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:21:39 -0800 Subject: [PATCH 03/12] proofs/exact-llm README: t3 on the M1, one pair with f64; adds per token from the kernel's counters --- proofs/exact-llm/README.md | 21 ++++- proofs/exact-llm/record/t3-count.json | 73 ++++++++++++++++ proofs/exact-llm/record/t3-m1.txt | 119 ++++++++++++++++++++++++++ proofs/exact-llm/t3_count.py | 84 ++++++++++++++++++ 4 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 proofs/exact-llm/record/t3-count.json create mode 100644 proofs/exact-llm/record/t3-m1.txt create mode 100644 proofs/exact-llm/t3_count.py diff --git a/proofs/exact-llm/README.md b/proofs/exact-llm/README.md index 5829c7c..2fea839 100644 --- a/proofs/exact-llm/README.md +++ b/proofs/exact-llm/README.md @@ -123,6 +123,24 @@ Each row narrows the exact model's product operands and is compared with the sam The large difference at 16-bit inputs is one outlier per vector setting its shift: peel 1 brings it to 0.175, peel 2 to 0.03 to 0.06, peel 4 adds nothing; peel 2 also takes 14-bit inputs and weights from 411 to 470. `measure16.py` (rows 1 to 11, against saved float32 runs), `record/exact16-table.txt`; last row `exact_llm.py hash`, `text` and `distance.py`. +## t3: the products by adds and subtracts + +A fourth route, `exact16-t3`, run where a C compiler is installed (`check --routes t3`; not in check's default three). Every product of exact-16 is computed by `t3.c`, whose arithmetic is add, subtract, shift by a constant and bitwise logic: the matrix products in `mm16` (q, k, v, o, gate, up, down, logits, the scores q16·k16, the context p·vh and p·vl), the peeled terms, and the element-wise products (the norm's squares and weight, rotary, the exp and SiLU interpolations, SiLU(gate)·up) through `Exact.mul`. Divisions (52,477 per token) and square roots (61) stay NumPy's `//` and float `sqrt` with their integer bracket checks, as on the other routes: looks, not products. + +| signed digits, `t3.c` | for m ≥ 0: `h = m >> 1`, `s = m + h`, `c = h ^ s`; plus `s & c`, minus `h & c`; m = plus − minus, non-adjacent form; a negative value by its magnitude with plus and minus exchanged | +| --- | --- | +| stream | a 16-bit word per instruction, `op << 14 \| i`; per output row, for digit position k = 15 down to 0: `ADD i` for each input with digit +1 at k, `SUB i` for each with −1, then `STEP` (`EMIT` at k = 0) | +| machine | per lane: `s` (int32), `acc` (int64). `ADD i`: s = s + x[i]. `SUB i`: s = s − x[i]. `STEP`: acc = acc + acc; acc = acc + s; s = 0. `EMIT`: `STEP`, then y[row] = acc, acc = 0. 8 lanes at a time | +| weights | each matrix compiled to its stream once (924,918,488 bytes for the 135M: 459,184,492 digits and 16 words for each of 204,672 output rows), kept in `$EXACT_LLM_T3` (default `.cache/t3`) | +| activations | the right operand recoded into digits at each product: k16 for the scores, vh and vl for the context, the smaller-magnitude operand element-wise and for the peeled terms | + +| Apple M1, 64 prompts, batch 8, one `check` process | logits | continuations | seconds | +| --- | --- | --- | --- | +| `exact16-f64` | `b0cc875f…0b4460` | `a800ebad…7613cc` | 437 | +| `exact16-t3`, 8 threads | `b0cc875f…0b4460` | `a800ebad…7613cc` | 1,456 | + +MATCH on both, 0 clamped. The machine was shared with other jobs during this run (f64 took 80 s in the run of the table above). Per token at the last position (context 64, `t3_count.py`, the kernel's counters): 459,184,492 digit adds and subtracts in the weight products, as the digit census predicts, and 6,549,504 Horner adds; 1,408,559 and 6,442,304 in the peeled terms; 11,875,560 and 1,658,880 in the attention (scores 5,007,459, p·vh 900,330, p·vl 5,967,771, Horner 552,960 each); 1,271,526 and 4,451,136 element-wise; 492,841,961 adds and subtracts in all. `t3_nomul.py`: the kernel's disassembly holds no multiply, multiply-add or divide instruction (arm64: 0 of 511). `t3_rv32.c`: the stream machine, freestanding, compiled for rv32i (no M extension) by LLVM: 45 instructions, none a multiply, no library multiply. + ## Larger models One code path: the sizes come from each model's pinned `config.json`, the clamps by rule from its own weights (above), the bound per route from the width. `python exact_llm.py check --model SmolLM2-360M` runs exact-16 on its three routes, float32 and onnxruntime. @@ -360,7 +378,8 @@ Commands: | `expected-16-SmolLM2-360M.sha256`, `record/exact16-360m-m1.txt`, `record/ci-43754428621.txt` | the 360M's pair; every line of its M1 run; every line of the twelve jobs (135M and 360M on six runners) | | `expected-16-SmolLM2-1.7B.sha256`, `record/ci-43759765827.txt` | the 1.7B's pair; every line of its 14 jobs (one route each), with each job's end and wall time | | `record/peaks-16.json`, `record/peaks-16-SmolLM2-360M.json` | each site's peak, clamp, margin and clamped count, from `check --peaks` | -| `routes_test.py` | the three routes against int64 products | +| `routes_test.py` | the three routes against int64 products; t3 too where a C compiler is installed, with every 16-bit value through the stream and 200,081 element-wise pairs | +| `t3.c`, `t3_nomul.py`, `t3_count.py`, `t3_rv32.c` | the t3 kernel; its disassembly checked for multipliers; its counters per token; the stream machine for rv32i | | `spare_row.py`, `spare_row_test.py`, `record/spare-m1.txt`, `record/spare-campaign.json` | exact-16 with every product and look checked, the fault campaign; its test on synthetic operands; every line of the M1 runs; every fault of the campaign | | `count_ops.py`, `report_census.py`; `count_spare.py`, `report_spare.py`, `record/census-spare.json`, `record/census-spare.md` | the census of every NumPy operation per token, sorted into steps and looks: of exact and exact-16; of exact-16 with and without the spare-row checks, and its tables | | `measure16.py`, `record/exact16-table.txt` | the width table, one row per run | diff --git a/proofs/exact-llm/record/t3-count.json b/proofs/exact-llm/record/t3-count.json new file mode 100644 index 0000000..7cb42d1 --- /dev/null +++ b/proofs/exact-llm/record/t3-count.json @@ -0,0 +1,73 @@ +{ + "model": "SmolLM2-135M", + "window": "last position, context 64, batch 1", + "next token": 376, + "per kind": { + "weights": { + "digit adds and subtracts": 459184492, + "Horner adds": 6549504, + "values recoded": 0, + "recoding operations": 0 + }, + "peeled": { + "digit adds and subtracts": 1408559, + "Horner adds": 6442304, + "values recoded": 409344, + "recoding operations": 2247228 + }, + "attention": { + "digit adds and subtracts": 11875560, + "Horner adds": 1658880, + "values recoded": 1105920, + "recoding operations": 5900826 + }, + "element-wise": { + "digit adds and subtracts": 1271526, + "Horner adds": 4451136, + "values recoded": 289152, + "recoding operations": 1515461 + } + }, + "attention split": { + "scores q16.k16": { + "digit adds and subtracts": 5007459, + "Horner adds": 552960 + }, + "context p.vh": { + "digit adds and subtracts": 900330, + "Horner adds": 552960 + }, + "context p.vl": { + "digit adds and subtracts": 5967771, + "Horner adds": 552960 + } + }, + "weight streams": { + "nonzero digits": 459184492, + "output rows": 204672, + "words": 462459244, + "bytes": 924918488 + }, + "weight streams, built once (this run)": { + "digit adds and subtracts": 0, + "Horner adds": 0, + "values recoded": 129429504, + "recoding operations": 711700313 + }, + "seconds": { + "load and prefill of 63": 30.0, + "the counted forward": 2.7 + }, + "totals": { + "weight products: digit adds": 459184492, + "weight products: Horner adds": 6549504, + "peeled terms: digit adds": 1408559, + "peeled terms: Horner adds": 6442304, + "variable by variable (attention): digit adds": 11875560, + "variable by variable (attention): Horner adds": 1658880, + "element-wise: digit adds": 1271526, + "element-wise: Horner adds": 4451136, + "adds and subtracts in all products": 492841961, + "values recoded at run time (a sign look each)": 1804416 + } +} diff --git a/proofs/exact-llm/record/t3-m1.txt b/proofs/exact-llm/record/t3-m1.txt new file mode 100644 index 0000000..113a8e8 --- /dev/null +++ b/proofs/exact-llm/record/t3-m1.txt @@ -0,0 +1,119 @@ +Sun Nov 15 16:41:44 PST 2026 +== routes +i8 product engine: i8-onnxruntime-1.32.0 +t3 kernel: $EXACT_LLM_T3/t3-f6d3700bf905.dylib +cases 42 (7 shapes x 6 kinds, int64 and int16 right operands), routes 4, different 0 +digit extremes, value (i8 digits) (f32 digits): 32767 (255, 255) (128, -1); -32767 (0, 1) (-128, 1); 32640 (255, 128) (128, -128); -32640 (0, 128) (-127, -128); 32639 (255, 127) (127, 127); -32641 (0, 127) (-128, 127); 32512 (255, 0) (127, 0); -32513 (0, 255) (-127, -1) +f32 digits: high -128..128, low -128..127 (int16 split the same: True); i8 digits of a + 2^15: high 0..255, low 0..255 (from int16 the same: True) +peel_index of [5, -9, 9, 3, -9, 1] k=2: [1, 2] (expect [1, 2]) +t3 stream, every value -32767..32767: exact; digit adds 356806, census formula 356806, at most 8 per value +t3 element-wise products, 200081 pairs and a broadcast: exact + 2.35 real 2.12 user 0.28 sys +exit 0 + 336134144 maximum resident set size +== nomul +object $EXACT_LLM_T3/t3-f6d3700bf905.dylib +_t3_compile_masks 76 instructions; multiply or divide: none +_t3_compile_fill 169 instructions; multiply or divide: none +_t3_run 89 instructions; multiply or divide: none +_run_rows 109 instructions; multiply or divide: none +_t3_mul 53 instructions; multiply or divide: none +__stubs 15 instructions; multiply or divide: none +histogram, all functions (511 instructions): add 68, mov 57, cmp 38, b.eq 33, b 28, ldr 28, stp 21, strh 21, tst 21, ldp 20, sub 19, b.lt 13, ldrh 12, orr 10, adrp 9, csel 9, ldurh 9, and 8, str 7, bl 6, lsr 6, movi.2d 6, add.2d 5, br 5, lsl 5, ret 5, subs 5, cbz 4, cinc 4, b.ge 3, b.ne 3, add.4s 2, b.hi 2, b.hs 2, cbnz 2, cneg 2, eor 2, saddw.2d 2, saddw2.2d 2, sub.4s 2, b.lo 1, blr 1, ldur 1, neg 1, stur 1 +multiply, multiply-add or divide instructions: 0 +NO MULTIPLIER + 0.68 real 0.19 user 0.15 sys +exit 0 + 32440320 maximum resident set size +== count +{ + "model": "SmolLM2-135M", + "window": "last position, context 64, batch 1", + "next token": 376, + "per kind": { + "weights": { + "digit adds and subtracts": 459184492, + "Horner adds": 6549504, + "values recoded": 0, + "recoding operations": 0 + }, + "peeled": { + "digit adds and subtracts": 1408559, + "Horner adds": 6442304, + "values recoded": 409344, + "recoding operations": 2247228 + }, + "attention": { + "digit adds and subtracts": 11875560, + "Horner adds": 1658880, + "values recoded": 1105920, + "recoding operations": 5900826 + }, + "element-wise": { + "digit adds and subtracts": 1271526, + "Horner adds": 4451136, + "values recoded": 289152, + "recoding operations": 1515461 + } + }, + "attention split": { + "scores q16.k16": { + "digit adds and subtracts": 5007459, + "Horner adds": 552960 + }, + "context p.vh": { + "digit adds and subtracts": 900330, + "Horner adds": 552960 + }, + "context p.vl": { + "digit adds and subtracts": 5967771, + "Horner adds": 552960 + } + }, + "weight streams": { + "nonzero digits": 459184492, + "output rows": 204672, + "words": 462459244, + "bytes": 924918488 + }, + "weight streams, built once (this run)": { + "digit adds and subtracts": 0, + "Horner adds": 0, + "values recoded": 129429504, + "recoding operations": 711700313 + }, + "seconds": { + "load and prefill of 63": 30.0, + "the counted forward": 2.7 + }, + "totals": { + "weight products: digit adds": 459184492, + "weight products: Horner adds": 6549504, + "peeled terms: digit adds": 1408559, + "peeled terms: Horner adds": 6442304, + "variable by variable (attention): digit adds": 11875560, + "variable by variable (attention): Horner adds": 1658880, + "element-wise: digit adds": 1271526, + "element-wise: Horner adds": 4451136, + "adds and subtracts in all products": 492841961, + "values recoded at run time (a sign look each)": 1804416 + } +} + 34.01 real 19.26 user 2.67 sys +exit 0 + 247906304 maximum resident set size +== check +versions: Python 3.11.18, NumPy 2.4.6, onnxruntime 1.32.0; darwin arm64 +model SmolLM2-135M @ 93efa2f: 30 layers, width 576, 9/3 heads, feed-forward 1536, rotary base 100000, 134515008 weights +exact logits 30ed613b6036d08046fbdbf273a0fb7b8cc04fb284ca49e0296980eed9cee773 continuations a800ebadc848514e02bebbb8077d531cf0493acd2a4c173403ba1db43d7613cc MATCH + clamped values: 0 +exact16 16-bit weights, int32 embedding and norms, weight statistics: ready in 0 s +exact16 logits b0cc875fde396181a70e2349ca35a60c3ae91b23f1c1012083797352d30b4460 continuations a800ebadc848514e02bebbb8077d531cf0493acd2a4c173403ba1db43d7613cc MATCH route f64, 437 s, clamped 0 + peaks (real) / clamps: attention input 11.429 / 131072; down input 2630.720 / 4194304; gate 50.741 / 32768; k 22.910 / 1024; logits input 53.630 / 8388608; mlp input 28.742 / 8388608; o input 9.531 / 8388608; q 22.075 / 1024; residual 19509.485 / 33554432; up 72.026 / 32768; v 12.972 / 8192 +exact16 logits b0cc875fde396181a70e2349ca35a60c3ae91b23f1c1012083797352d30b4460 continuations a800ebadc848514e02bebbb8077d531cf0493acd2a4c173403ba1db43d7613cc MATCH route t3, 1456 s, clamped 0 + t3 kernel t3-f6d3700bf905.dylib; digit adds, Horner adds, values recoded, recoding operations: weights 1,653,064,171,200, 23,578,214,400, 0, 0; peeled 5,079,175,319, 23,547,135,488, 1,473,638,400, 7,993,295,032; attention 19,327,619,286, 4,893,696,000, 1,738,506,240, 9,276,986,651; element-wise 4,326,722,524, 15,590,066,688, 973,555,200, 5,113,624,092; weight streams, built once 0, 0, 0, 0 +check every pair MATCH, 2250 s + 2250.89 real 788.97 user 229.70 sys +exit 0 + 793690112 maximum resident set size +Sun Nov 15 17:19:52 PST 2026 diff --git a/proofs/exact-llm/t3_count.py b/proofs/exact-llm/t3_count.py new file mode 100644 index 0000000..2a3ccd4 --- /dev/null +++ b/proofs/exact-llm/t3_count.py @@ -0,0 +1,84 @@ +"""The t3 route's own counters, per token at the last position (context 64), as the census counts it: a prefill of +63 tokens, then one forward of one token with the key-value cache; the counters of that one forward. + +usage: python t3_count.py [OUT.json] + +Tokens: the first 64 of the evaluation text (text()). The weights' counts do not depend on the tokens (every stream +word runs once per token); the attention's and the element-wise counts do (the digits of activations). +Kinds (T3.KINDS): weights (the 16-bit matrix products, from the streams), peeled (the 2 whole inputs per product +input vector times their 16-bit weight rows), attention (q16.k16, p.vh, p.vl: one operand recoded at run time), +element-wise (norm squares, norm weight, rotary, the exp and SiLU interpolations, SiLU(gate) times up). +Per kind: digit adds and subtracts, Horner adds (acc + acc, then + s), values recoded, recoding operations. +""" +import json +import os +import sys +import time + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import exact_llm as X # noqa: E402 + + +def streams_digits(k): + """Nonzero digits and output rows over every weight stream compiled so far.""" + digits = rows = 0 + for key, ro in k.rowoffs.items(): + rows += ro.shape[0] - 1 + digits += int(ro[-1]) - (X.TOP16 + 1) * (ro.shape[0] - 1) + return digits, rows + + +def main(argv): + t0 = time.time() + m = X.Exact16("t3") + k = m.t3 + L = 64 + ids = np.array(X.tokenizer().encode(X.text())[:L], dtype=np.int64)[None] + pos = np.arange(L)[None] + i = np.arange(L) + kvalid = (i[None, None, :] <= i[None, :, None]) + _, cache = m.forward(ids[:, :L - 1], pos[:, :L - 1], None, kvalid[:, :L - 1, :L - 1]) + t1 = time.time() + before = {kind: c.copy() for kind, c in k.cnt.items()} + k.log = [] + t2 = time.time() + logits, _ = m.forward(ids[:, L - 1:], pos[:, L - 1:], cache, kvalid[:, L - 1:, :]) + t3 = time.time() + d = {kind: [int(v) for v in (k.cnt[kind] - before[kind])] for kind in k.KINDS} + att = [e for e in k.log if e[0] == "attention"] + per = 3 * X.KV # per layer: KV scores, KV p.vh, KV p.vl, in that order + assert len(att) == per * X.LAYERS + split = {"scores q16.k16": [0, 0], "context p.vh": [0, 0], "context p.vl": [0, 0]} + for j, e in enumerate(att): + name = list(split)[(j % per) // X.KV] + split[name][0] += e[4] + split[name][1] += e[5] + digits, rows = streams_digits(k) + names = ["digit adds and subtracts", "Horner adds", "values recoded", "recoding operations"] + out = {"model": X.MODEL, "window": "last position, context %d, batch 1" % L, + "next token": int(np.argmax(logits[0, 0])), + "per kind": {kind: dict(zip(names, d[kind])) for kind in k.KINDS if kind != "weight streams, built once"}, + "attention split": {n: {"digit adds and subtracts": v[0], "Horner adds": v[1]} for n, v in split.items()}, + "weight streams": {"nonzero digits": digits, "output rows": rows, + "words": digits + (X.TOP16 + 1) * rows, "bytes": 2 * (digits + (X.TOP16 + 1) * rows)}, + "weight streams, built once (this run)": dict(zip(names, [int(v) for v in k.cnt["weight streams, built once"]])), + "seconds": {"load and prefill of 63": round(t1 - t0, 1), "the counted forward": round(t3 - t2, 2)}} + w, p, a, e = (d[x] for x in ("weights", "peeled", "attention", "element-wise")) + out["totals"] = { + "weight products: digit adds": w[0], "weight products: Horner adds": w[1], + "peeled terms: digit adds": p[0], "peeled terms: Horner adds": p[1], + "variable by variable (attention): digit adds": a[0], "variable by variable (attention): Horner adds": a[1], + "element-wise: digit adds": e[0], "element-wise: Horner adds": e[1], + "adds and subtracts in all products": w[0] + w[1] + p[0] + p[1] + a[0] + a[1] + e[0] + e[1], + "values recoded at run time (a sign look each)": p[2] + a[2] + e[2]} + s = json.dumps(out, indent=1) + print(s) + if len(argv) > 1: + open(argv[1], "w").write(s + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) From 2cb68b3c16b72595090e8d135807d7f593635c50 Mon Sep 17 00:00:00 2001 From: lmtlssss <261705307+lmtlssss@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:43:28 -0800 Subject: [PATCH 04/12] proofs/exact-llm: t3 weight streams in one folder per t3.c and model; at most 64 threads; a stream only for whole columns The weight streams were kept by matrix name, first column and width in one folder for every model: the embedding blocks (model.embed_tokens.weight, columns of 8,192) have the same names in the 135M, 360M and 1.7B, so a second model's t3 run in the same cache read the first model's logits streams. They are now kept in T3_DIR/streams-/; the hash also keeps a stream of an older t3.c from being read by a newer one. register() replaces the matrices registered before and records the model. t3.c runs at most 64 threads and leaves the rows past the 64th share unset; T3_THREADS is now at most 64. weight_key() gave a block the whole matrix's stream when the block began in row 0 with the matrix's row stride, whatever its rows; it now requires all the matrix's rows and columns within it. No caller passed such a block. --- proofs/exact-llm/exact_llm.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/proofs/exact-llm/exact_llm.py b/proofs/exact-llm/exact_llm.py index 4441623..7f4441f 100644 --- a/proofs/exact-llm/exact_llm.py +++ b/proofs/exact-llm/exact_llm.py @@ -765,22 +765,22 @@ def __call__(self, a, b): T3_DIR = os.environ.get("EXACT_LLM_T3", os.path.join(CACHE, "t3")) # the compiled kernel and the weight streams -T3_THREADS = int(os.environ.get("EXACT_LLM_T3_THREADS", "0")) or os.cpu_count() or 1 +T3_THREADS = min(64, int(os.environ.get("EXACT_LLM_T3_THREADS", "0")) or os.cpu_count() or 1) # t3.c: at most 64 class T3: """The t3 route's kernel, t3.c, compiled once with the C compiler `cc` (or $CC) into T3_DIR and opened with ctypes. Matrix products (mm) and element-wise products (mul) by add, subtract, shift by a constant and bitwise - logic. A registered weight matrix is compiled to its stream once and kept in T3_DIR (2 bytes per nonzero digit - plus 16 words per output row); any other right operand is recoded at each product. Counters per kind: digit - adds and subtracts, Horner adds, values recoded, recoding operations.""" + logic. A registered weight matrix is compiled to its stream once and kept in T3_DIR/streams-/ + (2 bytes per nonzero digit plus 16 words per output row); any other right operand is recoded at each product. + Counters per kind: digit adds and subtracts, Horner adds, values recoded, recoding operations.""" KINDS = ("weights", "peeled", "attention", "element-wise", "weight streams, built once") def __init__(self): import ctypes import subprocess src = os.path.join(HERE, "t3.c") - tag = hashlib.sha256(open(src, "rb").read()).hexdigest()[:12] + tag = self.tag = hashlib.sha256(open(src, "rb").read()).hexdigest()[:12] os.makedirs(T3_DIR, exist_ok=True) self.path = os.path.join(T3_DIR, "t3-%s.%s" % (tag, "dylib" if sys.platform == "darwin" else "so")) if not os.path.exists(self.path): @@ -795,7 +795,7 @@ def __init__(self): L.t3_run.argtypes = [p, p, i64, i64, i64, p, p, i64, p, p] L.t3_mul.argtypes = [i64, p, p, p, ctypes.c_uint64, p] self.cnt = {k: np.zeros(4, dtype=np.uint64) for k in self.KINDS} - self.mats = [] # registered weights: (first byte, last byte, name, row stride) + self.model, self.mats = None, [] # registered weights: (first byte, last byte, name, rows, cols) self.rowoffs = {} self.buf = np.empty(0, np.uint16) self.log = None # a list: one entry per product, when tracing @@ -805,18 +805,21 @@ def ptr(a): return a.ctypes.data def register(self, W): - """The int16 weight matrices [in, out] (memory-mapped) whose streams are compiled once and kept.""" - for name, w in W.items(): - if w.dtype == np.int16 and w.ndim == 2: - self.mats.append((self.ptr(w), self.ptr(w) + w.nbytes, name, w.shape[1])) + """The current model's int16 weight matrices [in, out] (memory-mapped), whose streams are compiled once and + kept; they replace those registered before.""" + self.model = MODEL + self.mats = [(self.ptr(w), self.ptr(w) + w.nbytes, name, w.shape[0], w.shape[1]) + for name, w in W.items() if w.dtype == np.int16 and w.ndim == 2] def weight_key(self, b): + """(model, matrix, first column, columns) when b is a block of whole columns of a registered matrix.""" if b.dtype != np.int16 or b.ndim != 2 or b.strides[1] != 2: return None at = self.ptr(b) - for lo, hi, name, ld in self.mats: - if lo <= at < hi and b.strides[0] == 2 * ld and (at - lo) // 2 < ld: - return name, (at - lo) // 2, b.shape[1] + for lo, hi, name, rows, ld in self.mats: + c0 = (at - lo) // 2 + if lo <= at < hi and b.strides[0] == 2 * ld and b.shape[0] == rows and c0 + b.shape[1] <= ld: + return self.model, name, c0, b.shape[1] return None def compile(self, b, kind): @@ -833,8 +836,8 @@ def compile(self, b, kind): return stream, rowoff def weight_stream(self, key, b): - name, c0, M = key - base = os.path.join(T3_DIR, "streams", "%s.c%d.m%d" % (name, c0, M)) + model, name, c0, M = key # the stream format is t3.c's: one folder per t3.c and model + base = os.path.join(T3_DIR, "streams-" + self.tag, model, "%s.c%d.m%d" % (name, c0, M)) if key not in self.rowoffs: if not (os.path.exists(base + ".words") and os.path.exists(base + ".rowoff.npy")): os.makedirs(os.path.dirname(base), exist_ok=True) From 63725f5058ec6cab419f113d3c9c54fd202e260c Mon Sep 17 00:00:00 2001 From: lmtlssss <261705307+lmtlssss@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:43:29 -0800 Subject: [PATCH 05/12] proofs/exact-llm: t3_nomul.py finds library calls in Mach-O; README: what t3 leaves to NumPy, no M1 seconds t3_nomul.py matched library multiply and divide calls with \b__mul..., which a Mach-O symbol's extra leading underscore defeats: a 128-bit divide built on arm64 macOS (a call to ___divti3) passed as NO MULTIPLIER. The pattern now takes one more underscore and the ARM EABI routines (__aeabi_*mul, *div). Also listed: arm64 fmlal, fmlal2, fmlsl, fmlsl2, fcmla, fdot, bfmlal; x86-64 pclmul, vpclmul, dpps, dppd, vdpp. The kernel built on the M1 (t3-f6d3700bf905) still holds none: 511 instructions, 496 in the five functions of t3.c and 15 in its call stubs. Reported: a*b on arm64 (mul), t3.c without its KEEP barriers (mul in run_rows), a*b for rv32i (__mulsi3), the 128-bit divide (___divti3). README, t3: "every product of exact-16's forward pass"; the division and square-root bracket checks multiply in NumPy, and the shifts by a computed amount stay NumPy's; the streams' folder; the M1 table without seconds (the M1 was shared during the run; record/t3-m1.txt keeps them); the 511 instructions as functions and stubs, and what t3_nomul.py reports; t3_rv32.c compiled only, not run. --- proofs/exact-llm/README.md | 14 +++++++------- proofs/exact-llm/t3_nomul.py | 10 ++++++---- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/proofs/exact-llm/README.md b/proofs/exact-llm/README.md index 2fea839..c3e4291 100644 --- a/proofs/exact-llm/README.md +++ b/proofs/exact-llm/README.md @@ -125,21 +125,21 @@ The large difference at 16-bit inputs is one outlier per vector setting its shif ## t3: the products by adds and subtracts -A fourth route, `exact16-t3`, run where a C compiler is installed (`check --routes t3`; not in check's default three). Every product of exact-16 is computed by `t3.c`, whose arithmetic is add, subtract, shift by a constant and bitwise logic: the matrix products in `mm16` (q, k, v, o, gate, up, down, logits, the scores q16·k16, the context p·vh and p·vl), the peeled terms, and the element-wise products (the norm's squares and weight, rotary, the exp and SiLU interpolations, SiLU(gate)·up) through `Exact.mul`. Divisions (52,477 per token) and square roots (61) stay NumPy's `//` and float `sqrt` with their integer bracket checks, as on the other routes: looks, not products. +A fourth route, `exact16-t3`, run where a C compiler is installed (`check --routes t3`; not in check's default three). Every product of exact-16's forward pass is computed by `t3.c`, whose arithmetic is add, subtract, shift by a constant and bitwise logic: the matrix products in `mm16` (q, k, v, o, gate, up, down, logits, the scores q16·k16, the context p·vh and p·vl), the peeled terms, and the element-wise products (the norm's squares and weight, rotary, the exp and SiLU interpolations, SiLU(gate)·up) through `Exact.mul`. Divisions (52,477 per token) and square roots (61) stay as on the other routes: NumPy's `//` and float `sqrt`, each checked by an integer bracket that NumPy multiplies (q·d ≤ n < (q + 1)·d, s² ≤ n < (s + 1)²). Shifts by a computed amount (the block shift, the row scale) stay NumPy's too. | signed digits, `t3.c` | for m ≥ 0: `h = m >> 1`, `s = m + h`, `c = h ^ s`; plus `s & c`, minus `h & c`; m = plus − minus, non-adjacent form; a negative value by its magnitude with plus and minus exchanged | | --- | --- | | stream | a 16-bit word per instruction, `op << 14 \| i`; per output row, for digit position k = 15 down to 0: `ADD i` for each input with digit +1 at k, `SUB i` for each with −1, then `STEP` (`EMIT` at k = 0) | | machine | per lane: `s` (int32), `acc` (int64). `ADD i`: s = s + x[i]. `SUB i`: s = s − x[i]. `STEP`: acc = acc + acc; acc = acc + s; s = 0. `EMIT`: `STEP`, then y[row] = acc, acc = 0. 8 lanes at a time | -| weights | each matrix compiled to its stream once (924,918,488 bytes for the 135M: 459,184,492 digits and 16 words for each of 204,672 output rows), kept in `$EXACT_LLM_T3` (default `.cache/t3`) | +| weights | each matrix compiled to its stream once (924,918,488 bytes for the 135M: 459,184,492 digits and 16 words for each of 204,672 output rows), kept in `$EXACT_LLM_T3` (default `.cache/t3`), one folder per `t3.c` and model | | activations | the right operand recoded into digits at each product: k16 for the scores, vh and vl for the context, the smaller-magnitude operand element-wise and for the peeled terms | -| Apple M1, 64 prompts, batch 8, one `check` process | logits | continuations | seconds | -| --- | --- | --- | --- | -| `exact16-f64` | `b0cc875f…0b4460` | `a800ebad…7613cc` | 437 | -| `exact16-t3`, 8 threads | `b0cc875f…0b4460` | `a800ebad…7613cc` | 1,456 | +| Apple M1, 64 prompts, batch 8, one `check` process | logits | continuations | +| --- | --- | --- | +| `exact16-f64` | `b0cc875f…0b4460` | `a800ebad…7613cc` | +| `exact16-t3`, 8 threads | `b0cc875f…0b4460` | `a800ebad…7613cc` | -MATCH on both, 0 clamped. The machine was shared with other jobs during this run (f64 took 80 s in the run of the table above). Per token at the last position (context 64, `t3_count.py`, the kernel's counters): 459,184,492 digit adds and subtracts in the weight products, as the digit census predicts, and 6,549,504 Horner adds; 1,408,559 and 6,442,304 in the peeled terms; 11,875,560 and 1,658,880 in the attention (scores 5,007,459, p·vh 900,330, p·vl 5,967,771, Horner 552,960 each); 1,271,526 and 4,451,136 element-wise; 492,841,961 adds and subtracts in all. `t3_nomul.py`: the kernel's disassembly holds no multiply, multiply-add or divide instruction (arm64: 0 of 511). `t3_rv32.c`: the stream machine, freestanding, compiled for rv32i (no M extension) by LLVM: 45 instructions, none a multiply, no library multiply. +MATCH on both, 0 clamped. The M1 was shared with other jobs during this run, so its seconds are not a measure of either route. Per token at the last position (context 64, `t3_count.py`, the kernel's counters): 459,184,492 digit adds and subtracts in the weight products, as the digit census predicts, and 6,549,504 Horner adds; 1,408,559 and 6,442,304 in the peeled terms; 11,875,560 and 1,658,880 in the attention (scores 5,007,459, p·vh 900,330, p·vl 5,967,771, Horner 552,960 each); 1,271,526 and 4,451,136 element-wise; 492,841,961 adds and subtracts in all. `t3_nomul.py`: the kernel's disassembly holds no multiply, multiply-add or divide instruction and calls no library multiply or divide (Apple clang, arm64: 0 of 511 instructions, 496 in the five functions of `t3.c` and 15 in its call stubs). It does report the `mul` in `t3.c` built without its `KEEP` barriers, and the calls `__mulsi3` (a multiply built for rv32i) and `___divti3` (a 128-bit divide on arm64 macOS). `t3_rv32.c`: the stream machine, freestanding, compiled for rv32i (no M extension) by LLVM: 45 instructions, none a multiply, no library multiply; compiled only, not run. ## Larger models diff --git a/proofs/exact-llm/t3_nomul.py b/proofs/exact-llm/t3_nomul.py index 7bfed84..761233c 100644 --- a/proofs/exact-llm/t3_nomul.py +++ b/proofs/exact-llm/t3_nomul.py @@ -16,14 +16,16 @@ ARM64_MUL = set("""mul madd msub mneg smull umull smulh umulh smaddl smsubl umaddl umsubl smnegl umnegl mla mls smull2 umull2 smlal smlal2 smlsl smlsl2 umlal umlal2 umlsl umlsl2 sqdmulh sqrdmulh sqdmull sqdmull2 sqdmlal sqdmlal2 sqdmlsl sqdmlsl2 sqrdmlah sqrdmlsh pmul pmull pmull2 sdot udot usdot sudot smmla ummla usmmla - fmul fmulx fmla fmls fnmul fmadd fmsub fnmadd fnmsub bfdot bfmmla bfmlalb bfmlalt fmmla""".split()) + fmul fmulx fmla fmls fnmul fmadd fmsub fnmadd fnmsub bfdot bfmmla bfmlalb bfmlalt fmmla + fmlal fmlal2 fmlsl fmlsl2 fcmla fdot bfmlal""".split()) ARM64_DIV = {"sdiv", "udiv", "fdiv", "fsqrt", "frecpe", "frecps", "frsqrte", "frsqrts"} X86_MUL = ("mul", "imul", "mulx", "pmul", "vpmul", "pmadd", "vpmadd", "vfmadd", "vfmsub", "vfnmadd", "vfnmsub", - "vpdp", "mulss", "mulsd", "mulps", "mulpd", "vmul") + "vpdp", "mulss", "mulsd", "mulps", "mulpd", "vmul", "pclmul", "vpclmul", "dpps", "dppd", "vdpp") X86_DIV = ("div", "idiv", "divss", "divsd", "divps", "divpd", "vdiv", "sqrt", "vsqrt") # RISC-V M extension (and its 64-bit word forms); on rv32i a C multiply becomes a call to a library routine instead RISCV_MUL = {"mulh", "mulhsu", "mulhu", "mulw", "divu", "rem", "remu", "divw", "divuw", "remw", "remuw"} -LIBCALLS = ("__mul", "__div", "__udiv", "__mod", "__umod") +# library multiply and divide routines (libgcc, compiler-rt, the ARM EABI); Mach-O writes one more leading underscore +LIBCALL = re.compile(r"(? Date: Sun, 15 Nov 2026 17:46:17 -0800 Subject: [PATCH 06/12] proofs/exact-llm README, t3: known limits, the unchecked thread start and the 14-bit input index t3_run does not check what pthread_create returns: a thread that fails to start leaves its rows unset, which check prints as DIFFERENT. A stream word's input index is 14 bits: a product has at most 16,383 inputs, which for the context products is 16,383 positions. --- proofs/exact-llm/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proofs/exact-llm/README.md b/proofs/exact-llm/README.md index c3e4291..c7d7362 100644 --- a/proofs/exact-llm/README.md +++ b/proofs/exact-llm/README.md @@ -141,6 +141,8 @@ A fourth route, `exact16-t3`, run where a C compiler is installed (`check --rout MATCH on both, 0 clamped. The M1 was shared with other jobs during this run, so its seconds are not a measure of either route. Per token at the last position (context 64, `t3_count.py`, the kernel's counters): 459,184,492 digit adds and subtracts in the weight products, as the digit census predicts, and 6,549,504 Horner adds; 1,408,559 and 6,442,304 in the peeled terms; 11,875,560 and 1,658,880 in the attention (scores 5,007,459, p·vh 900,330, p·vl 5,967,771, Horner 552,960 each); 1,271,526 and 4,451,136 element-wise; 492,841,961 adds and subtracts in all. `t3_nomul.py`: the kernel's disassembly holds no multiply, multiply-add or divide instruction and calls no library multiply or divide (Apple clang, arm64: 0 of 511 instructions, 496 in the five functions of `t3.c` and 15 in its call stubs). It does report the `mul` in `t3.c` built without its `KEEP` barriers, and the calls `__mulsi3` (a multiply built for rv32i) and `___divti3` (a 128-bit divide on arm64 macOS). `t3_rv32.c`: the stream machine, freestanding, compiled for rv32i (no M extension) by LLVM: 45 instructions, none a multiply, no library multiply; compiled only, not run. +Known limits of t3: `t3_run` does not check what `pthread_create` returns, so a thread that fails to start leaves its rows unset, which `check` prints as DIFFERENT; an input index is 14 bits, so a product has at most 16,383 inputs (for the context products p·vh and p·vl, 16,383 positions). + ## Larger models One code path: the sizes come from each model's pinned `config.json`, the clamps by rule from its own weights (above), the bound per route from the width. `python exact_llm.py check --model SmolLM2-360M` runs exact-16 on its three routes, float32 and onnxruntime. From 7dd91d6342195774bd89a45667e61e67cd0a7cc5 Mon Sep 17 00:00:00 2001 From: lmtlssss <261705307+lmtlssss@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:22:33 -0800 Subject: [PATCH 07/12] exact-llm workflow: the t3 route on the 135M on ubuntu-latest, ubuntu-24.04-arm and macos-latest, then t3_nomul.py Three jobs, one route each in the matrix form of the 1.7B jobs: check --model SmolLM2-135M --routes t3 --no-float --no-onnx, then t3_nomul.py on the kernel check built. Linux x86-64 and arm64 build t3.c with gcc, macOS arm64 with Apple clang; the Windows runners have no C compiler. 120 minutes each (the 135M's other jobs keep 90). The base matrix gets routes: [''] (check's default routes). Without it, an include entry whose model and runner match a combination of the cross product would be merged into that job (GitHub adds an include's pairs to every combination whose original values they do not overwrite) and turn the 135M's default job on that runner into a t3 job. With it, each t3 entry is a job of its own: 27 jobs, the 24 before and these 3. --- .github/workflows/exact-llm.yml | 15 ++++++++++++++- proofs/exact-llm/README.md | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/exact-llm.yml b/.github/workflows/exact-llm.yml index 88acb98..ac3c428 100644 --- a/.github/workflows/exact-llm.yml +++ b/.github/workflows/exact-llm.yml @@ -12,12 +12,15 @@ jobs: hashes: name: hashes (${{ matrix.model }}, ${{ matrix.os }}${{ matrix.routes && format(', {0}', matrix.routes) || '' }}) runs-on: ${{ matrix.os }} - timeout-minutes: ${{ matrix.model == 'SmolLM2-135M' && 90 || 120 }} + timeout-minutes: ${{ (matrix.model == 'SmolLM2-135M' && matrix.routes != 't3') && 90 || 120 }} strategy: fail-fast: false matrix: os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, macos-15-intel, windows-latest, windows-11-arm] model: [SmolLM2-135M, SmolLM2-360M] + # check's default routes. An include entry with a route of its own would overwrite this value, so it becomes + # a job of its own instead of being merged into the default job of the same model and runner + routes: [''] # SmolLM2-1.7B, one route per job, each inside 120 minutes in run 43759765827: f64 on the six runners, float32 # beside it on ubuntu-24.04-arm alone (53 min; 86 to over 120 elsewhere); f32 and i8 on the non-Windows # runners, except f32 on macos-latest and macos-15-intel (each ran past 120 minutes: macos-latest in run 43759765827, @@ -35,6 +38,11 @@ jobs: - {model: SmolLM2-1.7B, os: ubuntu-latest, routes: i8, args: "--routes i8 --no-float"} - {model: SmolLM2-1.7B, os: ubuntu-24.04-arm, routes: i8, args: "--routes i8 --no-float"} - {model: SmolLM2-1.7B, os: macos-15-intel, routes: i8, args: "--routes i8 --no-float"} + # the t3 route (t3.c, built by the runner's C compiler; the Windows runners have none) on the 135M: Linux + # x86-64 and arm64 with gcc, macOS arm64 with Apple clang; then t3_nomul.py on the kernel check built + - {model: SmolLM2-135M, os: ubuntu-latest, routes: t3, args: "--routes t3 --no-float --no-onnx"} + - {model: SmolLM2-135M, os: ubuntu-24.04-arm, routes: t3, args: "--routes t3 --no-float --no-onnx"} + - {model: SmolLM2-135M, os: macos-latest, routes: t3, args: "--routes t3 --no-float --no-onnx"} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 @@ -57,3 +65,8 @@ jobs: shell: bash working-directory: proofs/exact-llm run: python -u exact_llm.py check --model ${{ matrix.model }} ${{ matrix.args }} + - name: The t3 kernel that check built, disassembled; fails on any multiply, multiply-add or divide instruction or library call + if: matrix.routes == 't3' + shell: bash + working-directory: proofs/exact-llm + run: python t3_nomul.py diff --git a/proofs/exact-llm/README.md b/proofs/exact-llm/README.md index c7d7362..8e2a917 100644 --- a/proofs/exact-llm/README.md +++ b/proofs/exact-llm/README.md @@ -141,6 +141,8 @@ A fourth route, `exact16-t3`, run where a C compiler is installed (`check --rout MATCH on both, 0 clamped. The M1 was shared with other jobs during this run, so its seconds are not a measure of either route. Per token at the last position (context 64, `t3_count.py`, the kernel's counters): 459,184,492 digit adds and subtracts in the weight products, as the digit census predicts, and 6,549,504 Horner adds; 1,408,559 and 6,442,304 in the peeled terms; 11,875,560 and 1,658,880 in the attention (scores 5,007,459, p·vh 900,330, p·vl 5,967,771, Horner 552,960 each); 1,271,526 and 4,451,136 element-wise; 492,841,961 adds and subtracts in all. `t3_nomul.py`: the kernel's disassembly holds no multiply, multiply-add or divide instruction and calls no library multiply or divide (Apple clang, arm64: 0 of 511 instructions, 496 in the five functions of `t3.c` and 15 in its call stubs). It does report the `mul` in `t3.c` built without its `KEEP` barriers, and the calls `__mulsi3` (a multiply built for rv32i) and `___divti3` (a 128-bit divide on arm64 macOS). `t3_rv32.c`: the stream machine, freestanding, compiled for rv32i (no M extension) by LLVM: 45 instructions, none a multiply, no library multiply; compiled only, not run. +On the runners, the workflow runs `check --model SmolLM2-135M --routes t3 --no-float --no-onnx`, then `t3_nomul.py` on the kernel that check built, on three runners: ubuntu-latest (x86-64, gcc), ubuntu-24.04-arm (arm64, gcc) and macos-latest (arm64, Apple clang), 120 minutes each. On every Linux and macOS runner, `routes_test.py` also builds `t3.c` and tests it against int64 products. + Known limits of t3: `t3_run` does not check what `pthread_create` returns, so a thread that fails to start leaves its rows unset, which `check` prints as DIFFERENT; an input index is 14 bits, so a product has at most 16,383 inputs (for the context products p·vh and p·vl, 16,383 positions). ## Larger models From 76126a91fea29b0c5a062c815e9927f430ee75dd Mon Sep 17 00:00:00 2001 From: lmtlssss <261705307+lmtlssss@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:34:11 -0800 Subject: [PATCH 08/12] proofs/exact-llm: t3 rerun on the M1 with the kernel and streams built afresh; exact and t3 MATCH, no multiplier record/t3-m1-rerun.txt (paths under the home folder written as ~): this code (run from 2f5e961, the same files as this branch's) on the Apple M1 with an empty kernel and stream folder. routes_test.py: 42 cases on four routes, 0 different, every 16-bit value through the stream exact. check --routes t3 --no-float --no-onnx: exact 30ed613b... MATCH; exact-16 on t3 b0cc875f...0b4460 / a800ebad...7613cc MATCH, 0 clamped; 1,653,064,171,200 weight digit adds over the 64 prompts. t3_nomul.py on the new kernel: 511 instructions, no multiply or divide. The README names this record and the first run's (record/t3-m1.txt, record/t3-count.json). --- proofs/exact-llm/README.md | 6 +++- proofs/exact-llm/record/t3-m1-rerun.txt | 41 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 proofs/exact-llm/record/t3-m1-rerun.txt diff --git a/proofs/exact-llm/README.md b/proofs/exact-llm/README.md index 8e2a917..965d142 100644 --- a/proofs/exact-llm/README.md +++ b/proofs/exact-llm/README.md @@ -139,7 +139,10 @@ A fourth route, `exact16-t3`, run where a C compiler is installed (`check --rout | `exact16-f64` | `b0cc875f…0b4460` | `a800ebad…7613cc` | | `exact16-t3`, 8 threads | `b0cc875f…0b4460` | `a800ebad…7613cc` | -MATCH on both, 0 clamped. The M1 was shared with other jobs during this run, so its seconds are not a measure of either route. Per token at the last position (context 64, `t3_count.py`, the kernel's counters): 459,184,492 digit adds and subtracts in the weight products, as the digit census predicts, and 6,549,504 Horner adds; 1,408,559 and 6,442,304 in the peeled terms; 11,875,560 and 1,658,880 in the attention (scores 5,007,459, p·vh 900,330, p·vl 5,967,771, Horner 552,960 each); 1,271,526 and 4,451,136 element-wise; 492,841,961 adds and subtracts in all. `t3_nomul.py`: the kernel's disassembly holds no multiply, multiply-add or divide instruction and calls no library multiply or divide (Apple clang, arm64: 0 of 511 instructions, 496 in the five functions of `t3.c` and 15 in its call stubs). It does report the `mul` in `t3.c` built without its `KEEP` barriers, and the calls `__mulsi3` (a multiply built for rv32i) and `___divti3` (a 128-bit divide on arm64 macOS). `t3_rv32.c`: the stream machine, freestanding, compiled for rv32i (no M extension) by LLVM: 45 instructions, none a multiply, no library multiply; compiled only, not run. +MATCH on both, 0 clamped (`record/t3-m1.txt`). The M1 was shared with other jobs during this run, so its seconds are not a measure of either route. Rerun of this code on the M1, with the kernel built and every weight stream compiled afresh from the weights (`record/t3-m1-rerun.txt`): +- `routes_test.py`: 42 cases on four routes, 0 different; +- `check --routes t3`: exact MATCH, and the t3 route MATCH with 0 clamped; +- `t3_nomul.py` on that kernel: no multiply or divide in 511 instructions. Per token at the last position (context 64, `t3_count.py`, the kernel's counters): 459,184,492 digit adds and subtracts in the weight products, as the digit census predicts, and 6,549,504 Horner adds; 1,408,559 and 6,442,304 in the peeled terms; 11,875,560 and 1,658,880 in the attention (scores 5,007,459, p·vh 900,330, p·vl 5,967,771, Horner 552,960 each); 1,271,526 and 4,451,136 element-wise; 492,841,961 adds and subtracts in all. `t3_nomul.py`: the kernel's disassembly holds no multiply, multiply-add or divide instruction and calls no library multiply or divide (Apple clang, arm64: 0 of 511 instructions, 496 in the five functions of `t3.c` and 15 in its call stubs). It does report the `mul` in `t3.c` built without its `KEEP` barriers, and the calls `__mulsi3` (a multiply built for rv32i) and `___divti3` (a 128-bit divide on arm64 macOS). `t3_rv32.c`: the stream machine, freestanding, compiled for rv32i (no M extension) by LLVM: 45 instructions, none a multiply, no library multiply; compiled only, not run. On the runners, the workflow runs `check --model SmolLM2-135M --routes t3 --no-float --no-onnx`, then `t3_nomul.py` on the kernel that check built, on three runners: ubuntu-latest (x86-64, gcc), ubuntu-24.04-arm (arm64, gcc) and macos-latest (arm64, Apple clang), 120 minutes each. On every Linux and macOS runner, `routes_test.py` also builds `t3.c` and tests it against int64 products. @@ -384,6 +387,7 @@ Commands: | `record/peaks-16.json`, `record/peaks-16-SmolLM2-360M.json` | each site's peak, clamp, margin and clamped count, from `check --peaks` | | `routes_test.py` | the three routes against int64 products; t3 too where a C compiler is installed, with every 16-bit value through the stream and 200,081 element-wise pairs | | `t3.c`, `t3_nomul.py`, `t3_count.py`, `t3_rv32.c` | the t3 kernel; its disassembly checked for multipliers; its counters per token; the stream machine for rv32i | +| `record/t3-m1.txt`, `record/t3-m1-rerun.txt`, `record/t3-count.json` | every line of the t3 check on the M1, and of its rerun with the kernel and streams built afresh; the counters per token from `t3_count.py` | | `spare_row.py`, `spare_row_test.py`, `record/spare-m1.txt`, `record/spare-campaign.json` | exact-16 with every product and look checked, the fault campaign; its test on synthetic operands; every line of the M1 runs; every fault of the campaign | | `count_ops.py`, `report_census.py`; `count_spare.py`, `report_spare.py`, `record/census-spare.json`, `record/census-spare.md` | the census of every NumPy operation per token, sorted into steps and looks: of exact and exact-16; of exact-16 with and without the spare-row checks, and its tables | | `measure16.py`, `record/exact16-table.txt` | the width table, one row per run | diff --git a/proofs/exact-llm/record/t3-m1-rerun.txt b/proofs/exact-llm/record/t3-m1-rerun.txt new file mode 100644 index 0000000..b819d6a --- /dev/null +++ b/proofs/exact-llm/record/t3-m1-rerun.txt @@ -0,0 +1,41 @@ +Shortened after the run: the t3 kernel's path, to its file name, in two lines below. Nothing else changed. +branch t3-rebase-trial 2f5e961f4d6ec857e5fb09bebfbce8d0b9955613 (proofs/exact-llm: t3_nomul.py finds library calls in Mach-O; README: what t3 leaves to NumPy, no M1 seconds), on main 27d865e +== routes, load { 12.43 11.14 10.32 } +i8 product engine: i8-onnxruntime-1.32.0 +t3 kernel: t3-f6d3700bf905.dylib +cases 42 (7 shapes x 6 kinds, int64 and int16 right operands), routes 4, different 0 +digit extremes, value (i8 digits) (f32 digits): 32767 (255, 255) (128, -1); -32767 (0, 1) (-128, 1); 32640 (255, 128) (128, -128); -32640 (0, 128) (-127, -128); 32639 (255, 127) (127, 127); -32641 (0, 127) (-128, 127); 32512 (255, 0) (127, 0); -32513 (0, 255) (-127, -1) +f32 digits: high -128..128, low -128..127 (int16 split the same: True); i8 digits of a + 2^15: high 0..255, low 0..255 (from int16 the same: True) +peel_index of [5, -9, 9, 3, -9, 1] k=2: [1, 2] (expect [1, 2]) +t3 stream, every value -32767..32767: exact; digit adds 356806, census formula 356806, at most 8 per value +t3 element-wise products, 200081 pairs and a broadcast: exact + 2.09 real 2.02 user 0.38 sys +exit 0 + 347389952 maximum resident set size +== check, load { 8.46 10.08 10.37 } +versions: Python 3.11.18, NumPy 2.4.6, onnxruntime 1.32.0; darwin arm64 +model SmolLM2-135M @ 93efa2f: 30 layers, width 576, 9/3 heads, feed-forward 1536, rotary base 100000, 134515008 weights +exact logits 30ed613b6036d08046fbdbf273a0fb7b8cc04fb284ca49e0296980eed9cee773 continuations a800ebadc848514e02bebbb8077d531cf0493acd2a4c173403ba1db43d7613cc MATCH + clamped values: 0 +exact16 16-bit weights, int32 embedding and norms, weight statistics: ready in 0 s +exact16 logits b0cc875fde396181a70e2349ca35a60c3ae91b23f1c1012083797352d30b4460 continuations a800ebadc848514e02bebbb8077d531cf0493acd2a4c173403ba1db43d7613cc MATCH route t3, 610 s, clamped 0 + t3 kernel t3-f6d3700bf905.dylib; digit adds, Horner adds, values recoded, recoding operations: weights 1,653,064,171,200, 23,578,214,400, 0, 0; peeled 5,079,175,319, 23,547,135,488, 1,473,638,400, 7,993,295,032; attention 19,327,619,286, 4,893,696,000, 1,738,506,240, 9,276,986,651; element-wise 4,326,722,524, 15,590,066,688, 973,555,200, 5,113,624,092; weight streams, built once 0, 0, 134,479,872, 739,466,822 + peaks (real) / clamps: attention input 11.429 / 131072; down input 2630.720 / 4194304; gate 50.741 / 32768; k 22.910 / 1024; logits input 53.630 / 8388608; mlp input 28.742 / 8388608; o input 9.531 / 8388608; q 22.075 / 1024; residual 19509.485 / 33554432; up 72.026 / 32768; v 12.972 / 8192 +check every pair MATCH, 709 s + 709.65 real 581.74 user 141.57 sys +exit 0 + 817692672 maximum resident set size +== nomul, load { 8.46 10.08 10.37 } +object t3-f6d3700bf905.dylib +_t3_compile_masks 76 instructions; multiply or divide: none +_t3_compile_fill 169 instructions; multiply or divide: none +_t3_run 89 instructions; multiply or divide: none +_run_rows 109 instructions; multiply or divide: none +_t3_mul 53 instructions; multiply or divide: none +__stubs 15 instructions; multiply or divide: none +histogram, all functions (511 instructions): add 68, mov 57, cmp 38, b.eq 33, b 28, ldr 28, stp 21, strh 21, tst 21, ldp 20, sub 19, b.lt 13, ldrh 12, orr 10, adrp 9, csel 9, ldurh 9, and 8, str 7, bl 6, lsr 6, movi.2d 6, add.2d 5, br 5, lsl 5, ret 5, subs 5, cbz 4, cinc 4, b.ge 3, b.ne 3, add.4s 2, b.hi 2, b.hs 2, cbnz 2, cneg 2, eor 2, saddw.2d 2, saddw2.2d 2, sub.4s 2, b.lo 1, blr 1, ldur 1, neg 1, stur 1 +multiply, multiply-add or divide instructions: 0 +NO MULTIPLIER + 0.37 real 0.13 user 0.13 sys +exit 0 + 35553280 maximum resident set size From 579f79cda5cad4bbf0d0cff46b6df969c732d7de Mon Sep 17 00:00:00 2001 From: lmtlssss <261705307+lmtlssss@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:01:20 -0800 Subject: [PATCH 09/12] proofs/exact-llm/record: the 360M's M1 record names its gate script by file name --- proofs/exact-llm/record/exact16-360m-m1.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proofs/exact-llm/record/exact16-360m-m1.txt b/proofs/exact-llm/record/exact16-360m-m1.txt index 1d6c863..8fa4e81 100644 --- a/proofs/exact-llm/record/exact16-360m-m1.txt +++ b/proofs/exact-llm/record/exact16-360m-m1.txt @@ -1,5 +1,6 @@ # SmolLM2-360M on the Apple M1 (macOS 27.2, Python 3.11.18, NumPy 2.4.6 with Accelerate, onnxruntime 1.32.0). -# One process per step, under one turn of the machine's job gate (Work/.build.nosync/llm/src/e19_360m.sh). Elapsed: /usr/bin/time real seconds. +# One process per step, under one turn of the machine's job gate (e19_360m.sh). Elapsed: /usr/bin/time real seconds. +# Shortened after the run: the gate script's path, to its file name, in the line above. Nothing else changed. # This run came before expected-16-SmolLM2-360M.sha256 existed: the pair was pinned from its lines, so check printed DIFFERENT (nothing to compare) and exited 1. == check versions: Python 3.11.18, NumPy 2.4.6, onnxruntime 1.32.0; darwin arm64 From ce47bc5740cda916b3dd3ddcfbf1315a0410ffb3 Mon Sep 17 00:00:00 2001 From: lmtlssss <261705307+lmtlssss@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:08:06 -0800 Subject: [PATCH 10/12] proofs/exact-llm: t3 is not built on Windows; routes_test.py says so instead of loading a kernel the runner cannot open --- .github/workflows/exact-llm.yml | 2 +- proofs/exact-llm/README.md | 4 ++-- proofs/exact-llm/exact_llm.py | 15 ++++++++++++++- proofs/exact-llm/routes_test.py | 7 +++---- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.github/workflows/exact-llm.yml b/.github/workflows/exact-llm.yml index ac3c428..14eefbf 100644 --- a/.github/workflows/exact-llm.yml +++ b/.github/workflows/exact-llm.yml @@ -38,7 +38,7 @@ jobs: - {model: SmolLM2-1.7B, os: ubuntu-latest, routes: i8, args: "--routes i8 --no-float"} - {model: SmolLM2-1.7B, os: ubuntu-24.04-arm, routes: i8, args: "--routes i8 --no-float"} - {model: SmolLM2-1.7B, os: macos-15-intel, routes: i8, args: "--routes i8 --no-float"} - # the t3 route (t3.c, built by the runner's C compiler; the Windows runners have none) on the 135M: Linux + # the t3 route (t3.c, built by the runner's C compiler; not on Windows) on the 135M: Linux # x86-64 and arm64 with gcc, macOS arm64 with Apple clang; then t3_nomul.py on the kernel check built - {model: SmolLM2-135M, os: ubuntu-latest, routes: t3, args: "--routes t3 --no-float --no-onnx"} - {model: SmolLM2-135M, os: ubuntu-24.04-arm, routes: t3, args: "--routes t3 --no-float --no-onnx"} diff --git a/proofs/exact-llm/README.md b/proofs/exact-llm/README.md index 965d142..bbd847c 100644 --- a/proofs/exact-llm/README.md +++ b/proofs/exact-llm/README.md @@ -125,7 +125,7 @@ The large difference at 16-bit inputs is one outlier per vector setting its shif ## t3: the products by adds and subtracts -A fourth route, `exact16-t3`, run where a C compiler is installed (`check --routes t3`; not in check's default three). Every product of exact-16's forward pass is computed by `t3.c`, whose arithmetic is add, subtract, shift by a constant and bitwise logic: the matrix products in `mm16` (q, k, v, o, gate, up, down, logits, the scores q16·k16, the context p·vh and p·vl), the peeled terms, and the element-wise products (the norm's squares and weight, rotary, the exp and SiLU interpolations, SiLU(gate)·up) through `Exact.mul`. Divisions (52,477 per token) and square roots (61) stay as on the other routes: NumPy's `//` and float `sqrt`, each checked by an integer bracket that NumPy multiplies (q·d ≤ n < (q + 1)·d, s² ≤ n < (s + 1)²). Shifts by a computed amount (the block shift, the row scale) stay NumPy's too. +A fourth route, `exact16-t3`, run where a C compiler is installed, not on Windows (`check --routes t3`; not in check's default three). Every product of exact-16's forward pass is computed by `t3.c`, whose arithmetic is add, subtract, shift by a constant and bitwise logic: the matrix products in `mm16` (q, k, v, o, gate, up, down, logits, the scores q16·k16, the context p·vh and p·vl), the peeled terms, and the element-wise products (the norm's squares and weight, rotary, the exp and SiLU interpolations, SiLU(gate)·up) through `Exact.mul`. Divisions (52,477 per token) and square roots (61) stay as on the other routes: NumPy's `//` and float `sqrt`, each checked by an integer bracket that NumPy multiplies (q·d ≤ n < (q + 1)·d, s² ≤ n < (s + 1)²). Shifts by a computed amount (the block shift, the row scale) stay NumPy's too. | signed digits, `t3.c` | for m ≥ 0: `h = m >> 1`, `s = m + h`, `c = h ^ s`; plus `s & c`, minus `h & c`; m = plus − minus, non-adjacent form; a negative value by its magnitude with plus and minus exchanged | | --- | --- | @@ -385,7 +385,7 @@ Commands: | `expected-16-SmolLM2-360M.sha256`, `record/exact16-360m-m1.txt`, `record/ci-43754428621.txt` | the 360M's pair; every line of its M1 run; every line of the twelve jobs (135M and 360M on six runners) | | `expected-16-SmolLM2-1.7B.sha256`, `record/ci-43759765827.txt` | the 1.7B's pair; every line of its 14 jobs (one route each), with each job's end and wall time | | `record/peaks-16.json`, `record/peaks-16-SmolLM2-360M.json` | each site's peak, clamp, margin and clamped count, from `check --peaks` | -| `routes_test.py` | the three routes against int64 products; t3 too where a C compiler is installed, with every 16-bit value through the stream and 200,081 element-wise pairs | +| `routes_test.py` | the three routes against int64 products; t3 too where a C compiler is installed, not on Windows, with every 16-bit value through the stream and 200,081 element-wise pairs | | `t3.c`, `t3_nomul.py`, `t3_count.py`, `t3_rv32.c` | the t3 kernel; its disassembly checked for multipliers; its counters per token; the stream machine for rv32i | | `record/t3-m1.txt`, `record/t3-m1-rerun.txt`, `record/t3-count.json` | every line of the t3 check on the M1, and of its rerun with the kernel and streams built afresh; the counters per token from `t3_count.py` | | `spare_row.py`, `spare_row_test.py`, `record/spare-m1.txt`, `record/spare-campaign.json` | exact-16 with every product and look checked, the fault campaign; its test on synthetic operands; every line of the M1 runs; every fault of the campaign | diff --git a/proofs/exact-llm/exact_llm.py b/proofs/exact-llm/exact_llm.py index 7f4441f..157b6ae 100644 --- a/proofs/exact-llm/exact_llm.py +++ b/proofs/exact-llm/exact_llm.py @@ -25,6 +25,7 @@ import json import math import os +import shutil import struct import sys import time @@ -668,7 +669,7 @@ def logits(self, h): CHUNK32 = 1024 # f32 route: inputs per chunk, 1,024 x 2^14 = 2^24 N_I8 = 33025 # i8 route: n 255^2 < 2^31 for n <= 33,025 ROUTES = ("f64", "f32", "i8") # check's default -ROUTES_T3 = ("t3",) # by adds and subtracts (t3.c), where a C compiler is installed: --routes t3 +ROUTES_T3 = ("t3",) # by adds and subtracts (t3.c), where a C compiler is installed, not on Windows: --routes t3 TOP16 = 15 # t3: highest signed-digit position of a 16-bit value; 16 STEP words per output row @@ -768,6 +769,16 @@ def __call__(self, a, b): T3_THREADS = min(64, int(os.environ.get("EXACT_LLM_T3_THREADS", "0")) or os.cpu_count() or 1) # t3.c: at most 64 +def t3_missing(): + """Why t3.c is not built here, or None. It is built with a C compiler (`cc` or $CC) as a POSIX shared library + with threads; not on Windows.""" + if sys.platform == "win32": + return "not built on Windows" + if shutil.which(os.environ.get("CC", "cc")) is None: + return "no C compiler" + return None + + class T3: """The t3 route's kernel, t3.c, compiled once with the C compiler `cc` (or $CC) into T3_DIR and opened with ctypes. Matrix products (mm) and element-wise products (mul) by add, subtract, shift by a constant and bitwise @@ -779,6 +790,8 @@ class T3: def __init__(self): import ctypes import subprocess + if t3_missing(): + raise SystemExit("t3: " + t3_missing()) src = os.path.join(HERE, "t3.c") tag = self.tag = hashlib.sha256(open(src, "rb").read()).hexdigest()[:12] os.makedirs(T3_DIR, exist_ok=True) diff --git a/proofs/exact-llm/routes_test.py b/proofs/exact-llm/routes_test.py index 4002048..288b7ab 100644 --- a/proofs/exact-llm/routes_test.py +++ b/proofs/exact-llm/routes_test.py @@ -7,7 +7,6 @@ # with the products negative. "digit extremes" mixes, position by position, the eight values whose digits sit at the # ends of both routes' digit ranges. A kernel that saturates or rounds anywhere fails one of these. import os -import shutil import sys import numpy as np @@ -18,10 +17,10 @@ L = X.LIM16 mmi = X.MatMulInteger() print("i8 product engine:", mmi.name) -# t3 (t3.c, by adds and subtracts) where a C compiler is installed -T3 = shutil.which(os.environ.get("CC", "cc")) is not None +# t3 (t3.c, by adds and subtracts) where a C compiler is installed, not on Windows +T3 = X.t3_missing() is None ROUTES = X.ROUTES + (X.ROUTES_T3 if T3 else ()) -print("t3 kernel:", X.t3_kernel().path if T3 else "no C compiler: not tested") +print("t3 kernel:", X.t3_kernel().path if T3 else X.t3_missing() + ": not tested") shapes = [((5, 576), (576, 300)), ((3, 1023), (1023, 7)), ((3, 1024), (1024, 7)), ((3, 1025), (1025, 7)), ((4, 1536), (1536, 64)), ((2, 3, 27, 64), (2, 3, 64, 40)), ((2, 3, 27, 600), (2, 3, 600, 64))] # digits (i8: of a + 2^15, high and low; f32: balanced high and low): 32767 (255, 255; 128, -1), -32767 (0, 1; From 272a6d80aa254d15b82c42e3a66fddbbed6a3647 Mon Sep 17 00:00:00 2001 From: lmtlssss <261705307+lmtlssss@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:05:45 -0800 Subject: [PATCH 11/12] proofs/exact-llm: t3_compile_masks counts by an add per value behind a barrier, so gcc folds no multiply into it; t3_nomul.py prints each flagged instruction --- proofs/exact-llm/t3.c | 9 +++++++-- proofs/exact-llm/t3_nomul.py | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/proofs/exact-llm/t3.c b/proofs/exact-llm/t3.c index 63611c0..4ecfd87 100644 --- a/proofs/exact-llm/t3.c +++ b/proofs/exact-llm/t3.c @@ -51,12 +51,15 @@ static inline uint64_t recode(int64_t v, uint64_t *pl, uint64_t *mi) { int64_t t3_compile_masks(int64_t n_in, int64_t M, const int32_t *W, int64_t ld, uint16_t *Pm, uint16_t *Mm, int64_t *rowoff, uint64_t *cnt) { const int32_t *wr = W; + uint64_t vals = 0, ops = 0; for (int64_t i = 0; i < n_in; i++) { uint16_t *pp = Pm + i, *mm = Mm + i; for (int64_t j = 0; j < M; j++) { uint64_t p, q; - cnt[3] += recode(wr[j], &p, &q); - cnt[2] += 1; + ops += recode(wr[j], &p, &q); + vals += 1; + KEEP(ops); /* an add per value; without the barriers gcc */ + KEEP(vals); /* folds these counts into one n_in * M */ *pp = (uint16_t)p; *mm = (uint16_t)q; pp += n_in; @@ -64,6 +67,8 @@ int64_t t3_compile_masks(int64_t n_in, int64_t M, const int32_t *W, int64_t ld, } wr += ld; } + cnt[2] += vals; + cnt[3] += ops; int64_t off = 0; const uint16_t *pp = Pm, *mm = Mm; for (int64_t j = 0; j < M; j++) { diff --git a/proofs/exact-llm/t3_nomul.py b/proofs/exact-llm/t3_nomul.py index 761233c..e81b7ae 100644 --- a/proofs/exact-llm/t3_nomul.py +++ b/proofs/exact-llm/t3_nomul.py @@ -3,8 +3,8 @@ usage: python t3_nomul.py [OBJECT] (default: the kernel exact_llm.py builds and loads, t3-.dylib or .so) -Reads `objdump -d` (LLVM's or GNU's). Prints each function's instruction histogram and the whole object's; exit 1 -if any function holds an instruction of the lists below. arm64 and x86-64 mnemonics; Apple's objdump writes a +Reads `objdump -d` (LLVM's or GNU's). Prints each function's instruction count, each flagged instruction with the +three on either side, and the whole object's histogram; exit 1 if any function holds an instruction of the lists below. arm64 and x86-64 mnemonics; Apple's objdump writes a vector arrangement into the mnemonic (mul.4s), GNU's into the operands, and both are read by the stem. """ import collections @@ -46,17 +46,19 @@ def disassemble(path): calls = sorted(set(LIBCALL.findall(out))) if calls: raise SystemExit("library multiply or divide referenced: %s\nMULTIPLIER FOUND" % ", ".join(calls)) - funcs, name = collections.OrderedDict(), None + funcs, lines, name = collections.OrderedDict(), {}, None for line in out.splitlines(): m = re.match(r"^[0-9a-fA-F]+ <(.+)>:$", line.strip()) if m: name = m.group(1) funcs.setdefault(name, collections.Counter()) + lines.setdefault(name, []) continue m = re.match(r"^\s*[0-9a-fA-F]+:\s+([a-z][a-z0-9.]*)", line) # address, then the mnemonic if m and name is not None: funcs[name][m.group(1)] += 1 - return funcs + lines[name].append((m.group(1), line.strip())) + return funcs, lines def main(argv): @@ -66,7 +68,7 @@ def main(argv): sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import exact_llm as X path = X.t3_kernel().path - funcs = disassemble(path) + funcs, lines = disassemble(path) total, bad = collections.Counter(), [] print("object %s" % path) for name, h in funcs.items(): @@ -77,6 +79,11 @@ def main(argv): bad += [(name, m, h[m]) for m in hits] print("%-22s %5d instructions; multiply or divide: %s" % ( name, sum(h.values()), ", ".join("%s %d" % (m, h[m]) for m in hits) or "none")) + ins = lines[name] + for k, (mn, _) in enumerate(ins): # each flagged instruction, with three on either side + if flagged(mn): + for kk in range(max(0, k - 3), min(len(ins), k + 4)): + print(" %s %s" % ("=>" if kk == k else " ", ins[kk][1])) print("histogram, all functions (%d instructions): %s" % ( sum(total.values()), ", ".join("%s %d" % kv for kv in sorted(total.items(), key=lambda kv: (-kv[1], kv[0]))))) print("multiply, multiply-add or divide instructions: %d%s" % ( From 1bdc959babf1fb4d540902d3306614297c7326a0 Mon Sep 17 00:00:00 2001 From: lmtlssss <261705307+lmtlssss@users.noreply.github.com> Date: Mon, 16 Nov 2026 08:15:40 -0800 Subject: [PATCH 12/12] proofs/exact-llm: t3 rerun on the M1 of this code, kernel and streams built afresh: exact and t3 MATCH, counters as before, 0 multipliers in 517 instructions --- proofs/exact-llm/README.md | 2 +- proofs/exact-llm/record/t3-m1-rerun.txt | 35 ++++++++++++------------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/proofs/exact-llm/README.md b/proofs/exact-llm/README.md index bbd847c..317b83d 100644 --- a/proofs/exact-llm/README.md +++ b/proofs/exact-llm/README.md @@ -142,7 +142,7 @@ A fourth route, `exact16-t3`, run where a C compiler is installed, not on Window MATCH on both, 0 clamped (`record/t3-m1.txt`). The M1 was shared with other jobs during this run, so its seconds are not a measure of either route. Rerun of this code on the M1, with the kernel built and every weight stream compiled afresh from the weights (`record/t3-m1-rerun.txt`): - `routes_test.py`: 42 cases on four routes, 0 different; - `check --routes t3`: exact MATCH, and the t3 route MATCH with 0 clamped; -- `t3_nomul.py` on that kernel: no multiply or divide in 511 instructions. Per token at the last position (context 64, `t3_count.py`, the kernel's counters): 459,184,492 digit adds and subtracts in the weight products, as the digit census predicts, and 6,549,504 Horner adds; 1,408,559 and 6,442,304 in the peeled terms; 11,875,560 and 1,658,880 in the attention (scores 5,007,459, p·vh 900,330, p·vl 5,967,771, Horner 552,960 each); 1,271,526 and 4,451,136 element-wise; 492,841,961 adds and subtracts in all. `t3_nomul.py`: the kernel's disassembly holds no multiply, multiply-add or divide instruction and calls no library multiply or divide (Apple clang, arm64: 0 of 511 instructions, 496 in the five functions of `t3.c` and 15 in its call stubs). It does report the `mul` in `t3.c` built without its `KEEP` barriers, and the calls `__mulsi3` (a multiply built for rv32i) and `___divti3` (a 128-bit divide on arm64 macOS). `t3_rv32.c`: the stream machine, freestanding, compiled for rv32i (no M extension) by LLVM: 45 instructions, none a multiply, no library multiply; compiled only, not run. +- `t3_nomul.py` on that kernel: no multiply or divide in 517 instructions. Per token at the last position (context 64, `t3_count.py`, the kernel's counters): 459,184,492 digit adds and subtracts in the weight products, as the digit census predicts, and 6,549,504 Horner adds; 1,408,559 and 6,442,304 in the peeled terms; 11,875,560 and 1,658,880 in the attention (scores 5,007,459, p·vh 900,330, p·vl 5,967,771, Horner 552,960 each); 1,271,526 and 4,451,136 element-wise; 492,841,961 adds and subtracts in all. `t3_nomul.py`: the kernel's disassembly holds no multiply, multiply-add or divide instruction and calls no library multiply or divide (Apple clang, arm64: 0 of 517 instructions, 502 in the five functions of `t3.c` and 15 in its call stubs). It does report the `mul` in `t3.c` built without its `KEEP` barriers, and the calls `__mulsi3` (a multiply built for rv32i) and `___divti3` (a 128-bit divide on arm64 macOS). `t3_rv32.c`: the stream machine, freestanding, compiled for rv32i (no M extension) by LLVM: 45 instructions, none a multiply, no library multiply; compiled only, not run. On the runners, the workflow runs `check --model SmolLM2-135M --routes t3 --no-float --no-onnx`, then `t3_nomul.py` on the kernel that check built, on three runners: ubuntu-latest (x86-64, gcc), ubuntu-24.04-arm (arm64, gcc) and macos-latest (arm64, Apple clang), 120 minutes each. On every Linux and macOS runner, `routes_test.py` also builds `t3.c` and tests it against int64 products. diff --git a/proofs/exact-llm/record/t3-m1-rerun.txt b/proofs/exact-llm/record/t3-m1-rerun.txt index b819d6a..06adddf 100644 --- a/proofs/exact-llm/record/t3-m1-rerun.txt +++ b/proofs/exact-llm/record/t3-m1-rerun.txt @@ -1,41 +1,40 @@ -Shortened after the run: the t3 kernel's path, to its file name, in two lines below. Nothing else changed. -branch t3-rebase-trial 2f5e961f4d6ec857e5fb09bebfbce8d0b9955613 (proofs/exact-llm: t3_nomul.py finds library calls in Mach-O; README: what t3 leaves to NumPy, no M1 seconds), on main 27d865e -== routes, load { 12.43 11.14 10.32 } +branch t3-rebase-trial 272a6d80aa254d15b82c42e3a66fddbbed6a3647 (proofs/exact-llm: t3_compile_masks counts by an add per value behind a barrier, so gcc folds no multiply into it; t3_nomul.py prints each flagged instruction), on main 3d7e738 +== routes, load { 3.89 3.99 3.92 } i8 product engine: i8-onnxruntime-1.32.0 -t3 kernel: t3-f6d3700bf905.dylib +t3 kernel: t3/t3-21a279be7ff4.dylib cases 42 (7 shapes x 6 kinds, int64 and int16 right operands), routes 4, different 0 digit extremes, value (i8 digits) (f32 digits): 32767 (255, 255) (128, -1); -32767 (0, 1) (-128, 1); 32640 (255, 128) (128, -128); -32640 (0, 128) (-127, -128); 32639 (255, 127) (127, 127); -32641 (0, 127) (-128, 127); 32512 (255, 0) (127, 0); -32513 (0, 255) (-127, -1) f32 digits: high -128..128, low -128..127 (int16 split the same: True); i8 digits of a + 2^15: high 0..255, low 0..255 (from int16 the same: True) peel_index of [5, -9, 9, 3, -9, 1] k=2: [1, 2] (expect [1, 2]) t3 stream, every value -32767..32767: exact; digit adds 356806, census formula 356806, at most 8 per value t3 element-wise products, 200081 pairs and a broadcast: exact - 2.09 real 2.02 user 0.38 sys + 2.40 real 1.86 user 0.29 sys exit 0 - 347389952 maximum resident set size -== check, load { 8.46 10.08 10.37 } + 329383936 maximum resident set size +== check, load { 7.20 6.27 5.09 } versions: Python 3.11.18, NumPy 2.4.6, onnxruntime 1.32.0; darwin arm64 model SmolLM2-135M @ 93efa2f: 30 layers, width 576, 9/3 heads, feed-forward 1536, rotary base 100000, 134515008 weights exact logits 30ed613b6036d08046fbdbf273a0fb7b8cc04fb284ca49e0296980eed9cee773 continuations a800ebadc848514e02bebbb8077d531cf0493acd2a4c173403ba1db43d7613cc MATCH clamped values: 0 exact16 16-bit weights, int32 embedding and norms, weight statistics: ready in 0 s -exact16 logits b0cc875fde396181a70e2349ca35a60c3ae91b23f1c1012083797352d30b4460 continuations a800ebadc848514e02bebbb8077d531cf0493acd2a4c173403ba1db43d7613cc MATCH route t3, 610 s, clamped 0 - t3 kernel t3-f6d3700bf905.dylib; digit adds, Horner adds, values recoded, recoding operations: weights 1,653,064,171,200, 23,578,214,400, 0, 0; peeled 5,079,175,319, 23,547,135,488, 1,473,638,400, 7,993,295,032; attention 19,327,619,286, 4,893,696,000, 1,738,506,240, 9,276,986,651; element-wise 4,326,722,524, 15,590,066,688, 973,555,200, 5,113,624,092; weight streams, built once 0, 0, 134,479,872, 739,466,822 +exact16 logits b0cc875fde396181a70e2349ca35a60c3ae91b23f1c1012083797352d30b4460 continuations a800ebadc848514e02bebbb8077d531cf0493acd2a4c173403ba1db43d7613cc MATCH route t3, 429 s, clamped 0 + t3 kernel t3-21a279be7ff4.dylib; digit adds, Horner adds, values recoded, recoding operations: weights 1,653,064,171,200, 23,578,214,400, 0, 0; peeled 5,079,175,319, 23,547,135,488, 1,473,638,400, 7,993,295,032; attention 19,327,619,286, 4,893,696,000, 1,738,506,240, 9,276,986,651; element-wise 4,326,722,524, 15,590,066,688, 973,555,200, 5,113,624,092; weight streams, built once 0, 0, 134,479,872, 739,466,822 peaks (real) / clamps: attention input 11.429 / 131072; down input 2630.720 / 4194304; gate 50.741 / 32768; k 22.910 / 1024; logits input 53.630 / 8388608; mlp input 28.742 / 8388608; o input 9.531 / 8388608; q 22.075 / 1024; residual 19509.485 / 33554432; up 72.026 / 32768; v 12.972 / 8192 -check every pair MATCH, 709 s - 709.65 real 581.74 user 141.57 sys +check every pair MATCH, 504 s + 503.85 real 493.93 user 109.30 sys exit 0 - 817692672 maximum resident set size -== nomul, load { 8.46 10.08 10.37 } -object t3-f6d3700bf905.dylib -_t3_compile_masks 76 instructions; multiply or divide: none + 796868608 maximum resident set size +== nomul, load { 7.20 6.27 5.09 } +object t3/t3-21a279be7ff4.dylib +_t3_compile_masks 82 instructions; multiply or divide: none _t3_compile_fill 169 instructions; multiply or divide: none _t3_run 89 instructions; multiply or divide: none _run_rows 109 instructions; multiply or divide: none _t3_mul 53 instructions; multiply or divide: none __stubs 15 instructions; multiply or divide: none -histogram, all functions (511 instructions): add 68, mov 57, cmp 38, b.eq 33, b 28, ldr 28, stp 21, strh 21, tst 21, ldp 20, sub 19, b.lt 13, ldrh 12, orr 10, adrp 9, csel 9, ldurh 9, and 8, str 7, bl 6, lsr 6, movi.2d 6, add.2d 5, br 5, lsl 5, ret 5, subs 5, cbz 4, cinc 4, b.ge 3, b.ne 3, add.4s 2, b.hi 2, b.hs 2, cbnz 2, cneg 2, eor 2, saddw.2d 2, saddw2.2d 2, sub.4s 2, b.lo 1, blr 1, ldur 1, neg 1, stur 1 +histogram, all functions (517 instructions): add 70, mov 61, cmp 38, b.eq 33, b 28, ldr 28, stp 21, strh 21, tst 21, ldp 20, sub 19, b.lt 13, ldrh 12, orr 10, adrp 9, csel 9, ldurh 9, and 8, str 7, bl 6, lsr 6, movi.2d 6, add.2d 5, br 5, lsl 5, ret 5, subs 5, cbz 4, cinc 4, b.ge 3, b.ne 3, add.4s 2, b.hi 2, b.hs 2, cbnz 2, cneg 2, eor 2, saddw.2d 2, saddw2.2d 2, sub.4s 2, b.lo 1, blr 1, ldur 1, neg 1, stur 1 multiply, multiply-add or divide instructions: 0 NO MULTIPLIER - 0.37 real 0.13 user 0.13 sys + 0.30 real 0.12 user 0.10 sys exit 0 - 35553280 maximum resident set size + 35356672 maximum resident set size