diff --git a/.github/workflows/exact-llm.yml b/.github/workflows/exact-llm.yml index 88acb98..14eefbf 100644 --- a/.github/workflows/exact-llm.yml +++ b/.github/workflows/exact-llm.yml @@ -12,12 +12,15 @@ jobs: hashes: name: hashes (${{ matrix.model }}, ${{ matrix.os }}${{ matrix.routes && format(', {0}', matrix.routes) || '' }}) runs-on: ${{ matrix.os }} - timeout-minutes: ${{ matrix.model == 'SmolLM2-135M' && 90 || 120 }} + timeout-minutes: ${{ (matrix.model == 'SmolLM2-135M' && matrix.routes != 't3') && 90 || 120 }} strategy: fail-fast: false matrix: os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, macos-15-intel, windows-latest, windows-11-arm] model: [SmolLM2-135M, SmolLM2-360M] + # check's default routes. An include entry with a route of its own would overwrite this value, so it becomes + # a job of its own instead of being merged into the default job of the same model and runner + routes: [''] # SmolLM2-1.7B, one route per job, each inside 120 minutes in run 43759765827: f64 on the six runners, float32 # beside it on ubuntu-24.04-arm alone (53 min; 86 to over 120 elsewhere); f32 and i8 on the non-Windows # runners, except f32 on macos-latest and macos-15-intel (each ran past 120 minutes: macos-latest in run 43759765827, @@ -35,6 +38,11 @@ jobs: - {model: SmolLM2-1.7B, os: ubuntu-latest, routes: i8, args: "--routes i8 --no-float"} - {model: SmolLM2-1.7B, os: ubuntu-24.04-arm, routes: i8, args: "--routes i8 --no-float"} - {model: SmolLM2-1.7B, os: macos-15-intel, routes: i8, args: "--routes i8 --no-float"} + # the t3 route (t3.c, built by the runner's C compiler; not on Windows) on the 135M: Linux + # x86-64 and arm64 with gcc, macOS arm64 with Apple clang; then t3_nomul.py on the kernel check built + - {model: SmolLM2-135M, os: ubuntu-latest, routes: t3, args: "--routes t3 --no-float --no-onnx"} + - {model: SmolLM2-135M, os: ubuntu-24.04-arm, routes: t3, args: "--routes t3 --no-float --no-onnx"} + - {model: SmolLM2-135M, os: macos-latest, routes: t3, args: "--routes t3 --no-float --no-onnx"} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 @@ -57,3 +65,8 @@ jobs: shell: bash working-directory: proofs/exact-llm run: python -u exact_llm.py check --model ${{ matrix.model }} ${{ matrix.args }} + - name: The t3 kernel that check built, disassembled; fails on any multiply, multiply-add or divide instruction or library call + if: matrix.routes == 't3' + shell: bash + working-directory: proofs/exact-llm + run: python t3_nomul.py diff --git a/proofs/exact-llm/README.md b/proofs/exact-llm/README.md index 5829c7c..317b83d 100644 --- a/proofs/exact-llm/README.md +++ b/proofs/exact-llm/README.md @@ -123,6 +123,31 @@ Each row narrows the exact model's product operands and is compared with the sam The large difference at 16-bit inputs is one outlier per vector setting its shift: peel 1 brings it to 0.175, peel 2 to 0.03 to 0.06, peel 4 adds nothing; peel 2 also takes 14-bit inputs and weights from 411 to 470. `measure16.py` (rows 1 to 11, against saved float32 runs), `record/exact16-table.txt`; last row `exact_llm.py hash`, `text` and `distance.py`. +## t3: the products by adds and subtracts + +A fourth route, `exact16-t3`, run where a C compiler is installed, not on Windows (`check --routes t3`; not in check's default three). Every product of exact-16's forward pass is computed by `t3.c`, whose arithmetic is add, subtract, shift by a constant and bitwise logic: the matrix products in `mm16` (q, k, v, o, gate, up, down, logits, the scores q16·k16, the context p·vh and p·vl), the peeled terms, and the element-wise products (the norm's squares and weight, rotary, the exp and SiLU interpolations, SiLU(gate)·up) through `Exact.mul`. Divisions (52,477 per token) and square roots (61) stay as on the other routes: NumPy's `//` and float `sqrt`, each checked by an integer bracket that NumPy multiplies (q·d ≤ n < (q + 1)·d, s² ≤ n < (s + 1)²). Shifts by a computed amount (the block shift, the row scale) stay NumPy's too. + +| signed digits, `t3.c` | for m ≥ 0: `h = m >> 1`, `s = m + h`, `c = h ^ s`; plus `s & c`, minus `h & c`; m = plus − minus, non-adjacent form; a negative value by its magnitude with plus and minus exchanged | +| --- | --- | +| stream | a 16-bit word per instruction, `op << 14 \| i`; per output row, for digit position k = 15 down to 0: `ADD i` for each input with digit +1 at k, `SUB i` for each with −1, then `STEP` (`EMIT` at k = 0) | +| machine | per lane: `s` (int32), `acc` (int64). `ADD i`: s = s + x[i]. `SUB i`: s = s − x[i]. `STEP`: acc = acc + acc; acc = acc + s; s = 0. `EMIT`: `STEP`, then y[row] = acc, acc = 0. 8 lanes at a time | +| weights | each matrix compiled to its stream once (924,918,488 bytes for the 135M: 459,184,492 digits and 16 words for each of 204,672 output rows), kept in `$EXACT_LLM_T3` (default `.cache/t3`), one folder per `t3.c` and model | +| activations | the right operand recoded into digits at each product: k16 for the scores, vh and vl for the context, the smaller-magnitude operand element-wise and for the peeled terms | + +| Apple M1, 64 prompts, batch 8, one `check` process | logits | continuations | +| --- | --- | --- | +| `exact16-f64` | `b0cc875f…0b4460` | `a800ebad…7613cc` | +| `exact16-t3`, 8 threads | `b0cc875f…0b4460` | `a800ebad…7613cc` | + +MATCH on both, 0 clamped (`record/t3-m1.txt`). The M1 was shared with other jobs during this run, so its seconds are not a measure of either route. Rerun of this code on the M1, with the kernel built and every weight stream compiled afresh from the weights (`record/t3-m1-rerun.txt`): +- `routes_test.py`: 42 cases on four routes, 0 different; +- `check --routes t3`: exact MATCH, and the t3 route MATCH with 0 clamped; +- `t3_nomul.py` on that kernel: no multiply or divide in 517 instructions. Per token at the last position (context 64, `t3_count.py`, the kernel's counters): 459,184,492 digit adds and subtracts in the weight products, as the digit census predicts, and 6,549,504 Horner adds; 1,408,559 and 6,442,304 in the peeled terms; 11,875,560 and 1,658,880 in the attention (scores 5,007,459, p·vh 900,330, p·vl 5,967,771, Horner 552,960 each); 1,271,526 and 4,451,136 element-wise; 492,841,961 adds and subtracts in all. `t3_nomul.py`: the kernel's disassembly holds no multiply, multiply-add or divide instruction and calls no library multiply or divide (Apple clang, arm64: 0 of 517 instructions, 502 in the five functions of `t3.c` and 15 in its call stubs). It does report the `mul` in `t3.c` built without its `KEEP` barriers, and the calls `__mulsi3` (a multiply built for rv32i) and `___divti3` (a 128-bit divide on arm64 macOS). `t3_rv32.c`: the stream machine, freestanding, compiled for rv32i (no M extension) by LLVM: 45 instructions, none a multiply, no library multiply; compiled only, not run. + +On the runners, the workflow runs `check --model SmolLM2-135M --routes t3 --no-float --no-onnx`, then `t3_nomul.py` on the kernel that check built, on three runners: ubuntu-latest (x86-64, gcc), ubuntu-24.04-arm (arm64, gcc) and macos-latest (arm64, Apple clang), 120 minutes each. On every Linux and macOS runner, `routes_test.py` also builds `t3.c` and tests it against int64 products. + +Known limits of t3: `t3_run` does not check what `pthread_create` returns, so a thread that fails to start leaves its rows unset, which `check` prints as DIFFERENT; an input index is 14 bits, so a product has at most 16,383 inputs (for the context products p·vh and p·vl, 16,383 positions). + ## Larger models One code path: the sizes come from each model's pinned `config.json`, the clamps by rule from its own weights (above), the bound per route from the width. `python exact_llm.py check --model SmolLM2-360M` runs exact-16 on its three routes, float32 and onnxruntime. @@ -360,7 +385,9 @@ Commands: | `expected-16-SmolLM2-360M.sha256`, `record/exact16-360m-m1.txt`, `record/ci-43754428621.txt` | the 360M's pair; every line of its M1 run; every line of the twelve jobs (135M and 360M on six runners) | | `expected-16-SmolLM2-1.7B.sha256`, `record/ci-43759765827.txt` | the 1.7B's pair; every line of its 14 jobs (one route each), with each job's end and wall time | | `record/peaks-16.json`, `record/peaks-16-SmolLM2-360M.json` | each site's peak, clamp, margin and clamped count, from `check --peaks` | -| `routes_test.py` | the three routes against int64 products | +| `routes_test.py` | the three routes against int64 products; t3 too where a C compiler is installed, not on Windows, with every 16-bit value through the stream and 200,081 element-wise pairs | +| `t3.c`, `t3_nomul.py`, `t3_count.py`, `t3_rv32.c` | the t3 kernel; its disassembly checked for multipliers; its counters per token; the stream machine for rv32i | +| `record/t3-m1.txt`, `record/t3-m1-rerun.txt`, `record/t3-count.json` | every line of the t3 check on the M1, and of its rerun with the kernel and streams built afresh; the counters per token from `t3_count.py` | | `spare_row.py`, `spare_row_test.py`, `record/spare-m1.txt`, `record/spare-campaign.json` | exact-16 with every product and look checked, the fault campaign; its test on synthetic operands; every line of the M1 runs; every fault of the campaign | | `count_ops.py`, `report_census.py`; `count_spare.py`, `report_spare.py`, `record/census-spare.json`, `record/census-spare.md` | the census of every NumPy operation per token, sorted into steps and looks: of exact and exact-16; of exact-16 with and without the spare-row checks, and its tables | | `measure16.py`, `record/exact16-table.txt` | the width table, one row per run | diff --git a/proofs/exact-llm/exact_llm.py b/proofs/exact-llm/exact_llm.py index 80feb70..157b6ae 100644 --- a/proofs/exact-llm/exact_llm.py +++ b/proofs/exact-llm/exact_llm.py @@ -14,9 +14,10 @@ float32 baseline and onnxruntime where an export is pinned. usage: - python exact_llm.py check [--model M] [--routes f64,f32,i8] [--no-float] [--no-onnx] [--peaks FILE] + python exact_llm.py check [--model M] [--routes f64,f32,i8,t3] [--no-float] [--no-onnx] [--peaks FILE] digests, the hashes against expected*.sha256, baselines, distance - python exact_llm.py hash --engine exact|exact16-f64|exact16-f32|exact16-i8|float32|onnx [--batch N] [--threads N] + python exact_llm.py hash --engine exact|exact16-f64|exact16-f32|exact16-i8|exact16-t3|float32|onnx [--batch N] + [--threads N] [--model M] """ import gzip @@ -24,6 +25,7 @@ import json import math import os +import shutil import struct import sys import time @@ -515,6 +517,10 @@ def __init__(self, W=None, keep_f64=False, clamps=None): self.f64 = {} self.eps = [int(round(EPS * 2 ** (2 * A - 2 * s))) for s in range(A + 1)] # eps at 2^-(32 - 2s) + def mul(self, a, b, kind="element-wise"): + """The element-wise integer product a * b (int64): NumPy's here; the t3 route's by adds.""" + return a * b + def clamp(self, x, site, layer=None): c = self.C[site] a = np.abs(x) @@ -554,14 +560,14 @@ def rms(self, x, wname): top = np.abs(x).max(-1, keepdims=True) sh = np.maximum(0, np.frexp(top.astype(np.float64))[1] - NORM_BITS).astype(np.int64) # bit length - NORM_BITS xs = x >> sh - ss = (xs * xs).sum(-1, keepdims=True) # at 2^-(32 - 2 sh), below 2^62 + ss = self.mul(xs, xs).sum(-1, keepdims=True) # at 2^-(32 - 2 sh), below 2^62 ms = fdiv(ss, HIDDEN) assert (sh <= A).all() eps = np.array(self.eps, dtype=np.int64)[sh] r = isqrt(ms + np.maximum(eps, 1)) # sqrt(mean x^2 + eps) at 2^-(16 - sh) y = fdiv(x << (A - sh), r) # x / r at 2^-16 assert (np.abs(y) <= RMS_Y << A).all() # |x_i| <= sqrt(HIDDEN) rms: bounds.py - return rshift_round(y * np.asarray(self.W[wname]), A) + return rshift_round(self.mul(y, np.asarray(self.W[wname])), A) def rope_tables(self, pos): """cos and sin of each position, read once per forward pass: [B, 1, L, 32] at 2^-20.""" @@ -570,19 +576,20 @@ def rope_tables(self, pos): def rope(self, x, cs): c, s = cs x1, x2 = x[..., :HEAD // 2], x[..., HEAD // 2:] - return np.concatenate([rshift_round(x1 * c - x2 * s, ROPE), rshift_round(x2 * c + x1 * s, ROPE)], -1) + m = self.mul + return np.concatenate([rshift_round(m(x1, c) - m(x2, s), ROPE), rshift_round(m(x2, c) + m(x1, s), ROPE)], -1) def exp(self, t): k = np.minimum(t >> EXP_STEP, EXP_N) r = t & ((1 << EXP_STEP) - 1) - return (self.Et[k] * ((1 << EXP_STEP) - r) + self.Et[np.minimum(k + 1, EXP_N)] * r) >> EXP_STEP + return (self.mul(self.Et[k], (1 << EXP_STEP) - r) + self.mul(self.Et[np.minimum(k + 1, EXP_N)], r)) >> EXP_STEP def silu(self, x): top = SILU_N << SILU_STEP u = x + (SILU_LO << A) uc = np.clip(u, 0, top - 1) k, r = uc >> SILU_STEP, uc & ((1 << SILU_STEP) - 1) - g = (self.St[k] * ((1 << SILU_STEP) - r) + self.St[k + 1] * r) >> SILU_STEP + g = (self.mul(self.St[k], (1 << SILU_STEP) - r) + self.mul(self.St[k + 1], r)) >> SILU_STEP return np.where((u >= 0) & (u < top), g, np.where(u < 0, 0, x)) def cache_entry(self, k, v): @@ -631,7 +638,7 @@ def forward(self, ids, pos, cache, kvalid): h = self.prepare(self.rms(x, p + "post_attention_layernorm.weight").reshape(B * L, HIDDEN), "mlp input") g = self.clamp(self.linear(h, p + "mlp.gate_proj.weight", "mlp input"), "gate") u = self.clamp(self.linear(h, p + "mlp.up_proj.weight", "mlp input"), "up") - a = self.prepare(rshift_round(self.silu(g) * u, A), "down input") + a = self.prepare(rshift_round(self.mul(self.silu(g), u), A), "down input") x = x + self.linear(a, p + "mlp.down_proj.weight", "down input").reshape(B, L, HIDDEN) x = self.clamp(x, "residual", LAYERS) h = self.rms(x, "model.norm.weight").reshape(B * L, HIDDEN) @@ -661,7 +668,9 @@ def logits(self, h): PEEL16 = 2 CHUNK32 = 1024 # f32 route: inputs per chunk, 1,024 x 2^14 = 2^24 N_I8 = 33025 # i8 route: n 255^2 < 2^31 for n <= 33,025 -ROUTES = ("f64", "f32", "i8") +ROUTES = ("f64", "f32", "i8") # check's default +ROUTES_T3 = ("t3",) # by adds and subtracts (t3.c), where a C compiler is installed, not on Windows: --routes t3 +TOP16 = 15 # t3: highest signed-digit position of a 16-bit value; 16 STEP words per output row def shift_round(v, k): @@ -756,6 +765,181 @@ def __call__(self, a, b): return self.s.run(None, {"A": a, "B": b})[0].astype(np.int64) +T3_DIR = os.environ.get("EXACT_LLM_T3", os.path.join(CACHE, "t3")) # the compiled kernel and the weight streams +T3_THREADS = min(64, int(os.environ.get("EXACT_LLM_T3_THREADS", "0")) or os.cpu_count() or 1) # t3.c: at most 64 + + +def t3_missing(): + """Why t3.c is not built here, or None. It is built with a C compiler (`cc` or $CC) as a POSIX shared library + with threads; not on Windows.""" + if sys.platform == "win32": + return "not built on Windows" + if shutil.which(os.environ.get("CC", "cc")) is None: + return "no C compiler" + return None + + +class T3: + """The t3 route's kernel, t3.c, compiled once with the C compiler `cc` (or $CC) into T3_DIR and opened with + ctypes. Matrix products (mm) and element-wise products (mul) by add, subtract, shift by a constant and bitwise + logic. A registered weight matrix is compiled to its stream once and kept in T3_DIR/streams-/ + (2 bytes per nonzero digit plus 16 words per output row); any other right operand is recoded at each product. + Counters per kind: digit adds and subtracts, Horner adds, values recoded, recoding operations.""" + KINDS = ("weights", "peeled", "attention", "element-wise", "weight streams, built once") + + def __init__(self): + import ctypes + import subprocess + if t3_missing(): + raise SystemExit("t3: " + t3_missing()) + src = os.path.join(HERE, "t3.c") + tag = self.tag = hashlib.sha256(open(src, "rb").read()).hexdigest()[:12] + os.makedirs(T3_DIR, exist_ok=True) + self.path = os.path.join(T3_DIR, "t3-%s.%s" % (tag, "dylib" if sys.platform == "darwin" else "so")) + if not os.path.exists(self.path): + cc = os.environ.get("CC", "cc") + subprocess.run([cc, "-O2", "-shared", "-fPIC", "-pthread", src, "-o", self.path + ".part"], check=True) + os.replace(self.path + ".part", self.path) + L = self.lib = ctypes.CDLL(self.path) + i64, p = ctypes.c_int64, ctypes.c_void_p + L.t3_compile_masks.argtypes = [i64, i64, p, i64, p, p, p, p] + L.t3_compile_masks.restype = i64 + L.t3_compile_fill.argtypes = [i64, i64, p, p, p] + L.t3_run.argtypes = [p, p, i64, i64, i64, p, p, i64, p, p] + L.t3_mul.argtypes = [i64, p, p, p, ctypes.c_uint64, p] + self.cnt = {k: np.zeros(4, dtype=np.uint64) for k in self.KINDS} + self.model, self.mats = None, [] # registered weights: (first byte, last byte, name, rows, cols) + self.rowoffs = {} + self.buf = np.empty(0, np.uint16) + self.log = None # a list: one entry per product, when tracing + + @staticmethod + def ptr(a): + return a.ctypes.data + + def register(self, W): + """The current model's int16 weight matrices [in, out] (memory-mapped), whose streams are compiled once and + kept; they replace those registered before.""" + self.model = MODEL + self.mats = [(self.ptr(w), self.ptr(w) + w.nbytes, name, w.shape[0], w.shape[1]) + for name, w in W.items() if w.dtype == np.int16 and w.ndim == 2] + + def weight_key(self, b): + """(model, matrix, first column, columns) when b is a block of whole columns of a registered matrix.""" + if b.dtype != np.int16 or b.ndim != 2 or b.strides[1] != 2: + return None + at = self.ptr(b) + for lo, hi, name, rows, ld in self.mats: + c0 = (at - lo) // 2 + if lo <= at < hi and b.strides[0] == 2 * ld and b.shape[0] == rows and c0 + b.shape[1] <= ld: + return self.model, name, c0, b.shape[1] + return None + + def compile(self, b, kind): + """b [n_in, M] (|entries| <= 2^15 - 1) -> the stream (uint16) and the row offsets (int64 [M + 1]).""" + w = np.ascontiguousarray(b, dtype=np.int32) + n_in, M = w.shape + assert n_in < (1 << 14) + pm, mm = np.empty((M, n_in), np.uint16), np.empty((M, n_in), np.uint16) + rowoff = np.empty(M + 1, np.int64) + total = self.lib.t3_compile_masks(n_in, M, self.ptr(w), M, self.ptr(pm), self.ptr(mm), self.ptr(rowoff), + self.ptr(self.cnt[kind])) + stream = np.empty(total, np.uint16) + self.lib.t3_compile_fill(n_in, M, self.ptr(pm), self.ptr(mm), self.ptr(stream)) + return stream, rowoff + + def weight_stream(self, key, b): + model, name, c0, M = key # the stream format is t3.c's: one folder per t3.c and model + base = os.path.join(T3_DIR, "streams-" + self.tag, model, "%s.c%d.m%d" % (name, c0, M)) + if key not in self.rowoffs: + if not (os.path.exists(base + ".words") and os.path.exists(base + ".rowoff.npy")): + os.makedirs(os.path.dirname(base), exist_ok=True) + stream, rowoff = self.compile(b, "weight streams, built once") + with open(base + ".words.part", "wb") as f: + f.write(stream.astype(" 1: # rows split at equal shares of the stream's words + part = np.searchsorted(rowoff, np.linspace(0, rowoff[-1], threads + 1)).astype(np.int64) + part[0], part[-1] = 0, M + else: + part = np.array([0, M], np.int64) + before = self.cnt[kind].copy() + self.lib.t3_run(self.ptr(stream), self.ptr(rowoff), x.shape[0], Np, N, self.ptr(x), self.ptr(y), + part.size - 1, self.ptr(part), self.ptr(self.cnt[kind])) + if self.log is not None: + d = self.cnt[kind] - before + self.log.append((kind, x.shape[0], M, N, int(d[0]), int(d[1]))) + return y[:, :N] + + def mm2(self, a, b): + N, n_in = a.shape + M = b.shape[1] + if N == 0 or M == 0: + return np.zeros((N, M), np.int64) + x = np.ascontiguousarray(a.T, dtype=np.int32) + key = self.weight_key(b) + if key is not None: + stream, rowoff = self.weight_stream(key, b) + return self.run(stream, rowoff, x, "weights", T3_THREADS).T + stream, rowoff = self.compile(b, "attention") + return self.run(stream, rowoff, x, "attention", 1).T + + def mm(self, a, b): + """a @ b exactly (int64) for integer arrays with |entries| <= 2^15 - 1, a [..., N, n], b [..., n, M].""" + a, b = np.asarray(a), np.asarray(b) + if a.ndim == 2 and b.ndim == 2: + return self.mm2(a, b) + lead = np.broadcast_shapes(a.shape[:-2], b.shape[:-2]) + a = np.broadcast_to(a, lead + a.shape[-2:]) + b = np.broadcast_to(b, lead + b.shape[-2:]) + out = np.empty(lead + (a.shape[-2], b.shape[-1]), np.int64) + for i in np.ndindex(*lead): + out[i] = self.mm2(a[i], b[i]) + return out + + def mul(self, a, b, kind="element-wise"): + """a * b element-wise (int64, broadcast), the operand of smaller magnitude recoded into signed digits.""" + a, b = np.broadcast_arrays(np.asarray(a, dtype=np.int64), np.asarray(b, dtype=np.int64)) + out = np.empty(a.shape, np.int64) + if a.size == 0: + return out + ma, mb = int(np.abs(a).max()), int(np.abs(b).max()) + if ma < mb: + a, b, mb = b, a, ma + assert mb < (1 << 62) + a, b = np.ascontiguousarray(a), np.ascontiguousarray(b) + self.lib.t3_mul(a.size, self.ptr(a), self.ptr(b), self.ptr(out), 1 << mb.bit_length(), self.ptr(self.cnt[kind])) + return out + + +_T3 = [] + + +def t3_kernel(): + if not _T3: + _T3.append(T3()) + return _T3[0] + + def mm16(a, b, route, mmi=None): """a @ b exactly, as int64, for integer arrays with every |entry| <= 2^15 - 1 (a [..., N, n], b [..., n, M]). f64: one float64 product, |partial sums| <= n 2^30 < 2^53. @@ -764,10 +948,14 @@ def mm16(a, b, route, mmi=None): i8: unsigned: u = a + 2^15 in [1, 2^16 - 1], u = 256 u1 + u0 with u1, u0 in [0, 255] (uint8), the same for b; four int32 MatMulInteger products of uint8 by uint8, partial sums <= n 255^2 < 2^31; then, in int64, a b = u_a u_b - 2^15 (row sums of a) - 2^15 (column sums of b) - n 2^30. - Then a b (u_a u_b on the i8 route) = 2^16 G11 + 2^8 (G10 + G01) + G00, G_ij = a_i b_j, in int64.""" + Then a b (u_a u_b on the i8 route) = 2^16 G11 + 2^8 (G10 + G01) + G00, G_ij = a_i b_j, in int64. + t3: b in signed digits (non-adjacent form, positions 0 to 15), each column of b a stream of ADD i, SUB i and + STEP words; t3.c runs the stream on the columns of a with adds and subtracts only (class T3).""" n = a.shape[-1] if route == "f64": return (a.astype(np.float64) @ b.astype(np.float64)).astype(np.int64) + if route == "t3": + return t3_kernel().mm(a, b) # the digits in the operand's own type when it is int16 (the weights: no widening; every step below stays within # [-2^15, 2^15)), else in int64 a = np.asarray(a, dtype=np.int64) @@ -914,6 +1102,12 @@ def __init__(self, route="f64"): self.mmi = MatMulInteger() if route == "i8" else None self.extra = 0 # vectors whose rounding needed one more shift self.p_held = 0 # p values held at 2^15 - 1 + if route == "t3": + self.t3 = t3_kernel() + self.t3.register(self.W16) + + def mul(self, a, b, kind="element-wise"): + return self.t3.mul(a, b, kind) if self.route == "t3" else a * b def prepare(self, x, site): """A product input as every product that reads it needs it, made once: clamped at its site, its PEEL16 @@ -937,7 +1131,7 @@ def product(self, prep, name, dtype=np.int64): wc = w[:, c:c + 8192] # int16, memory-mapped T = np.left_shift(mm16(xb, wc, self.route, self.mmi), sh) for j in range(PEEL16): - T += xp[:, j:j + 1] * wc[idx[:, j]] # int64 times int16: int64 + T += self.mul(xp[:, j:j + 1], wc[idx[:, j]], "peeled") # int64 times int16: int64 blk = shift_round(T, g[None, c:c + 8192]) assert dtype == np.int64 or np.abs(blk).max() < (1 << 31) y[:, c:c + 8192] = blk @@ -1035,7 +1229,7 @@ def run_window(model, ids): def engine(name, threads=0): if name == "exact": return Exact() - if name in ("exact16-" + r for r in ROUTES): + if name in ("exact16-" + r for r in ROUTES + ROUTES_T3): return Exact16(name.split("-")[1]) if name == "float32": return Float32() @@ -1123,6 +1317,9 @@ def main(argv): if name.startswith("exact16"): print("route %s; vectors with one more shift %d; p held at 2^15 - 1: %d" % ( eng.mmi.name if eng.mmi else eng.route, eng.extra, eng.p_held)) + if eng.route == "t3": + print("t3 counters (digit adds, Horner adds, values recoded, recoding operations):", + {k: [int(v) for v in c] for k, c in eng.t3.cnt.items()}) return 0 if len(argv) > 1 and argv[1] == "text": name, save = arg("--engine", "exact"), arg("--save", None) @@ -1171,6 +1368,10 @@ def main(argv): print("exact16 logits %s continuations %s %s route %s, %.0f s, clamped %d" % ( h16l, h16c, "MATCH" if ok16 else "DIFFERENT", m16.mmi.name if m16.mmi else r, time.time() - t0, sum(m16.clamped.values()))) + if r == "t3": + print(" t3 kernel %s; digit adds, Horner adds, values recoded, recoding operations: %s" % ( + os.path.basename(m16.t3.path), "; ".join("%s %s" % (k, ", ".join("{:,}".format(int(v)) for v in c)) + for k, c in m16.t3.cnt.items()))) if not kept16: print(" peaks (real) / clamps: %s" % peaks_line(m16)) if arg("--peaks", None): diff --git a/proofs/exact-llm/record/exact16-360m-m1.txt b/proofs/exact-llm/record/exact16-360m-m1.txt index 1d6c863..8fa4e81 100644 --- a/proofs/exact-llm/record/exact16-360m-m1.txt +++ b/proofs/exact-llm/record/exact16-360m-m1.txt @@ -1,5 +1,6 @@ # SmolLM2-360M on the Apple M1 (macOS 27.2, Python 3.11.18, NumPy 2.4.6 with Accelerate, onnxruntime 1.32.0). -# One process per step, under one turn of the machine's job gate (Work/.build.nosync/llm/src/e19_360m.sh). Elapsed: /usr/bin/time real seconds. +# One process per step, under one turn of the machine's job gate (e19_360m.sh). Elapsed: /usr/bin/time real seconds. +# Shortened after the run: the gate script's path, to its file name, in the line above. Nothing else changed. # This run came before expected-16-SmolLM2-360M.sha256 existed: the pair was pinned from its lines, so check printed DIFFERENT (nothing to compare) and exited 1. == check versions: Python 3.11.18, NumPy 2.4.6, onnxruntime 1.32.0; darwin arm64 diff --git a/proofs/exact-llm/record/t3-count.json b/proofs/exact-llm/record/t3-count.json new file mode 100644 index 0000000..7cb42d1 --- /dev/null +++ b/proofs/exact-llm/record/t3-count.json @@ -0,0 +1,73 @@ +{ + "model": "SmolLM2-135M", + "window": "last position, context 64, batch 1", + "next token": 376, + "per kind": { + "weights": { + "digit adds and subtracts": 459184492, + "Horner adds": 6549504, + "values recoded": 0, + "recoding operations": 0 + }, + "peeled": { + "digit adds and subtracts": 1408559, + "Horner adds": 6442304, + "values recoded": 409344, + "recoding operations": 2247228 + }, + "attention": { + "digit adds and subtracts": 11875560, + "Horner adds": 1658880, + "values recoded": 1105920, + "recoding operations": 5900826 + }, + "element-wise": { + "digit adds and subtracts": 1271526, + "Horner adds": 4451136, + "values recoded": 289152, + "recoding operations": 1515461 + } + }, + "attention split": { + "scores q16.k16": { + "digit adds and subtracts": 5007459, + "Horner adds": 552960 + }, + "context p.vh": { + "digit adds and subtracts": 900330, + "Horner adds": 552960 + }, + "context p.vl": { + "digit adds and subtracts": 5967771, + "Horner adds": 552960 + } + }, + "weight streams": { + "nonzero digits": 459184492, + "output rows": 204672, + "words": 462459244, + "bytes": 924918488 + }, + "weight streams, built once (this run)": { + "digit adds and subtracts": 0, + "Horner adds": 0, + "values recoded": 129429504, + "recoding operations": 711700313 + }, + "seconds": { + "load and prefill of 63": 30.0, + "the counted forward": 2.7 + }, + "totals": { + "weight products: digit adds": 459184492, + "weight products: Horner adds": 6549504, + "peeled terms: digit adds": 1408559, + "peeled terms: Horner adds": 6442304, + "variable by variable (attention): digit adds": 11875560, + "variable by variable (attention): Horner adds": 1658880, + "element-wise: digit adds": 1271526, + "element-wise: Horner adds": 4451136, + "adds and subtracts in all products": 492841961, + "values recoded at run time (a sign look each)": 1804416 + } +} diff --git a/proofs/exact-llm/record/t3-m1-rerun.txt b/proofs/exact-llm/record/t3-m1-rerun.txt new file mode 100644 index 0000000..06adddf --- /dev/null +++ b/proofs/exact-llm/record/t3-m1-rerun.txt @@ -0,0 +1,40 @@ +branch t3-rebase-trial 272a6d80aa254d15b82c42e3a66fddbbed6a3647 (proofs/exact-llm: t3_compile_masks counts by an add per value behind a barrier, so gcc folds no multiply into it; t3_nomul.py prints each flagged instruction), on main 3d7e738 +== routes, load { 3.89 3.99 3.92 } +i8 product engine: i8-onnxruntime-1.32.0 +t3 kernel: t3/t3-21a279be7ff4.dylib +cases 42 (7 shapes x 6 kinds, int64 and int16 right operands), routes 4, different 0 +digit extremes, value (i8 digits) (f32 digits): 32767 (255, 255) (128, -1); -32767 (0, 1) (-128, 1); 32640 (255, 128) (128, -128); -32640 (0, 128) (-127, -128); 32639 (255, 127) (127, 127); -32641 (0, 127) (-128, 127); 32512 (255, 0) (127, 0); -32513 (0, 255) (-127, -1) +f32 digits: high -128..128, low -128..127 (int16 split the same: True); i8 digits of a + 2^15: high 0..255, low 0..255 (from int16 the same: True) +peel_index of [5, -9, 9, 3, -9, 1] k=2: [1, 2] (expect [1, 2]) +t3 stream, every value -32767..32767: exact; digit adds 356806, census formula 356806, at most 8 per value +t3 element-wise products, 200081 pairs and a broadcast: exact + 2.40 real 1.86 user 0.29 sys +exit 0 + 329383936 maximum resident set size +== check, load { 7.20 6.27 5.09 } +versions: Python 3.11.18, NumPy 2.4.6, onnxruntime 1.32.0; darwin arm64 +model SmolLM2-135M @ 93efa2f: 30 layers, width 576, 9/3 heads, feed-forward 1536, rotary base 100000, 134515008 weights +exact logits 30ed613b6036d08046fbdbf273a0fb7b8cc04fb284ca49e0296980eed9cee773 continuations a800ebadc848514e02bebbb8077d531cf0493acd2a4c173403ba1db43d7613cc MATCH + clamped values: 0 +exact16 16-bit weights, int32 embedding and norms, weight statistics: ready in 0 s +exact16 logits b0cc875fde396181a70e2349ca35a60c3ae91b23f1c1012083797352d30b4460 continuations a800ebadc848514e02bebbb8077d531cf0493acd2a4c173403ba1db43d7613cc MATCH route t3, 429 s, clamped 0 + t3 kernel t3-21a279be7ff4.dylib; digit adds, Horner adds, values recoded, recoding operations: weights 1,653,064,171,200, 23,578,214,400, 0, 0; peeled 5,079,175,319, 23,547,135,488, 1,473,638,400, 7,993,295,032; attention 19,327,619,286, 4,893,696,000, 1,738,506,240, 9,276,986,651; element-wise 4,326,722,524, 15,590,066,688, 973,555,200, 5,113,624,092; weight streams, built once 0, 0, 134,479,872, 739,466,822 + peaks (real) / clamps: attention input 11.429 / 131072; down input 2630.720 / 4194304; gate 50.741 / 32768; k 22.910 / 1024; logits input 53.630 / 8388608; mlp input 28.742 / 8388608; o input 9.531 / 8388608; q 22.075 / 1024; residual 19509.485 / 33554432; up 72.026 / 32768; v 12.972 / 8192 +check every pair MATCH, 504 s + 503.85 real 493.93 user 109.30 sys +exit 0 + 796868608 maximum resident set size +== nomul, load { 7.20 6.27 5.09 } +object t3/t3-21a279be7ff4.dylib +_t3_compile_masks 82 instructions; multiply or divide: none +_t3_compile_fill 169 instructions; multiply or divide: none +_t3_run 89 instructions; multiply or divide: none +_run_rows 109 instructions; multiply or divide: none +_t3_mul 53 instructions; multiply or divide: none +__stubs 15 instructions; multiply or divide: none +histogram, all functions (517 instructions): add 70, mov 61, cmp 38, b.eq 33, b 28, ldr 28, stp 21, strh 21, tst 21, ldp 20, sub 19, b.lt 13, ldrh 12, orr 10, adrp 9, csel 9, ldurh 9, and 8, str 7, bl 6, lsr 6, movi.2d 6, add.2d 5, br 5, lsl 5, ret 5, subs 5, cbz 4, cinc 4, b.ge 3, b.ne 3, add.4s 2, b.hi 2, b.hs 2, cbnz 2, cneg 2, eor 2, saddw.2d 2, saddw2.2d 2, sub.4s 2, b.lo 1, blr 1, ldur 1, neg 1, stur 1 +multiply, multiply-add or divide instructions: 0 +NO MULTIPLIER + 0.30 real 0.12 user 0.10 sys +exit 0 + 35356672 maximum resident set size diff --git a/proofs/exact-llm/record/t3-m1.txt b/proofs/exact-llm/record/t3-m1.txt new file mode 100644 index 0000000..113a8e8 --- /dev/null +++ b/proofs/exact-llm/record/t3-m1.txt @@ -0,0 +1,119 @@ +Sun Nov 15 16:41:44 PST 2026 +== routes +i8 product engine: i8-onnxruntime-1.32.0 +t3 kernel: $EXACT_LLM_T3/t3-f6d3700bf905.dylib +cases 42 (7 shapes x 6 kinds, int64 and int16 right operands), routes 4, different 0 +digit extremes, value (i8 digits) (f32 digits): 32767 (255, 255) (128, -1); -32767 (0, 1) (-128, 1); 32640 (255, 128) (128, -128); -32640 (0, 128) (-127, -128); 32639 (255, 127) (127, 127); -32641 (0, 127) (-128, 127); 32512 (255, 0) (127, 0); -32513 (0, 255) (-127, -1) +f32 digits: high -128..128, low -128..127 (int16 split the same: True); i8 digits of a + 2^15: high 0..255, low 0..255 (from int16 the same: True) +peel_index of [5, -9, 9, 3, -9, 1] k=2: [1, 2] (expect [1, 2]) +t3 stream, every value -32767..32767: exact; digit adds 356806, census formula 356806, at most 8 per value +t3 element-wise products, 200081 pairs and a broadcast: exact + 2.35 real 2.12 user 0.28 sys +exit 0 + 336134144 maximum resident set size +== nomul +object $EXACT_LLM_T3/t3-f6d3700bf905.dylib +_t3_compile_masks 76 instructions; multiply or divide: none +_t3_compile_fill 169 instructions; multiply or divide: none +_t3_run 89 instructions; multiply or divide: none +_run_rows 109 instructions; multiply or divide: none +_t3_mul 53 instructions; multiply or divide: none +__stubs 15 instructions; multiply or divide: none +histogram, all functions (511 instructions): add 68, mov 57, cmp 38, b.eq 33, b 28, ldr 28, stp 21, strh 21, tst 21, ldp 20, sub 19, b.lt 13, ldrh 12, orr 10, adrp 9, csel 9, ldurh 9, and 8, str 7, bl 6, lsr 6, movi.2d 6, add.2d 5, br 5, lsl 5, ret 5, subs 5, cbz 4, cinc 4, b.ge 3, b.ne 3, add.4s 2, b.hi 2, b.hs 2, cbnz 2, cneg 2, eor 2, saddw.2d 2, saddw2.2d 2, sub.4s 2, b.lo 1, blr 1, ldur 1, neg 1, stur 1 +multiply, multiply-add or divide instructions: 0 +NO MULTIPLIER + 0.68 real 0.19 user 0.15 sys +exit 0 + 32440320 maximum resident set size +== count +{ + "model": "SmolLM2-135M", + "window": "last position, context 64, batch 1", + "next token": 376, + "per kind": { + "weights": { + "digit adds and subtracts": 459184492, + "Horner adds": 6549504, + "values recoded": 0, + "recoding operations": 0 + }, + "peeled": { + "digit adds and subtracts": 1408559, + "Horner adds": 6442304, + "values recoded": 409344, + "recoding operations": 2247228 + }, + "attention": { + "digit adds and subtracts": 11875560, + "Horner adds": 1658880, + "values recoded": 1105920, + "recoding operations": 5900826 + }, + "element-wise": { + "digit adds and subtracts": 1271526, + "Horner adds": 4451136, + "values recoded": 289152, + "recoding operations": 1515461 + } + }, + "attention split": { + "scores q16.k16": { + "digit adds and subtracts": 5007459, + "Horner adds": 552960 + }, + "context p.vh": { + "digit adds and subtracts": 900330, + "Horner adds": 552960 + }, + "context p.vl": { + "digit adds and subtracts": 5967771, + "Horner adds": 552960 + } + }, + "weight streams": { + "nonzero digits": 459184492, + "output rows": 204672, + "words": 462459244, + "bytes": 924918488 + }, + "weight streams, built once (this run)": { + "digit adds and subtracts": 0, + "Horner adds": 0, + "values recoded": 129429504, + "recoding operations": 711700313 + }, + "seconds": { + "load and prefill of 63": 30.0, + "the counted forward": 2.7 + }, + "totals": { + "weight products: digit adds": 459184492, + "weight products: Horner adds": 6549504, + "peeled terms: digit adds": 1408559, + "peeled terms: Horner adds": 6442304, + "variable by variable (attention): digit adds": 11875560, + "variable by variable (attention): Horner adds": 1658880, + "element-wise: digit adds": 1271526, + "element-wise: Horner adds": 4451136, + "adds and subtracts in all products": 492841961, + "values recoded at run time (a sign look each)": 1804416 + } +} + 34.01 real 19.26 user 2.67 sys +exit 0 + 247906304 maximum resident set size +== check +versions: Python 3.11.18, NumPy 2.4.6, onnxruntime 1.32.0; darwin arm64 +model SmolLM2-135M @ 93efa2f: 30 layers, width 576, 9/3 heads, feed-forward 1536, rotary base 100000, 134515008 weights +exact logits 30ed613b6036d08046fbdbf273a0fb7b8cc04fb284ca49e0296980eed9cee773 continuations a800ebadc848514e02bebbb8077d531cf0493acd2a4c173403ba1db43d7613cc MATCH + clamped values: 0 +exact16 16-bit weights, int32 embedding and norms, weight statistics: ready in 0 s +exact16 logits b0cc875fde396181a70e2349ca35a60c3ae91b23f1c1012083797352d30b4460 continuations a800ebadc848514e02bebbb8077d531cf0493acd2a4c173403ba1db43d7613cc MATCH route f64, 437 s, clamped 0 + peaks (real) / clamps: attention input 11.429 / 131072; down input 2630.720 / 4194304; gate 50.741 / 32768; k 22.910 / 1024; logits input 53.630 / 8388608; mlp input 28.742 / 8388608; o input 9.531 / 8388608; q 22.075 / 1024; residual 19509.485 / 33554432; up 72.026 / 32768; v 12.972 / 8192 +exact16 logits b0cc875fde396181a70e2349ca35a60c3ae91b23f1c1012083797352d30b4460 continuations a800ebadc848514e02bebbb8077d531cf0493acd2a4c173403ba1db43d7613cc MATCH route t3, 1456 s, clamped 0 + t3 kernel t3-f6d3700bf905.dylib; digit adds, Horner adds, values recoded, recoding operations: weights 1,653,064,171,200, 23,578,214,400, 0, 0; peeled 5,079,175,319, 23,547,135,488, 1,473,638,400, 7,993,295,032; attention 19,327,619,286, 4,893,696,000, 1,738,506,240, 9,276,986,651; element-wise 4,326,722,524, 15,590,066,688, 973,555,200, 5,113,624,092; weight streams, built once 0, 0, 0, 0 +check every pair MATCH, 2250 s + 2250.89 real 788.97 user 229.70 sys +exit 0 + 793690112 maximum resident set size +Sun Nov 15 17:19:52 PST 2026 diff --git a/proofs/exact-llm/routes_test.py b/proofs/exact-llm/routes_test.py index 443b04e..288b7ab 100644 --- a/proofs/exact-llm/routes_test.py +++ b/proofs/exact-llm/routes_test.py @@ -17,6 +17,10 @@ L = X.LIM16 mmi = X.MatMulInteger() print("i8 product engine:", mmi.name) +# t3 (t3.c, by adds and subtracts) where a C compiler is installed, not on Windows +T3 = X.t3_missing() is None +ROUTES = X.ROUTES + (X.ROUTES_T3 if T3 else ()) +print("t3 kernel:", X.t3_kernel().path if T3 else X.t3_missing() + ": not tested") shapes = [((5, 576), (576, 300)), ((3, 1023), (1023, 7)), ((3, 1024), (1024, 7)), ((3, 1025), (1025, 7)), ((4, 1536), (1536, 64)), ((2, 3, 27, 64), (2, 3, 64, 40)), ((2, 3, 27, 600), (2, 3, 600, 64))] # digits (i8: of a + 2^15, high and low; f32: balanced high and low): 32767 (255, 255; 128, -1), -32767 (0, 1; @@ -41,14 +45,14 @@ a, b = rng.integers(0, L + 1, sa), rng.integers(-L, L + 1, sb) a, b = a.astype(np.int64), b.astype(np.int64) ref = np.matmul(a, b) # int64, exact: |sum| < 1536 * 2^30 - for r in X.ROUTES: + for r in ROUTES: for bb in (b, b.astype(np.int16)): # the weights come as int16 y = X.mm16(a, bb, r, mmi) if y.dtype != np.int64 or y.shape != ref.shape or not (y == ref).all(): bad += 1 print("DIFFERENT", r, sa, sb, kind, bb.dtype) print("cases %d (%d shapes x %d kinds, int64 and int16 right operands), routes %d, different %d" % ( - len(shapes) * len(KINDS), len(shapes), len(KINDS), len(X.ROUTES), bad)) + len(shapes) * len(KINDS), len(shapes), len(KINDS), len(ROUTES), bad)) # the extremes have the digits stated above u = EXTREMES + (1 << 15) h = (EXTREMES + 128) >> 8 @@ -72,4 +76,27 @@ # peel_index: ties go to the lower index x = np.array([[5, -9, 9, 3, -9, 1]], dtype=np.int64) print("peel_index of", x.tolist()[0], "k=2:", sorted(X.peel_index(x, 2)[0].tolist()), "(expect [1, 2])") +if T3: + k3 = X.t3_kernel() + # every 16-bit value through the stream: y = 1 . b for b = -32767 .. 32767, one input, 65,535 output rows; then + # the digits as the kernel forms them: nonzero digits per value against the census formula, popcount(h ^ (m + h)) + v = np.arange(-L, L + 1, dtype=np.int64) + y = k3.mm(np.ones((1, 1), np.int64), v[None]) + m = np.abs(v) + naf = np.bitwise_count(((m >> 1) ^ (m + (m >> 1))).astype(np.uint64)).astype(np.int64) + before = k3.cnt["attention"].copy() + k3.mm(np.ones((1, 1), np.int64), v[None]) + adds = int(k3.cnt["attention"][0] - before[0]) + ok3 = (y[0] == v).all() and adds == int(naf.sum()) + bad += 0 if ok3 else 1 + print("t3 stream, every value -32767..32767: %s; digit adds %d, census formula %d, at most %d per value" % ( + "exact" if (y[0] == v).all() else "DIFFERENT", adds, int(naf.sum()), int(naf.max()))) + # element-wise: random int64 pairs whose product fits, the extremes, and signs + a = rng.integers(-(1 << 31), 1 << 31, 200000) + b = rng.integers(-(1 << 31), 1 << 31, 200000) + e = np.array([0, 1, -1, L, -L, (1 << 31) - 1, -(1 << 31), (1 << 40) + 3, -(1 << 41) - 5]) + a, b = np.concatenate([a, np.repeat(e, e.size)]), np.concatenate([b, np.tile(e[::-1], e.size)]) + okm = (k3.mul(a, b) == a * b).all() and (k3.mul(b[:, None], a[None, :50]) == b[:, None] * a[None, :50]).all() + bad += 0 if okm else 1 + print("t3 element-wise products, %d pairs and a broadcast: %s" % (a.size, "exact" if okm else "DIFFERENT")) sys.exit(1 if bad else 0) diff --git a/proofs/exact-llm/t3.c b/proofs/exact-llm/t3.c new file mode 100644 index 0000000..4ecfd87 --- /dev/null +++ b/proofs/exact-llm/t3.c @@ -0,0 +1,243 @@ +/* t3.c: the t3 route of exact-16. Matrix products and element-wise products by add, subtract, shift by a constant + * and bitwise logic; no multiply instruction (t3_nomul.py reads the compiled object and checks). + * + * Signed digits. For m >= 0: h = m >> 1, s = m + h, c = h ^ s, plus = s & c, minus = h & c; then m = plus - minus, + * plus & minus = 0, and no two neighbouring positions are nonzero (the non-adjacent form, the fewest nonzero signed + * digits). A negative value is recoded as its magnitude 0 - v with plus and minus exchanged. |v| <= 2^15 - 1 has its + * digits at positions 0 to 15. + * + * The stream. A matrix W [n_in][M] (inputs by outputs) is compiled to 16-bit words, output row by output row; per + * row, for digit position k = 15 down to 0: ADD i for each input whose weight has digit +1 at k, then SUB i for each + * with -1 at k (i ascending), then STEP (k > 0) or EMIT (k = 0). + * word = op << 14 | i op 0 ADD, 1 SUB, 2 STEP, 3 EMIT; i < 2^14 + * The machine. State: N lanes (one per input vector), each with s (int32) and acc (int64); x[i][lane] the inputs. + * ADD i s = s + x[i] + * SUB i s = s - x[i] + * STEP acc = acc + acc; acc = acc + s; s = 0 (Horner over the digit positions: 2 adds) + * EMIT STEP, then y[row] = acc; acc = 0; row = row + 1 + * So y[row] = sum_k 2^k (sum of x over digit +1 at k - sum over digit -1 at k) = sum_i W[i][row] x[i], exactly: + * |s| <= n_in (2^15 - 1) < 2^31 for n_in < 2^16; after the STEP of position k, acc = sum_i x[i] (the digits of + * W[i][row] at positions k..15, as a number), each such number below 2^16 in magnitude: |acc| < n_in 2^31 < 2^47. + * The lanes run 8 at a time (N a multiple of 8, padded with zero inputs); the counters count the nreal lanes. + * + * Counters (uint64, added to, never reset here): cnt[0] digit adds and subtracts, lane by lane; cnt[1] Horner adds; + * cnt[2] values recoded; cnt[3] recoding operations (shift, add, xor, and, and, plus a subtract for a negative). + */ +#include +#include +#include + +#define OP_ADD 0u +#define OP_SUB 1u +#define OP_STEP 2u +#define OP_EMIT 3u +#define TOP 15 +/* An empty barrier on a register: the compiler may not fold a loop of adds around it into a multiply. */ +#define KEEP(v) __asm__ volatile("" : "+r"(v)) + +/* The signed digits of v (|v| < 2^62): *pl and *mi with v = *pl - *mi. Returns the operations used. */ +static inline uint64_t recode(int64_t v, uint64_t *pl, uint64_t *mi) { + uint64_t neg = v < 0; + uint64_t m = neg ? (uint64_t)0 - (uint64_t)v : (uint64_t)v; + uint64_t h = m >> 1, s = m + h, c = h ^ s; + uint64_t p = s & c, q = h & c; + *pl = neg ? q : p; + *mi = neg ? p : q; + return neg ? 6 : 5; +} + +/* Pass 1. W int32 [n_in][M], row stride ld (elements). Writes the masks Pm, Mm [M][n_in] (16 bits each) and + * rowoff[0..M] (word offsets of each row in the stream). Returns the stream length in words. */ +int64_t t3_compile_masks(int64_t n_in, int64_t M, const int32_t *W, int64_t ld, uint16_t *Pm, uint16_t *Mm, + int64_t *rowoff, uint64_t *cnt) { + const int32_t *wr = W; + uint64_t vals = 0, ops = 0; + for (int64_t i = 0; i < n_in; i++) { + uint16_t *pp = Pm + i, *mm = Mm + i; + for (int64_t j = 0; j < M; j++) { + uint64_t p, q; + ops += recode(wr[j], &p, &q); + vals += 1; + KEEP(ops); /* an add per value; without the barriers gcc */ + KEEP(vals); /* folds these counts into one n_in * M */ + *pp = (uint16_t)p; + *mm = (uint16_t)q; + pp += n_in; + mm += n_in; + } + wr += ld; + } + cnt[2] += vals; + cnt[3] += ops; + int64_t off = 0; + const uint16_t *pp = Pm, *mm = Mm; + for (int64_t j = 0; j < M; j++) { + rowoff[j] = off; + off += TOP + 1; /* the STEP and EMIT words */ + for (int64_t i = 0; i < n_in; i++) { + uint32_t d = (uint32_t)pp[i] | (uint32_t)mm[i]; + while (d) { /* one per nonzero digit */ + off += d & 1u; + d >>= 1; + } + } + pp += n_in; + mm += n_in; + } + rowoff[M] = off; + return off; +} + +/* Pass 2: the words, from the masks. */ +void t3_compile_fill(int64_t n_in, int64_t M, const uint16_t *Pm, const uint16_t *Mm, uint16_t *stream) { + uint16_t *o = stream; + const uint16_t *pp = Pm, *mm = Mm; + for (int64_t j = 0; j < M; j++) { + uint32_t bit = 1u << TOP; + while (bit) { + for (int64_t i = 0; i < n_in; i++) + if (pp[i] & bit) + *o++ = (uint16_t)((OP_ADD << 14) | (uint32_t)i); + for (int64_t i = 0; i < n_in; i++) + if (mm[i] & bit) + *o++ = (uint16_t)((OP_SUB << 14) | (uint32_t)i); + bit >>= 1; + *o++ = (uint16_t)((bit ? OP_STEP : OP_EMIT) << 14); + } + pp += n_in; + mm += n_in; + } +} + +/* The machine on rows [r0, r1) of a stream. x int32 [n_in][N]; y int64 [M][N]; N a multiple of 8. */ +typedef struct { + const uint16_t *stream; + const int64_t *rowoff; + int64_t r0, r1, n_in, N, nreal; + const int32_t *x; + int64_t *y; + uint64_t cnt[4]; +} job_t; + +typedef int32_t v8 __attribute__((vector_size(32))); /* 8 lanes of s */ +typedef int64_t v8l __attribute__((vector_size(64))); /* 8 lanes of acc */ + +static void *run_rows(void *arg) { + job_t *jb = (job_t *)arg; + int64_t N = jb->N; + const int32_t **xrow = (const int32_t **)malloc(((size_t)jb->n_in + 1) * sizeof(*xrow)); + const int32_t *xp = jb->x; + for (int64_t i = 0; i < jb->n_in; i++) { /* row addresses by repeated addition */ + xrow[i] = xp; + xp += N; + KEEP(xp); + } + int64_t *yrow = jb->y; + for (int64_t r = 0; r < jb->r0; r++) { + yrow += N; + KEEP(yrow); /* an add per row, not one multiply */ + } + uint64_t adds = 0, horner = 0; + for (int64_t r = jb->r0; r < jb->r1; r++) { + const uint16_t *start = jb->stream + jb->rowoff[r]; + int64_t left = jb->nreal; /* real lanes not yet run */ + for (int64_t c = 0; c < N; c += 8) { /* the row's words once per 8 lanes */ + uint64_t nl = left < 8 ? (uint64_t)left : 8u; + left -= 8; + const uint16_t *p = start; + v8 s = {0, 0, 0, 0, 0, 0, 0, 0}; + v8l acc = {0, 0, 0, 0, 0, 0, 0, 0}; + for (;;) { + uint32_t w = *p++; + uint32_t op = w >> 14; + if (op == OP_ADD) { + v8 xv; + __builtin_memcpy(&xv, xrow[w & 0x3FFFu] + c, sizeof xv); + s += xv; + adds += nl; + } else if (op == OP_SUB) { + v8 xv; + __builtin_memcpy(&xv, xrow[w & 0x3FFFu] + c, sizeof xv); + s -= xv; + adds += nl; + } else { + acc = acc + acc; + acc = acc + __builtin_convertvector(s, v8l); + s = (v8){0, 0, 0, 0, 0, 0, 0, 0}; + horner += nl; + horner += nl; + if (op == OP_EMIT) + break; + } + } + __builtin_memcpy(yrow + c, &acc, sizeof acc); + } + yrow += N; + } + jb->cnt[0] = adds; + jb->cnt[1] = horner; + free(xrow); + return NULL; +} + +/* y [M][N] = W^T x for the compiled stream, rows split at part[0..nthreads] (part[0] = 0, part[nthreads] = M). */ +void t3_run(const uint16_t *stream, const int64_t *rowoff, int64_t n_in, int64_t N, int64_t nreal, const int32_t *x, int64_t *y, + int64_t nthreads, const int64_t *part, uint64_t *cnt) { + job_t jobs[64]; + pthread_t th[64]; + if (nthreads > 64) + nthreads = 64; + for (int64_t t = 0; t < nthreads; t++) { + job_t *jb = &jobs[t]; + jb->stream = stream; + jb->rowoff = rowoff; + jb->r0 = part[t]; + jb->r1 = part[t + 1]; + jb->n_in = n_in; + jb->N = N; + jb->nreal = nreal; + jb->x = x; + jb->y = y; + if (nthreads == 1) + run_rows(jb); + else + pthread_create(&th[t], NULL, run_rows, jb); + } + for (int64_t t = 0; t < nthreads; t++) { + if (nthreads > 1) + pthread_join(th[t], NULL); + cnt[0] += jobs[t].cnt[0]; + cnt[1] += jobs[t].cnt[1]; + } +} + +/* out[e] = a[e] b[e] for n elements, b recoded at run time (|b| < 2^62, its digits at positions 0..K, top = 2^K): + * Horner from position K down: acc = acc + acc, then + a at a digit +1, - a at a digit -1. Arithmetic mod 2^64, as + * NumPy's int64 product. */ +void t3_mul(int64_t n, const int64_t *a, const int64_t *b, int64_t *out, uint64_t top, uint64_t *cnt) { + uint64_t adds = 0, horner = 0, ops = 0; + for (int64_t e = 0; e < n; e++) { + uint64_t pl, mi; + ops += recode(b[e], &pl, &mi); + uint64_t x = (uint64_t)a[e], acc = 0, bit = top; + while (bit) { + acc = acc + acc; + horner += 1; + KEEP(horner); + if (pl & bit) { + acc = acc + x; + adds += 1; + } + if (mi & bit) { + acc = acc - x; + adds += 1; + } + bit >>= 1; + } + out[e] = (int64_t)acc; + } + cnt[0] += adds; + cnt[1] += horner; + cnt[2] += (uint64_t)n; + cnt[3] += ops; +} diff --git a/proofs/exact-llm/t3_count.py b/proofs/exact-llm/t3_count.py new file mode 100644 index 0000000..2a3ccd4 --- /dev/null +++ b/proofs/exact-llm/t3_count.py @@ -0,0 +1,84 @@ +"""The t3 route's own counters, per token at the last position (context 64), as the census counts it: a prefill of +63 tokens, then one forward of one token with the key-value cache; the counters of that one forward. + +usage: python t3_count.py [OUT.json] + +Tokens: the first 64 of the evaluation text (text()). The weights' counts do not depend on the tokens (every stream +word runs once per token); the attention's and the element-wise counts do (the digits of activations). +Kinds (T3.KINDS): weights (the 16-bit matrix products, from the streams), peeled (the 2 whole inputs per product +input vector times their 16-bit weight rows), attention (q16.k16, p.vh, p.vl: one operand recoded at run time), +element-wise (norm squares, norm weight, rotary, the exp and SiLU interpolations, SiLU(gate) times up). +Per kind: digit adds and subtracts, Horner adds (acc + acc, then + s), values recoded, recoding operations. +""" +import json +import os +import sys +import time + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import exact_llm as X # noqa: E402 + + +def streams_digits(k): + """Nonzero digits and output rows over every weight stream compiled so far.""" + digits = rows = 0 + for key, ro in k.rowoffs.items(): + rows += ro.shape[0] - 1 + digits += int(ro[-1]) - (X.TOP16 + 1) * (ro.shape[0] - 1) + return digits, rows + + +def main(argv): + t0 = time.time() + m = X.Exact16("t3") + k = m.t3 + L = 64 + ids = np.array(X.tokenizer().encode(X.text())[:L], dtype=np.int64)[None] + pos = np.arange(L)[None] + i = np.arange(L) + kvalid = (i[None, None, :] <= i[None, :, None]) + _, cache = m.forward(ids[:, :L - 1], pos[:, :L - 1], None, kvalid[:, :L - 1, :L - 1]) + t1 = time.time() + before = {kind: c.copy() for kind, c in k.cnt.items()} + k.log = [] + t2 = time.time() + logits, _ = m.forward(ids[:, L - 1:], pos[:, L - 1:], cache, kvalid[:, L - 1:, :]) + t3 = time.time() + d = {kind: [int(v) for v in (k.cnt[kind] - before[kind])] for kind in k.KINDS} + att = [e for e in k.log if e[0] == "attention"] + per = 3 * X.KV # per layer: KV scores, KV p.vh, KV p.vl, in that order + assert len(att) == per * X.LAYERS + split = {"scores q16.k16": [0, 0], "context p.vh": [0, 0], "context p.vl": [0, 0]} + for j, e in enumerate(att): + name = list(split)[(j % per) // X.KV] + split[name][0] += e[4] + split[name][1] += e[5] + digits, rows = streams_digits(k) + names = ["digit adds and subtracts", "Horner adds", "values recoded", "recoding operations"] + out = {"model": X.MODEL, "window": "last position, context %d, batch 1" % L, + "next token": int(np.argmax(logits[0, 0])), + "per kind": {kind: dict(zip(names, d[kind])) for kind in k.KINDS if kind != "weight streams, built once"}, + "attention split": {n: {"digit adds and subtracts": v[0], "Horner adds": v[1]} for n, v in split.items()}, + "weight streams": {"nonzero digits": digits, "output rows": rows, + "words": digits + (X.TOP16 + 1) * rows, "bytes": 2 * (digits + (X.TOP16 + 1) * rows)}, + "weight streams, built once (this run)": dict(zip(names, [int(v) for v in k.cnt["weight streams, built once"]])), + "seconds": {"load and prefill of 63": round(t1 - t0, 1), "the counted forward": round(t3 - t2, 2)}} + w, p, a, e = (d[x] for x in ("weights", "peeled", "attention", "element-wise")) + out["totals"] = { + "weight products: digit adds": w[0], "weight products: Horner adds": w[1], + "peeled terms: digit adds": p[0], "peeled terms: Horner adds": p[1], + "variable by variable (attention): digit adds": a[0], "variable by variable (attention): Horner adds": a[1], + "element-wise: digit adds": e[0], "element-wise: Horner adds": e[1], + "adds and subtracts in all products": w[0] + w[1] + p[0] + p[1] + a[0] + a[1] + e[0] + e[1], + "values recoded at run time (a sign look each)": p[2] + a[2] + e[2]} + s = json.dumps(out, indent=1) + print(s) + if len(argv) > 1: + open(argv[1], "w").write(s + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/proofs/exact-llm/t3_nomul.py b/proofs/exact-llm/t3_nomul.py new file mode 100644 index 0000000..e81b7ae --- /dev/null +++ b/proofs/exact-llm/t3_nomul.py @@ -0,0 +1,96 @@ +"""The t3 kernel has no multiplier: the compiled object's disassembly, function by function, holds no multiply, +multiply-add or divide instruction, scalar or vector, integer or floating point. + +usage: python t3_nomul.py [OBJECT] (default: the kernel exact_llm.py builds and loads, t3-.dylib or .so) + +Reads `objdump -d` (LLVM's or GNU's). Prints each function's instruction count, each flagged instruction with the +three on either side, and the whole object's histogram; exit 1 if any function holds an instruction of the lists below. arm64 and x86-64 mnemonics; Apple's objdump writes a +vector arrangement into the mnemonic (mul.4s), GNU's into the operands, and both are read by the stem. +""" +import collections +import os +import re +import subprocess +import sys + +ARM64_MUL = set("""mul madd msub mneg smull umull smulh umulh smaddl smsubl umaddl umsubl smnegl umnegl + mla mls smull2 umull2 smlal smlal2 smlsl smlsl2 umlal umlal2 umlsl umlsl2 sqdmulh sqrdmulh sqdmull sqdmull2 + sqdmlal sqdmlal2 sqdmlsl sqdmlsl2 sqrdmlah sqrdmlsh pmul pmull pmull2 sdot udot usdot sudot smmla ummla usmmla + fmul fmulx fmla fmls fnmul fmadd fmsub fnmadd fnmsub bfdot bfmmla bfmlalb bfmlalt fmmla + fmlal fmlal2 fmlsl fmlsl2 fcmla fdot bfmlal""".split()) +ARM64_DIV = {"sdiv", "udiv", "fdiv", "fsqrt", "frecpe", "frecps", "frsqrte", "frsqrts"} +X86_MUL = ("mul", "imul", "mulx", "pmul", "vpmul", "pmadd", "vpmadd", "vfmadd", "vfmsub", "vfnmadd", "vfnmsub", + "vpdp", "mulss", "mulsd", "mulps", "mulpd", "vmul", "pclmul", "vpclmul", "dpps", "dppd", "vdpp") +X86_DIV = ("div", "idiv", "divss", "divsd", "divps", "divpd", "vdiv", "sqrt", "vsqrt") +# RISC-V M extension (and its 64-bit word forms); on rv32i a C multiply becomes a call to a library routine instead +RISCV_MUL = {"mulh", "mulhsu", "mulhu", "mulw", "divu", "rem", "remu", "divw", "divuw", "remw", "remuw"} +# library multiply and divide routines (libgcc, compiler-rt, the ARM EABI); Mach-O writes one more leading underscore +LIBCALL = re.compile(r"(?:$", line.strip()) + if m: + name = m.group(1) + funcs.setdefault(name, collections.Counter()) + lines.setdefault(name, []) + continue + m = re.match(r"^\s*[0-9a-fA-F]+:\s+([a-z][a-z0-9.]*)", line) # address, then the mnemonic + if m and name is not None: + funcs[name][m.group(1)] += 1 + lines[name].append((m.group(1), line.strip())) + return funcs, lines + + +def main(argv): + if len(argv) > 1: + path = argv[1] + else: + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import exact_llm as X + path = X.t3_kernel().path + funcs, lines = disassemble(path) + total, bad = collections.Counter(), [] + print("object %s" % path) + for name, h in funcs.items(): + if not h: + continue + total.update(h) + hits = sorted(m for m in h if flagged(m)) + bad += [(name, m, h[m]) for m in hits] + print("%-22s %5d instructions; multiply or divide: %s" % ( + name, sum(h.values()), ", ".join("%s %d" % (m, h[m]) for m in hits) or "none")) + ins = lines[name] + for k, (mn, _) in enumerate(ins): # each flagged instruction, with three on either side + if flagged(mn): + for kk in range(max(0, k - 3), min(len(ins), k + 4)): + print(" %s %s" % ("=>" if kk == k else " ", ins[kk][1])) + print("histogram, all functions (%d instructions): %s" % ( + sum(total.values()), ", ".join("%s %d" % kv for kv in sorted(total.items(), key=lambda kv: (-kv[1], kv[0]))))) + print("multiply, multiply-add or divide instructions: %d%s" % ( + sum(n for _, _, n in bad), "" if not bad else " (" + "; ".join("%s in %s x%d" % (m, f, n) for f, m, n in bad) + ")")) + print("NO MULTIPLIER" if not bad else "MULTIPLIER FOUND") + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/proofs/exact-llm/t3_rv32.c b/proofs/exact-llm/t3_rv32.c new file mode 100644 index 0000000..7605ad0 --- /dev/null +++ b/proofs/exact-llm/t3_rv32.c @@ -0,0 +1,39 @@ +/* t3_rv32.c: the t3 stream machine of t3.c, one lane, freestanding, for a core without a multiplier (rv32i). + * No libc, no threads, no vector types. The same words: op << 14 | i, op 0 ADD, 1 SUB, 2 STEP, 3 EMIT. + * ADD i s = s + x[i] + * SUB i s = s - x[i] + * STEP acc = acc + acc; acc = acc + s; s = 0 + * EMIT STEP, then y[row] = acc; acc = 0; row = row + 1 + * s is 32 bits, acc 64 bits (on rv32i two registers: add, sltu for the carry, add). Returns the words run. + * + * Build (Homebrew LLVM 23, no emulator here): + * clang --target=riscv32-unknown-elf -march=rv32i -mabi=ilp32 -O2 -ffreestanding -nostdlib -c t3_rv32.c + * python t3_nomul.py t3_rv32.o + */ +#include + +uint32_t t3_rv32_run(const uint16_t *p, int32_t rows, const int32_t *x, int64_t *y) { + int32_t s = 0; + int64_t acc = 0; + uint32_t words = 0; + while (rows > 0) { + uint32_t w = *p++; + uint32_t op = w >> 14; + words += 1; + if (op == 0u) { + s += x[w & 0x3FFFu]; + } else if (op == 1u) { + s -= x[w & 0x3FFFu]; + } else { + acc = acc + acc; + acc = acc + (int64_t)s; + s = 0; + if (op == 3u) { + *y++ = acc; + acc = 0; + rows -= 1; + } + } + } + return words; +}