diff --git a/.github/workflows/REVIEW.md b/.github/workflows/REVIEW.md index a385ce8870..012d13838b 100644 --- a/.github/workflows/REVIEW.md +++ b/.github/workflows/REVIEW.md @@ -23,6 +23,8 @@ lane's platform, in its PR body or commit message; a green run of that lane on t commit is that evidence.** A check that fails for a non-defect turns a green branch red for everyone. -**A step in `pages.yml` that names the deployed games spells the list as a `for g in ; do` -loop, never inline.** `examples/games/REVIEW.das` (repo root) reads the deployed list from -those loops; an inline list beside a surviving loop is one nothing cross-checks. +**A step in `pages.yml` that names more than one id under `examples/games/` spells them as a +`for g in ; do` loop, never inline.** `examples/games/REVIEW.das` (repo root) reads the +deployed list from those loops; an inline list beside a surviving loop is one nothing +cross-checks. An example outside `examples/games/` (a graphics showcase, a dasLLAMA app) is not +on that list and is named on its own. diff --git a/.github/workflows/extended_checks.yml b/.github/workflows/extended_checks.yml index 006b8589ca..690fe1bff7 100644 --- a/.github/workflows/extended_checks.yml +++ b/.github/workflows/extended_checks.yml @@ -327,7 +327,8 @@ jobs: ci/nano_arm_build.sh "$BIN/daslang" - name: "Build standalone executables" - # core: the formatter and lint steps below run the das-fmt / das-lint exes this builds + # core: the formatter step below runs the das-fmt exe this builds; das-lint is built so the + # shipped lint.exe is proven to compile (the bundle smoke test runs it on daslib) if: matrix.role != 'modules' run: | set -eux @@ -403,7 +404,6 @@ jobs: # outside the PR, and >=16 files parallelize into per-file ledgers - both produce # false stales. The nightly whole-tree -j 1 pass owns staleness. $BIN/daslang ./utils/lint/main.das -- "${CHANGED[@]}" --quiet --disable LINT019 - $BIN/das-lint.exe "${CHANGED[@]}" --quiet --disable LINT019 - name: "Run ast-verify on changed .das files" if: matrix.role != 'modules' diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 7843b44a5f..e81ace6350 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -115,7 +115,7 @@ jobs: # The workflow file itself is a build input (it carries the configure flags # for web/build64 below), so hash it too — a change to the build steps must # miss a stale cache rather than skip the rebuild and reuse old archives. - key: wasm-${{ runner.os }}-${{ hashFiles('.github/workflows/pages.yml', 'CMakeLists.txt', 'web/CMakeLists.txt', 'web/stage_playground_imgui_samples.cmake', 'src/**', 'include/**', 'modules/**', 'daslib/**', 'utils/daslang/**', 'utils/daspkg/**', 'examples/games/**', 'examples/graphics/**', 'examples/pathTracer/**', 'web/examples/ui/samples/**', 'tutorials/**', 'dastest/**') }} + key: wasm-${{ runner.os }}-${{ hashFiles('.github/workflows/pages.yml', 'CMakeLists.txt', 'web/CMakeLists.txt', 'web/stage_playground_imgui_samples.cmake', 'src/**', 'include/**', 'modules/**', 'daslib/**', 'utils/daslang/**', 'utils/daspkg/**', 'examples/games/**', 'examples/graphics/**', 'examples/pathTracer/**', 'examples/dasLLAMA/storyteller/**', 'web/examples/ui/samples/**', 'tutorials/**', 'dastest/**') }} # Host daslang — one games-capable build (dasLLVM for cross-compile + dasGlfw # + dasOpenGL shared modules for the games) that serves BOTH das2rst and the @@ -316,6 +316,18 @@ jobs: echo "WARNING: physarum_lab wasm build failed — its examples card will be skipped this deploy." fi + # 7. dasllama.io/examples storyteller — dasLLAMA itself compiled to wasm64 (the story + # model + KittenTTS, both served as prepared .dlim images the dasllama.io stage step + # downloads from the rolling `dasllama-web` release). Its .das_package turns the GPU + # modules off, so the host needs no Metal/Vulkan. Non-fatal like the games; the + # dasllama.io stage step stages the card's page only when all three outputs exist. + if ./bin/daslang utils/daspkg/main.das -- \ + release wasm --root examples/dasLLAMA/storyteller --out "$REPO/web/output64/examples"; then + echo "storyteller wasm build OK" + else + echo "WARNING: storyteller wasm build failed — the dasllama.io storyteller page will be a placeholder this deploy." + fi + - name: "Compile-gate the curated playground samples" # The per-sample wasm64 builds used to live here and doubled as the proof # that every curated sample still compiles. They now build on demand @@ -555,6 +567,8 @@ jobs: fi - name: "Stage dasllama.io for deployment" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # gh release download of the browser examples' model sets run: | set -eux mkdir -p _site_dasllama/files @@ -566,7 +580,33 @@ jobs: python3 site-dasllama/test_metadata.py cp site-dasllama/index.html _site_dasllama/ + cp site-dasllama/examples.html _site_dasllama/ cp site-dasllama/stories.html _site_dasllama/ + + # The browser examples: the wasm64 build from step 7 above plus its model set from the + # rolling `dasllama-web` release (prepared .dlim images + the phoneme packs, ~70 MB - + # build artifacts like the tutorial MP4s, never in git; re-minted and re-uploaded with + # --clobber on every IMAGE_VERSION bump). Served same-origin, so the page needs no CORS; + # the Caddy vhost adds the isolation headers under /examples/ (caddy.snippet). A missing + # build or set stages a placeholder page, so the card never 404s. + mkdir -p _site_dasllama/examples/storyteller + if [ -f web/output64/examples/storyteller/storyteller.html ] \ + && [ -f web/output64/examples/storyteller/storyteller.js ] \ + && [ -f web/output64/examples/storyteller/storyteller.wasm ] \ + && gh release download dasllama-web --repo GaijinEntertainment/daScript \ + --dir _site_dasllama/examples/storyteller/models --clobber \ + && test -s _site_dasllama/examples/storyteller/models/stories15M.dlim \ + && test -s _site_dasllama/examples/storyteller/models/kitten-nano.dlim \ + && test -s _site_dasllama/examples/storyteller/models/tts_g2p_en_us.bin \ + && test -s _site_dasllama/examples/storyteller/models/tts_postag.bin; then + cp web/output64/examples/storyteller/storyteller.html _site_dasllama/examples/storyteller/ + cp web/output64/examples/storyteller/storyteller.js _site_dasllama/examples/storyteller/ + cp web/output64/examples/storyteller/storyteller.wasm _site_dasllama/examples/storyteller/ + else + echo "WARNING: storyteller build or model set incomplete — staging a placeholder so the card doesn't 404." + rm -rf _site_dasllama/examples/storyteller/models + printf '%s' 'Storyteller — building

This example is being rebuilt and will be available shortly.

' > _site_dasllama/examples/storyteller/storyteller.html + fi mkdir -p _site_dasllama/stories cp -R site-dasllama/stories/. _site_dasllama/stories/ cp site-dasllama/ladder.html _site_dasllama/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 06a9a9b819..4b17962023 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1446,6 +1446,7 @@ if (NOT ${DAS_TESTS_DISABLED}) # AOT examples are enabled — gate the inclusion accordingly. if(NOT ${DAS_AOT_EXAMPLES_DISABLED}) include(examples/pathTracer/CMakeLists.txt) + include(examples/dasLLAMA/CMakeLists.txt) endif() if(NOT ${DAS_AOT_EXAMPLES_DISABLED} AND NOT (WIN32 AND CMAKE_SIZEOF_VOID_P EQUAL 4)) include(tests/aot/CMakeLists.txt) diff --git a/daslib/ARCHITECTURE.md b/daslib/ARCHITECTURE.md index 97fe1f9703..9dba5c40eb 100644 --- a/daslib/ARCHITECTURE.md +++ b/daslib/ARCHITECTURE.md @@ -222,3 +222,25 @@ Three companions carry a concern each; a section number is unique across all fou - **`_table_index_and_init` exists for infer's `default_init_containers` rewrite of non-store `tab[key]`** - it has no daslib call site and is not dead. + +## 37. tune {#tune-shells-module-exists} + +- **Every tune annotation - `[tune]`, `[tuned]`, `[tune_perm]`, `[tune_scope]`, `[tune_policy]`, + `[llvm_code]` - registers HERE and nowhere else, as a shell that forwards to the framework in + `llvm/daslib/llvm_tune` when that module is visible and answers "unchanged" when it is not.** A + module's macro state is per module, so a shell and the `[tune_scope]` bank it reads have to + share one macro context; `tune_status` and `log_tune_status` are shells of the same kind. +- **The shells decide with `typeinfo module_exists(llvm_tune)`, a pair with the trait's C++ half + in `src/ast/ast_infer_type.cpp` (`module_exists`): visible from the compiling module, which is + what the `require ?llvm llvm/daslib/llvm_tune public` line above them decided.** Nothing fails + when the two sides drift - `tests/language/optional_require.das` pins the trait's visibility + half on fixtures, and `tests/daslib/test_tune_shells.das` is tier-agnostic by design - so the pair + is recorded here: a trait that answered off the process registry instead would take the + framework arm inside a tool's nested compile, where the framework is not mounted. +- **The `llvm/daslib/*` files resolve in every build that carries `modules/dasLLVM/`; only the + witness decides what `?llvm` and `module_exists(llvm_tune)` answer.** The framework's das files + compile without their C++ module, so a direct `require llvm/daslib/llvm_tune` in a build without + dasLLVM compiles with the shells inert and no `_variants()` registry - a program that + reads one is framework-only and says so with that direct require. `daslib/just_in_time` keeps + its direct require for the opposite reason: a static host that never registered the witness + still runs the JIT through the LLVM library, and the guard would switch it off. diff --git a/daslib/REVIEW.md b/daslib/REVIEW.md index 173f4ad68a..0dc0e20b8a 100644 --- a/daslib/REVIEW.md +++ b/daslib/REVIEW.md @@ -262,3 +262,17 @@ a value. marker annotation to the instance it finishes, and returns unchanged when it sees that marker.** A chain of template ancestors delivers one copy of the annotation per level, so a macro without the marker runs a second time over a structure it already rewrote. + +**A diff that adds or renames a public intrinsic in `aarch64_neon.das`, `x64_avx.das` or +`f16_cvt.das` changes its row in the lookup table of `modules/dasLLVM/daslib/llvm_jit_intrin.das` +(repo root) that is gated on the CPU feature the instruction needs, in the same change.** The JIT +recognizes these calls by function name alone, so a missing row runs the portable fallback on +every target with every test green, and a row in a table gated on a weaker feature emits an +instruction the generic machine cannot select - the `tests/jit_tests` twins compare hardware +against a reference body and catch a wrong emitter, never a missing or misplaced one. + +**A function in `aarch64_neon.das`, `x64_avx.das` or `f16_cvt.das` that is the wider- or +newer-tier variant of the same math another function in that file computes calls that function - +never a hand-written scalar loop.** A box whose CPU lacks the tier then runs the recognized +narrower path instead of scalar code. A tier function with no same-math sibling (`smmla`'s 2x8 +by 8x2 shape) keeps its own fallback body. diff --git a/modules/dasLLVM/daslib/aarch64_neon.das b/daslib/aarch64_neon.das similarity index 80% rename from modules/dasLLVM/daslib/aarch64_neon.das rename to daslib/aarch64_neon.das index 73ac587eb3..62e3dc225a 100644 --- a/modules/dasLLVM/daslib/aarch64_neon.das +++ b/daslib/aarch64_neon.das @@ -6,11 +6,13 @@ module aarch64_neon shared public // // Every function here computes the correct result on ANY target via its daslang body — so the // interpreter, AOT, and non-ARM JIT builds all just run that body. On an aarch64 JIT target the -// LLVM JIT recognizes the call BY NAME (see g_aarch64_intrin_lookup in llvm_jit_intrin.das, gated -// on g_target_is_aarch64) and emits the hardware NEON instruction in place of the call. There is +// LLVM JIT recognizes the call BY NAME (the g_aarch64_*_intrin_lookup tables in llvm_jit_intrin.das, +// gated on the target arch and on the instruction's feature: TBL is baseline, SDOT needs DotProd, +// SMMLA needs i8mm) and emits the hardware NEON instruction in place of the call. There is // nothing to annotate and nothing to require from the llvm module — the recognition is purely // name-based at JIT codegen time, so the fallback runs verbatim everywhere the intrinsic isn't -// emitted. Callers therefore get the fast path for free on arm64 and stay correct elsewhere. +// emitted: a foreign arch, or a generic-CPU aarch64 exe whose baseline machine lacks the feature. +// Callers therefore get the fast path for free on arm64 and stay correct elsewhere. //! Signed dot-product (`SDOT` / `@llvm.aarch64.neon.sdot`): load 16 int8 from each of `w` and `x` //! (16-wide unaligned loads), multiply elementwise, and accumulate consecutive groups of 4 products @@ -43,17 +45,16 @@ def sdot4_w(acc : int4; w : int4; x : int8 const?) : int4 { var s3 = 0 unsafe { for (e in range(4)) { - s0 += sxb(w.x, e) * int(x[e]) - s1 += sxb(w.y, e) * int(x[4 + e]) - s2 += sxb(w.z, e) * int(x[8 + e]) - s3 += sxb(w.w, e) * int(x[12 + e]) + s0 += sign_extend_byte(w.x, e) * int(x[e]) + s1 += sign_extend_byte(w.y, e) * int(x[4 + e]) + s2 += sign_extend_byte(w.z, e) * int(x[8 + e]) + s3 += sign_extend_byte(w.w, e) * int(x[12 + e]) } } return acc + int4(s0, s1, s2, s3) } -// sign-extend byte e (0..3, little-endian) of a packed int lane -def private sxb(lane : int; e : int) : int => (((lane >> (e * 8)) & 0xff) ^ 0x80) - 0x80 +def private sign_extend_byte(lane : int; e : int) : int => (((lane >> (e * 8)) & 0xff) ^ 0x80) - 0x80 //! Byte table lookup of LOW nibbles (`TBL` / `vqtbl1q_s8(table, nib & 0x0f)`): each of 16 `nib` //! bytes has its low nibble (masked to 0..15, never out of table) looked up in `table`; the 16 @@ -123,10 +124,10 @@ def sdot4_laneq_w(acc : int4; w : int4; x : int8 const?; lane : int) : int4 { let x1 = int(x[b + 1]) let x2 = int(x[b + 2]) let x3 = int(x[b + 3]) - r.x += sxb(w.x, 0) * x0 + sxb(w.x, 1) * x1 + sxb(w.x, 2) * x2 + sxb(w.x, 3) * x3 - r.y += sxb(w.y, 0) * x0 + sxb(w.y, 1) * x1 + sxb(w.y, 2) * x2 + sxb(w.y, 3) * x3 - r.z += sxb(w.z, 0) * x0 + sxb(w.z, 1) * x1 + sxb(w.z, 2) * x2 + sxb(w.z, 3) * x3 - r.w += sxb(w.w, 0) * x0 + sxb(w.w, 1) * x1 + sxb(w.w, 2) * x2 + sxb(w.w, 3) * x3 + r.x += sign_extend_byte(w.x, 0) * x0 + sign_extend_byte(w.x, 1) * x1 + sign_extend_byte(w.x, 2) * x2 + sign_extend_byte(w.x, 3) * x3 + r.y += sign_extend_byte(w.y, 0) * x0 + sign_extend_byte(w.y, 1) * x1 + sign_extend_byte(w.y, 2) * x2 + sign_extend_byte(w.y, 3) * x3 + r.z += sign_extend_byte(w.z, 0) * x0 + sign_extend_byte(w.z, 1) * x1 + sign_extend_byte(w.z, 2) * x2 + sign_extend_byte(w.z, 3) * x3 + r.w += sign_extend_byte(w.w, 0) * x0 + sign_extend_byte(w.w, 1) * x1 + sign_extend_byte(w.w, 2) * x2 + sign_extend_byte(w.w, 3) * x3 } return r } diff --git a/daslib/daspkg.das b/daslib/daspkg.das index 28e510f2c1..a380551d25 100644 --- a/daslib/daspkg.das +++ b/daslib/daspkg.das @@ -131,6 +131,7 @@ struct ReleaseSpec { emcc_args : array //!< extra emcc flags this module contributes to a `daspkg release wasm` link (e.g. "-sUSE_GLFW=3"); gathered from every referenced module embed_paths : array //!< "src@dst" embed pairs this module needs in the wasm MEMFS (e.g. a HUD font); gathered from every referenced module web_shell : string //!< optional emcc `--shell-file` HTML for `release wasm`; empty = daslang's minimal canvas shell + wasm_disabled_modules : array //!< host module folders (`dasvulkan`) kept unloaded on the `release wasm` cross-compile, so a guarded `require ?name` resolves as absent there wasm_build_command : string //!< shell command that builds this EXTERNAL module's wasm archives (run with cwd = module dir); empty = in-tree module built by `daspkg build --wasm` wasm_archives : array //!< archives required by wasm link; build outputs are module-relative, SDK archives may be named from the staged wasm lib dir requires_jit : bool //!< app is JIT-only (per-box [tune]/[llvm_code] kernels); a baked -exe would run broken, so `daspkg release` refuses it @@ -212,6 +213,13 @@ def release_embed_file(src, dst : string) { _release_spec.embed_paths |> push("{src}@{dst}") } +//! Keep a host module (folder name, case-insensitive) out of the `release wasm` cross-compile: +//! it runs on the host, where a module with no wasm build still satisfies a guarded `require ?name`. +//! dashv is disabled by the rail; an app reaching vulkan, das_metal or das_accelerate names those. +def release_wasm_disable_module(name : string) { + _release_spec.wasm_disabled_modules |> push(name) +} + //! Choose the emcc `--shell-file` HTML wrapper for `release wasm`. Empty (the //! default) ships daslang's minimal canvas-only shell. A site can point this at //! its own shell (e.g. the forge canvas frame) for in-page embedding. diff --git a/modules/dasLLVM/daslib/f16_cvt.das b/daslib/f16_cvt.das similarity index 96% rename from modules/dasLLVM/daslib/f16_cvt.das rename to daslib/f16_cvt.das index 009fa78303..629d5f7d1f 100644 --- a/modules/dasLLVM/daslib/f16_cvt.das +++ b/daslib/f16_cvt.das @@ -25,7 +25,7 @@ def f16_to_f32(h : uint) : float { } let sign = (h & 0x8000u) << 16u var bits : uint - if (exp == 0x1Fu) { // inf / nan + if (exp == 0x1Fu) { bits = sign | 0x7F800000u | (mant << 13u) } else { // normal: rebias exponent 15 -> 127 (+112), widen mantissa 10 -> 23 bits bits = sign | ((exp + 112u) << 23u) | (mant << 13u) @@ -39,7 +39,7 @@ def f32_to_f16(f : float) : uint { let bits = unsafe(reinterpret(f)) let sign = (bits >> 16u) & 0x8000u let absb = bits & 0x7FFFFFFFu - if (absb > 0x7F800000u) { // nan -> quiet nan + if (absb > 0x7F800000u) { return sign | 0x7E00u } if (absb >= 0x47800000u) { // inf, or >= 65536 (overflows past f16 max) diff --git a/daslib/tune.das b/daslib/tune.das new file mode 100644 index 0000000000..c6bf82d485 --- /dev/null +++ b/daslib/tune.das @@ -0,0 +1,145 @@ +options gen2 +options indenting = 4 +options no_aot = true + +module tune shared public + +require daslib/ast +require daslib/ast_boost + +require ?llvm llvm/daslib/llvm_code public +require ?llvm llvm/daslib/llvm_tune public + +//! The six kernel-tuning annotations - `[tune_perm]`, `[tune_companion]`, `[tune]`, +//! `[tune_scope]`, `[tune_policy]` and `[llvm_code]` - plus the default-policy infer pass, as +//! shells every program can spell. The framework that implements them lives in +//! `llvm/daslib/llvm_tune` and `llvm/daslib/llvm_code`, reached through the `?llvm` guards +//! above and re-exported, so `require daslib/tune` gives exactly what `require llvm/daslib/llvm_tune` +//! gave. The guard is the `llvm` C++ witness module, compiled in exactly when the build is +//! configured with dasLLVM - the das files being on disk never counts; the guarded require is +//! then what makes `module_exists` below answer for this compile (`module_exists`, not +//! `builtin_module_exists`: visible from this module, the same answer inside a tool's nested +//! compile). Every tune annotation registers HERE and nowhere else, because a +//! module's macro state is per-module: the `[tune_scope]` bank and the pass that reads it have +//! to run in one macro context. Without dasLLVM every apply is a no-op - no sidecar read, no +//! stamp, no `_variants()` registry, no tuner spawn - and the reference body IS the function. +//! Framework semantics, arguments and the sidecar contract: `skills/tune.md`. + +[function_macro(name = "tune_perm")] +class private TunePermShell : AstFunctionAnnotation { + def override apply(var func : FunctionPtr; var group : ModuleGroup; + args : AnnotationArgumentList; var errors : das_string) : bool { + static_if (typeinfo module_exists(llvm_tune)) { + return tune_perm_apply(func, group, args, errors) + } else { + return true + } + } +} + +[function_macro(name = "tune_companion")] +class private TuneCompanionShell : AstFunctionAnnotation { + def override apply(var func : FunctionPtr; var group : ModuleGroup; + args : AnnotationArgumentList; var errors : das_string) : bool { + static_if (typeinfo module_exists(llvm_tune)) { + return tune_companion_apply(func, group, args, errors) + } else { + return true + } + } +} + +[function_macro(name = "tune")] +class private TuneShell : AstFunctionAnnotation { + def override apply(var func : FunctionPtr; var group : ModuleGroup; + args : AnnotationArgumentList; var errors : das_string) : bool { + static_if (typeinfo module_exists(llvm_tune)) { + return tune_apply(func, group, args, errors) + } else { + return true + } + } +} + +[structure_macro(name = "tune_scope")] +class private TuneScopeShell : AstStructureAnnotation { + def override apply(var st : StructurePtr; var group : ModuleGroup; + args : AnnotationArgumentList; var errors : das_string) : bool { + static_if (typeinfo module_exists(llvm_tune)) { + return tune_scope_apply(st, group, args, errors) + } else { + return true + } + } +} + +[function_macro(name = "tune_policy")] +class private TunePolicyShell : AstFunctionAnnotation { + def override apply(var func : FunctionPtr; var group : ModuleGroup; + args : AnnotationArgumentList; var errors : das_string) : bool { + static_if (typeinfo module_exists(llvm_tune)) { + return tune_policy_apply(func, group, args, errors) + } else { + return true + } + } +} + +[function_macro(name = "llvm_code")] +class private LlvmCodeShell : AstFunctionAnnotation { + def override apply(var func : FunctionPtr; var group : ModuleGroup; + args : AnnotationArgumentList; var errors : das_string) : bool { + static_if (typeinfo module_exists(llvm_code)) { + return llvm_code_apply(func, group, args, errors) + } else { + return true + } + } +} + +[infer_macro] +class private TuneDefaultPolicyShell : AstPassMacro { + def override apply(prog : ProgramPtr; mod : Module?) : bool { + static_if (typeinfo module_exists(llvm_tune)) { + return tune_default_policy_apply(prog, mod) + } else { + return false + } + } +} + +//! Log the tune status table at `LOG_INFO`, prefixed by `appName`: `/ kernels tuned for +//! this box`, one line per function, a `--tune` hint when any kernel sits on a fallback tier. +//! Silent when the table is empty, and in a build without the framework - nothing was stamped. +[arch(at = "ARCHITECTURE.md#tune-shells-module-exists")] +def log_tune_status(appName : string) { + var inscope rows <- tune_status_rows() + if (empty(rows)) { + return + } + var tuned = 0 + for (r in rows) { + tuned += (r.source == "manifest" || r.source == "profile") ? 1 : 0 + } + to_log(LOG_INFO, "{appName}: tune status ({tuned}/{length(rows)} kernels tuned for this box):\n") + for (r in rows) { + to_log(LOG_INFO, " {r.fname}: {empty(r.suffix) ? "reference" : r.suffix} ({r.source})\n") + } + if (tuned < length(rows)) { + to_log(LOG_INFO, "{appName}: run with --tune to tune the remaining kernels for this box\n") + } +} + +//! The status table as plain strings - the framework's row type never crosses the guard, so the +//! caller above is one function in every build; empty without the framework. +def private tune_status_rows() : array> { + var rows : array> + static_if (typeinfo module_exists(llvm_tune)) { + var inscope st <- tune_status() + rows |> reserve(length(st)) + for (s in st) { + rows |> push((fname = s.fname, suffix = s.suffix, source = s.source)) + } + } + return <- rows +} diff --git a/modules/dasLLVM/daslib/x64_avx.das b/daslib/x64_avx.das similarity index 96% rename from modules/dasLLVM/daslib/x64_avx.das rename to daslib/x64_avx.das index f7b47b0db8..ac819ea90c 100644 --- a/modules/dasLLVM/daslib/x64_avx.das +++ b/daslib/x64_avx.das @@ -109,10 +109,6 @@ def mx4_dequant32(lut : int8 const?; wn : uint8 const?; var dst : int8?) { } } -// ===== AVX kernel matrix tiers (VNNI / AVX-512), each gated on its own g_target_x64_* flag ===== -// Every body DELEGATES to the established intrinsic of the same math, so a box without the tier -// degrades to the recognized ymm sign-trick path rather than to scalar code. - //! dot32 with the VNNI dot step: same signature, same exact int4 result, lane for lane. With //! 256-bit VNNI (AVX-VNNI, or AVX512-VNNI+VL) the VPMADDUBSW+VPMADDWD pair becomes ONE VPDPBUSD //! (u8xs8 quad dot into i32, no 16-bit intermediate); sign trick and high/low fold are unchanged. diff --git a/doc/REVIEW.md b/doc/REVIEW.md index 0b31b8883d..c3733304cb 100644 --- a/doc/REVIEW.md +++ b/doc/REVIEW.md @@ -1,9 +1,8 @@ # doc (the Sphinx manual) Code Review Checklist -**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** RST -mechanics are `skills/internal/documentation_rst.md` (repo root) and the prose itself is -`skills/internal/tutorial_prose.md` (repo root); how a page reads is reviewed there, not -here. +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture +doc: `skills/internal/documentation_rst.md` (repo root). The prose itself is +`skills/internal/tutorial_prose.md` (repo root); how a page reads is reviewed there, not here. **Weakening `REVIEW.das` (beside this file) is a defect - a diff that makes it read fewer files, or report less than it did before the change with the same `ALLOWED_HOSTS`, is weaker.** diff --git a/doc/reflections/das2rst.das b/doc/reflections/das2rst.das index aeed88d1b8..551f26c5ef 100644 --- a/doc/reflections/das2rst.das +++ b/doc/reflections/das2rst.das @@ -145,7 +145,7 @@ require strudel/strudel_sf2_voice require strudel/strudel_pattern require spirv/spirv_reflect require dasllama/dasllama -require llvm/daslib/llvm_tune // documentation context keeps [tune] fully inert +require ?llvm llvm/daslib/llvm_tune // nolint:STYLE030,LINT019 — the framework's is_building_documentation check keeps [tune] inert; a build without dasLLVM has no framework to keep inert def document_module_math(_root : string) { var mod = get_module("math") @@ -199,7 +199,7 @@ get_value|insert_clone|emplace_new|insert_default|emplace_default|get_with_defau group_by_regex("Binary serializer", mod, %regex~(binary_load|binary_save)$%%), group_by_regex("Path and command line", mod, %regex~(get_command_line_arguments|with_argv)$%%), group_by_regex("Time and date", mod, %regex~(get_time_usec|ref_time_ticks|get_clock|get_time_nsec|mktime|iso8601_now|format_time|clock)$%%), - group_by_regex("Platform queries", mod, %regex~(get_context_share_counter|das_is_dll_build|das_is_exceptions_enabled|is_standalone_exe|get_platform_name|get_running_platform_name|get_cross_platform_name|get_architecture_name|cpu_supports)$%%), + group_by_regex("Platform queries", mod, %regex~(get_context_share_counter|das_is_dll_build|das_is_exceptions_enabled|is_standalone_exe|get_platform_name|get_running_platform_name|get_cross_platform_name|get_architecture_name|get_target_triple|get_target_architecture_name|cpu_supports)$%%), group_by_regex("String formatting", mod, %regex~(fmt)$%%), hide_group(group_by_regex("Temp string reclaim", mod, %regex~(_temp_string_result)$%%)), group_by_regex("Argument consumption", mod, %regex~(consume_argument)$%%), @@ -401,7 +401,7 @@ def document_module_dasllama_tts(_root : string) { } var mod = [find_module("dasllama_tts"), find_module("dasllama_tts_types")] var groups <- array( - group_by_regex("Loading and capabilities", mod, %regex~(load_tts_model|caps|tts_voice_lang|finalize)$%%), + group_by_regex("Loading and capabilities", mod, %regex~(load_tts_model|g2p_pack_path|caps|tts_voice_lang|finalize)$%%), group_by_regex("Text front end", mod, %regex~(tts_normalize|tts_phonemize|tts_chunks)$%%), group_by_regex("Synthesis", mod, %regex~(synthesize|synthesize_stream)$%%), group_by_regex("The served lane", mod, %regex~(set_tts_q8|reset_tts_q8|tts_serves_q8)$%%), diff --git a/doc/source/reference/language/generic_programming.rst b/doc/source/reference/language/generic_programming.rst index 85bda003eb..c8f5655206 100644 --- a/doc/source/reference/language/generic_programming.rst +++ b/doc/source/reference/language/generic_programming.rst @@ -152,7 +152,8 @@ All ``typeinfo`` traits can operate on either an expression or a ``type`` arg * ``typeinfo builtin_function_exists(expr)`` — true if a ``@@function`` exists * ``typeinfo builtin_annotation_exists(expr)`` — true if an annotation type exists -* ``typeinfo builtin_module_exists(expr)`` — true if a module is loaded +* ``typeinfo builtin_module_exists(expr)`` — true if a module is registered in the process: a linked C++ module, or a shared das module promoted by a running script +* ``typeinfo module_exists(expr)`` — true if the module is visible from the compiling module: the module itself, one it requires, or one a require re-exports ``public``; the answer is the same inside a tool's nested compile, which is what a guarded ``require`` needs * ``typeinfo is_argument(expr)`` — true if the expression is a function argument * ``typeinfo mangled_name(expr)`` — returns the mangled name of a ``@@function`` diff --git a/doc/source/reference/language/modules.rst b/doc/source/reference/language/modules.rst index 36f43fcad2..d3d4335a4c 100644 --- a/doc/source/reference/language/modules.rst +++ b/doc/source/reference/language/modules.rst @@ -99,8 +99,16 @@ unavailable, the require is skipped silently. This enables the contributor patte in ``llvm/daslib/llvm_user_modules.das``: a ``require ? `` line pulls a das package's registration glue exactly when that package is mounted. ``typeinfo builtin_module_exists`` additionally sees **shared das modules** -(``module X shared``) compiled earlier in the program, so the usual ``static_if`` -guard works for das-module contributors too. +(``module X shared``) promoted by the running script - but a tool that compiles the +same program in a nested context (lint, the language server, a test harness) +promotes nothing, and there the trait answers false for every das target. Guard the +use of a guarded das target with ``typeinfo module_exists(target)`` instead: it asks +whether the target is visible from the compiling module, which is exactly what the +guarded ``require`` decided, so the answer is the same on both rails. Two edges of that +rule: the name must be the module's own name (an alias from ``require X as Y`` answers +false, like any name the library does not know), and inside a generic function the +compiling module is the one INSTANTIATING the generic, not the one that declared it - a +guard in a generic body reads the caller's view. Pair it with :ref:`typeinfo builtin_module_exists ` to guard code that uses the optional target's symbols — ``static_if`` drops the untaken diff --git a/doc/source/reference/tutorials/dasLLAMA_16_text_to_speech.rst b/doc/source/reference/tutorials/dasLLAMA_16_text_to_speech.rst index 1c5c11970e..7af66adfc2 100644 --- a/doc/source/reference/tutorials/dasLLAMA_16_text_to_speech.rst +++ b/doc/source/reference/tutorials/dasLLAMA_16_text_to_speech.rst @@ -38,11 +38,15 @@ One loader, the packs beside the model so no family name appears in the call. Two more files travel with the model: ``tts_g2p.bin`` and ``tts_postag.bin``, and the loader reads them from the model's directory. A file that is not a TTS family panics and names the -architecture it found. +architecture it found. ``g2p_pack_path`` names the phoneme pack the loader +takes from a directory: ``tts_g2p.bin`` when it is there, else the +American-only ``tts_g2p_en_us.bin`` - the smaller pack a browser build ships, +which reads no British voice. .. code-block:: das var m <- load_tts_model("kitten-nano.gguf") // tts_g2p.bin + tts_postag.bin sit beside it + print("phoneme pack: {base_name(g2p_pack_path(dir_name("kitten-nano.gguf")))}\n") caps(): ask, don't assume ========================= diff --git a/doc/source/stdlib/handmade/function-builtin-get_target_architecture_name-0x5a46cb8acf2fed30.rst b/doc/source/stdlib/handmade/function-builtin-get_target_architecture_name-0x5a46cb8acf2fed30.rst new file mode 100644 index 0000000000..e051e987f2 --- /dev/null +++ b/doc/source/stdlib/handmade/function-builtin-get_target_architecture_name-0x5a46cb8acf2fed30.rst @@ -0,0 +1 @@ +Returns the architecture component of the compile target: `"wasm64"`, `"wasm32"`, `"x86_64"`, `"arm64"` (for both the `aarch64-` and `arm64-` triples), `"x86"`, `"arm"`, or `"unknown"`; the host's own architecture when no `--jit-target` names another. Folded to a constant like `get_target_triple`, it is the switch a kernel uses to pick a per-target body. diff --git a/doc/source/stdlib/handmade/function-builtin-get_target_triple-0x6b5a8940c4c62c1.rst b/doc/source/stdlib/handmade/function-builtin-get_target_triple-0x6b5a8940c4c62c1.rst new file mode 100644 index 0000000000..9be8661cc0 --- /dev/null +++ b/doc/source/stdlib/handmade/function-builtin-get_target_triple-0x6b5a8940c4c62c1.rst @@ -0,0 +1 @@ +Returns the LLVM target triple the program is being compiled for, such as `"wasm64-unknown-emscripten"`, read from the command line's `--jit-target` (the last one wins); `""` for the host. The compiler folds the call to a constant, so a `static_if` or an `if` on it costs nothing at run time and the code for the other target is not compiled. diff --git a/examples/dasLLAMA/CMakeLists.txt b/examples/dasLLAMA/CMakeLists.txt new file mode 100644 index 0000000000..3503ec65aa --- /dev/null +++ b/examples/dasLLAMA/CMakeLists.txt @@ -0,0 +1,111 @@ +### dasLLAMA AOT host: the daslang command-line binary with the engine's CPU modules compiled +### to C++ ahead of time and linked in - the portable tier without the JIT. +### +### cmake --build build --target dasllama_aot +### bin/dasllama_aot -use-aot examples/dasLLAMA/run.das -- +### +### Opt-in (EXCLUDE_FROM_ALL): ~90 AOT TUs. The GPU tiers, the LLVM IR generators and the +### exchange server are not AOT'd - they interpret as scaffolding (the Vulkan classes alone +### emit 40 MB of C++), and a call from AOT'd code into them is inlined C++ regardless. +### +### The same target exists in the wasm64 web build (web/CMakeLists.txt adds this tree as a +### subdirectory): the repo's HOST daslang generates the C++ (the `daslang` target is a .wasm +### there), em++ compiles it, and the engine's das files embed into MEMFS beside daslib - +### web/output64/dasllama_aot.{js,wasm}, driven by examples/dasLLAMA/wasm/run_node.js. No +### LLVM module is embedded, so `require ?llvm` is false and no tune framework exists in it. +### wasm32 is out: the host bakes 8-byte-pointer layouts into the generated C++. + +IF(DAS_LLAMA_INCLUDED AND NOT ${DAS_AOT_EXAMPLES_DISABLED} AND NOT (WIN32 AND CMAKE_SIZEOF_VOID_P EQUAL 4) AND NOT (EMSCRIPTEN AND NOT DAS_WASM_MEMORY64)) + if(EMSCRIPTEN) + if(DEFINED DAS_HOST_DASLANG_OVERRIDE) + set(DASLLAMA_AOT_TOOL ${DAS_HOST_DASLANG_OVERRIDE}) + else() + set(DASLLAMA_AOT_TOOL ${PROJECT_SOURCE_DIR}/bin/daslang) + endif() + else() + set(DASLLAMA_AOT_TOOL daslang) + endif() + + FILE(GLOB DASLLAMA_AOT_ENGINE_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "modules/dasLLAMA/dasllama/*.das") + list(FILTER DASLLAMA_AOT_ENGINE_FILES EXCLUDE REGEX "dasllama_(vulkan|metal|math_vulkan|gemm_gen|gemm_register|exchange)") + if(NOT DAS_ACCELERATE_INCLUDED) + # the Accelerate driver's C++ calls the das_accelerate binding by symbol; without the + # module (every non-Apple build, wasm included) the guarded require never loads it + list(FILTER DASLLAMA_AOT_ENGINE_FILES EXCLUDE REGEX "dasllama_math_accelerate") + endif() + + # the daslib modules the CPU path calls at run time that libDaScriptAot does not carry + SET(DASLLAMA_AOT_DASLIB_FILES + daslib/aarch64_neon.das + daslib/x64_avx.das + daslib/f16_cvt.das + daslib/jobque_boost.das + daslib/jobque_profile.das + daslib/fio.das + daslib/array_boost.das + daslib/math_bits.das + daslib/sha_256.das + daslib/archive.das + daslib/clargs.das + daslib/env_registry.das + daslib/tty.das + daslib/toml.das + daslib/safe_addr.das + ) + + # an edit to any engine module changes its dependents' semantic hashes: every engine TU + # regenerates on every engine edit (the daslib glob is a global dependency already). The + # generator reads each C++ module's aotRequire out of the loaded shared module, so those + # are inputs too - a module without the override AOT-disables its whole require closure + FILE(GLOB DASLLAMA_AOT_ALL_ENGINE_FILES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/modules/dasLLAMA/dasllama/*.das") + SET(DAS_AOT_EXTRA_DEPENDS ${DASLLAMA_AOT_ALL_ENGINE_FILES}) + if(NOT EMSCRIPTEN) + LIST(APPEND DAS_AOT_EXTRA_DEPENDS ${DAS_DYN_MODULES_LIBS}) + endif() + # one generated set serves the native and the wasm64 build (same host tool, same sources, + # 8-byte pointers on both): the two trees regenerate the same files + add_custom_target(dasllama_aot_engine) + SET(DASLLAMA_AOT_ENGINE_GENERATED_SRC) + DAS_AOT_LIB("${DASLLAMA_AOT_ENGINE_FILES}" DASLLAMA_AOT_ENGINE_GENERATED_SRC dasllama_aot_engine ${DASLLAMA_AOT_TOOL}) + add_custom_target(dasllama_aot_daslib) + SET(DASLLAMA_AOT_DASLIB_GENERATED_SRC) + DAS_AOT_LIB("${DASLLAMA_AOT_DASLIB_FILES}" DASLLAMA_AOT_DASLIB_GENERATED_SRC dasllama_aot_daslib ${DASLLAMA_AOT_TOOL}) + unset(DAS_AOT_EXTRA_DEPENDS) + SOURCE_GROUP_FILES("aot generated" DASLLAMA_AOT_ENGINE_GENERATED_SRC) + SOURCE_GROUP_FILES("aot generated" DASLLAMA_AOT_DASLIB_GENERATED_SRC) + + add_executable(dasllama_aot EXCLUDE_FROM_ALL ${DAS_DASCRIPT_MAIN_SRC} + ${DASLLAMA_AOT_ENGINE_GENERATED_SRC} + ${DASLLAMA_AOT_DASLIB_GENERATED_SRC} + ) + target_sources(dasllama_aot PRIVATE ${PROJECT_SOURCE_DIR}/src/misc/alloc_tracker_overrides.cpp) + if(EMSCRIPTEN) + # libDaScriptAot's stubs are generated by the `daslang` target, a .wasm here - the thirteen + # daslib modules it carries interpret in this host instead (none is on the inference path) + TARGET_LINK_LIBRARIES(dasllama_aot libDaScript ${SRC_LIBRARIES} ${DAS_MODULES_LIBS}) + ADD_DEPENDENCIES(dasllama_aot dasllama_aot_engine dasllama_aot_daslib) + set_target_properties(dasllama_aot PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${DAS_WEB_OUTPUT_DIR}) + # the engine's das files and its descriptor ride MEMFS beside daslib (the web build embeds + # daslib and the other module trees for every executable in its scope) + target_link_options(dasllama_aot PRIVATE + "SHELL:--embed-file ${PROJECT_SOURCE_DIR}/modules/dasLLAMA/dasllama@modules/dasLLAMA/dasllama" + "SHELL:--embed-file ${PROJECT_SOURCE_DIR}/modules/dasLLAMA/.das_module@modules/dasLLAMA/.das_module" + # ENV joins the web build's callMain,FS: the node driver sets the engine's knobs + # (DASLLAMA_IMAGE=0 - a mint under the wasm identity must not land beside the model) + -sEXPORTED_RUNTIME_METHODS=callMain,FS,ENV + # the engine's ~100 AOT TUs carry their type tables as static data on top of the + # embedded sources: the web build's 128 MB initial heap is short of it (growth stays on) + -sINITIAL_MEMORY=536870912) + else() + TARGET_LINK_LIBRARIES(dasllama_aot libDaScriptAot ${SRC_LIBRARIES} ${DAS_MODULES_LIBS}) + ADD_DEPENDENCIES(dasllama_aot libDaScriptAot dasllama_aot_engine dasllama_aot_daslib) + endif() + # the engine's AOT TUs include the C++ headers of every module in their require closure + # (dasAudio's miniaudio, dasMinfft, dasVulkan's volk) - test_aot's dirs and defines plus volk + target_include_directories(dasllama_aot PRIVATE ${NEED_MODULES_PATH} ${AUDIO_INCLUDE_DIR} ${CIPIC_HRTF_INCLUDE_DIR} + ${PROJECT_SOURCE_DIR}/modules/dasMinfft/minfft + ${PROJECT_SOURCE_DIR}/modules/dasVulkan/vendor/volk + ${PROJECT_SOURCE_DIR}/modules/dasVulkan/vendor/Vulkan-Headers/include) + target_compile_definitions(dasllama_aot PRIVATE MINFFT_SINGLE HRTF_SAMPLE_RATE=${DAS_AUDIO_SAMPLE_RATE}) + SETUP_CPP11(dasllama_aot) +ENDIF() diff --git a/examples/dasLLAMA/speak.das b/examples/dasLLAMA/speak.das new file mode 100644 index 0000000000..c77343cfa6 --- /dev/null +++ b/examples/dasLLAMA/speak.das @@ -0,0 +1,77 @@ +options gen2 +options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) +options _jit_fast_math = true // ggml-parity FP laxity (non-bit-exact, ~+10%); tests stay bit-exact + +require dasllama/dasllama // the facade: the TTS entry points, the WAV writer, the jobque setup +require daslib/clargs +require daslib/jobque_boost +require daslib/strings_boost +require strings + +// Speak a line locally: load a TTS GGUF (KittenTTS or Kokoro, with tts_g2p.bin and +// tts_postag.bin beside it), synthesize, write a WAV, and report audio seconds against the +// seconds the synthesis took. The voice defaults to the last one the model offers. +// +// bin/daslang -jit examples/dasLLAMA/speak.das -- --model --text "Hello." --out hello.wav + +[CommandLineArgs] +struct SpeakArgs { + @clarg_short = "m" + @clarg_doc = "The TTS model (GGUF or a prepared .dlim)" + model : string + + @clarg_short = "t" + @clarg_doc = "The text to speak" + text : string = "Once upon a time, there was a little girl named Lily. She loved to play outside in the sunshine." + + @clarg_short = "o" + @clarg_doc = "The WAV file to write" + out : string = "speech.wav" + + @clarg_short = "v" + @clarg_doc = "Voice name or alias (default: the model's last voice)" + voice : string + + @clarg_short = "s" + @clarg_doc = "Speech speed, 0.25 to 4.0" + speed : float = 1.0 + + @clarg_short = "?" + @clarg_name = "show-help" + @clarg_doc = "Show this help and exit" + help : bool +} + +[export] +def main : int { + var inscope r <- parse_args(type) + if (r |> is_err) { + print("error: {r |> unwrap_err}\n\n") + print_help(get_command_info(type), "speak") + return 1 + } + var inscope cfg <- r |> move_unwrap + if (cfg.help || empty(cfg.model)) { + print_help(get_command_info(type), "speak") + return cfg.help ? 0 : 1 + } + var inscope m <- load_tts_model(cfg.model) + allow_cpu_prefill() + var inscope c <- caps(m) + let voice = empty(cfg.voice) ? c.voices[length(c.voices) - 1] : cfg.voice + print("{length(c.voices)} voices: {join(c.voices, ", ")}; speaking as {voice}\n") + var rc = 0 + with_job_que() { + setup_dasllama_jobque() + var inscope a <- synthesize(m, cfg.text, voice, cfg.speed) + let audio_s = float(length(a.pcm)) / float(a.sample_rate) + let wall_s = float(a.timings.total_us) / 1000000.0 + if (!write_wav_pcm16(cfg.out, a.pcm, a.sample_rate)) { + print("cannot write {cfg.out}\n") + rc = 1 + } else { + print("{cfg.out}: {audio_s} s of audio at {a.sample_rate} Hz, synthesized in {wall_s} s - {audio_s / wall_s}x real time\n") + } + } + return rc +} diff --git a/examples/dasLLAMA/storyteller/.das_package b/examples/dasLLAMA/storyteller/.das_package new file mode 100644 index 0000000000..f9213f2e25 --- /dev/null +++ b/examples/dasLLAMA/storyteller/.das_package @@ -0,0 +1,19 @@ +options gen2 + +require daslib/daspkg + +[export] +def package() { + package_name("storyteller") + package_description("Storyteller: a tiny story LLM writes on screen while KittenTTS reads it aloud - dasGlfw + dasOpenGL + dasAudio over dasLLAMA") +} + +[export] +def release() { + release_main("main.das") // one source for the desktop run AND wasm64 (daspkg release wasm) + release_web_shell("web_shell.html") // fetches the models into MEMFS, then a click starts the program (audio needs the gesture) + // host-only GPU tiers: absent from the wasm build, so their guarded requires resolve as absent + release_wasm_disable_module("dasvulkan") + release_wasm_disable_module("dasmetal") + release_wasm_disable_module("dasaccelerate") +} diff --git a/examples/dasLLAMA/storyteller/main.das b/examples/dasLLAMA/storyteller/main.das new file mode 100644 index 0000000000..2d38b2d32a --- /dev/null +++ b/examples/dasLLAMA/storyteller/main.das @@ -0,0 +1,396 @@ +options gen2 +options persistent_heap +options stack = 524288 // every dasLLAMA program root takes this budget (options stack does not unify up from libs) + +require dasllama/dasllama // the facade: model, session, eval / sample / piece, the TTS entry points +require daslib/jobque_boost +require daslib/strings_boost +require daslib/clargs +require daslib/fio +require daslib/archive +require glfw/glfw_boost +require live/glfw_live +require opengl/opengl_boost +require opengl/opengl_cache +require opengl/opengl_ttf +require audio/audio_boost +require live_host +require math +require daslib/math_boost // ortho_rh / compose for the text layer +require strings + +// Storyteller. A tiny story model (Karpathy's stories15M, Q8) writes a children's story on +// screen a few tokens per frame, and KittenTTS reads each finished sentence aloud in a kid's +// voice while the next one is still being written. Space starts a new story, Escape quits. +// +// bin/daslang -jit examples/dasLLAMA/storyteller/main.das -- --models +// +// holds the two models and the front-end packs - tts_g2p.bin (or its American-only twin +// tts_g2p_en_us.bin, which the web set ships) and tts_postag.bin. A model +// is its gguf (the default names) or a prepared .dlim image baked for the running build's +// identity (dasllama-convert --config) - the web build ships images only, the shell names them. +// The language model runs on the frame thread in per-frame token budgets, so the loop never +// blocks; speech synthesis runs on its own thread, fed sentences through one stream and +// answering with PCM through another, and the frame thread plays the clips back to back. + +[CommandLineArgs] +struct StoryArgs { + @clarg_short = "m" + @clarg_doc = "Directory holding the story model, the TTS model and its phoneme packs" + models : string = "/Users/borisbatkin/Work/llama.cpp/models" + + @clarg_doc = "The story model file inside --models" + story_model : string = "stories15M-Q8_0.gguf" + + @clarg_doc = "The TTS model file inside --models" + tts_model : string = "kitten-nano.gguf" + + @clarg_doc = "Voice name or alias (default: the model's last voice)" + voice : string + + @clarg_doc = "Start a story at once (the browser shell's start button) - the loop then runs on, space for the next" + autoplay : bool + + @clarg_doc = "Start a story at once and quit when it has been read out - the smoke rail" + smoke : bool + + @clarg_doc = "Stop after this many frames (0 = never) - the smoke rail" + max_frames : int + + @clarg_short = "?" + @clarg_name = "show-help" + @clarg_doc = "Show this help and exit" + help : bool +} + +let PROMPT = "Once upon a time" +let MAX_TOKENS = 220 +let STORY_CONTEXT = 256l // prompt + MAX_TOKENS must fit; stories15M was trained at 256 +let TOKENS_PER_FRAME = 2 // ~120 tokens a second at 60 Hz: readable typing, never a stall +let BOS_TOKEN = 1l // llama2 vocabulary; the TinyStories corpus separates stories with BOS, so it is the stop word too +let EOS_TOKEN = 2l +let WRAP_CHARS = 74 // droidsansmono is monospace: characters are the wrap unit +let VISIBLE_LINES = 15 // what fits between the title and the status line at the reference size +let TEXT_SIZE = 0.72 +let SMALL_SIZE = 0.52 + +enum Phase { + idle + writing + told +} + +struct Line { + text : string //! a sentence for the speech thread; "" asks it to stop + story : int //! which story it belongs to: a clip from an older story is dropped unheard +} + +struct Speech { + pcm : array + rate : int + story : int +} + +var g_args = StoryArgs() +var g_model = Model() +var g_session = Session() +var g_params = SamplingParams(temp = 0.8, top_k = 40l, penalty = 1.1) +var g_phase = Phase.idle +var g_wrapped_lines : array +var g_sentence_pieces : array +var g_ntokens = 0 +var g_stories = 0 + +var g_say : Stream? // frame thread -> speech thread: sentences +var g_speech : Stream? // speech thread -> frame thread: clips +var g_speech_done : Channel? +var g_clips : array +var g_elapsed_s = 0.0 +var g_speaking_until = 0.0 +var g_unplayed_sentences = 0 +var g_audio_initialized = false +var g_asch : AudioSystemChannels +var g_font : Font? +var g_frames = 0 +var g_space_was = false +var display_w = 0 +var display_h = 0 + +def start_speech_thread(tts_path, voice_pick : string; var say, speech : Stream?; var done : Channel?) { + new_thread() <| @capture(= tts_path, = voice_pick, = say, = speech, = done) { + setup_dasllama_jobque() // the fork-context pool is per context: without it every parallel kernel clones the program + var inscope m <- load_tts_model(tts_path) + var inscope c <- caps(m) + let voice = empty(voice_pick) ? c.voices[length(c.voices) - 1] : voice_pick + var running = true + while (running) { + say |> pop_archive() $(var l : Line&) { + if (empty(l.text)) { + running = false + return + } + var inscope a <- synthesize(m, l.text, voice, 1.0) + var clip = Speech(pcm := a.pcm, rate = a.sample_rate, story = l.story) + speech |> push_archive(clip) + } + } + say |> release() + speech |> release() + done |> notify_and_release() + } +} + +def say(text : string) { + var l = Line(text = text, story = g_stories) + g_say |> push_archive(l) + g_unplayed_sentences++ +} + +def poll_speech() { + g_speech |> try_pop() $(bytes) { + var clip : Speech + mem_archive_load(bytes, clip) + if (clip.story == g_stories) { + g_clips |> emplace(clip) + } + } + if (g_elapsed_s >= g_speaking_until && !empty(g_clips)) { + let seconds = float(length(g_clips[0].pcm)) / float(max(g_clips[0].rate, 1)) + var pcm <- g_clips[0].pcm + play_sound_from_pcm(g_clips[0].rate, 1, pcm) + g_clips |> erase(0) + g_speaking_until = g_elapsed_s + seconds + g_unplayed_sentences-- + } +} + +def start_story() { + delete g_session + g_session <- create_session(g_model) + set_seed(g_session, int(ref_time_ticks() % 2147483647l)) // a fresh session samples the same story every time otherwise + g_wrapped_lines |> clear() + g_wrapped_lines |> push("") + g_sentence_pieces |> clear() + g_sentence_pieces |> reserve(64) + g_ntokens = 0 + g_stories++ + g_clips |> clear() + g_unplayed_sentences = 0 + g_speaking_until = g_elapsed_s + var prompt <- encode(g_model, PROMPT, true) + eval(g_model, g_session, prompt) + append_text(PROMPT) + g_sentence_pieces |> push(PROMPT) + g_phase = Phase.writing +} + +// the corpus writes curly quotes; the screen font carries ASCII, so they would vanish +def plain_quotes(piece : string) : string { + return piece |> replace("“", "\"") |> replace("”", "\"") |> replace("‘", "'") |> replace("’", "'") +} + +def append_text(piece : string) { + var inscope parts <- split(piece, "\n") + for (i, part in iter_range(parts), parts) { + if (i > 0) { + g_wrapped_lines |> push("") + } + let li = length(g_wrapped_lines) - 1 + g_wrapped_lines[li] = "{g_wrapped_lines[li]}{part}" + wrap_last_line() + } +} + +def wrap_last_line() { + while (length(g_wrapped_lines[length(g_wrapped_lines) - 1]) > WRAP_CHARS) { + let li = length(g_wrapped_lines) - 1 + var head = "" + var tail = "" + peek_data(g_wrapped_lines[li]) $(d) { + var cut = rfind(slice(d, 0, WRAP_CHARS), " ") + if (cut <= 0) { + cut = WRAP_CHARS + } + head = clone_string(slice(d, 0, cut)) + tail = clone_string(slice(d, cut)) + } + g_wrapped_lines[li] = head + g_wrapped_lines |> push(strip(tail)) + } +} + +def sentence_ends(piece : string) : bool { + let t = strip(piece) + return ends_with(t, ".") || ends_with(t, "!") || ends_with(t, "?") || ends_with(t, "\"") && (find(t, ".") >= 0 || find(t, "!") >= 0 || find(t, "?") >= 0) +} + +def write_step() { + var one : array + one |> resize(1) + for (_i in range(TOKENS_PER_FRAME)) { + let tok = sample(g_session, g_params) + if (tok == EOS_TOKEN || tok == BOS_TOKEN || g_ntokens >= MAX_TOKENS) { + finish_story() + return + } + g_ntokens++ + let text = plain_quotes(piece(g_model, tok)) + append_text(text) + g_sentence_pieces |> push(text) + if (sentence_ends(text)) { + say(strip(join(g_sentence_pieces, ""))) + g_sentence_pieces |> clear() + } + one[0] = tok + eval(g_model, g_session, one) + } +} + +def finish_story() { + let rest = strip(join(g_sentence_pieces, "")) + if (!empty(rest)) { + say(rest) + } + g_sentence_pieces |> clear() + g_phase = Phase.told +} + +//! the text layer's coordinates are design pixels against a 1280x720 reference +def hud_scale() : float { + return max(float(display_h) / 720.0, 0.5) +} + +def text_mvp(x, y, scale : float) : float4x4 { + let projection = ortho_rh(0.0, float(display_w), float(display_h), 0.0, -1.0, 1.0) + let model = compose(float3(x, y, 0.0), float4(0.0, 0.0, 0.0, 1.0), float3(scale, scale, 1.0)) + return projection * model +} + +def draw_text(text : string; x, y : float; size : float; tint : float3) { + return if (g_font == null || empty(text)) + var quads <- (*g_font) |> create_quads(text) + let s = hud_scale() * size + (*g_font) |> draw_quads(quads, text_mvp(x * hud_scale(), y * hud_scale(), s), tint) + delete quads +} + +def draw_story() { + let ink = float3(0.93, 0.9, 0.82) + let dim = float3(0.55, 0.52, 0.48) + let accent = float3(0.98, 0.75, 0.35) + draw_text("storyteller", 60.0, 60.0, 0.6, dim) + var y = 130.0 + let step = 34.0 * TEXT_SIZE / 0.72 + let first = max(0, length(g_wrapped_lines) - VISIBLE_LINES) // a long story scrolls: the newest lines stay on screen + for (i in range(first, length(g_wrapped_lines))) { + draw_text(g_wrapped_lines[i], 60.0, y, TEXT_SIZE, ink) + y += step + } + let status = (g_phase == Phase.idle ? "press SPACE for a story" + : g_phase == Phase.writing ? "writing..." + : g_unplayed_sentences > 0 || g_elapsed_s < g_speaking_until ? "reading aloud..." + : "the end - press SPACE for another") + draw_text(status, 60.0, 660.0, SMALL_SIZE, g_phase == Phase.idle ? accent : dim) +} + +[export] +def init() { + var inscope r <- parse_args(type) + if (r |> is_err) { + panic("storyteller: {r |> unwrap_err}") + } + g_args <- r |> move_unwrap + live_create_window("Storyteller", 1280, 720) + cache_ttf_objects() + g_font = cache_font("{get_das_root()}/modules/dasStbImage/fonts/droidsansmono.ttf") + if (!g_audio_initialized) { + g_asch = audio_system_create() + g_audio_initialized = true + } + create_job_que() + setup_dasllama_jobque() + delete g_model + g_model <- load_model(path_join(g_args.models, g_args.story_model), QuantMode.q8) + g_model.config.seq_len = max(g_model.config.seq_len, STORY_CONTEXT) // the session's KV cache is sized to it + allow_cpu_prefill() + g_say = unsafe(stream_create()) + g_speech = unsafe(stream_create()) + g_speech_done = unsafe(channel_create()) + g_speech_done |> append(1) + start_speech_thread(path_join(g_args.models, g_args.tts_model), g_args.voice, g_say, g_speech, g_speech_done) + if (g_args.autoplay || g_args.smoke) { + start_story() + } +} + +[export] +def update() { + if (!live_begin_frame()) { + return + } + g_frames++ + g_elapsed_s += get_dt() + live_get_framebuffer_size(display_w, display_h) + glViewport(0, 0, display_w, display_h) + glClearColor(0.07, 0.06, 0.09, 1.0) + glClear(GL_COLOR_BUFFER_BIT) + glDisable(GL_DEPTH_TEST) + glEnable(GL_BLEND) + glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA) + + let space = glfwGetKey(live_window, GLFW_KEY_SPACE) == GLFW_PRESS + let space_just = space && !g_space_was + g_space_was = space + if (glfwGetKey(live_window, GLFW_KEY_ESCAPE) == GLFW_PRESS) { + glfwSetWindowShouldClose(live_window, 1) + } + if (space_just && g_phase != Phase.writing) { + start_story() + } + if (g_phase == Phase.writing) { + write_step() + } + poll_speech() + draw_story() + live_end_frame() +} + +[export] +def shutdown() { + say("") + g_speech_done |> join() + unsafe { + channel_remove(g_speech_done) + stream_remove(g_say) + stream_remove(g_speech) + } + if (g_audio_initialized) { + audio_system_finalize(g_asch.command, g_asch.next_sid) + g_audio_initialized = false + } + delete g_session + delete g_model + destroy_job_que() + live_destroy_window() +} + +def done_for_smoke() : bool { + return true if (g_args.max_frames > 0 && g_frames >= g_args.max_frames) + return g_args.smoke && g_phase == Phase.told && g_unplayed_sentences == 0 && g_elapsed_s >= g_speaking_until +} + +// eval_main_loop drives the block once per frame: a blocking while-loop natively, the +// browser's requestAnimationFrame on the web - one main for both +[export] +def main() { + init() + eval_main_loop() { + update() + return false if (done_for_smoke()) + return !exit_requested() + } + shutdown() + if (g_args.smoke) { + print("{join(g_wrapped_lines, "\n")}\n") + } +} diff --git a/examples/dasLLAMA/storyteller/web_shell.html b/examples/dasLLAMA/storyteller/web_shell.html new file mode 100644 index 0000000000..9952430861 --- /dev/null +++ b/examples/dasLLAMA/storyteller/web_shell.html @@ -0,0 +1,143 @@ + + + + + +storyteller — dasllama.io + + + + + + + + + + + + + +
+ +
+

storyteller

+

fetching the models...

+
+ +
+
+ +{{{ SCRIPT }}} + + diff --git a/examples/dasLLAMA/wasm/run_node.js b/examples/dasLLAMA/wasm/run_node.js new file mode 100644 index 0000000000..8ef2694e88 --- /dev/null +++ b/examples/dasLLAMA/wasm/run_node.js @@ -0,0 +1,54 @@ +// Runs the wasm64 dasllama_aot host under node: the repo mounts at /repo and the model's +// directory at /models through NODEFS, daslib and the engine's das files are embedded in +// the module, and main runs with -use-aot so the engine's AOT stubs bind. +// +// node examples/dasLLAMA/wasm/run_node.js [args...] +// +// output-dir - the directory holding dasllama_aot.js + dasllama_aot.wasm (web/output64) +// script.das - a path inside the repo (examples/dasLLAMA/run.das) +// model.gguf - a host path; its directory is mounted read-write, so the image rail is +// held off (DASLLAMA_IMAGE=0): a mint under the wasm identity would land +// beside the model, and the image GC would reap the box's own image +// args - the script's own arguments; with none, the mounted model path is passed as +// the one positional argument (run.das), otherwise they are passed verbatim +// with every `@model` replaced by that path (speak.das takes --model @model) +const path = require('path'); + +const outputDir = process.argv[2] && path.resolve(process.argv[2]); +const script = process.argv[3]; +const model = process.argv[4] && path.resolve(process.argv[4]); +const extra = process.argv.slice(5); +if (!outputDir || !script || !model) { + console.error('usage: node run_node.js [args...]'); + process.exit(1); +} +const repoRoot = path.resolve(__dirname, '../../..'); + +// emscripten implements exit() by throwing ExitStatus, which surfaces as an unhandled rejection +process.on('unhandledRejection', (reason) => { + if (reason && reason.name === 'ExitStatus') { + process.exit(reason.status); + } + console.error('WASM error:', reason); + process.exit(1); +}); + +// the module script adopts a pre-existing global Module: preRun is the one hook that runs before +// emscripten builds the C environment from ENV, so the knobs land there, not after init +const Module = { + preRun: [function() { + Module.ENV.DASLLAMA_IMAGE = '0'; + Module.ENV.DASLLAMA_ALLOW_UNTUNED = '1'; + Module.FS.mkdir('/repo'); + Module.FS.mount(Module.FS.filesystems.NODEFS, { root: repoRoot }, '/repo'); + Module.FS.mkdir('/models'); + Module.FS.mount(Module.FS.filesystems.NODEFS, { root: path.dirname(model) }, '/models'); + }], + onRuntimeInitialized: function() { + const mounted = path.posix.join('/models', path.basename(model)); + const scriptArgs = extra.length ? extra.map((a) => a.replace(/@model/g, mounted)) : [mounted]; + Module.callMain(['-use-aot', path.posix.join('/repo', script), '--', ...scriptArgs]); + }, +}; +global.Module = Module; +require(path.join(outputDir, 'dasllama_aot.js')); diff --git a/include/daScript/simulate/REVIEW.md b/include/daScript/simulate/REVIEW.md index d408bf4545..5c80f23622 100644 --- a/include/daScript/simulate/REVIEW.md +++ b/include/daScript/simulate/REVIEW.md @@ -32,7 +32,10 @@ checklist on its own. folder that generated code executes per evaluated expression. Such a diff - including one an optimized build flattens to nothing - lands its entry under `ARCHITECTURE.md`'s sanctioned hot-path additions in the same diff: what was added, where, why correctness - required it, and the alternative that was rejected. + required it, and the alternative that was rejected. Cost is judged in the build the repo + ships: a change that costs more only under a relaxed-math or otherwise non-default compiler + flag states which flavor and how much in its PR description, and ledgers only when the + shipped build pays. - **A diff that changes the layout of a `debug_info.h` struct - a field added, removed, reordered, or retyped, or a base changed - states a per-consumer verdict (updated / no diff --git a/include/daScript/simulate/aot_builtin.h b/include/daScript/simulate/aot_builtin.h index 3a04da7b5f..d9e67cc4b5 100644 --- a/include/daScript/simulate/aot_builtin.h +++ b/include/daScript/simulate/aot_builtin.h @@ -22,6 +22,8 @@ namespace das { DAS_API bool is_compiling ( ); DAS_API bool is_compiling_macros ( ); DAS_API bool is_standalone_exe ( ); + DAS_API bool das_is_jit_function ( const Func func ); + DAS_API bool das_is_aot_function ( const Func func ); DAS_API uint64_t get_context_share_counter ( Context * context ); DAS_API char * builtin_das_root ( Context * context, LineInfoArg * at ); @@ -301,6 +303,8 @@ namespace das { const char * das_get_platform_name(); const char * das_get_cross_platform_name(); const char * das_get_architecture_name(); + const char * das_get_target_triple(); + const char * das_get_target_architecture_name(); DAS_API bool das_cpu_supports ( const char * feature ); DAS_API char * fmt_i8 ( const char * fmt, int8_t value, Context * context, LineInfoArg * at ); diff --git a/include/daScript/simulate/aot_builtin_jobque.h b/include/daScript/simulate/aot_builtin_jobque.h index 4397436649..447d1256c8 100644 --- a/include/daScript/simulate/aot_builtin_jobque.h +++ b/include/daScript/simulate/aot_builtin_jobque.h @@ -276,6 +276,8 @@ namespace das { DAS_API int32_t get_jobque_worker_limit ( Context * context, LineInfoArg * at ); DAS_API void set_jobque_team_rank_gate ( bool on, Context * context, LineInfoArg * at ); DAS_API bool get_jobque_team_rank_gate ( Context * context, LineInfoArg * at ); + DAS_API void setJobqueAffinity ( int32_t mode ); + DAS_API int32_t getJobqueAffinity (); DAS_API uint64_t count_jobque_leaks(); DAS_API void new_job_invoke ( Lambda lambda, Func fn, int32_t lambdaSize, Context * context, LineInfoArg * lineinfo ); DAS_API void set_jobque_fork_pool ( bool keep, bool skipInit, Context * context, LineInfoArg * at ); diff --git a/modules/dasAccelerate/src/dasAccelerate.cpp b/modules/dasAccelerate/src/dasAccelerate.cpp index f2724bf583..5fe8965f04 100644 --- a/modules/dasAccelerate/src/dasAccelerate.cpp +++ b/modules/dasAccelerate/src/dasAccelerate.cpp @@ -7,8 +7,9 @@ #include #include "daScript/daScript.h" +#include "dasAccelerate.h" -using namespace das; +namespace das { // BLASSetThreading/BLASGetThreading exist only in the macOS 15 / iOS 18 SDK headers — the // __builtin_available checks below gate RUNTIME, but an older SDK fails at COMPILE time @@ -37,10 +38,10 @@ static inline void accel_pin_single_thread() { // C[m x n] = A[m x k] * B[n x k]^T, row-major — the exact call shape ggml-blas.cpp uses for // mul_mat (activations A stay token-major, weights B stay row-major, no pre-transpose). -static void accel_sgemm_nt(int32_t m, int32_t n, int32_t k, - const float * a, int32_t lda, - const float * b, int32_t ldb, - float * c, int32_t ldc) { +void accel_sgemm_nt(int32_t m, int32_t n, int32_t k, + const float * a, int32_t lda, + const float * b, int32_t ldb, + float * c, int32_t ldc) { accel_pin_single_thread(); cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, m, n, k, @@ -51,9 +52,9 @@ static void accel_sgemm_nt(int32_t m, int32_t n, int32_t k, // y[n] = W[n x k] * x[k], row-major — the decode-lane probe (uncrowned on M1; here for the // M4/M5-SME applicability sweep, never gated out). -static void accel_sgemv_n(int32_t n, int32_t k, - const float * w, int32_t ldw, - const float * x, float * y) { +void accel_sgemv_n(int32_t n, int32_t k, + const float * w, int32_t ldw, + const float * x, float * y) { accel_pin_single_thread(); cblas_sgemv(CblasRowMajor, CblasNoTrans, n, k, 1.0f, w, ldw, x, 1, 0.0f, y, 1); } @@ -64,10 +65,10 @@ static void accel_sgemv_n(int32_t n, int32_t k, // Filter create+destroy is ~0.2us (bnns_hgemm_probe), so per-call creation is free and no // pointer-lifetime coupling exists. rc: 0 = ok, nonzero = BNNS unavailable or refused — // the caller falls back to the portable path. -static int32_t accel_bnns_hgemm_nt(int32_t m, int32_t n, int32_t k, - const uint16_t * a, int32_t lda, - const uint16_t * b, int32_t ldb, - float * c, int32_t ldc) { +int32_t accel_bnns_hgemm_nt(int32_t m, int32_t n, int32_t k, + const uint16_t * a, int32_t lda, + const uint16_t * b, int32_t ldb, + float * c, int32_t ldc) { if (__builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *)) { #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdeprecated-declarations" // classic BNNS: deprecated 15.0 for BNNSGraph, still the only C-callable dynamic matmul @@ -99,7 +100,7 @@ static int32_t accel_bnns_hgemm_nt(int32_t m, int32_t n, int32_t k, // Calling thread's effective BLAS threading mode after the pin: 0 = multi, 1 = single, // -1 = BLASSetThreading unavailable (pre-macOS-15). Diagnostics for the contention rig. -static int32_t accel_threading_mode() { +int32_t accel_threading_mode() { accel_pin_single_thread(); #if DAS_ACCEL_HAS_BLAS_THREADING if (__builtin_available(macOS 15.0, iOS 18.0, tvOS 18.0, watchOS 11.0, *)) { @@ -109,8 +110,6 @@ static int32_t accel_threading_mode() { return -1; } -namespace das { - class Module_DasAccelerate : public Module { public: Module_DasAccelerate() : Module("das_accelerate") { @@ -131,6 +130,10 @@ class Module_DasAccelerate : public Module { addExtern(*this, lib, "accel_threading_mode", SideEffects::modifyExternal, "accel_threading_mode"); } + virtual ModuleAotType aotRequire ( TextWriter & tw ) const override { + tw << "#include \"../modules/dasAccelerate/src/dasAccelerate.h\"\n"; + return ModuleAotType::cpp; + } }; REGISTER_DYN_MODULE(Module_DasAccelerate, Module_DasAccelerate); diff --git a/modules/dasAccelerate/src/dasAccelerate.h b/modules/dasAccelerate/src/dasAccelerate.h new file mode 100644 index 0000000000..2580b57efc --- /dev/null +++ b/modules/dasAccelerate/src/dasAccelerate.h @@ -0,0 +1,20 @@ +#pragma once + +#include "daScript/misc/platform.h" + +// the das_accelerate binding surface, declared for AOT-generated C++ (Module::aotRequire) + +namespace das { + DAS_MOD_API void accel_sgemm_nt(int32_t m, int32_t n, int32_t k, + const float * a, int32_t lda, + const float * b, int32_t ldb, + float * c, int32_t ldc); + DAS_MOD_API void accel_sgemv_n(int32_t n, int32_t k, + const float * w, int32_t ldw, + const float * x, float * y); + DAS_MOD_API int32_t accel_bnns_hgemm_nt(int32_t m, int32_t n, int32_t k, + const uint16_t * a, int32_t lda, + const uint16_t * b, int32_t ldb, + float * c, int32_t ldc); + DAS_MOD_API int32_t accel_threading_mode(); +} diff --git a/modules/dasLLAMA/ARCHITECTURE.md b/modules/dasLLAMA/ARCHITECTURE.md index f3b5df8d1e..be6a80ebed 100644 --- a/modules/dasLLAMA/ARCHITECTURE.md +++ b/modules/dasLLAMA/ARCHITECTURE.md @@ -66,11 +66,13 @@ re-transcoding `$LCPP/src/unicode-data.cpp`). rig, the tune gate, the sanctioned instrumentation rails, kernel-race fidelity, the gemv's own tune seat, the CPU kernel bench's fixture conditions, the speculative round's ruler record, the `[tuned]` perm precedence, and the mint wall in the sidecar's provenance. -- `ARCHITECTURE_CPU_KERNELS.md` - sec.2.22-2.24: the sub-block-packed k3/k6 planes, the grid - formats' panel and row-group decodes, and the VBMI symbol lattice. -- `ARCHITECTURE_TTS.md` - sec.2.28-2.35: the TTS block home's two layouts, tap stacking, the +- `ARCHITECTURE_CPU_KERNELS.md` - sec.2.22-2.24, 2.42: the sub-block-packed k3/k6 planes, the grid + formats' panel and row-group decodes, the VBMI symbol lattice, and the tier that selects on the + target rather than the host. +- `ARCHITECTURE_TTS.md` - sec.2.28-2.35, 2.43: the TTS block home's two layouts, tap stacking, the padded concat width, the per-synthesis carrier, the served carrier on the image rail, the - sine source's operation order, the heteronym context, and the rig's scoring. + sine source's operation order, the heteronym context, the rig's scoring, and the two-tier + phoneme pack. ## 3. Inherited invariants diff --git a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md index 9a32bf3b44..6c0b32dacd 100644 --- a/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md +++ b/modules/dasLLAMA/ARCHITECTURE_CPU_KERNELS.md @@ -43,3 +43,15 @@ code bytes in its qword, `VPMULTISHIFTQB` spreads the symbols into bytes, one `v magnitudes, and the signs ride the activation copy as a mask `(x ^ m) - m`. The lattice row shares its tile body and planes with the 512/mr16 row, so only the gemv differs - what the gemv's own seat (`ARCHITECTURE_MEASUREMENT.md` sec.2.26) races. + +### 2.42 A CPU tier selects on the TARGET, not the host {#cpu-tier-target-select} + +The arm64 SDOT tier registers its backends only under the JIT and only for an arm64 TARGET - the +artifact's architecture, never the running host's. Off the JIT the `sdot4` family runs its scalar +fallback bodies, which are slower than the portable `dot_q8q8` the vectorizer handles, so the +portable tier stays selected wherever hardware SDOT is not emitted. + +The portable tier picks its dot form on the target at compile time. On wasm SIMD128 the +auto-vectorized template dot is the slow form - the ISA carries no int8 dot for LLVM to find, +while the `idot4` builtin lowers there to the ISA's own widening multiply-adds - so a wasm target +takes `dot_q8q8_idot4_ps`. Every other target keeps the template. diff --git a/modules/dasLLAMA/ARCHITECTURE_ENGINE.md b/modules/dasLLAMA/ARCHITECTURE_ENGINE.md index 32f123475e..7f6ecdd8b5 100644 --- a/modules/dasLLAMA/ARCHITECTURE_ENGINE.md +++ b/modules/dasLLAMA/ARCHITECTURE_ENGINE.md @@ -187,7 +187,12 @@ trunk-only and trunk+head images never collide and one image file serves both tr generator, the layout/perm schema shared by generator and runtime, and the `[tune]` family registration. A hand-written tile that the generator could emit belongs in the generator. - **`dasllama_tune.das`** - the per-box loop-hint tuner (`[tuned]` / `[dasllama_grid]`). Tuning - POLICY lives here; tuned VALUES live in the box's sidecar, never in source. + POLICY lives here; tuned VALUES live in the box's sidecar, never in source. The framework is + OPTIONAL: the `llvm_tune` require is `?llvm`-guarded and every use of it sits behind + `static_if (typeinfo module_exists(llvm_tune))`, so a build without dasLLVM opens no + sidecar, reads no policy env and reports nothing - `[tuned]` stamps the declared `fallback=` + (or `DEFAULT_PERM`) and the AST rewrite is unchanged. `box_profile_verdict_at` answers the + reason NAME rather than llvm_tune's enum for the same reason: a signature cannot be gated. - **`dasllama_tune_scope.das`** - the one `[tune_scope]` declaration (tuner, covered modules, version pin, shipped defaults), in a module every kernel module requires: a kernel reads its class entry in the defaults profile at its own compile, so the scope must be on the AST first. @@ -247,7 +252,12 @@ file builds an `ArchDesc` (name * `configure` * the `ArchBlocks` fn-ptr quad * ` explicit `exchange_*` config counts as the expressed choice, otherwise the `.consent` sidecar-sibling file governs - unset asks on a real terminal, or emits `@sidecar consent state=needed` for the watchdog dialog / control page, and no request - leaves until a surface records "accepted". + leaves until a surface records "accepted". The client is meaningless without a sidecar, so + every requirer takes it through the guard `require ?llvm dasllama/dasllama_exchange` (`llvm` + is the C++ witness module dasLLVM compiles in exactly when the build is configured with it) + and wraps each use in `static_if (typeinfo module_exists(dasllama_exchange))`: with no tune + framework in the build the server's `/exchange` routes answer `not_found`, the boot resolver + and submit check are never registered, and `lcpp_bench --submit` refuses before it measures. - **`benchmarks/asr/mem_census.sh`** - the peak-memory census (`/usr/bin/time -l` around one asr_bench process per cell; macOS only) - the interim footprint instrument until a footprint leg lands in `gen_bench_records`; its numbers live in `PERF_LEDGER.md`, never the stores. diff --git a/modules/dasLLAMA/ARCHITECTURE_IMAGE.md b/modules/dasLLAMA/ARCHITECTURE_IMAGE.md index d2d97a3af4..110951f8c0 100644 --- a/modules/dasLLAMA/ARCHITECTURE_IMAGE.md +++ b/modules/dasLLAMA/ARCHITECTURE_IMAGE.md @@ -77,7 +77,9 @@ A gguf load's O(model) transform loops run about ten times slower interpreted - takes 53 s against 5.5 s jitted, and a 69 GB hybrid extrapolates to an hour (the repack itself is native tune kernels and costs the same either way). A prepared image costs nothing interpreted, because mapping and borrowing planes runs no such loop. That asymmetry is why the guard fires on -the gguf path and never on the image path. +the gguf path and never on the image path. The guard passes on every compiled tier - the JIT, a +standalone exe, and a host that linked the engine's AOT stubs (`aot_kernels_linked`, the probe +`guard_interp_inference` shares) - because the transform loops are native on each. ### 2.1e Publishing an image {#image-publish} diff --git a/modules/dasLLAMA/ARCHITECTURE_INVARIANTS.md b/modules/dasLLAMA/ARCHITECTURE_INVARIANTS.md index 9fb572e231..947e982d67 100644 --- a/modules/dasLLAMA/ARCHITECTURE_INVARIANTS.md +++ b/modules/dasLLAMA/ARCHITECTURE_INVARIANTS.md @@ -9,17 +9,22 @@ Durable "why it is built this way" facts harvested from the design docs archived **From `x64_arch.md`:** -- **The LIBRARY never runs interpreted or AOT - the scaffolding may.** Inference - anything that - runs a model (eval / decode / prefill / generate) - is `-jit`-only, hard stop: an interpreted or - AOT model run is not "slow but correct", it is out of contract, and the library panics on it. - Loop hints and intrinsic lowering exist only in the JIT, and the JIT tier is the only one the - oracles gate. **Scaffolding is exempt:** conversion utilities (`.dlim` bake, image processing), - debug scripts, and batch/driver tools that spawn the real runner as a child may run interpreted. - Enforcement is `guard_interp_gguf_load` (`dasllama_image.das` - big-load panic + - `DASLLAMA_ALLOW_INTERP_LOAD` escape for the conversion path) plus `guard_interp_inference` - (`dasllama_math.das`), which panics at the library seams: `make_run_state`, ASR session - creation, and `vad_chunk_prob`. Intrinsic fallback bodies exist for off-ARCH correctness - *inside* the JIT, not as runnable tiers. +- **The LIBRARY never runs interpreted - the scaffolding may.** Inference - anything that runs + a model (eval / decode / prefill / generate) - runs on a compiled tier, hard stop: an + interpreted model run is not "slow but correct", it is out of contract, and the library panics + on it. Three tiers pass: the JIT (the crown - loop hints, intrinsic lowering and the tuned + kernels exist only there, and it is the tier the oracles gate), a standalone `-exe`, and a host + that linked the engine's AOT stubs (the portable tier: the reference bodies as the C++ the AOT + emitter produced, `examples/dasLLAMA`, `-use-aot`). **Scaffolding is exempt:** conversion + utilities (`.dlim` bake, image processing), debug scripts, and batch/driver tools that spawn + the real runner as a child may run interpreted. Enforcement is `guard_interp_gguf_load` + (`dasllama_image.das` - big-load panic + `DASLLAMA_ALLOW_INTERP_LOAD` escape for the + conversion path) plus `guard_interp_inference` (`dasllama_math.das`), which panics at the + library seams: `make_run_state`, ASR session creation, and `vad_chunk_prob`. Both read the + AOT tier through `aot_kernels_linked` - a runtime probe of one kernel's binding, never + `aot_enabled()`, which folds at compile time and desyncs the semantic hash between AOT + generation and the AOT run. Intrinsic fallback bodies are what the AOT tier runs where the + JIT would lower an instruction; inside the JIT they exist for off-ARCH correctness. - **Correctness before speed, token-for-token.** The engine is validated against external oracles (llama2.c + upstream `simple_ids`) plus per-arch parity fixtures. A new kernel passes the suite *and* the oracles with the new backend active before any perf claim. diff --git a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md index d13ea20828..2af245da35 100644 --- a/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md +++ b/modules/dasLLAMA/ARCHITECTURE_MEASUREMENT.md @@ -242,6 +242,10 @@ profile (`performance/defaults`), the annotation's `fallback` `;`-chain, then `D FILE its answer came from, so a verbose compile names which of the two stamped each kernel. A box the shipped profile covers therefore compiles tuned kernels without racing anything, and a box it does not cover falls to the fallback chain - never to another box's winners. +With no tune framework in the build the first three steps do not exist - no policy env, no sidecar, +no profile - so the ladder is the `perm=` pin, then the `fallback` chain's first UNCONDITIONAL entry +(a `suffix:requires` seat cannot be judged with no feature probe), then `DEFAULT_PERM`, and the +compile reports nothing. ### 2.41 The mint's own wall rides in the sidecar's provenance {#mint-wall-provenance} diff --git a/modules/dasLLAMA/ARCHITECTURE_TTS.md b/modules/dasLLAMA/ARCHITECTURE_TTS.md index b91f1950a2..1582b47487 100644 --- a/modules/dasLLAMA/ARCHITECTURE_TTS.md +++ b/modules/dasLLAMA/ARCHITECTURE_TTS.md @@ -1,7 +1,7 @@ # dasLLAMA architecture - text to speech The companion `ARCHITECTURE.md` indexes: the TTS file charters (sec.1.7c) and the mechanisms the -TTS files implement (sec.2.28-2.35). `ARCHITECTURE_COMMON.md` (repo root) is the contract. +TTS files implement (sec.2.28-2.35, 2.43). `ARCHITECTURE_COMMON.md` (repo root) is the contract. ## 1. File charters @@ -44,7 +44,8 @@ TTS files implement (sec.2.28-2.35). `ARCHITECTURE_COMMON.md` (repo root) is the the DRESS vowel before a linking rhotic as SQUARE, having nothing in the string to tell merry from Mary. A vowel the two lexicons give no evidence for before a dropped rhotic keeps that rhotic rather than losing it. The bath-trap split reaches only lexicon - words. Loads `tts_g2p.bin`, pack + words. Loads a phoneme pack - `tts_g2p.bin` (both dialect tiers) or `tts_g2p_en_us.bin` (the + American tier alone, sec.2.43) - pack version 2 (`harness/build_g2p_data.py`: the gold tier extended by `harness/g2p_local_additions.json`, the US and GB keys merged into one string table per tier, the GRU stored as f16, CMUdict pruned of the words both dialects' lexicons carry - @@ -56,7 +57,9 @@ TTS files implement (sec.2.28-2.35). `ARCHITECTURE_COMMON.md` (repo root) is the parity rails for all three stages. - **`dasllama_tts_types.das`** - the TTS floor: `TtsCaps`, `TtsAudio` (f32 PCM + rate), `TtsNoise` (the source noise a synthesis consumed - captured from the oracle, or drawn into a reused - carrier). Family files require this, never each other. + carrier), and the two family data records the shared carrier serializes - `KittenFamily` (speed + priors, voice aliases) and `KokoroFamily` (the symbol vocabulary) - plain data, no family logic. + Family files require this, never each other. - **`dasllama_tts_blocks.das`** - the StyleTTS2-lineage block home, the TTS twin of `dasllama_tower.das`, in the two layouts of sec.2.28: Conv1d (dense, depthwise, forward and transposed), the dense layer, LayerNorm over rows and over channels, InstanceNorm and AdaIN, @@ -85,23 +88,27 @@ TTS files implement (sec.2.28-2.35). `ARCHITECTURE_COMMON.md` (repo root) is the rows-form input, the style and the source spectrum as rows and answers with the waveform or declines; the SineGen phase chain and the harmonic STFT stay on the CPU in both routes (sec.2.33), the trace rail keeps the CPU chain, and engage is read from the counters. No - driver fills the slot yet. The assembly names no family; a family's quirk lives in its family - file. + driver fills the slot yet. The carrier also holds each family's DATA - the `KittenFamily` / + `KokoroFamily` records of `dasllama_tts_types.das`, read from the GGUF's `kitten.*` / + `kokoro.symbol_*` metadata by `stage_family_data` - because the image meta serializes them and a + `.dlim` load has no GGUF to read them from (sec.2.32); the family LOGIC that interprets those + records lives in the family files. - **`dasllama_kitten.das`** - the KittenTTS family (nano and mini): the reference driver's symbol - table, re-spacing rule and style-row rule (the chunk's character count), its speed priors and - voice aliases (`kitten.*` metadata), its 5000-sample tail trim, and the rewrite of the front - end's inventory into the espeak-style IPA these models consume. A phoneme the symbol table + table, re-spacing rule and style-row rule (the chunk's character count), how the speed priors + and voice aliases of its `KittenFamily` record are applied, its 5000-sample tail trim, and the + rewrite of the front end's inventory into the espeak-style IPA these models consume. A phoneme the symbol table does not carry takes its separator space with it, so a dropped symbol never leaves a doubled space token where the model was trained on none. - **`dasllama_kokoro.das`** - the Kokoro family (Kokoro-82M): the reference pipeline's - vocabulary (`kokoro.symbol_*` metadata - the front end's own inventory, no rewrite), its token - wrapping and style-row rule (the phoneme string's character count less one). Fifty-four voice + vocabulary (the `KokoroFamily` symbol record - the front end's own inventory, no rewrite), its + token wrapping and style-row rule (the phoneme string's character count less one). Fifty-four voice packs of 510 rows, each named `_`; a pack's language is read from that shape and only from it, because the language letters are ordinary first letters of ordinary names, so a name of any other shape has no language rather than the one its first letter spells. - **`dasllama_tts.das`** - the TTS facade: `load_tts_model` (the shared model plus the family - picked by `general.architecture`; `tts_g2p.bin` and `tts_postag.bin` read from the GGUF's - directory; the packs it leaves out are named once in the log), `caps` (the voices the front + picked by `general.architecture` - from a GGUF or from a prepared `.dlim`; the phoneme pack + and `tts_postag.bin` read from the model's directory, the full pack preferred over the + American-only twin, sec.2.43; the packs it leaves out are named once in the log), `caps` (the voices the front end can drive - a Kokoro pack's name carries its language, and only the languages the family declares are listed or accepted; the rest refuse with the language in the message, or, where the name carried none, with the fact that the front end cannot phonemize it), `tts_voice_lang` @@ -223,8 +230,14 @@ control). Two lanes, each its own image: `tts-q8`, the served default - the rows weights as Q8_0 quants in an int8 plane repacked for the box's backend, the tag config-bound - and `tts-f32`, the file's planes under a config-free tag, the reference lane the parity rail and the block test hold against, which carries no optimization duty: the rig held q8 at f32 -quality, and f32 costs memory for nothing. The meta blob carries the scalars, the spans and the -voice roster through the leaf structs' own `serialize` overloads; the loader binds every weight +quality, and f32 costs memory for nothing. The meta blob carries the scalars, the spans, the +voice roster and the family's driver data (Kitten's per-voice speed priors and alias names, +Kokoro's symbol table - staged from the GGUF's `kitten.*` / `kokoro.*` keys with the weights) +through the leaf structs' own `serialize` overloads, so an image serves without its GGUF: a +`.dlim` path maps the lane the file was baked on, whatever the policy asks (`load_styletts2` +tries the policy's tag first, then the other), and `dasllama-convert` bakes one offline +(`bake_styletts2_image`, `-o` anywhere) under a supplied `--config` for another box's identity. +The loader binds every weight array as a borrowed view over the mapped plane after the parse (post-load runs before the planes bind), so the carrier and every struct holding it take an explicit `finalize`. The layout fingerprint refuses a struct-shape change by name; a served-layout change that keeps the @@ -266,3 +279,13 @@ exposes `expected` so the normalizer cell and the rig read the one correction; t (`harness/mint_tts_g2p_fixture.py`) carries the table so a re-mint reproduces it. The reference line is the reference arms' own WAVs re-scored from the experiment's transcripts on the same forms with the same scorer, the clock suffix ("a m", "am") counted as one word on both sides. + +### 2.43 The phoneme pack ships in two tiers {#tts-g2p-pack-tiers} + +`build_g2p_data.py` mints two packs from one source: `tts_g2p.bin`, carrying both dialect tiers, +and `tts_g2p_en_us.bin` under `--dialect us`, the American tier alone - four megabytes smaller, +and what the web serving set ships. A pack names its own tier in its source line, which `load_g2p` +reads into `G2pModel.us_only`. `load_tts_model` takes the full pack from the model's directory when +it is there and the American-only twin otherwise, and panics when neither is. On the American-only +pack `g2p_phonemize` refuses a British request by name and `caps` offers no British voice, so a +Kokoro `bf_*` / `bm_*` pack is simply not on the list. diff --git a/modules/dasLLAMA/CLAUDE.md b/modules/dasLLAMA/CLAUDE.md index e7b5d330bd..6e38212224 100644 --- a/modules/dasLLAMA/CLAUDE.md +++ b/modules/dasLLAMA/CLAUDE.md @@ -16,8 +16,10 @@ directly). A changed file in a model suite (decode | mtp | prefill | matrix | im area - `--changed` names it and runs nothing for it; run that suite with `--arm` scoped to the change. Before a PR: `--suite model-free`, then `--suite stocked` on a box with models - a PR's `stocked` run carries no `--exclude`; `--exclude test_ple_modes` is the between-PR iteration -form. The runner's `REPORT` and `NEXT` lines say what ran, what the run skipped (a filtered arm, -an absent device, or a missing model), and what is still owed. The rest: `tests/CLAUDE.md`. +form. `--no-tune` runs any suite but the image suites on the reference kernel bodies +(`DAS_TUNE_POLICY=reference`, the image rail off) - the portable tier's arm. The runner's `REPORT` +and `NEXT` lines say what ran, what the run skipped (a filtered arm, an absent device, or a +missing model), and what is still owed. The rest: `tests/CLAUDE.md`. Follow the daslang **gen2** conventions - the root `CLAUDE.md` rules apply to every `.das` file here. diff --git a/modules/dasLLAMA/CMakeLists.txt b/modules/dasLLAMA/CMakeLists.txt index b6c7606089..5043067217 100644 --- a/modules/dasLLAMA/CMakeLists.txt +++ b/modules/dasLLAMA/CMakeLists.txt @@ -4,7 +4,8 @@ IF(NOT DAS_LLAMA_INCLUDED) ADD_MODULE_DAS_FROM_DESCRIPTOR(dasllama dasllama) - # dasLLAMA IS NEVER AOT'D — -jit only; no AOT source/depends exports (see tests/aot/CMakeLists.txt). + # dasLLAMA's tests are never in test_aot (see tests/aot/CMakeLists.txt); the engine's AOT + # tier is the opt-in examples/dasLLAMA target, which AOTs the CPU modules itself. install(DIRECTORY ${PROJECT_SOURCE_DIR}/modules/dasLLAMA/dasllama DESTINATION ${DAS_INSTALL_MODULESDIR}/dasLLAMA diff --git a/modules/dasLLAMA/REVIEW.das b/modules/dasLLAMA/REVIEW.das index 64ecada2b7..47b633896e 100644 --- a/modules/dasLLAMA/REVIEW.das +++ b/modules/dasLLAMA/REVIEW.das @@ -545,7 +545,7 @@ let private IMAGE_FILE = "modules/dasLLAMA/dasllama/dasllama_image.das" // in file order. A closure change with IMAGE_VERSION unmoved is red; the finding prints the // value to re-stamp with. let private IMAGE_LAYOUT_STAMP_VERSION = 33 -let private IMAGE_LAYOUT_STAMP_HASH = 0x16cc08ecdcac17c4ul +let private IMAGE_LAYOUT_STAMP_HASH = 0x18e58f5384bf57e0ul // The helpers that decide WHERE bytes land: the page pad, the plane and total sizing, the // writer's append / zero-fill / header patch, and the header's scalar stores. Changing one diff --git a/modules/dasLLAMA/REVIEW_IMAGE.md b/modules/dasLLAMA/REVIEW_IMAGE.md index 4f2883f99b..06158d13b5 100644 --- a/modules/dasLLAMA/REVIEW_IMAGE.md +++ b/modules/dasLLAMA/REVIEW_IMAGE.md @@ -53,10 +53,13 @@ property shapes never does.** A config-bound identity on a property-free family image per tune state and reaps the others on every switch; a config-free identity on a property-shaped family serves the wrong bytes under a changed property. -**A diff that moves a byte in a `.dlim`, adds or drops a serialized field, or changes what an -image at an UNCHANGED path contains, bumps `IMAGE_VERSION` (`dasllama/dasllama_image.das`) in -the same change; weakening the `REVIEW.das` layout-stamp check - narrowing its closure or -dropping the cell - is a defect.** Without the bump a stale image stays structurally valid and +**A diff that changes what an image at an UNCHANGED path contains without changing the meta +closure's struct shape - a moved byte, a re-meaning of a serialized field, a serializer body +change that the layout fingerprint cannot see - bumps `IMAGE_VERSION` (`dasllama/dasllama_image.das`) +in the same change; weakening the `REVIEW.das` layout-stamp check - narrowing its closure or +dropping the cell - is a defect.** A struct field added or dropped moves `layout_fingerprint()`, +which the load compares by name, so that image refuses loudly and a re-stamp of the closure hash +discharges it; without the fingerprint's help a stale image stays structurally valid and silently serves a different model. What the stamp covers is the layout stamp section of `ARCHITECTURE_IMAGE.md`. diff --git a/modules/dasLLAMA/REVIEW_MEASUREMENT.md b/modules/dasLLAMA/REVIEW_MEASUREMENT.md index 7a253eef13..5629a1a1dc 100644 --- a/modules/dasLLAMA/REVIEW_MEASUREMENT.md +++ b/modules/dasLLAMA/REVIEW_MEASUREMENT.md @@ -47,11 +47,12 @@ decision in the report of the run that took it and in the PR that lands the kern winner enters the ledger only through a re-measured board cell. **A diff that makes the engine serve tokens by a route no board cell exercises mints that cell -in the same change.** A route is what a request takes end to end: a weight format, a modality, -a family, a backend, a serving lane (q8 or f32), a GPU tower (a GPU-run vision or audio -encoder), a sampler class (greedy, sampled) through the speculative round (a decode step that -drafts several tokens and verifies them in one pass), or the path a run with no flags and no -environment overrides takes. +in the same change.** A route is anything that changes which code serves the tokens end to end: +a weight format, a modality, a family, a backend, a serving lane (q8 or f32), a GPU tower (a +GPU-run vision or audio encoder), a sampler class (greedy, sampled) through the speculative +round (a decode step that drafts several tokens and verifies them in one pass), a compile tier +(JIT, AOT, interpreted reference bodies), a cross target (wasm64), or the path a run with no +flags and no environment overrides takes. **A diff that claims to make an already-served path faster, from an author whose box mints that path, re-mints a board row (`performance/records/.json`) that exercises that path, in @@ -83,3 +84,9 @@ board cell or ruler record produced names the record and row instead of the harn sits in the figure's own sentence, in a table heading that covers the table's rows, in a section-level provenance line that covers the paragraphs under it, or in a citation of the `PERF_LEDGER.md` entry whose provenance line covers it. + +**A diff that pins the kernel backend - `pin_kernel_backend`, `select_kernel_backend`, or +`DASLLAMA_PIN_BACKEND` - pins it before the load it governs, and on a name the box has not +registered refuses or skips rather than running on the default.** The loader repacks weights +into the pinned backend's layout, so a pin set after the load never reaches them, and a +misspelled pin that falls through measures the default backend under the pinned one's name. diff --git a/modules/dasLLAMA/REVIEW_TTS.md b/modules/dasLLAMA/REVIEW_TTS.md index e65974f7f4..74c8d3d293 100644 --- a/modules/dasLLAMA/REVIEW_TTS.md +++ b/modules/dasLLAMA/REVIEW_TTS.md @@ -65,7 +65,8 @@ family's quirk goes in its family file.** **A change to a kernel, a lexicon, a normalizer rule, or a phoneme rule ships the WER and UTMOS of `harness/tts_rig.py`, before and after, on every model the change reaches at the -rig's voice, in the PR body.** +rig's voice, in the PR body - unless a corpus-identity cell in `tests/test_tts_g2p.das` proves +the change moves no phoneme of the 200-sentence corpus, which pins the audio bit for bit.** **A text normalization or grapheme-to-phoneme error `harness/tts_rig.py`'s transcripts expose lands as a failing-first case in `tests/test_tts_textnorm.das` or diff --git a/modules/dasLLAMA/THIRD_PARTY_NOTICES.md b/modules/dasLLAMA/THIRD_PARTY_NOTICES.md index 31916a541b..b5a7b38993 100644 --- a/modules/dasLLAMA/THIRD_PARTY_NOTICES.md +++ b/modules/dasLLAMA/THIRD_PARTY_NOTICES.md @@ -35,7 +35,7 @@ derive from the Silero VAD project (https://github.com/snakers4/silero-vad), Cop ## Text-to-speech front end The TTS front end and the two TTS families reimplement, and pack data from, the following -works. The packed files (`tts_g2p.bin`, `tts_postag.bin`, `kitten-nano.gguf`, +works. The packed files (`tts_g2p.bin` and its American-only twin `tts_g2p_en_us.bin`, `tts_postag.bin`, `kitten-nano.gguf`, `kitten-mini.gguf`, `kokoro-82m.gguf`, the `tts_oracle/` dumps) are built by the `harness/` scripts into the model store (`performance/build_tts_data.das`), each beside a license file naming these sources; nothing under this repository redistributes the data itself. diff --git a/modules/dasLLAMA/benchmarks/REVIEW.md b/modules/dasLLAMA/benchmarks/REVIEW.md index e5b635f1f4..825b800031 100644 --- a/modules/dasLLAMA/benchmarks/REVIEW.md +++ b/modules/dasLLAMA/benchmarks/REVIEW.md @@ -71,10 +71,11 @@ no attribution across stages - is not a difference. and corpus - or withdraws the affected rows and names the withdrawal and its reason in the PR body.** A board cell is a timed cell of the published results board: one `../performance/gen_bench_records.das` spawns, or a manual `lcpp_bench.das` cell with its own -`../PROFILE.md` section. What a cell times changes when a change to its code, to its input -corpus, or to the pinned reference build - anything that decides which reference binary or -reference environment the run measures, `DEFAULT_REF_SHA` in `setup_lcpp_ref.das` included - -moves the measured quantity. The re-mint or withdrawal lands in +`../PROFILE.md` section. What a cell times changes when a change inside its timed body, to its +input corpus, or to the pinned reference build (`DEFAULT_REF_SHA` in `setup_lcpp_ref.das`, or +anything else deciding which reference binary or environment the run measures) moves the +measured quantity; a change outside the timed body - a flag, a require, the submit path - does +not. The re-mint or withdrawal lands in `../performance/records/.json`, the file the affected rows live in. **A diff that adds or changes an instrument makes it exit non-zero on a run that ends diff --git a/modules/dasLLAMA/benchmarks/lcpp_bench.das b/modules/dasLLAMA/benchmarks/lcpp_bench.das index e7c248b933..dc8c80efc2 100644 --- a/modules/dasLLAMA/benchmarks/lcpp_bench.das +++ b/modules/dasLLAMA/benchmarks/lcpp_bench.das @@ -31,8 +31,8 @@ require daslib/module_path // get_this_module_dir — the default pro require daslib/jobque_profile // JOBQUE_PROFILING_ENABLED — --prof refuses on a build without the marker rail require ?das_accelerate dasllama/dasllama_math_accelerate // the --accel leg; Apple-only C++ module require ../performance/profile_common.das // profile_threads + affinity_on + bench records — the standing per-box methodology -require dasllama/dasllama_exchange // --submit: POST the stamped records to the exchange -require llvm/daslib/llvm_tune // tune_app_script — the exe-vs-script discriminator behind --for-debug-purposes +require ?llvm dasllama/dasllama_exchange // nolint:STYLE030,LINT019 — OPTIONAL: --submit POSTs the stamped records to the exchange, which rides the tune framework +require ?llvm llvm/daslib/llvm_tune // nolint:STYLE030,LINT019 — OPTIONAL: tune_app_script, the exe-vs-script discriminator behind --for-debug-purposes require daslib/jobque_boost require daslib/clargs require daslib/fio @@ -1270,6 +1270,14 @@ def private tests_of(rows : array) : table { // flavor is stamped debug-jit so no board or oracle can mistake it for measurement var private g_debug_run = false +def private exchange_compiled_in() : bool { + static_if (typeinfo module_exists(dasllama_exchange)) { + return true + } else { + return false + } +} + def private flavor_stamp(base : string) : string { let b = tune_gate_bypassed() ? "untuned:{base}" : base return g_debug_run ? "debug-jit:{b}" : b @@ -1298,7 +1306,11 @@ def main() : int { } // measurement exists only from the released exe: a -jit SCRIPT run reads whatever sidecar // sits beside the script and re-keys nothing — its numbers must never enter records - g_debug_run = !empty(tune_app_script()) + static_if (typeinfo module_exists(llvm_tune)) { + g_debug_run = !empty(tune_app_script()) + } else { + g_debug_run = true // no framework, no released exe: a reference-kernel run is never a record + } if (g_debug_run && !cfg.for_debug_purposes) { print("lcpp_bench: this is a -jit SCRIPT run - measurement runs the released exe (daspkg release).\n") print(" pass --for-debug-purposes to proceed as a debug instrument; output is stamped debug-jit and is not record-grade\n") @@ -1351,6 +1363,10 @@ def main() : int { print("error: --submit refused on a debug -jit run - the numbers are debug-grade; submit from the released exe\n") return 1 } + if (cfg.submit && !exchange_compiled_in()) { + print("error: --submit refused - this build carries no sidecar exchange client (no tune framework)\n") + return 1 + } // the standing per-box methodology, enforced here rather than left to the environment // (gen_profile's pattern): cap the future que's workers (computing main is the last lane) and // hard-pin lanes to distinct physical cores on the boxes that pin. -t is the explicit ask, but @@ -1844,13 +1860,15 @@ def main() : int { } print("{text}\n") if (cfg.submit) { - let url = empty(g_env_exchange.exchange_url) ? EXCHANGE_URL_DEFAULT : g_env_exchange.exchange_url - let serr = exchange_submit_records_doc(url, text) - if (!empty(serr)) { - say(fmt, "records NOT submitted to {url}: {serr}\n") - return 1 + static_if (typeinfo module_exists(dasllama_exchange)) { + let url = empty(g_env_exchange.exchange_url) ? EXCHANGE_URL_DEFAULT : g_env_exchange.exchange_url + let serr = exchange_submit_records_doc(url, text) + if (!empty(serr)) { + say(fmt, "records NOT submitted to {url}: {serr}\n") + return 1 + } + say(fmt, "records submitted to {url}\n") } - say(fmt, "records submitted to {url}\n") } return 0 } diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_gemv_kernels.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_gemv_kernels.das index 0877b19b22..fdbbb4ba26 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_gemv_kernels.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_gemv_kernels.das @@ -11,7 +11,7 @@ require ?das_metal dasllama/dasllama_metal_gemm // the production 32x32 GEMM MS require dasllama/dasllama_metal_kernels // the production GEMV MSL (v0 baseline) require daslib/typemacro_boost // [template_struct_instance] - the simdgroup-matrix lab stamps require ?das_metal dasllama/dasllama_metal_prefill // MetalKqMulMmK45TensorT - the Metal-4 tensor GEMM stamped at verify widths -require llvm/daslib/f16_cvt // f16 bit converts for the K-quant superblock fills +require daslib/f16_cvt // f16 bit converts for the K-quant superblock fills require math // used only inside the Apple static_if half require strings diff --git a/modules/dasLLAMA/benchmarks/matmul/bench_metal_moe_lab.das b/modules/dasLLAMA/benchmarks/matmul/bench_metal_moe_lab.das index 4883b8237b..93918eb9fe 100644 --- a/modules/dasLLAMA/benchmarks/matmul/bench_metal_moe_lab.das +++ b/modules/dasLLAMA/benchmarks/matmul/bench_metal_moe_lab.das @@ -10,7 +10,7 @@ require dasllama/dasllama_env // the g_env_* knob globals ([EnvConfig] declarat require metal/msl_shader require dasllama/dasllama_metal_prefill // the production MoE mul_mm MSL (zero copy drift) require dasllama/dasllama_metal_kernels // the production MoE GEMV + geglu MSL (zero copy drift) -require llvm/daslib/f16_cvt +require daslib/f16_cvt require daslib/fio // used only inside the Apple static_if half require math // used only inside the Apple static_if half require daslib/math_bits // uint_bits_to_float in the lcppe kernel diff --git a/modules/dasLLAMA/benchmarks/matmul/matmul_variants.das b/modules/dasLLAMA/benchmarks/matmul/matmul_variants.das index 85918af16f..91f846d17d 100644 --- a/modules/dasLLAMA/benchmarks/matmul/matmul_variants.das +++ b/modules/dasLLAMA/benchmarks/matmul/matmul_variants.das @@ -4,7 +4,7 @@ module matmul_variants shared public require dasllama/dasllama_lint public require daslib/jobque_boost -require llvm/daslib/aarch64_neon // sdot4: JIT-emitted ARM SDOT (scalar fallback elsewhere) +require daslib/aarch64_neon // sdot4: JIT-emitted ARM SDOT (scalar fallback elsewhere) require math // mad: fused multiply-add in the decode-GEMV flush variants // Experiment ledger for the matmul kernel. Every optimization attempt lives here as diff --git a/modules/dasLLAMA/dasllama/dasllama.das b/modules/dasLLAMA/dasllama/dasllama.das index 256d2d3ff2..dcf7c152c2 100644 --- a/modules/dasLLAMA/dasllama/dasllama.das +++ b/modules/dasLLAMA/dasllama/dasllama.das @@ -47,9 +47,9 @@ require dasllama/dasllama_gpu_resident // resident slot-switch/pin surfa // ===== Model, session, tokenizer ===== def setup_dasllama_jobque() { - //! Configure the job queue for dasLLAMA's fork/join matmul dispatch: pooled fork contexts, - //! batched dispatch, and the worker spin-before-park window (``jobque_spin_us``; 0 disables). - //! Call INSIDE ``with_job_que()``, before the first ``generate``/``eval``. + //! Configure the job queue for dasLLAMA's fork/join matmul dispatch: pooled fork contexts, batched dispatch, the worker spin-before-park window (``jobque_spin_us``; 0 disables). + //! Call INSIDE ``with_job_que()``, before the first ``generate``/``eval``. The fork-context pool is PER CONTEXT: + //! a ``new_thread`` body that runs inference calls this again, or every parallel kernel clones the whole program (90x slower). setup_dasllama_jobque_() } diff --git a/modules/dasLLAMA/dasllama/dasllama_common.das b/modules/dasLLAMA/dasllama/dasllama_common.das index 74148544ee..117445e47a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_common.das +++ b/modules/dasLLAMA/dasllama/dasllama_common.das @@ -18,9 +18,10 @@ require daslib/jobque_profile // JOBQUE_PROFILING_ENABLED — the compile-time require dasllama/dasllama_kv_codec public // the KV-cache codec families — kv_store_row/kv_load_row/kv_dot/kv_axpy dispatch into them require dasllama/dasllama_convert // cvt_f32_to_f16/cvt_f16_to_f32 — the f16 rows (wscale plane + the f16 KV codec) require dasllama/dasllama_math_default // registers the portable Q8·Q8 backend at [init] (the fallback everywhere) -require dasllama/dasllama_math_aarch64_neon // registers the arm64 SDOT Q8·Q8 backend at [init]; no-op off-ARM -require ?llvm/daslib/llvm_tune dasllama/dasllama_math_gen // [init]-only: registers "arm64-gen" (the generated GEMM family, arm64 load-select tier). PATH guard: the module hard-requires llvm_tune, so it loads exactly when dasLLVM is mounted -require ?llvm/daslib/llvm_tune llvm/daslib/llvm_tune public // re-export ONLY llvm_tune (not math_gen's whole public surface — its jobque_boost would flood requirers with ambiguities): the auto-policy pass and tune_status reach every app that requires this module +require dasllama/dasllama_math_aarch64_neon // registers the arm64 SDOT Q8·Q8 backend at [init] under arm64 + JIT alone; compiles everywhere (daslib/aarch64_neon carries scalar fallbacks) +require dasllama/dasllama_math_gen // [init]-only: registers "arm64-gen" / "x64-gen" (the generated GEMM family) under the JIT alone; compiles everywhere - its reference bodies are the portable tier's kq kernels +require daslib/tune public // the tune annotations + the auto-policy pass, inert without dasLLVM; PUBLIC for the same reach as the line below +require ?llvm llvm/daslib/llvm_tune public // re-export ONLY llvm_tune (not math_gen's whole public surface — its jobque_boost would flood requirers with ambiguities): tune_status reaches every app that requires this module require ?das_metal dasllama/dasllama_metal_gemm // [init]-only: registers "metal" (the GPU prefill-GEMM batch donor). STRICT guard: the C++ das_metal module exists on Apple builds only, so non-Apple lanes never compile the Metal driver require ?das_accelerate dasllama/dasllama_math_accelerate // [init]-only: arms the "+AMX" float-batch override when DASLLAMA_ACCEL=1. Same Apple-only guard shape as the metal driver require dasllama/dasllama_gguf @@ -1476,16 +1477,16 @@ def apply_box_profile_runtime_checked(path : string) { g_box_profile_runtime_applied = false // both flags describe THIS apply, never an earlier one g_box_profile_runtime_declined = "" let reason = box_profile_verdict_at(path) - if (reason == TuneSidecarReason.absent) { - g_box_profile_runtime_declined = "{reason}" + if (reason == "absent") { + g_box_profile_runtime_declined = reason return } - if (reason == TuneSidecarReason.foreign_box || reason == TuneSidecarReason.unreadable) { - g_box_profile_runtime_declined = "{reason}" + if (reason == "foreign_box" || reason == "unreadable") { + g_box_profile_runtime_declined = reason to_log(LOG_INFO, "dasLLAMA: tune sidecar {path} not applied ({reason}) - its runtime knobs are the minting box's state\n") return } - if (reason == TuneSidecarReason.stale_binary) { + if (reason == "stale_binary") { to_log(LOG_INFO, "dasLLAMA: tune sidecar predates the binary - kernel winners fall back; the runtime knobs are box state and apply\n") } apply_box_profile_runtime_at(path) diff --git a/modules/dasLLAMA/dasllama/dasllama_convert.das b/modules/dasLLAMA/dasllama/dasllama_convert.das index e84a957117..5e243178fa 100644 --- a/modules/dasLLAMA/dasllama/dasllama_convert.das +++ b/modules/dasLLAMA/dasllama/dasllama_convert.das @@ -6,7 +6,7 @@ module dasllama_convert shared public require dasllama/dasllama_lint public require math -require llvm/daslib/f16_cvt // f16<->f32 halfword codecs — the Q5_K encoder + every f16 scale read +require daslib/f16_cvt // f16<->f32 halfword codecs — the Q5_K encoder + every f16 scale read require daslib/jobque_boost public // quantize_k5_plane threads its superblock loop require dasllama/dasllama_par // maybe_parallel_for (+ its emitted dispatch counters) require dasllama/dasllama_math // get_dispatch_lanes — the requant wrappers shape their fan-out diff --git a/modules/dasLLAMA/dasllama/dasllama_g2p.das b/modules/dasLLAMA/dasllama/dasllama_g2p.das index 63f34e7367..b9df30d5ef 100644 --- a/modules/dasLLAMA/dasllama/dasllama_g2p.das +++ b/modules/dasLLAMA/dasllama/dasllama_g2p.das @@ -10,7 +10,7 @@ require strings require daslib/strings_boost require daslib/fio require daslib/utf8_utils -require llvm/daslib/f16_cvt // the pack stores the GRU as f16; the reader widens it once at load +require daslib/f16_cvt // the pack stores the GRU as f16; the reader widens it once at load require dasllama/dasllama_unicode require dasllama/dasllama_postag require dasllama/dasllama_textnorm @@ -38,6 +38,7 @@ struct LexTable { struct G2pModel { data : array source : string + us_only : bool //! the pack carries the American tier alone (build_g2p_data.py --dialect us): British is refused by name gold : LexTable silver : LexTable cmu : LexTable @@ -237,6 +238,7 @@ def load_g2p(path : string) : G2pModel { let src_len = rd_u16(g.data, o) o += 2l g.source = bytes_str(g.data, o, src_len) + g.us_only = find(g.source, "dialect us") >= 0 o += int64(src_len) g.gold = read_lex_table(g.data, o) g.silver = read_lex_table(g.data, o) @@ -1756,6 +1758,9 @@ def g2p_tokens(g : G2pModel; tagged : array; british : bool = false) : //! Phonemes for normalized text, in the inventory `british` names: tokens joined with the //! source spacing. def g2p_phonemize(g : G2pModel; tg : PosTagger; text : string; british : bool = false) : string { + if (british && g.us_only) { + panic("dasLLAMA tts: this phoneme pack carries the American tier only ({g.source}) - British English needs the full tts_g2p.bin") + } var inscope tagged <- pos_tokenize_tag(tg, text) var inscope toks <- g2p_tokens(g, tagged, british) return build_string() $(var w) { diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das index e2640f7f5f..de690f75ed 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_gen.das @@ -1502,15 +1502,6 @@ def private grid_rows_path(te : TileEmit) : bool { return false } -//! eight +-1 bytes for sign byte `sb`, lane k negative when bit k is set -def public pm1_of(sb : int) : uint64 { // test_kquant round-trips the sign expansion - var v = 0ul - for (k in range(8)) { - v |= uint64(((sb >> k) & 1) != 0 ? 0xFF : 0x01) << uint64(8 * k) - } - return v -} - //! the row-pair decode's sign table: the 128 ksigns codes with the code's parity as the 8th bit //! (ksigns[i] == i | parity(i) << 7), each as its eight +-1 bytes def private sign_pm1_emit_globals(var te : TileEmit; mod : LLVMOpaqueModule?) { @@ -1876,13 +1867,6 @@ def private vbmi_push_iq2(te : TileEmit; var p0, p1 : array; w p1 |> push(LLVMConstInt(te.types.t_int8, uint64(vbmi_pack_word(w1, alpha, 2, 4)), 0)) } -//! the grid's symbol alphabet: the halved iq3xxs magnitudes, iq3s's odd magnitudes, the iq2 family's three -def public vbmi_alphabet(kq : int) : array { // test_kquant round-trips the code planes - if (kq == 34) return <- [2, 6, 10, 14, 18, 22, 26, 31] - if (kq == 33) return <- [1, 3, 5, 7, 9, 11, 13, 15] - return <- [8, 25, 43] -} - //! the grid gemv's module-level tables: the format's grid, the ksigns +-1 table when the row form reads it, the lattice's def private grid_emit_globals(var te : TileEmit; fmt : int; mod : LLVMOpaqueModule?) { if (fmt == 33) { @@ -1907,20 +1891,6 @@ def private vbmi_tag(kq : int) : string => kq == 25 ? "iq2xxs" : (kq == 24 ? "iq //! grid entries per format: the compact plane's byte count def private vbmi_entries(kq : int) : int => (kq == 25 || kq == 34) ? 256 : (kq == 23 ? 1024 : 512) -//! a grid word's low `nbytes` weight bytes as packed symbols (weight t at bits t*bits); a byte outside the alphabet is a broken table -def public vbmi_pack_word(word : uint; alpha : array; bits, nbytes : int) : int { - var code = 0 - for (t in range(nbytes)) { - let wb = int((word >> uint(8 * t)) & 0xFFu) - let si = alpha |> find_index(wb) - if (si < 0) { - panic("vbmi lattice: grid byte {wb} is outside the alphabet") - } - code |= si << (bits * t) - } - return code -} - def private vbmi_const_bytes(te : TileEmit; mod : LLVMOpaqueModule?; name : string; var elems : array) : LLVMOpaqueValue? { var g = LLVMAddGlobal(mod, LLVMArrayType(te.types.t_int8, uint(length(elems))), name) LLVMSetInitializer(g, LLVMConstArray(te.types.t_int8, array_data_ptr(elems), uint(length(elems)))) diff --git a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das index f476357100..16ee216ab5 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das +++ b/modules/dasLLAMA/dasllama/dasllama_gemm_schema.das @@ -104,3 +104,33 @@ def kq_ssb(fmt : int) : int64 { def kq_reads_packed_planes(fmt : int64) : bool { return fmt == 4l || fmt == 40l || fmt == 44l || fmt == 3l || fmt == 45l || fmt == 2l } + +//! eight +-1 bytes for sign byte `sb`, lane k negative when bit k is set +def pm1_of(sb : int) : uint64 { + var v = 0ul + for (k in range(8)) { + v |= uint64(((sb >> k) & 1) != 0 ? 0xFF : 0x01) << uint64(8 * k) + } + return v +} + +//! the grid's symbol alphabet: the halved iq3xxs magnitudes, iq3s's odd magnitudes, the iq2 family's three +def vbmi_alphabet(kq : int) : array { + if (kq == 34) return <- [2, 6, 10, 14, 18, 22, 26, 31] + if (kq == 33) return <- [1, 3, 5, 7, 9, 11, 13, 15] + return <- [8, 25, 43] +} + +//! a grid word's low `nbytes` weight bytes as packed symbols (weight t at bits t*bits); a byte outside the alphabet is a broken table +def vbmi_pack_word(word : uint; alpha : array; bits, nbytes : int) : int { + var code = 0 + for (t in range(nbytes)) { + let wb = int((word >> uint(8 * t)) & 0xFFu) + let si = alpha |> find_index(wb) + if (si < 0) { + panic("vbmi lattice: grid byte {wb} is outside the alphabet") + } + code |= si << (bits * t) + } + return code +} diff --git a/modules/dasLLAMA/dasllama/dasllama_gguf.das b/modules/dasLLAMA/dasllama/dasllama_gguf.das index 743b88a045..d033a9d8ab 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gguf.das +++ b/modules/dasLLAMA/dasllama/dasllama_gguf.das @@ -12,7 +12,7 @@ require daslib/fio // shard-path expansion + mapping (split GGU require daslib/strings_boost // trim_suffix / pad_left (split shard names) require daslib/strings_convert // try_to_int (validating shard-index parse) require dasllama/dasllama_env // g_env_engine.mtp_head — the split NextN head sidecar override -require llvm/daslib/f16_cvt public // f16_to_f32 / f32_to_f16 (re-export: whisper/vad/parakeet/audio callers) +require daslib/f16_cvt public // f16_to_f32 / f32_to_f16 (re-export: whisper/vad/parakeet/audio callers) require dasllama/dasllama_convert public // the pure codec layer (readers/dequants/transcodes/encoder) — split out 2026-07-29 require dasllama/dasllama_par // maybe_parallel_for (+ its emitted dispatch counters) — the transcode hot loops thread when a jobque is up diff --git a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das index 8341f857bd..44e83908c3 100644 --- a/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das +++ b/modules/dasLLAMA/dasllama/dasllama_gpu_resident.das @@ -17,7 +17,7 @@ require dasllama/dasllama_kqformat require dasllama/dasllama_math require dasllama/dasllama_rope // build_rope_row_packed — the one source for every resident rope row require dasllama/dasllama_gemm_schema // Q8_BLOCK_ELEMS / KQ_SUPERBLOCK_ELEMS — device block sizing -require llvm/daslib/f16_cvt // f32_to_f16 — the trim pass re-packs the emb scale plane +require daslib/f16_cvt // f32_to_f16 — the trim pass re-packs the emb scale plane require dasllama/dasllama_par require dasllama/dasllama_env require math diff --git a/modules/dasLLAMA/dasllama/dasllama_image.das b/modules/dasLLAMA/dasllama/dasllama_image.das index a92440a09b..ae6c997a64 100644 --- a/modules/dasLLAMA/dasllama/dasllama_image.das +++ b/modules/dasLLAMA/dasllama/dasllama_image.das @@ -1525,7 +1525,7 @@ def private cache_via_image_(var m; var out; img : string; tag : string; quant : [arch(at = "../ARCHITECTURE_IMAGE.md#image-interp-load")] def private guard_interp_gguf_load(path : string) { - if (jit_enabled() || is_standalone_exe()) { + if (jit_enabled() || is_standalone_exe() || aot_kernels_linked()) { return } var st : FStat diff --git a/modules/dasLLAMA/dasllama/dasllama_kitten.das b/modules/dasLLAMA/dasllama/dasllama_kitten.das index 01753361b3..c472c81f56 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kitten.das +++ b/modules/dasLLAMA/dasllama/dasllama_kitten.das @@ -7,10 +7,9 @@ module dasllama_kitten shared public require dasllama/dasllama_lint public require math require strings -require daslib/fio -require dasllama/dasllama_gguf require dasllama/dasllama_common require dasllama/dasllama_unicode +require dasllama/dasllama_tts_types require dasllama/dasllama_styletts2 //! The KittenTTS family (nano, mini): the reference driver's token table, re-spacing and @@ -95,37 +94,6 @@ def kitten_style_row(norm_text : string; rows : int) : int { return clamp(length(cps), 0, rows - 1) } -//! Kitten's driver data beside the shared model: per-voice speed priors and the alias names. -struct KittenFamily { - speed_priors : table - aliases : table -} - -//! Read the `kitten.*` metadata of a converted KittenTTS GGUF. -def load_kitten_family(path : string) : KittenFamily { - var inscope fam = KittenFamily() - let f = fopen(path, "rb") - if (f == null) { - panic("dasLLAMA kitten: cannot open model '{path}'") - } - fmap(f) $(var bytes : array#) { - var inscope m <- parse_gguf_meta(bytes) - var inscope voices <- gguf_str_array(m, bytes, "styletts2.voices") - for (v in voices) { - if (gguf_has(m, "kitten.speed_prior.{v}")) { - fam.speed_priors[v] = gguf_f32(m, bytes, "kitten.speed_prior.{v}") - } - } - var inscope anames <- gguf_str_array(m, bytes, "kitten.alias_names") - var inscope avoices <- gguf_str_array(m, bytes, "kitten.alias_voices") - for (an, av in anames, avoices) { - fam.aliases[an] = av - } - } - fclose(f) - return <- fam -} - //! Resolve a voice name or alias to a canonical voice; "" when unknown. def kitten_voice(fam : KittenFamily; m : StyleTts2Model; name : string) : string { if (styletts2_has_voice(m, name)) { diff --git a/modules/dasLLAMA/dasllama/dasllama_kokoro.das b/modules/dasLLAMA/dasllama/dasllama_kokoro.das index 8c2b02b834..62e3f01b6a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kokoro.das +++ b/modules/dasLLAMA/dasllama/dasllama_kokoro.das @@ -7,38 +7,13 @@ module dasllama_kokoro shared public require dasllama/dasllama_lint public require math require strings -require daslib/fio -require dasllama/dasllama_gguf require dasllama/dasllama_unicode +require dasllama/dasllama_tts_types //! The Kokoro family (Kokoro-82M): the reference pipeline's symbol table (the front end's own //! inventory - no rewrite), its token wrapping and style-row rule. The model itself is the //! shared StyleTTS2 assembly; the voice packs carry one style row per phoneme count. -//! Kokoro's driver data beside the shared model: codepoint -> token id. -struct KokoroFamily { - sym : table -} - -//! Read the `kokoro.*` metadata of a converted Kokoro GGUF. -def load_kokoro_family(path : string) : KokoroFamily { - var inscope fam = KokoroFamily() - let f = fopen(path, "rb") - if (f == null) { - panic("dasLLAMA kokoro: cannot open model '{path}'") - } - fmap(f) $(var bytes : array#) { - var inscope m <- parse_gguf_meta(bytes) - var inscope cpts <- gguf_int_array(m, bytes, "kokoro.symbol_cpts") - var inscope ids <- gguf_int_array(m, bytes, "kokoro.symbol_ids") - for (c, i in cpts, ids) { - fam.sym[int(c)] = int(i) - } - } - fclose(f) - return <- fam -} - //! The token ids the model reads for a phoneme string in the front end's inventory: every //! symbol the vocabulary knows, in order, wrapped in the pad token on both sides. def kokoro_token_ids(fam : KokoroFamily; ps : string) : array { diff --git a/modules/dasLLAMA/dasllama/dasllama_kv_codec.das b/modules/dasLLAMA/dasllama/dasllama_kv_codec.das index fc90442883..42dfc43e52 100644 --- a/modules/dasLLAMA/dasllama/dasllama_kv_codec.das +++ b/modules/dasLLAMA/dasllama/dasllama_kv_codec.das @@ -6,7 +6,7 @@ module dasllama_kv_codec shared public require dasllama/dasllama_lint public require math -require llvm/daslib/f16_cvt // per-element f16 scale reads/writes inside the hot loops +require daslib/f16_cvt // per-element f16 scale reads/writes inside the hot loops require dasllama/dasllama_math // dot/axpy/copy_floats — the f32 arms of the dispatch below require dasllama/dasllama_convert // cvt_f16_to_f32 — the bulk f16 row dequant require dasllama/dasllama_tune // [tuned]: reconstitute kernels from templates with per-box loop hints diff --git a/modules/dasLLAMA/dasllama/dasllama_load.das b/modules/dasLLAMA/dasllama/dasllama_load.das index 60bd97f697..04748cfeb9 100644 --- a/modules/dasLLAMA/dasllama/dasllama_load.das +++ b/modules/dasLLAMA/dasllama/dasllama_load.das @@ -22,7 +22,7 @@ require dasllama/dasllama_tune require dasllama/dasllama_env require dasllama/dasllama_par require dasllama/dasllama_gpu_tier -require llvm/daslib/f16_cvt +require daslib/f16_cvt require daslib/json_boost require daslib/array_boost require dasllama/dasllama_tokenizer // load_tokenizer_auto — load_model_ populates the Model's tokenizer diff --git a/modules/dasLLAMA/dasllama/dasllama_math.das b/modules/dasLLAMA/dasllama/dasllama_math.das index a6070c3e36..8bab029d7b 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math.das +++ b/modules/dasLLAMA/dasllama/dasllama_math.das @@ -16,7 +16,7 @@ require dasllama/dasllama_tune // [tuned]: reconstitute dot from dot_template w require dasllama/dasllama_tune_scope // nolint:STYLE030 — the dasllama [tune_scope] must be on the AST before this module's [tuned] kernels pick their perms require dasllama/dasllama_gemm_schema // kq_qsb/kq_ssb — the fmt-id kq plane strides require dasllama/dasllama_gpu_tier public // the device-cooperation seam (split out 2026-07-29) — re-exported for the 112 requirers -require llvm/daslib/f16_cvt // f16<->f32 converts: JIT half intrinsics on aarch64 / x64+F16C, exact bit-twiddle elsewhere +require daslib/f16_cvt // f16<->f32 converts: JIT half intrinsics on aarch64 / x64+F16C, exact bit-twiddle elsewhere // dasLLAMA numeric primitives — naive fp32 baseline (curriculum step 1). // Correctness first; optimization (blocking, float4, threading) comes later. @@ -25,11 +25,20 @@ require llvm/daslib/f16_cvt // f16<->f32 converts: JIT half intrinsics on aarch // can be passed zero-copy as views into one big model blob and outputs can be written // straight into a view (e.g. the KV cache). Dims/indices are int64 for large models. -//! The library-seam ``-jit`` stop: dasLLAMA inference NEVER runs interpreted (kernels 10-40x -//! slower). Inference seams call it; scaffolding that only parses or peeks never crosses it. +//! True when the engine's kernels are the C++ the AOT emitter produced - the host linked this +//! module's stubs (examples/dasLLAMA, `-use-aot`). A runtime probe of one kernel's binding, not +//! `aot_enabled()`: that one folds at compile time, differently under AOT generation and the +//! AOT-consuming run, and would desync every caller's semantic hash (error[50101]). +def aot_kernels_linked() : bool { + return is_aot_function(@@ < (var x : float?; size : int64) : void > silu) +} + +//! The library-seam compiled-tier stop: dasLLAMA inference NEVER runs interpreted (kernels 10-40x +//! slower). The JIT, a standalone exe, and linked AOT stubs each pass; inference seams call it, +//! scaffolding that only parses or peeks never crosses it. def guard_interp_inference(what : string) { - if (!jit_enabled() && !is_standalone_exe()) { - panic("dasLLAMA: {what} requires -jit (or a compiled exe) - interpreted inference runs the kernels 10-40x slower") + if (!jit_enabled() && !is_standalone_exe() && !aot_kernels_linked()) { + panic("dasLLAMA: {what} requires -jit, a compiled exe, or an AOT host with the engine's stubs (-use-aot) - interpreted inference runs the kernels 10-40x slower") } } @@ -287,7 +296,7 @@ def dasllama_jobque_threads_cap() { // Elsewhere mode 2 is a REAL mask — stays opt-in. DAS_JOBQUE_AFFINITY and app calls override. [init] def dasllama_jobque_affinity_default() { - if (get_platform_name() == "darwin" && get_jobque_affinity() < 0) { + if (get_platform_name() == "darwin" && empty(get_target_triple()) && get_jobque_affinity() < 0) { // the HOST's darwin, never a cross artifact's set_jobque_affinity(2) } } @@ -2046,9 +2055,9 @@ def matmul_q8q8(var y : array; wq : array; ws : array; woff } } -//! Batched Q8·Q8 GEMM for prefill: same weight-stationary nest as matmul_batch over the SDOT kernel. -//! Activations PRE-quantized per token into (xq [ntok x n], xs [ntok x n/32]). Y token-major -//! [ntok x d]. Bit-for-bit ntok× matmul_q8q8. Dispatches to the active backend. n % 32 == 0. +//! Batched Q8·Q8 GEMM for prefill over PRE-quantized activations (xq [ntok x n], xs [ntok x n/32]), +//! Y token-major [ntok x d], n % 32 == 0. Bit-for-bit ntok× matmul_q8q8 on the portable backend; +//! a generated backend's tile and token-tail perms fold differently (test_q8q8_family holds them within the dot's envelope). def matmul_q8q8_batch(var y : array; wq : array | #; ws : array | #; xq : array; xs : array; n, d, ntok : int64) { unsafe { invoke(g_mm_q8q8_batch, addr(y[0]), diff --git a/modules/dasLLAMA/dasllama/dasllama_math_aarch64_neon.das b/modules/dasLLAMA/dasllama/dasllama_math_aarch64_neon.das index d1b2f268d7..371471177e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_aarch64_neon.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_aarch64_neon.das @@ -9,8 +9,8 @@ require dasllama/dasllama_math require dasllama/dasllama_math_default // dot_q8q8_idot4x4/_f16s — the plain-das GEMV dots this backend dispatches require dasllama/dasllama_tune // [tuned]: reconstitute the laneq4x4 tile from its template with per-box loop hints require dasllama/dasllama_tune_scope // nolint:STYLE030 — the dasllama [tune_scope] must be on the AST before this module's [tuned] kernels pick their perms -require llvm/daslib/aarch64_neon // tbl16/sdot4_w/sdot4_laneq: the mx4 + laneq tiles' JIT intrinsics -require llvm/daslib/f16_cvt // f16_to_f32 — the wscale_f16 twins widen the binary16 scale in-loop +require daslib/aarch64_neon // tbl16/sdot4_w/sdot4_laneq: the mx4 + laneq tiles' JIT intrinsics +require daslib/f16_cvt // f16_to_f32 — the wscale_f16 twins widen the binary16 scale in-loop require daslib/jobque_boost public // kernels expand parallel_for; consumers need its symbols require dasllama/dasllama_par // maybe_parallel_for @@ -451,12 +451,9 @@ def mx4q8_rows_kernel_neon(var yp : float?; wn : uint8 const?; we : uint8 const? } } -[init] +[init, arch(at = "../ARCHITECTURE_CPU_KERNELS.md#cpu-tier-target-select")] def dasllama_math_aarch64_neon_register() { - // Only register the SDOT backends under JIT on arm64. Off-JIT (interpreter or AOT) sdot4/sdot4_laneq - // run their SCALAR fallback body — slower than the portable dot_q8q8, which the vectorizer handles — - // so the portable backend stays selected everywhere the hardware SDOT isn't emitted. - if (get_architecture_name() == "arm64" && jit_enabled()) { + if (get_target_architecture_name() == "arm64" && jit_enabled()) { // arm64-sdot: idot4x4 on row-major weights (idot4 → SDOT) — the auto-selected default for direct callers // (tests, benchmarks). priority 10 out-ranks portable (0). register_kernel_backend(KernelBackend(name = "arm64-sdot", mm = @@q8q8_kernel_neon_idot4x4, diff --git a/modules/dasLLAMA/dasllama/dasllama_math_accelerate.das b/modules/dasLLAMA/dasllama/dasllama_math_accelerate.das index 992d2aab34..9d1a460acd 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_accelerate.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_accelerate.das @@ -10,7 +10,7 @@ require dasllama/dasllama_convert // cvt_f32_to_f16 — the BNNS f16 batch prep require dasllama/dasllama_env // the g_env_* knob globals ([EnvConfig] declarations; ENVIRONMENT.md is generated from them) require dasllama/dasllama_par require das_accelerate -require llvm/daslib/f16_cvt +require daslib/f16_cvt require math require daslib/math_bits diff --git a/modules/dasLLAMA/dasllama/dasllama_math_default.das b/modules/dasLLAMA/dasllama/dasllama_math_default.das index df55a1fce2..93f1120af2 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_default.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_default.das @@ -9,7 +9,7 @@ require dasllama/dasllama_math // KernelBackend + register_kernel_backend require dasllama/dasllama_tune // [tuned]: reconstitute dot_q8q8 from its template with per-box loop hints require dasllama/dasllama_tune_scope // nolint:STYLE030 — the dasllama [tune_scope] must be on the AST before this module's [tuned] kernels pick their perms require dasllama/dasllama_gemm_schema // kq_qsb/kq_ssb — the fmt-id kq plane strides -require llvm/daslib/f16_cvt // dot_q8q8_f16s widens the binary16 weight scale in-loop +require daslib/f16_cvt // dot_q8q8_f16s widens the binary16 weight scale in-loop require daslib/jobque_boost public // parallel_for / notify_and_release (the kernels expand parallel_for) require dasllama/dasllama_par // maybe_parallel_for require math @@ -146,6 +146,92 @@ def dot_q8q8_idot4x4_f16s(wq : int8 const?; ws : uint16 const?; xq : int8 const? return f0 + f1 + f2 + f3 } +//! The vector-epilogue idot4 dot: per 32-block one idot4 pair, then float4 accumulate under the block's +//! scale - no horizontal sum until the row ends, two rotating float4 accumulators. The wasm SIMD128 form +//! (dot_q8q8_idot4x4's per-block hsum is four lane extracts there). Called from lifted lambdas. +[hint(unsafe_range_check, noalias = wq, noalias = xq, noalias = ws, noalias = xs)] +def dot_q8q8_idot4_ps(wq : int8 const?; ws : float const?; xq : int8 const?; xs : float const?; n : int64) : float { + let nb = n / 32l + var f0 = float4(0.0, 0.0, 0.0, 0.0) + var f1 = float4(0.0, 0.0, 0.0, 0.0) + var bi = 0l + unsafe { + let wv = reinterpret(wq) + let xv = reinterpret(xq) + while (bi + 2l <= nb) { + let v = bi * 2l + var a0 = idot4(wv[v], xv[v]) + a0 = idot4(a0, wv[v + 1l], xv[v + 1l]) + var a1 = idot4(wv[v + 2l], xv[v + 2l]) + a1 = idot4(a1, wv[v + 3l], xv[v + 3l]) + let s0 = ws[bi] * xs[bi] + let s1 = ws[bi + 1l] * xs[bi + 1l] + f0 += float4(a0) * float4(s0, s0, s0, s0) + f1 += float4(a1) * float4(s1, s1, s1, s1) + bi += 2l + } + while (bi < nb) { + let v = bi * 2l + var a0 = idot4(wv[v], xv[v]) + a0 = idot4(a0, wv[v + 1l], xv[v + 1l]) + let s0 = ws[bi] * xs[bi] + f0 += float4(a0) * float4(s0, s0, s0, s0) + bi++ + } + } + let f = f0 + f1 + return f.x + f.y + f.z + f.w +} + +//! The wscale_f16 twin of dot_q8q8_idot4_ps: the per-block weight scale is a raw binary16 halfword +//! widened in-loop. Not `private`: called from lifted worker lambdas. +[hint(unsafe_range_check, noalias = wq, noalias = xq, noalias = ws, noalias = xs)] +def dot_q8q8_idot4_ps_f16s(wq : int8 const?; ws : uint16 const?; xq : int8 const?; xs : float const?; n : int64) : float { + let nb = n / 32l + var f0 = float4(0.0, 0.0, 0.0, 0.0) + var f1 = float4(0.0, 0.0, 0.0, 0.0) + var bi = 0l + unsafe { + let wv = reinterpret(wq) + let xv = reinterpret(xq) + while (bi + 2l <= nb) { + let v = bi * 2l + var a0 = idot4(wv[v], xv[v]) + a0 = idot4(a0, wv[v + 1l], xv[v + 1l]) + var a1 = idot4(wv[v + 2l], xv[v + 2l]) + a1 = idot4(a1, wv[v + 3l], xv[v + 3l]) + let s0 = f16_to_f32(uint(ws[bi])) * xs[bi] + let s1 = f16_to_f32(uint(ws[bi + 1l])) * xs[bi + 1l] + f0 += float4(a0) * float4(s0, s0, s0, s0) + f1 += float4(a1) * float4(s1, s1, s1, s1) + bi += 2l + } + while (bi < nb) { + let v = bi * 2l + var a0 = idot4(wv[v], xv[v]) + a0 = idot4(a0, wv[v + 1l], xv[v + 1l]) + let s0 = f16_to_f32(uint(ws[bi])) * xs[bi] + f0 += float4(a0) * float4(s0, s0, s0, s0) + bi++ + } + } + let f = f0 + f1 + return f.x + f.y + f.z + f.w +} + +//! a def, not a module global: team lanes run no global initializers, and the fold below is a +//! compile-time constant either way +def private wasm_target() : bool => get_target_architecture_name() == "wasm64" || get_target_architecture_name() == "wasm32" + +[arch(at = "../ARCHITECTURE_CPU_KERNELS.md#cpu-tier-target-select")] +def private dot_q8q8_p(wq : int8 const?; ws : float const?; xq : int8 const?; xs : float const?; n : int64) : float { + return wasm_target() ? dot_q8q8_idot4_ps(wq, ws, xq, xs, n) : dot_q8q8(wq, ws, xq, xs, n) +} + +def private dot_q8q8_f16s_p(wq : int8 const?; ws : uint16 const?; xq : int8 const?; xs : float const?; n : int64) : float { + return wasm_target() ? dot_q8q8_idot4_ps_f16s(wq, ws, xq, xs, n) : dot_q8q8_f16s(wq, ws, xq, xs, n) +} + // ----- parallel_for dispatch profiler (off by default) ----- // Records each worker's start time (rel. to pf-start) into g_pf_ws[rb]; a large first/last-worker gap // = workers trickling in (job pool waking from sleep) — the suspected prefill stall. @@ -185,7 +271,7 @@ def private q8q8_kernel(var yp : float?; wp : int8 const?; sp : float const?; xq maybe_parallel_for(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(rb, re) { unsafe { for (i in range(rb, re)) { - myp[i] = dot_q8q8(wp + int64(i) * n, sp + int64(i) * nb, xqp, xsp, n) + myp[i] = dot_q8q8_p(wp + int64(i) * n, sp + int64(i) * nb, xqp, xsp, n) } } } @@ -213,7 +299,7 @@ def private q8q8_batch_kernel(var yp : float?; wp : int8 const?; sp : float cons let wrow = wp + int64(i) * n let srow = sp + int64(i) * nb for (tk in range64(ntok)) { - myp[tk * d + int64(i)] = dot_q8q8(wrow, srow, xqp + tk * n, xsp + tk * nb, n) + myp[tk * d + int64(i)] = dot_q8q8_p(wrow, srow, xqp + tk * n, xsp + tk * nb, n) } } } @@ -242,7 +328,7 @@ def private q8q8_batch_kernel(var yp : float?; wp : int8 const?; sp : float cons let wrow = wp + int64(i) * n let srow = sp + int64(i) * nb for (tk in range64(ntok)) { - myp[tk * d + int64(i)] = dot_q8q8(wrow, srow, xqp + tk * n, xsp + tk * nb, n) + myp[tk * d + int64(i)] = dot_q8q8_p(wrow, srow, xqp + tk * n, xsp + tk * nb, n) } } } @@ -256,7 +342,7 @@ def q8q8_rows_kernel(var yp : float?; wp : int8 const?; sp : float const?; xqp : let nb = n / 32l unsafe { for (i in range64(rb, re)) { - yp[i] = dot_q8q8(wp + i * n, sp + i * nb, xqp, xsp, n) + yp[i] = dot_q8q8_p(wp + i * n, sp + i * nb, xqp, xsp, n) } } } @@ -273,13 +359,13 @@ def private q8q8_group3_kernel(var y0p : float?; var y1p : float?; var y2p : flo for (gi in range(rb, re)) { let g = int64(gi) if (g < d0) { - w0[g] = dot_q8q8(wp + woff0 + g * n, sp + woff0 / 32l + g * nb, xqp, xsp, n) + w0[g] = dot_q8q8_p(wp + woff0 + g * n, sp + woff0 / 32l + g * nb, xqp, xsp, n) } elif (g < d0 + d1) { let r = g - d0 - w1[r] = dot_q8q8(wp + woff1 + r * n, sp + woff1 / 32l + r * nb, xqp, xsp, n) + w1[r] = dot_q8q8_p(wp + woff1 + r * n, sp + woff1 / 32l + r * nb, xqp, xsp, n) } else { let r = g - d0 - d1 - w2[r] = dot_q8q8(wp + woff2 + r * n, sp + woff2 / 32l + r * nb, xqp, xsp, n) + w2[r] = dot_q8q8_p(wp + woff2 + r * n, sp + woff2 / 32l + r * nb, xqp, xsp, n) } } } @@ -295,7 +381,7 @@ def private q8q8_kernel_s16(var yp : float?; wp : int8 const?; sp : uint16 const maybe_parallel_for(0, int(d), matmul_chunks_gemv(int(d), 1, n * d)) $(rb, re) { unsafe { for (i in range(rb, re)) { - myp[i] = dot_q8q8_f16s(wp + int64(i) * n, sp + int64(i) * nb, xqp, xsp, n) + myp[i] = dot_q8q8_f16s_p(wp + int64(i) * n, sp + int64(i) * nb, xqp, xsp, n) } } } @@ -310,7 +396,7 @@ def private q8q8_batch_kernel_s16(var yp : float?; wp : int8 const?; sp : uint16 let wrow = wp + int64(i) * n let srow = sp + int64(i) * nb for (tk in range64(ntok)) { - myp[tk * d + int64(i)] = dot_q8q8_f16s(wrow, srow, xqp + tk * n, xsp + tk * nb, n) + myp[tk * d + int64(i)] = dot_q8q8_f16s_p(wrow, srow, xqp + tk * n, xsp + tk * nb, n) } } } @@ -322,7 +408,7 @@ def q8q8_rows_kernel_s16(var yp : float?; wp : int8 const?; sp : uint16 const?; let nb = n / 32l unsafe { for (i in range64(rb, re)) { - yp[i] = dot_q8q8_f16s(wp + i * n, sp + i * nb, xqp, xsp, n) + yp[i] = dot_q8q8_f16s_p(wp + i * n, sp + i * nb, xqp, xsp, n) } } } @@ -339,13 +425,13 @@ def private q8q8_group3_kernel_s16(var y0p : float?; var y1p : float?; var y2p : for (gi in range(rb, re)) { let g = int64(gi) if (g < d0) { - w0[g] = dot_q8q8_f16s(wp + woff0 + g * n, sp + woff0 / 32l + g * nb, xqp, xsp, n) + w0[g] = dot_q8q8_f16s_p(wp + woff0 + g * n, sp + woff0 / 32l + g * nb, xqp, xsp, n) } elif (g < d0 + d1) { let r = g - d0 - w1[r] = dot_q8q8_f16s(wp + woff1 + r * n, sp + woff1 / 32l + r * nb, xqp, xsp, n) + w1[r] = dot_q8q8_f16s_p(wp + woff1 + r * n, sp + woff1 / 32l + r * nb, xqp, xsp, n) } else { let r = g - d0 - d1 - w2[r] = dot_q8q8_f16s(wp + woff2 + r * n, sp + woff2 / 32l + r * nb, xqp, xsp, n) + w2[r] = dot_q8q8_f16s_p(wp + woff2 + r * n, sp + woff2 / 32l + r * nb, xqp, xsp, n) } } } @@ -363,7 +449,7 @@ def q8q8_groupn_kernel_s16(var yp : float?; wp : int8 const?; sp : uint16 const? let row = ii % d let woff = offs[r * 2l] let xoff = offs[r * 2l + 1l] - var v = dot_q8q8_f16s(wp + woff + row * n, sp + woff / 32l + row * nb, xqp + xoff, xsp + xoff / 32l, n) + var v = dot_q8q8_f16s_p(wp + woff + row * n, sp + woff / 32l + row * nb, xqp + xoff, xsp + xoff / 32l, n) if (bp != null) { v += bp[boffs[r] + row] } @@ -1570,7 +1656,7 @@ def q8q8_groupn_kernel(var yp : float?; wp : int8 const?; sp : float const?; off let row = ii % d let woff = offs[r * 2l] let xoff = offs[r * 2l + 1l] - var v = dot_q8q8(wp + woff + row * n, sp + woff / 32l + row * nb, xqp + xoff, xsp + xoff / 32l, n) + var v = dot_q8q8_p(wp + woff + row * n, sp + woff / 32l + row * nb, xqp + xoff, xsp + xoff / 32l, n) if (bp != null) { v += bp[boffs[r] + row] } diff --git a/modules/dasLLAMA/dasllama/dasllama_math_gen.das b/modules/dasLLAMA/dasllama/dasllama_math_gen.das index 3df6053d3e..475a901694 100644 --- a/modules/dasLLAMA/dasllama/dasllama_math_gen.das +++ b/modules/dasLLAMA/dasllama/dasllama_math_gen.das @@ -10,9 +10,10 @@ require dasllama/dasllama_math require dasllama/dasllama_math_aarch64_neon require dasllama/dasllama_math_default // dot_q8q8_f16s — the row-major tails of the s16 twins require dasllama/dasllama_repack // the grp repack family + kq panel unpack (pure transforms; the _gen/_bake twins below feed them numbers) -require llvm/daslib/f16_cvt // the kq reference bodies widen the f16 (s, o) scale pairs -require llvm/daslib/llvm_tune public // [tune]/[tune_perm]/[tune_companion] + [llvm_code]; PUBLIC so the auto-policy infer pass and tune_status reach app roots that require this module -require dasllama/dasllama_tune_scope // nolint:STYLE030 — the dasllama [tune_scope] must be on the AST before this module's [tune] kernels pick their perms +require daslib/f16_cvt // the kq reference bodies widen the f16 (s, o) scale pairs +require daslib/tune public // [tune]/[tune_perm]/[tune_companion] + [llvm_code], inert without dasLLVM; PUBLIC so the auto-policy infer pass reaches app roots that require this module +require ?llvm llvm/daslib/llvm_tune public // PUBLIC so tune_status reaches app roots that require this module; witness guard: the one framework call below sits under static_if +require dasllama/dasllama_tune_scope // nolint:STYLE029,LINT019 — an ORDER dependency, not a symbol use: the dasllama [tune_scope] must be on the AST before this module's [tune] kernels pick their perms require dasllama/dasllama_gemm_schema require daslib/jobque_boost public require dasllama/dasllama_par @@ -44,7 +45,7 @@ require math // ISA-agnostic slot traversals: "arm64-gen" (priority 25, the arm64 load-select tier since // slice C) and "x64-gen" (priority 25, witness-gated — only selectable when the stamped // family emitted). dasllama_common pulls it in behind a path guard -// (`require ?llvm/daslib/llvm_tune`), so any Q8/mx4 model load on a gen-capable box runs the +// (`require ?llvm`), so any Q8/mx4 model load on a gen-capable box runs the // generated family. The generators live in dasllama/dasllama_gemm_gen.das (llvm-jit context, // wired via llvm/daslib/llvm_user_modules.das). @@ -2748,8 +2749,12 @@ def dasllama_math_gen_register() { // untuned box still pays the minutes-long auto-tune before the gate ever runs. The log // line is unconditional-when-set — active overrides announce themselves if (g_env_bench.allow_untuned) { - to_log(LOG_WARNING, "dasLLAMA: DASLLAMA_ALLOW_UNTUNED=1 - tune mint suppressed; untuned kernels run fallback, numbers are not publishable\n") - tune_suppress_mint("DASLLAMA_ALLOW_UNTUNED=1") + static_if (typeinfo module_exists(llvm_tune)) { + to_log(LOG_WARNING, "dasLLAMA: DASLLAMA_ALLOW_UNTUNED=1 - tune mint suppressed; untuned kernels run fallback, numbers are not publishable\n") + tune_suppress_mint("DASLLAMA_ALLOW_UNTUNED=1") + } else { + to_log(LOG_WARNING, "dasLLAMA: DASLLAMA_ALLOW_UNTUNED=1 - no tune framework in this build; every kernel runs its reference body, numbers are not publishable\n") + } } // the cross-box override's q51 reach: x64-gen keeps disk-order q51 slots, so the generic // grp q51 repack registers here for bakes whose TARGET has grp q51 planes @@ -2760,7 +2765,7 @@ def dasllama_math_gen_register() { // needs_repack keeps this out of auto-select (direct callers stay on arm64-sdot). q8_layout // must be evaluated at selection time, not [init] — [init]s run before the JIT installs // generated code (the slice C mr8 bug); arm64 needs no availability predicate. - if (get_architecture_name() == "arm64" && jit_enabled()) { + if (get_target_architecture_name() == "arm64" && jit_enabled()) { register_kernel_backend(KernelBackend(name = "arm64-gen", mm = @@q8q8_kernel_gen, batch = @@q8q8_batch_kernel_neon_laneq_gen, group3 = @@q8q8_group3_gen, repack = @@repack_q8q8_gen, @@ -2792,7 +2797,7 @@ def dasllama_math_gen_register() { // x64 twin: the ONLY x64-specific wiring is the availability predicate — the witness must // be true only if the family actually emitted (a declined stamp falls back to slow NEON // scalar bodies here). avx2 is the family's minimum tier. - if (get_architecture_name() == "x86_64" && jit_enabled() && cpu_supports("avx2")) { + if (get_target_architecture_name() == "x86_64" && jit_enabled() && cpu_supports("avx2")) { register_kernel_backend(KernelBackend(name = "x64-gen", mm = @@q8q8_kernel_gen, batch = @@q8q8_batch_kernel_neon_laneq_gen, group3 = @@q8q8_group3_gen, repack = @@repack_q8q8_gen, diff --git a/modules/dasLLAMA/dasllama/dasllama_styletts2.das b/modules/dasLLAMA/dasllama/dasllama_styletts2.das index da09b0050c..97cfbae80e 100644 --- a/modules/dasLLAMA/dasllama/dasllama_styletts2.das +++ b/modules/dasLLAMA/dasllama/dasllama_styletts2.das @@ -248,6 +248,8 @@ struct StyleTts2Model { decoder : St2Decoder = St2Decoder() source : SineSourceCfg = SineSourceCfg() q8 : bool // the lane this image was minted on: the rows GEMMs serve Q8_0 quants + kitten : KittenFamily + kokoro : KokoroFamily blob : PlaneF = PlaneF() qblob : PlaneI8 = PlaneI8() image_map : void? @@ -266,9 +268,9 @@ def weights_walk(var io : TtsBlobIo; var m : StyleTts2Model) { } } -// serialize_image_meta covers 14 fields; voices (views bound from voice_spans), image_map and +// serialize_image_meta covers 16 fields; voices (views bound from voice_spans), image_map and // image_bytes are the deliberate skips -let private ST2_META_FIELDS = 14 + 3 +let private ST2_META_FIELDS = 16 + 3 let ST2_IMAGE_TAG = "tts-f32" let ST2_IMAGE_TAG_Q8 = "tts-q8" @@ -288,6 +290,12 @@ def private serialize_image_meta(var arch : Archive; var m : StyleTts2Model) { arch |> serialize(m.decoder) arch |> serialize(m.source) arch |> serialize_raw(m.q8) + // the family records are hand-listed too, so each pins its count - an unwritten field reads back zero + verify(count_meta_fields(m.kitten) == 2) // grew KittenFamily? extend this list + verify(count_meta_fields(m.kokoro) == 1) // grew KokoroFamily? extend this list + arch |> serialize(m.kitten.speed_priors) + arch |> serialize(m.kitten.aliases) + arch |> serialize(m.kokoro.sym) } [unused_argument(m)] @@ -357,6 +365,8 @@ def finalize(var m : StyleTts2Model) { delete m.text_encoder delete m.predictor delete m.decoder + delete m.kitten + delete m.kokoro delete m.blob delete m.qblob if (m.image_map != null) { @@ -679,8 +689,8 @@ def private read_weight(m : GGUFMeta; bytes : array | #; name : string) : } //! Read a converted StyleTTS2-lineage GGUF (harness/convert_kitten.py, harness/convert_kokoro.py) -//! into the mint-side carrier: served layouts minted, voices and STFT convention read, every -//! weight moved into the staging blob. Family keys are the family file's to read. +//! into the mint-side carrier: served layouts minted, voices, STFT convention and the family's +//! driver keys read, every weight moved into the staging blob. [arch(at = "../ARCHITECTURE_TTS.md#tts-image-rail")] def stage_styletts2(path : string; q8 : bool = false) : St2Staging { var inscope st = St2Staging() @@ -712,6 +722,7 @@ def stage_styletts2(path : string; q8 : bool = false) : St2Staging { for (v in st.e.voice_names) { st.e.voices[v] <- read_arr(m, bytes, "voice.{v}") } + stage_family_data(m, bytes, st.e) read_albert(m, bytes, st.e.albert) read_text_encoder(m, bytes, st.e.text_encoder) read_predictor(m, bytes, st.e.predictor) @@ -723,6 +734,27 @@ def stage_styletts2(path : string; q8 : bool = false) : St2Staging { return <- st } +def private stage_family_data(m : GGUFMeta; bytes : array#; var e : StyleTts2Model) { + if (e.family == "kitten-tts") { + for (v in e.voice_names) { + if (gguf_has(m, "kitten.speed_prior.{v}")) { + e.kitten.speed_priors[v] = gguf_f32(m, bytes, "kitten.speed_prior.{v}") + } + } + var inscope anames <- gguf_str_array(m, bytes, "kitten.alias_names") + var inscope avoices <- gguf_str_array(m, bytes, "kitten.alias_voices") + for (an, av in anames, avoices) { + e.kitten.aliases[an] = av + } + } elif (e.family == "kokoro") { + var inscope cpts <- gguf_int_array(m, bytes, "kokoro.symbol_cpts") + var inscope ids <- gguf_int_array(m, bytes, "kokoro.symbol_ids") + for (c, i in cpts, ids) { + e.kokoro.sym[int(c)] = int(i) + } + } +} + //! Mint a served model from a staged read without touching disk - the in-memory image rail, //! the off-rail control for suites. The staged read's lane picks the tag. def mint_styletts2(var st : St2Staging; var out : StyleTts2Model) { @@ -731,13 +763,36 @@ def mint_styletts2(var st : St2Staging; var out : StyleTts2Model) { st2_bind(out) } -//! Load a converted StyleTTS2-lineage GGUF through its prepared image on the current lane: the -//! sibling `.dlim` maps when it exists, else the GGUF stages, bakes and maps -//! (``DASLLAMA_IMAGE=0`` mints in memory). +//! Bake a GGUF's prepared image to `out` on the lane `q8` names - the converter's TTS arm. The +//! box's backend pin (the converter's ``--config``) decides the q8 plane's repack. false = not written. +def bake_styletts2_image(path, out : string; q8 : bool) : bool { + var inscope st <- stage_styletts2(path, q8) + var inscope m = StyleTts2Model() + return cache_via_image_staged(st, m, out, q8 ? ST2_IMAGE_TAG_Q8 : ST2_IMAGE_TAG) +} + +//! Load a converted StyleTTS2-lineage GGUF through its prepared image on the current lane (the sibling `.dlim` maps when it +//! exists, else the GGUF stages, bakes and maps; ``DASLLAMA_IMAGE=0`` mints in memory), or a `.dlim` path directly: the +//! file names its lane, and the gguf is not needed. [arch(at = "../ARCHITECTURE_TTS.md#tts-image-rail"), arch(at = "../ARCHITECTURE_IMAGE.md#image-identity-backend-order")] def load_styletts2(path : string) : StyleTts2Model { var inscope m = StyleTts2Model() let q8 = st2_serve_q8() + if (path |> ends_with(".dlim")) { + apply_box_profile_runtime() + let ts = ref_time_ticks() + let preferred_tag = q8 ? ST2_IMAGE_TAG_Q8 : ST2_IMAGE_TAG + let other_tag = q8 ? ST2_IMAGE_TAG : ST2_IMAGE_TAG_Q8 + if (!stat(path).is_valid) { + panic("dasLLAMA styletts2: '{path}' does not exist") + } + if (!load_image(path, m, preferred_tag) && !load_image(path, m, other_tag)) { + panic("dasLLAMA styletts2: '{path}' is not a prepared TTS image for this box (identity {image_identity(ST2_IMAGE_TAG_Q8)} / {image_identity(ST2_IMAGE_TAG)}) - bake it from the source gguf with dasllama-convert") + } + st2_bind(m) + to_log(LOG_INFO, "dasLLAMA styletts2: prepared image mapped in {get_time_usec(ts) / 1000} ms on its {m.q8 ? "q8" : "f32"} lane - {path}\n") + return <- m + } let tag = st2_tag() st2_announce_lane() if (g_env_engine.image) { diff --git a/modules/dasLLAMA/dasllama/dasllama_tts.das b/modules/dasLLAMA/dasllama/dasllama_tts.das index 2f8745c79c..fca143ae95 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tts.das +++ b/modules/dasLLAMA/dasllama/dasllama_tts.das @@ -31,10 +31,8 @@ struct TtsModel { //! A loaded TTS model: the shared assembly, the family's driver data, and the front-end packs //! read beside the GGUF. Load with ``load_tts_model``; ``delete`` frees every plane and pack. kind : TtsKind //! the family the GGUF's architecture named - model : StyleTts2Model = StyleTts2Model() //! the shared StyleTTS2-lineage assembly and its voices + model : StyleTts2Model = StyleTts2Model() //! the shared StyleTTS2-lineage assembly, its voices and the family's driver data scratch : St2Scratch = St2Scratch() //! the activation carrier every synthesis reuses - kitten : KittenFamily //! the KittenTTS driver data (symbol table, aliases, speed priors) - kokoro : KokoroFamily //! the Kokoro driver data (vocabulary) g2p : G2pModel //! ``tts_g2p.bin``: lexicons, rules and the fallback chain tagger : PosTagger //! ``tts_postag.bin``: the tokenizer and part-of-speech tagger } @@ -45,34 +43,43 @@ def finalize(var m : TtsModel) { //! Free the model's planes, the carrier and both packs; ``delete m`` runs it. delete m.model delete m.scratch - delete m.kitten - delete m.kokoro delete m.g2p delete m.tagger } def load_tts_model(path : string) : TtsModel { - //! Load a TTS GGUF (KittenTTS or Kokoro, picked from its architecture); ``tts_g2p.bin`` and - //! ``tts_postag.bin`` come from the same directory. The rows GEMMs serve as the lane - //! ``tts_serves_q8`` names at the time of the load. + //! Load a TTS model: a converted GGUF (KittenTTS or Kokoro, picked from its architecture) or its prepared + //! ``.dlim`` image, which serves without the GGUF; ``tts_g2p.bin`` and ``tts_postag.bin`` come from the same directory. + //! A GGUF serves the rows GEMMs on the lane ``tts_serves_q8`` names at load time; an image serves the lane it was baked on. var inscope m = TtsModel() m.model <- load_styletts2(path) if (m.model.family == "kitten-tts") { m.kind = TtsKind.kitten - m.kitten <- load_kitten_family(path) } elif (m.model.family == "kokoro") { m.kind = TtsKind.kokoro - m.kokoro <- load_kokoro_family(path) } else { panic("dasLLAMA tts: '{path}' has architecture '{m.model.family}', which is not a TTS family") } let dir = dir_name(path) - m.g2p <- load_g2p(path_join(dir, "tts_g2p.bin")) + m.g2p <- load_g2p(g2p_pack_path(dir)) m.tagger <- load_pos_tagger(path_join(dir, "tts_postag.bin")) log_dropped_voices(m) return <- m } +//! The phoneme pack a model directory serves: the full pack when present, else the American-only +//! twin; neither is a panic naming the directory. +[arch(at = "../ARCHITECTURE_TTS.md#tts-g2p-pack-tiers")] +def g2p_pack_path(dir : string) : string { + let full = path_join(dir, "tts_g2p.bin") + return full if (stat(full).is_valid) + let us = path_join(dir, "tts_g2p_en_us.bin") + if (!stat(us).is_valid) { + panic("dasLLAMA tts: no phoneme pack beside the model - neither {full} nor {us}") + } + return us +} + [cold_path] def private log_dropped_voices(m : TtsModel) { //! Name, once per load, the voice packs ``caps`` leaves out: the front end phonemizes none @@ -105,12 +112,12 @@ def tts_serves_q8() : bool { } def private front_end_langs(m : TtsModel) : array { - //! The languages the front end phonemizes for this model, as the reference pipeline's - //! codes. Both families ship American English; Kokoro adds British, which its ``bf_*`` and - //! ``bm_*`` packs speak. A family speaks a language only once its lexicon rung lands here. + //! The languages the front end phonemizes for this model, as the reference pipeline's codes. Both + //! families ship American English; Kokoro adds British (its ``bf_*`` / ``bm_*`` packs) when the phoneme + //! pack carries the British tier. A family speaks a language only once its lexicon rung lands here. var inscope langs : array langs |> push("en-us") - if (m.kind == TtsKind.kokoro) { + if (m.kind == TtsKind.kokoro && !m.g2p.us_only) { langs |> push("en-gb") } return <- langs @@ -329,9 +336,9 @@ def private synthesize_chunk(var m : TtsModel; norm : string; vname : string; sp let clock = ref_time_ticks() let ps_misaki = g2p_phonemize(m.g2p, m.tagger, norm, voice_lang(m, vname) == "en-gb") let is_kitten = m.kind == TtsKind.kitten - var inscope ids <- is_kitten ? kitten_token_ids(kitten_phonemes_from_misaki(ps_misaki)) : kokoro_token_ids(m.kokoro, ps_misaki) + var inscope ids <- is_kitten ? kitten_token_ids(kitten_phonemes_from_misaki(ps_misaki)) : kokoro_token_ids(m.model.kokoro, ps_misaki) let row = is_kitten ? kitten_style_row(norm, m.model.style_rows) : kokoro_style_row(ps_misaki, m.model.style_rows) - let sp = is_kitten ? speed * (m.kitten.speed_priors?[vname] ?? 1.0) : speed + let sp = is_kitten ? speed * (m.model.kitten.speed_priors?[vname] ?? 1.0) : speed var inscope style : array styletts2_style(m.model, vname, row, style) out.timings.front_end_us = int64(get_time_usec(clock)) @@ -345,7 +352,7 @@ def private synthesize_chunk(var m : TtsModel; norm : string; vname : string; sp } def private resolve_voice(m : TtsModel; voice : string) : string { - let vname = m.kind == TtsKind.kitten ? kitten_voice(m.kitten, m.model, voice) : (styletts2_has_voice(m.model, voice) ? voice : "") + let vname = m.kind == TtsKind.kitten ? kitten_voice(m.model.kitten, m.model, voice) : (styletts2_has_voice(m.model, voice) ? voice : "") if (empty(vname)) { panic("dasLLAMA tts: unknown voice '{voice}'") } diff --git a/modules/dasLLAMA/dasllama/dasllama_tts_types.das b/modules/dasLLAMA/dasllama/dasllama_tts_types.das index c18e62eec1..007d0efed1 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tts_types.das +++ b/modules/dasLLAMA/dasllama/dasllama_tts_types.das @@ -62,3 +62,15 @@ struct TtsNoise { @scratch normal : array //! [samples x harmonics], sample-major captured : bool //! supplied by an oracle dump: a synthesis reads it as is and never redraws } + +//! Kitten's driver data beside the shared model: per-voice speed priors and the alias names. +//! Staged from the GGUF's ``kitten.*`` keys into the carrier, so it rides the prepared image. +struct KittenFamily { + speed_priors : table + aliases : table +} + +//! Kokoro's driver data beside the shared model: codepoint -> token id, from ``kokoro.*``. +struct KokoroFamily { + sym : table +} diff --git a/modules/dasLLAMA/dasllama/dasllama_tune.das b/modules/dasLLAMA/dasllama/dasllama_tune.das index 80825c2d8e..c3d006771a 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tune.das +++ b/modules/dasLLAMA/dasllama/dasllama_tune.das @@ -11,7 +11,8 @@ require daslib/rtti require daslib/ast_boost require daslib/templates_boost require daslib/json_boost // read_json / read_json_field (re-exports daslib/json) -require llvm/daslib/llvm_tune // the per-app tune sidecar (path, staleness, kernels read) +require daslib/strings_boost // nolint:STYLE030,LINT019 - split/find, reached only by the no-framework fallback-chain scan +require ?llvm llvm/daslib/llvm_tune // OPTIONAL: the per-app tune sidecar (path, staleness, kernels read) // ===== Tune spec ===== @@ -277,24 +278,36 @@ def private tuned_template_name(func : FunctionPtr; args : AnnotationArgumentLis return "{func.name}_template" // convention: dot -> dot_template } -//! The ONE per-app tune sidecar (`.tune.json` beside the root script/binary; -//! `DAS_TUNE_MANIFEST` env overrides) — shared by `[tuned]` compile-time reads, runtime apply, -//! `[tune]` generator winners, and the tuner harness write. Loop hints live under `"kernels"`, knobs under `"runtime"`. +//! The ONE per-app tune sidecar (`.tune.json` beside the root script/binary; `DAS_TUNE_MANIFEST` +//! env overrides) — shared by `[tuned]` compile-time reads, runtime apply, `[tune]` generator winners, +//! and the tuner harness write. Hints under `"kernels"`, knobs under `"runtime"`; "" with no framework. def box_profile_path() : string { - return tune_manifest_path() + static_if (typeinfo module_exists(llvm_tune)) { + return tune_manifest_path() + } else { + return "" + } } //! The sidecar is older than the running binary — its measurements are dead (the runtime //! apply skips it; the compile-time reads already treat it as absent). def box_profile_stale() : bool { - return tune_sidecar_stale(tune_manifest_path()) + static_if (typeinfo module_exists(llvm_tune)) { + return tune_sidecar_stale(tune_manifest_path()) + } else { + return false + } } -//! Why the sidecar at `path` would not serve this box: `stale_binary` (a rebuild killed the -//! kernel winners; the runtime knobs are still this box's state), `foreign_box` / `unreadable` -//! (another box's state, or not a sidecar), `absent`; anything else reads as fresh and ours. -def box_profile_verdict_at(path : string) : TuneSidecarReason { - return tune_sidecar_verdict(path, "runtime").reason +//! Why the sidecar at `path` would not serve this box, as the `TuneSidecarReason` NAME - `stale_binary` +//! (a rebuild killed the kernel winners; the runtime knobs are still this box's state), `foreign_box` / +//! `unreadable`, `absent`; anything else reads as fresh and ours, and with no framework it is `absent`. +def box_profile_verdict_at(path : string) : string { // nolint:LINT012,LINT019 - `path` is read only in the framework arm + static_if (typeinfo module_exists(llvm_tune)) { + return "{tune_sidecar_verdict(path, "runtime").reason}" + } else { + return "absent" + } } @@ -306,15 +319,24 @@ def private fallback_perm(args : AnnotationArgumentList) : string { let dv = find_arg(args, "fallback") if (dv is tString) { // ';'-chain of `suffix` or `suffix:requires` — one annotation, a per-ISA default - let picked = tune_pick_fallback(dv as tString) - if (!empty(picked)) return picked + static_if (typeinfo module_exists(llvm_tune)) { + let picked = tune_pick_fallback(dv as tString) + if (!empty(picked)) return picked + } else { + // no CPU-feature probe here, so take the first unconditional entry - the chain's ISA-neutral tail + for (entry in split(dv as tString, ";")) { + if (!empty(entry) && find(entry, ":") < 0) return entry + } + } } return DEFAULT_PERM } -def private report_perm(fname, perm, source : string) : string { - if (tune_verbosity() == "verbose") { - print("dasllama_tune: {fname} <- {perm} ({source})\n") +def private report_perm(fname, perm, source : string) : string { // nolint:LINT012,LINT019 - `fname`/`source` are read only in the framework arm + static_if (typeinfo module_exists(llvm_tune)) { + if (tune_verbosity() == "verbose") { + print("dasllama_tune: {fname} <- {perm} ({source})\n") + } } return perm } @@ -325,19 +347,25 @@ def private resolve_perm(func : FunctionPtr; args : AnnotationArgumentList) : st let fname = "{func.name}" let fb = fallback_perm(args) g_fallbacks[fname] = fb - if (tune_policy_env() == "reference") { - // the true kill switch, [tune]'s twin: the template's own loops, no hints stamped - return report_perm(fname, "plain", "reference policy") - } - let av = find_arg(args, "perm") - if (av is tString) return av as tString - var pickSource = "" - let cfg = tune_kernel_pick(fname, pickSource) - if (!empty(cfg)) { - return report_perm(fname, cfg, pickSource) + static_if (typeinfo module_exists(llvm_tune)) { + if (tune_policy_env() == "reference") { + // the true kill switch, [tune]'s twin: the template's own loops, no hints stamped + return report_perm(fname, "plain", "reference policy") + } + let av = find_arg(args, "perm") + if (av is tString) return av as tString + var pickSource = "" + let cfg = tune_kernel_pick(fname, pickSource) + if (!empty(cfg)) { + return report_perm(fname, cfg, pickSource) + } + let declared = find_arg(args, "fallback") is tString + return report_perm(fname, fb, declared ? "fallback" : "default") + } else { + // no policy env and no sidecar here: the `perm=` pin still names a row, else the chain's answer + let av = find_arg(args, "perm") + return av is tString ? av as tString : fb } - let declared = find_arg(args, "fallback") is tString - return report_perm(fname, fb, declared ? "fallback" : "default") } // Template lives in the compiling module for production `[tuned]`; the profiler harness diff --git a/modules/dasLLAMA/dasllama/dasllama_tune_scope.das b/modules/dasLLAMA/dasllama/dasllama_tune_scope.das index e1ba93711d..2ffc1d69db 100644 --- a/modules/dasLLAMA/dasllama/dasllama_tune_scope.das +++ b/modules/dasLLAMA/dasllama/dasllama_tune_scope.das @@ -5,7 +5,8 @@ options _dasllama_internal = true module dasllama_tune_scope shared public require dasllama/dasllama_lint public -require llvm/daslib/llvm_tune public // the scope annotation; PUBLIC so the auto-policy infer pass and tune_status reach app roots +require daslib/tune public // the [tune_scope] annotation, inert without dasLLVM; PUBLIC so the auto-policy infer pass reaches app roots +require ?llvm llvm/daslib/llvm_tune public // OPTIONAL; PUBLIC so the auto-policy infer pass and tune_status reach app roots require dasllama/dasllama_version // nolint:STYLE030 — version_of= below names DASLLAMA_RELEASE by string; the module must be in this compile //! The scope reaches the AST before any covered module compiles - the scope apply refuses a covered diff --git a/modules/dasLLAMA/harness/REVIEW.md b/modules/dasLLAMA/harness/REVIEW.md index f567b1389c..fe00d1405a 100644 --- a/modules/dasLLAMA/harness/REVIEW.md +++ b/modules/dasLLAMA/harness/REVIEW.md @@ -6,12 +6,11 @@ the Metal backend, `../followup_vulkan.md` for anything about the Vulkan backend `../PERF_LEDGER.md` for any other performance followup, `../followup_general.md` for everything else. -**A diff that adds or moves a timed unit in `tune_kernels.das` - a row in the `benches` list, -or a race that runs after the list - keeps `dot_q8q8_laneq4x4` last among the timings that use -the CPU matmul backend.** Running that bench pins one matmul backend for the rest of the -process, so a CPU timing after it runs against the pinned backend instead of the one it would -have picked. The Metal crown race after the list uses no CPU matmul backend and is out of the -rule's scope. +**A diff that adds or moves a timed unit in `tune_kernels.das` keeps `dot_q8q8_laneq4x4` the +LAST row of the `benches` list and adds no CPU timing after it; the Metal crown race after the +list is the one thing allowed there.** Running that bench pins one matmul backend for the rest of +the process, so a CPU timing after it runs against the pinned backend instead of the one it would +have picked. **A diff that points a `dashv` call in this folder at the sidecar exchange - dasllama.io's tune-sidecar service: sidecar lookup, download, or submit - is a defect; it calls the exchange diff --git a/modules/dasLLAMA/harness/accel_contention_probe.das b/modules/dasLLAMA/harness/accel_contention_probe.das index 5608973df0..fe54094831 100644 --- a/modules/dasLLAMA/harness/accel_contention_probe.das +++ b/modules/dasLLAMA/harness/accel_contention_probe.das @@ -6,7 +6,7 @@ require das_accelerate require ?das_metal metal/das_metal_boost // the live-Metal-device ingredient (env METAL=1) require dasllama/dasllama_math require dasllama/dasllama_par -require llvm/daslib/f16_cvt +require daslib/f16_cvt require daslib/jobque_boost require math require dasllama/dasllama_env // nolint:STYLE030,LINT019 — the read sites sit in the apple static_if half; the linux lint flavor sees the require as unused diff --git a/modules/dasLLAMA/harness/build_g2p_data.py b/modules/dasLLAMA/harness/build_g2p_data.py index 13d559982f..42c39bc359 100644 --- a/modules/dasLLAMA/harness/build_g2p_data.py +++ b/modules/dasLLAMA/harness/build_g2p_data.py @@ -158,6 +158,9 @@ def main(): ap.add_argument("--local-additions", help="json {word: misaki phonemes} merged over the gold tier") ap.add_argument("--focus-words", help="also write the tag-keyed gold words, one per line, lowercased - the " "tagger's silver prose is sampled around them (mint_postag_silver.py --focus-words)") + ap.add_argument("--dialect", choices=["both", "us"], default="both", + help="'us' packs the American tier alone (the web serving set): the GB lexicon values and " + "GB-only keys are dropped, CMUdict and the speller stay; the reader refuses British on it by name") a = ap.parse_args() sys.path.insert(0, os.path.join(a.root, "scripts")) os.environ.setdefault("NLTK_DATA", os.path.join(a.root, ".nltk")) @@ -169,6 +172,8 @@ def main(): tables = load_lexicons(data_dir, a.local_additions, a.focus_words) gold, silver = tables["us_gold"], tables["us_silver"] gb_gold, gb_silver = tables["gb_gold"], tables["gb_silver"] + if a.dialect == "us": + gb_gold, gb_silver = {}, {} cmu_path = os.path.join(os.environ["NLTK_DATA"], "corpora", "cmudict", "cmudict") cmu = read_cmudict(cmu_path) @@ -178,7 +183,9 @@ def main(): # and only then this table. That is what makes the drop safe - a glued group sends its WHOLE # surface to the fallback as soon as one piece is unresolvable, so such a word DOES reach it # ("water-tzarina"), and a CMUdict row would only have offered the coarser reading. - covered = (set(gold) | set(silver)) & (set(gb_gold) | set(gb_silver)) + # an American-only pack is asked for American alone, so every word its lexicon carries is covered + covered = ((set(gold) | set(silver)) if a.dialect == "us" + else (set(gold) | set(silver)) & (set(gb_gold) | set(gb_silver))) dropped = sum(1 for w in cmu if w in covered) cmu_misaki = {w: G.arpabet_to_ipa(G.flap(ph, calib["flap"]), "misaki", calib["opts"]) for w, ph in cmu.items() if w not in covered} @@ -202,16 +209,19 @@ def main(): ("gru", tensor_section(arrays, graphemes, phonemes))] buf = bytearray() buf += struct.pack("misaki table; g2p_en 2.1.0 checkpoint20").encode("utf8") buf += struct.pack("= 0 + if (!known) { + print("error: unknown backend '{name}' - registered: {join(names, ", ")}\n") + } + delete names + if (known) { + pin_kernel_backend(name) + } + return known +} + def private parse_ids(s : string) : array { var ids <- [for (part in split(s, ",")); to_int64(part, false); where !empty(part)] return <- ids @@ -87,6 +109,7 @@ def main : int { if (cfg.ngl != 0) { // before load_model — pins the portable backend, activates the metal overrides set_metal_mode(MetalMode.required) } + return 2 if (!apply_backend_pin(cfg.backend)) var t <- load_gguf(cfg.model, cfg.quant) t.tok <- load_tokenizer_auto(cfg.model) // load_gguf skips the tokenizer; GEN_TEXT needs it if (cfg.ngl != 0 && !convert_model_to_metal_blob(t)) { // the Metal drivers serve blob-form models only @@ -97,8 +120,8 @@ def main : int { if (t.config.seq_len > 2048l) { t.config.seq_len = 2048l } - let c = t.config - var s = make_run_state(c, cfg.kv, cfg.kv) + print("GEN_BACKEND: {active_kernel_backend()} (batch: {active_batch_backend()})\n") + var s = make_run_state(t.config, cfg.kv, cfg.kv) var rc = 0 with_job_que() { setup_dasllama_jobque_() diff --git a/modules/dasLLAMA/harness/publish_tts_hf.py b/modules/dasLLAMA/harness/publish_tts_hf.py index ac22716a3e..0c9ed5fcc5 100644 --- a/modules/dasLLAMA/harness/publish_tts_hf.py +++ b/modules/dasLLAMA/harness/publish_tts_hf.py @@ -14,7 +14,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) MODULE = os.path.dirname(HERE) CARD = os.path.join(HERE, "tts_model_card.md") -FILES = ["kitten-nano.gguf", "kitten-mini.gguf", "kokoro-82m.gguf", "tts_g2p.bin", "tts_postag.bin"] +FILES = ["kitten-nano.gguf", "kitten-mini.gguf", "kokoro-82m.gguf", "tts_g2p.bin", "tts_g2p_en_us.bin", "tts_postag.bin"] LICENCES = ["LICENSE.APACHE-2.0", "LICENSE.CMUDICT", "LICENSE.UD_EWT", "LICENSE.SPACY", "LICENSE.STYLETTS2"] diff --git a/modules/dasLLAMA/harness/smmla_probe.das b/modules/dasLLAMA/harness/smmla_probe.das index 78e397de0d..2e00799e59 100644 --- a/modules/dasLLAMA/harness/smmla_probe.das +++ b/modules/dasLLAMA/harness/smmla_probe.das @@ -1,7 +1,7 @@ options gen2 options _dasllama_internal = true require daslib/jobque_boost -require llvm/daslib/aarch64_neon +require daslib/aarch64_neon require dasllama/dasllama_env // SMMLA (i8mm) single-core scout: correctness + ALU-throughput ratio vs SDOT on the same diff --git a/modules/dasLLAMA/harness/tq4_basis_probe.das b/modules/dasLLAMA/harness/tq4_basis_probe.das index 412759d89e..bbd67dd612 100644 --- a/modules/dasLLAMA/harness/tq4_basis_probe.das +++ b/modules/dasLLAMA/harness/tq4_basis_probe.das @@ -9,7 +9,7 @@ require dasllama/dasllama_transformer // umbrella fires each arch [init] regis require dasllama/dasllama_math require dasllama/dasllama_kv_codec // the tq4 codec + FWHT rotation under probe require dasllama/dasllama_tokenizer -require llvm/daslib/f16_cvt +require daslib/f16_cvt require dasllama/dasllama_env // Basis A/B on REAL cache rows: run SmolLM2 over real text with an f32 KV cache, then re-quantize diff --git a/modules/dasLLAMA/harness/tq4_order_probe.das b/modules/dasLLAMA/harness/tq4_order_probe.das index 652bc77c04..c58a9c07c3 100644 --- a/modules/dasLLAMA/harness/tq4_order_probe.das +++ b/modules/dasLLAMA/harness/tq4_order_probe.das @@ -9,7 +9,7 @@ require dasllama/dasllama_transformer // umbrella fires each arch [init] regis require dasllama/dasllama_math require dasllama/dasllama_kv_codec // quantize_tq4kv_row/cvt_tq4kv_to_f32 — the codec under probe require dasllama/dasllama_tokenizer -require llvm/daslib/f16_cvt +require daslib/f16_cvt require dasllama/dasllama_env // Is one channel order enough, or is it MoE-shaped? Run SmolLM2 over 5 text domains, learn a diff --git a/modules/dasLLAMA/harness/tts_model_card.md b/modules/dasLLAMA/harness/tts_model_card.md index 5a46d91f30..844f07da9b 100644 --- a/modules/dasLLAMA/harness/tts_model_card.md +++ b/modules/dasLLAMA/harness/tts_model_card.md @@ -35,10 +35,11 @@ no phonemizer: the front end is data, and the data is in the two packs. | `kitten-mini.gguf` | KittenTTS mini 0.8, f32 | 295975008 | `e127a95c6ffa390eccc5956e7156ff055e9f30078cf16361cbcd0eea6c21fdb5` | | `kokoro-82m.gguf` | Kokoro-82M v1.0 with its 54 voice packs, f32 | 352965024 | `e0d6584b5b650730bd62e7b105a2e9502384bbe3fbebf3030113560355e6232f` | | `tts_g2p.bin` | the grapheme-to-phoneme pack: misaki's gold and silver lexicons in both English dialects (one merged table per tier), CMUdict 0.7a rendered into the American inventory, the g2p_en GRU spelling model | 14011554 | `d7b6afea7a0901a877d10531054d3f967845c58f415f0ea937decd8a3081ccd5` | +| `tts_g2p_en_us.bin` | the same pack with the American tier alone - no British values, CMUdict pruned of every word the American lexicon carries; the web serving set's pack, British voices are not offered on it | 10257455 | `6f69d2e74565bd7d876b8d1f4042bf8c1c5b615387fa26ff45215cf447932154` | | `tts_postag.bin` | the tokenizer exception table and the averaged-perceptron PTB tagger | 12566510 | `38c2e85f7fef3e57d561d2aa0af25fccda4276376ba1993c3dbc2ae0ebfa57b4` | -The two packs sit beside whichever GGUF you load; the loader reads them from the model's -directory. The GGUFs carry f32 weights: dasLLAMA quantizes the served layouts to Q8_0 at first +The packs sit beside whichever GGUF you load; the loader reads them from the model's +directory - `tts_g2p.bin` when it is there, else `tts_g2p_en_us.bin`. The GGUFs carry f32 weights: dasLLAMA quantizes the served layouts to Q8_0 at first load and keeps the result beside the file as a prepared image, so the f32 file is also the reference lane. diff --git a/modules/dasLLAMA/harness/tune_confirm_prefill.das b/modules/dasLLAMA/harness/tune_confirm_prefill.das index 0e15b0ae36..9fc5124887 100644 --- a/modules/dasLLAMA/harness/tune_confirm_prefill.das +++ b/modules/dasLLAMA/harness/tune_confirm_prefill.das @@ -6,7 +6,8 @@ options _dasllama_internal = true require dasllama/dasllama_transformer // umbrella fires each arch [init] registration; requiring dasllama_common directly would drop them require dasllama/dasllama_math // setup_dasllama_jobque_ + apply_box_profile_runtime -require llvm/daslib/llvm_tune // [tune_policy]: the confirm arm measures WHAT THE MANIFEST SAYS — never self-tune +require daslib/tune // [tune_policy]: the confirm arm measures WHAT THE MANIFEST SAYS — never self-tune +require llvm/daslib/llvm_tune // the framework the annotation forwards to, and the visibility the auto-policy pass needs require daslib/jobque_boost require strings require math diff --git a/modules/dasLLAMA/harness/tune_kernels.das b/modules/dasLLAMA/harness/tune_kernels.das index 0d80684342..8f3c4ee60b 100644 --- a/modules/dasLLAMA/harness/tune_kernels.das +++ b/modules/dasLLAMA/harness/tune_kernels.das @@ -23,12 +23,13 @@ require dasllama/dasllama_common // the runtime-knob getters (the profile's "run require dasllama/dasllama_mtp_gemma // gemma_drafter_sidecar - the assistant-depth confirm's vehicle test require dasllama/dasllama_tune // [dasllama_grid] emitter require llvm/daslib/llvm_tune // tune_sidecar_merge — the app-sidecar upsert writer +require daslib/tune // [tune_policy] - inert without dasLLVM require ?das_metal dasllama/dasllama_metal_prefill // metal_tensor_race (Apple static_if half) require ?das_metal dasllama/dasllama_metal_kernels // metal_tensor_race_decode (Apple static_if half) require tuner_cli // the shared tuner CLI require tuner_noise // the noise gate + the measured floor, shared with the other half -require llvm/daslib/aarch64_neon // sdot4_laneq (referenced by the cloned laneq template bodies) -require llvm/daslib/f16_cvt // f16 references + the cloned f16-kernel template bodies +require daslib/aarch64_neon // sdot4_laneq (referenced by the cloned laneq template bodies) +require daslib/f16_cvt // f16 references + the cloned f16-kernel template bodies require math // mad (referenced by the cloned template bodies) require daslib/json require daslib/json_boost diff --git a/modules/dasLLAMA/performance/REVIEW.md b/modules/dasLLAMA/performance/REVIEW.md index e78c1330de..64263fd50c 100644 --- a/modules/dasLLAMA/performance/REVIEW.md +++ b/modules/dasLLAMA/performance/REVIEW.md @@ -95,10 +95,11 @@ beside it: a projector, a draft head, an assistant sidecar, an image fixture - p `companions` of the row that pins its carrier, and names it from every other row that consumes it.** Uniqueness itself is `../tests/test_model_specs.das`'s to enforce. -**A diff that adds or changes a `serve_*` field on a row of `model_specs()`, or any `serve_*` -function in `model_specs.das`, lands `utils/dasllama-server/test_model_catalog.das` (repo -root) green in the same change** - the serving catalog is a view over these rows, so its gates -red on a row this folder ships. +**A diff that changes what any `serve_*` function in `model_specs.das` returns - a `serve_*` +field on a row, the function's body, or a `companions` entry with a `url` on a row a `serve_*` +view reads - lands `utils/dasllama-server/test_model_catalog.das` (repo root) green in the same +change** - the serving catalog is a view over these rows, so its gates red on a row this folder +ships. **A convert, a bench, or a tune-state write reached from `fetch_models.das --fetch` is a defect - `--fetch` downloads only.** Each has its own home: a conversion recipe runs under diff --git a/modules/dasLLAMA/performance/build_tts_data.das b/modules/dasLLAMA/performance/build_tts_data.das index 532f0cbfe4..0830abfdae 100644 --- a/modules/dasLLAMA/performance/build_tts_data.das +++ b/modules/dasLLAMA/performance/build_tts_data.das @@ -53,8 +53,11 @@ def main() : int { let fixture = path_join(get_das_root(), "modules/dasLLAMA/tests/_tts_fixtures/g2p_corpus.json") let silver = path_join(args.root, "work/silver_focus") let focus = path_join(args.root, "work/focus_words.txt") + let additions = path_join(harness, "g2p_local_additions.json") return 1 if (!step("pack the G2P data", [py, path_join(harness, "build_g2p_data.py"), args.root, "--out", path_join(store, "tts_g2p.bin"), - "--local-additions", path_join(harness, "g2p_local_additions.json"), "--focus-words", focus])) + "--local-additions", additions, "--focus-words", focus]) + || !step("pack the American-only G2P data", [py, path_join(harness, "build_g2p_data.py"), args.root, "--out", path_join(store, "tts_g2p_en_us.bin"), + "--local-additions", additions, "--dialect", "us"])) if (args.silver_sents > 0) { var inscope prose : array dir(path_join(args.root, "work/gutenberg")) $(name) { diff --git a/modules/dasLLAMA/performance/establish_baselines.das b/modules/dasLLAMA/performance/establish_baselines.das index b479ca08ac..cd89cad411 100644 --- a/modules/dasLLAMA/performance/establish_baselines.das +++ b/modules/dasLLAMA/performance/establish_baselines.das @@ -14,7 +14,8 @@ options _dasllama_internal = true require profile_common require daslib/clargs require strings -require llvm/daslib/llvm_tune // [tune_policy]: this tool benches the reference only — never auto-tune +require daslib/tune // [tune_policy]: this tool benches the reference only — never auto-tune +require llvm/daslib/llvm_tune // the framework the annotation forwards to, and the visibility the auto-policy pass needs require dasllama/dasllama_env [CommandLineArgs] diff --git a/modules/dasLLAMA/performance/fetch_models.das b/modules/dasLLAMA/performance/fetch_models.das index 81c36cd2fd..257090c3a5 100644 --- a/modules/dasLLAMA/performance/fetch_models.das +++ b/modules/dasLLAMA/performance/fetch_models.das @@ -17,7 +17,8 @@ require daslib/clargs require daslib/fio require strings require profile_common -require llvm/daslib/llvm_tune // [tune_policy]: provisioning only — never auto-tune +require daslib/tune // [tune_policy]: provisioning only — never auto-tune +require llvm/daslib/llvm_tune // the framework the annotation forwards to, and the visibility the auto-policy pass needs require dasllama/dasllama_env [CommandLineArgs] diff --git a/modules/dasLLAMA/performance/gen_bench_records.das b/modules/dasLLAMA/performance/gen_bench_records.das index c366381688..65cf66119a 100644 --- a/modules/dasLLAMA/performance/gen_bench_records.das +++ b/modules/dasLLAMA/performance/gen_bench_records.das @@ -7,7 +7,8 @@ options _dasllama_internal = true require profile_common require dasllama/dasllama_env // the g_env_* knob globals ([EnvConfig] declarations; ENVIRONMENT.md is generated from them) require dasllama/dasllama_image // dlim_wipe + dlim_inventory — the batch lifecycle and the re-mint tripwire -require llvm/daslib/llvm_tune // [tune_policy]: the orchestrator never measures — it must never self-tune +require daslib/tune // [tune_policy]: the orchestrator never measures — it must never self-tune +require llvm/daslib/llvm_tune // the framework the annotation forwards to, and the visibility the auto-policy pass needs require daslib/clargs require daslib/fio require math diff --git a/modules/dasLLAMA/performance/gen_profile.das b/modules/dasLLAMA/performance/gen_profile.das index 5ac8e7ee6d..71c1b49a16 100644 --- a/modules/dasLLAMA/performance/gen_profile.das +++ b/modules/dasLLAMA/performance/gen_profile.das @@ -14,7 +14,8 @@ options _dasllama_internal = true require dasllama/dasllama_transformer // the umbrella fires each arch [init] registration; requiring dasllama_common directly would drop them require dasllama/dasllama_math // setup_dasllama_jobque_ + apply_box_profile_runtime knobs (engine-level, not re-exported) require dasllama/dasllama_tune // box_profile_path (the applied per-box manifest, recorded in the platform block) -require llvm/daslib/llvm_tune // [tune_policy] + log_tune_status — the box tunes itself before measuring +require daslib/tune // [tune_policy] - inert without dasLLVM +require llvm/daslib/llvm_tune // log_tune_status — the box tunes itself before measuring require daslib/jobque_boost require profile_common require daslib/clargs diff --git a/modules/dasLLAMA/performance/gen_site_records.das b/modules/dasLLAMA/performance/gen_site_records.das index 64f70bdf58..5cd922f697 100644 --- a/modules/dasLLAMA/performance/gen_site_records.das +++ b/modules/dasLLAMA/performance/gen_site_records.das @@ -5,7 +5,8 @@ options _dasllama_internal = true require daslib/clargs require daslib/fio require profile_common -require llvm/daslib/llvm_tune // [tune_policy]: provisioning only — never auto-tune +require daslib/tune // [tune_policy]: provisioning only — never auto-tune +require llvm/daslib/llvm_tune // the framework the annotation forwards to, and the visibility the auto-policy pass needs // Merge every per-box record store (performance/records/.json) into the one file the site // renders: site/files/dasllama/bench_records.json. Boxes never collide on the run key (box is part diff --git a/modules/dasLLAMA/performance/model_specs.das b/modules/dasLLAMA/performance/model_specs.das index 36754c514e..2e371cb2ef 100644 --- a/modules/dasLLAMA/performance/model_specs.das +++ b/modules/dasLLAMA/performance/model_specs.das @@ -280,7 +280,7 @@ let REF_Q8_RECIPE = "derive: setup_asr_rig.das --refs quantizes it from the fp b let PARAKEET_V2_NOTE = "historic nemo-venv conversion (rail retired); present only on boxes that converted it - v3 is the fetched carrier" let CANARY_ENC_RECIPE = "convert: modules/dasLLAMA/harness/convert_canary_to_ggml.py over nvidia/canary-qwen-2.5b (nemo venv; encoder is a repack - sha canonical everywhere)" let CANARY_DEC_RECIPE = "convert: modules/dasLLAMA/harness/convert_canary_to_ggml.py over nvidia/canary-qwen-2.5b (nemo venv), then llama-quantize Q8_0 (a Q8_0 disk embedding lets the tied classifier serve cls_q8 on the Metal rail); bytes are per-arch (fp16 LoRA merge) - gate via asr_bench --text over jfk/jfk3/gb1 vs benchmarks/asr/canary_transcripts.expected" -let TTS_HF = "{HF}/borisbat/dasllama-tts/resolve/c5247896344155d36ce1d01f74ebfd9fef9dd862" // the published set, commit-pinned +let TTS_HF = "{HF}/borisbat/dasllama-tts/resolve/a469827cfb344d6a4c2dbfe6e259245af7020734" // the published set, commit-pinned let TTS_KITTEN_RECIPE = "convert: modules/dasLLAMA/performance/build_tts_data.das -- --root --out runs harness/convert_kitten.py over KittenML/kitten-tts-nano-0.8 @ 7a1db645b1f3ab9420761d87428e042b9cec3f26 and kitten-tts-mini-0.8 @ c02725660cea441db4c383af69f1f26f5cd00947 (the ONNX weights into GGUF, f32); THIRD_PARTY_NOTICES.md (repo root) carries the terms" let TTS_KOKORO_RECIPE = "convert: modules/dasLLAMA/performance/build_tts_data.das -- --root --out runs harness/convert_kokoro.py over hexgrad/Kokoro-82M @ f3ff3571791e39611d31c381e3a41a3af07b4987 (kokoro-v1_0.pth + the voice packs into GGUF, f32); THIRD_PARTY_NOTICES.md (repo root) carries the terms" let TTS_PACKS_RECIPE = "mint: modules/dasLLAMA/performance/build_tts_data.das -- --root --out runs harness/build_g2p_data.py (misaki 0.9.4 gold/silver + CMUdict 0.7a + the g2p_en 2.1.0 GRU + harness/g2p_local_additions.json into tts_g2p.bin) and harness/train_postag.py (UD English-EWT + spaCy-labelled silver prose into tts_postag.bin)" @@ -557,6 +557,9 @@ def model_specs() : array { // nolint:STYLE038 — flat model-set t ProvEntry(name = "tts_g2p.bin", root = "llm", recipe = TTS_PACKS_RECIPE, url = "{TTS_HF}/tts_g2p.bin", bytes = 14011554l, sha256 = "d7b6afea7a0901a877d10531054d3f967845c58f415f0ea937decd8a3081ccd5"), + ProvEntry(name = "tts_g2p_en_us.bin", root = "llm", recipe = TTS_PACKS_RECIPE, + url = "{TTS_HF}/tts_g2p_en_us.bin", bytes = 10257455l, + sha256 = "6f69d2e74565bd7d876b8d1f4042bf8c1c5b615387fa26ff45215cf447932154"), ProvEntry(name = "tts_postag.bin", root = "llm", recipe = TTS_PACKS_RECIPE, url = "{TTS_HF}/tts_postag.bin", bytes = 12566510l, sha256 = "38c2e85f7fef3e57d561d2aa0af25fccda4276376ba1993c3dbc2ae0ebfa57b4"), @@ -574,10 +577,11 @@ def model_specs() : array { // nolint:STYLE038 — flat model-set t ] } -//! The text-to-speech set the serving catalog offers: the three converted GGUFs and the two -//! front-end packs every one of them loads, as pinned on the borisbat/dasllama-tts repository - -//! a view over the rows above (the packs ride the kitten-nano row's companions). Every entry -//! carries url + bytes + sha256; `pack` tells the two apart from the models. +//! The text-to-speech set the serving catalog offers: the three converted GGUFs and the three +//! front-end packs (the full phoneme pack, its American-only twin, the tagger), as pinned on the +//! borisbat/dasllama-tts repository - a view over the rows above (the packs ride the kitten-nano +//! row's companions). Every entry carries url + bytes + sha256; `pack` tells the packs apart from +//! the models. def serve_tts_set() : array> { var out : array> var all <- model_specs() diff --git a/modules/dasLLAMA/performance/profile_common.das b/modules/dasLLAMA/performance/profile_common.das index e0ef0f8070..083cca45ad 100644 --- a/modules/dasLLAMA/performance/profile_common.das +++ b/modules/dasLLAMA/performance/profile_common.das @@ -21,7 +21,7 @@ require daslib/json public // JV(...) for logger field payloads require daslib/json_boost // ?[] / ?? navigation — the sidecar provenance reader require dasllama/dasllama_common public // QuantMode require dasllama/dasllama_math // active_kernel_backend — the exec_fmt layout receipt -require llvm/daslib/llvm_tune // tune_status — stamp provenance for the platform block +require ?llvm llvm/daslib/llvm_tune // nolint:STYLE030,LINT019 — tune_status, the stamp provenance; empty without the framework require model_specs public // the model-set table + its llm/official views require box_ident // derive_box_name — the hardware-keyed box tag @@ -474,7 +474,8 @@ def private platform_os() : string { return strip("{run_capture(". /etc/os-release && echo $PRETTY_NAME", true)} {run_capture("uname -r", true)}") } -def private tune_row_str(r : TuneStatus) : string { +//! generic on purpose: the row type lives in the framework, and this file compiles without it +def private tune_row_str(r) : string { let sfx = empty(r.suffix) ? "reference" : r.suffix return "{r.fname}={sfx} ({r.source})" } @@ -482,12 +483,16 @@ def private tune_row_str(r : TuneStatus) : string { //! One-line gen-kernel stamp provenance — "" (omitted from JSON) when the driver declares no //! `[tune_policy]` (tune_status is empty then). def tune_summary() : string { - var st <- tune_status() - var parts <- [for (r in st); tune_row_str(r)] - let s = join(parts, "; ") - delete parts - delete st - return s + static_if (typeinfo module_exists(llvm_tune)) { + var st <- tune_status() + var parts <- [for (r in st); tune_row_str(r)] + let s = join(parts, "; ") + delete parts + delete st + return s + } else { + return "" + } } //! The bench cells' provenance gate: REFUSE to measure on fallback kernel winners. A @@ -499,7 +504,9 @@ def tune_summary() : string { //! * **``DAS_TUNE_MANIFEST`` pinned**: stamps apply, status stays empty (framework gap, //! ledgered) — gate on the FILE: present, newer than the binary, carrying kernels. //! * **plain script**: the policy rail ran, so every row must stamp a manifest winner. -//! ``DASLLAMA_ALLOW_UNTUNED=1`` is the dev-run escape in all three (it also suppresses the +//! A build without the framework (no dasLLVM, the portable tier) has no winners at all - every +//! kernel is its reference body - and the gate refuses by name. +//! ``DASLLAMA_ALLOW_UNTUNED=1`` is the dev-run escape in every world (it also suppresses the //! auto-mint at ``[init]`` — see ``dasllama_math_gen_register`` — so the gate is reached at all). var private g_tune_gate_bypassed = false @@ -510,6 +517,30 @@ def tune_gate_bypassed() : bool { } def tune_gate() : bool { + static_if (typeinfo module_exists(llvm_tune)) { + return tune_gate_verdict(tune_gate_why(0)) + } else { + return tune_gate_verdict("this build carries no tune framework (dasLLVM absent) - every kernel runs its reference body") + } +} + +def private tune_gate_verdict(why : string) : bool { + if (empty(why)) { + return true + } + if (g_env_bench.allow_untuned) { + g_tune_gate_bypassed = true + to_log(LOG_WARNING, "tune gate: {why} - measuring anyway (DASLLAMA_ALLOW_UNTUNED=1, numbers are NOT publishable)\n") + return true + } + to_log(LOG_ERROR, "tune gate: {why}. Mint with: DAS_TUNE_MODE=tune bin/daslang -jit modules/dasLLAMA/harness/dasllama_tuner.das -dasroot ; DASLLAMA_ALLOW_UNTUNED=1 overrides for dev runs\n") + return false +} + +//! The three-world refusal text, "" when the cell may measure. Generic on purpose (the unused +//! parameter is what keeps it one): it names framework functions, so its body must instantiate +//! only from `tune_gate`'s framework arm - a plain function would compile in every build. +def private tune_gate_why(_) : string { var why = "" let mpath = g_env_core.tune_manifest if (is_standalone_exe()) { @@ -553,16 +584,7 @@ def tune_gate() : bool { why = "{bad}/{total} kernels stamp non-manifest winners (first: {sample}) - the sidecar is stale or absent; a class-profile stamp covers the kernels but is not this box's mint" } } - if (empty(why)) { - return true - } - if (g_env_bench.allow_untuned) { - g_tune_gate_bypassed = true - to_log(LOG_WARNING, "tune gate: {why} - measuring anyway (DASLLAMA_ALLOW_UNTUNED=1, numbers are NOT publishable)\n") - return true - } - to_log(LOG_ERROR, "tune gate: {why}. Mint with: DAS_TUNE_MODE=tune bin/daslang -jit modules/dasLLAMA/harness/dasllama_tuner.das -dasroot ; DASLLAMA_ALLOW_UNTUNED=1 overrides for dev runs\n") - return false + return why } //! True when a das run cannot name the winners it measured under — an empty ``tune`` or @@ -903,14 +925,25 @@ def private compute_sha256(path : string) : string { //! sha256 of this process's tune sidecar — the sidecar GENERATION a measurement runs under. //! Rows minted under different generations are not comparable; "" when the sidecar is absent. def tune_manifest_sha() : string { - return sidecar_generation_sha(tune_manifest_path()) + static_if (typeinfo module_exists(llvm_tune)) { + return sidecar_generation_sha(tune_manifest_path()) + } else { + return "" + } } //! The mint's noise verdict from the sidecar provenance ("ok" | "overridden"; "" when the -//! sidecar or its provenance is absent) — records copy it so a mint taken through a failing -//! noise gate is visible on the board, not only inside the sidecar. +//! sidecar or its provenance is absent, and in a build without the framework) — records copy it +//! so a mint taken through a failing noise gate is visible on the board, not only inside the sidecar. def tune_manifest_noise() : string { - let path = tune_manifest_path() + static_if (typeinfo module_exists(llvm_tune)) { + return sidecar_noise_verdict(tune_manifest_path()) + } else { + return "" + } +} + +def private sidecar_noise_verdict(path : string) : string { if (!stat(path).is_valid) { return "" } diff --git a/modules/dasLLAMA/performance/setup_asr_rig.das b/modules/dasLLAMA/performance/setup_asr_rig.das index 6c1d837c90..c945198059 100644 --- a/modules/dasLLAMA/performance/setup_asr_rig.das +++ b/modules/dasLLAMA/performance/setup_asr_rig.das @@ -4,6 +4,7 @@ options _dasllama_internal = true require profile_common require dasllama/dasllama_env +require daslib/tune // [tune_policy] - inert without dasLLVM require daslib/clargs require daslib/fio require strings diff --git a/modules/dasLLAMA/portable_tier_plan.md b/modules/dasLLAMA/portable_tier_plan.md new file mode 100644 index 0000000000..1aae804540 --- /dev/null +++ b/modules/dasLLAMA/portable_tier_plan.md @@ -0,0 +1,509 @@ +# The portable tier - dasLLAMA without the JIT crowns: no-LLVM build, AOT, WASM64 - plan + +**Status: stage 0 landed (2026-09-04) - the coverage additions (a)-(h) are in `tests/` and green +on this box, `test_parity` reproduces the frozen ids on the portable backend, and the four owed +rule fixes are in; stage 1 in progress.** Decisions settled in the 2026-09-04 discussion. The arc has one +product goal, reached in three stages that each prove the next one's premise: first a regular +`-jit` build with the `[tune]` framework short-circuited (the portable tier running on JIT-compiled +reference bodies), then an AOT compilation target under `examples/dasLLAMA/` on a very small +model, and only then WASM64. Main targets are ASR and TTS; a small LLM rides along, and the +model set may go smaller still. Vectorizing the default kernels is a separate, measured leg +between the stages (sec. "The vectorization leg"). The `-jit`-only rule in +`ARCHITECTURE_INVARIANTS.md` sec.3 is adjusted when the AOT stage reaches it, not before. + +Companion evidence: the four inventories the 2026-09-04 session produced (the LLVM dependency +census, the tune-framework coupling, the WASM build state, the require-guard grammar, and the +kernel test coverage) are summarized here with their file:line cites; nothing below is a guess. + +--- + +## The frame + +dasLLAMA is pure daslang (108 files under `dasllama/`, no C++ of its own). Its whole +dependency on LLVM is one file, `dasllama/dasllama_gemm_gen.das` (3756 lines), which builds +LLVM IR through the `llvm_boost` / `llvm_jit_code` / `llvm_jit_common` bindings; 40 generator +keys, permutations over `dot` (sdot, maddubs, vpdpbusd, vpdpbssd, smmla, amx_int8) and `width` +(128, 256, 512). It is reached only through path guards (`dasllama_common.das:22`, +`modules/dasLLVM/daslib/llvm_user_modules.das:8`). Everything else that spells `llvm/` is +packaging: 33 requires of `llvm/daslib/f16_cvt` and 4 of `llvm/daslib/aarch64_neon` (pure +das, zero requires of their own, recognized by the JIT by mangled name), and 27 of +`llvm/daslib/llvm_tune` (2759 lines, not one LLVM call). The `llvm/` path root exists only when +`DAS_LLVM_DISABLED=OFF` (`modules/dasLLVM/CMakeLists.txt:5`), and a source tree still resolves +it through the `.das_module` folder scan, which is why the module "works" today with LLVM +disabled by accident and fails in an installed SDK. + +`[tune]` never rewrites a body: it stamps `[llvm_code]`, and the tuned kernel IS the IR the +generator emits (`llvm_tune.das:1436-1448`, `llvm_jit.das:7961`). Every `[tune]` family keeps a +runnable das reference body; the `portable` backend registers unconditionally at priority 0 +(`dasllama_math_default.das:1613`); `arm64-gen`, `x64-gen` and `arm64-sdot` are +`jit_enabled()`-gated (`dasllama_math_gen.das:2775, 2807`, `dasllama_math_aarch64_neon.das:458`). +das `float4`/`int4` lower to 128-bit SIMD in every tier; 256/512-bit exists only as JIT +name-recognized intrinsics and the IR generators. There is no mechanism that freezes a chosen +permutation as das or C++; `policies.tune_frozen` means de-tuned. + +What the portable tier therefore is: the reference bodies, on whichever compiler the tier has. +Under the JIT they are auto-vectorized (the NEON file prefers the portable `dot_q8q8` over its +own scalar `sdot4` fallback for that reason, `dasllama_math_aarch64_neon.das:455-457`). Under +AOT they are C++ through clang or gcc. Under WASM they are that C++ through emcc with +`-msimd128`, or LLVM IR from the host emitter through the `-exe` cross path. + +The WASM side is further along than assumed: a wasm64 + pthreads runtime builds and ships +(`web/CMakeLists.txt` with `DAS_WASM_MEMORY64` and `DAS_WASM_PTHREADS`, deployed by +`pages.yml:260`); the cross-compile is `daslang -exe --jit-target=wasm64-unknown-emscripten` +with LLVM on the host only; wasm32 cross is unsupported by design (pointer-width layout bake, +`web/CMakeLists.txt:363-372`). + +--- + +## Settled decisions + +- **The cut.** Annotations that are 100% inert without the framework move to daslib; the + framework stays in dasLLVM whole. With no LLVM there is no tune, no sidecar, no exchange, + no tuner spawn - as if nothing happened. Not the read/mint split first proposed. +- **`f16_cvt` moves to `daslib/f16_cvt.das`.** The JIT recognizes it by mangled name from a + target-independent table (`llvm_jit_intrin.das:232-234`), so the move changes nothing for + the JIT; 33 require spellings change. +- **`aarch64_neon` and `x64_avx` move to daslib too** (ruled 2026-09-04, landed). The first + cut kept them in dasLLVM behind a path guard on the NEON tier; the no-LLVM gate then showed + the generated families' reference bodies in `dasllama_math_gen.das` call the NEON module's + `dot_q8q8_laneq4` and `idot4`, so the whole generated tier and the six tests that require it + fell out of a no-LLVM build. Both intrinsic modules are pure das with zero requires; the JIT + recognizes them by module name, unchanged. With them in daslib the NEON tier and `math_gen` + compile everywhere, their `[init]` registrations self-gate on `jit_enabled()` and the arch, + and the kq ladder runs on the no-LLVM tier. The three pure table builders `test_kquant` + borrowed from the IR generator (`pm1_of`, `vbmi_alphabet`, `vbmi_pack_word`) moved to + `dasllama_gemm_schema.das` for the same reason. +- **`require ?G target` names a guard module, never the target** (`ds2_parser.ypp:849-861`, + `parser_impl.cpp:1225-1279`). The guard for everything LLVM is `?llvm`: dasLLVM is a pure-das + dasbind package, so it now carries one C++ witness module named `llvm` + (`modules/dasLLVM/src/dasLLVM.cpp`), compiled in exactly when the build is configured with + dasLLVM. The build configuration decides, never the files on disk - a wasm build never + compiles the witness, and a console build with the LLVM sources present but configured out + loads no framework, no sidecar, no IR generator (ruled 2026-09-04; the path-guard spelling + `?llvm/daslib/llvm_tune` that preceded it satisfied itself off the filesystem). `get_platform_name()`, + `get_cross_platform_name()` and `get_architecture_name()` fold at compile time + (`module_builtin_runtime.cpp:2887-2907`), so a synthetic `wasm` module buys nothing for code + gating. `require !?X` is about five parser lines plus the hand-written pre-parse scanner + mirror (`ast_parse.cpp:229-268`); reach for it only if a module must not LOAD when X exists. +- **The routes.** AOT C++ through emcc is the LLVM-free WASM route. The `-exe` cross route + keeps tuned kernels and needs host LLVM; it is the later option, not the first. + +--- + +## Stage 0 - the coverage audit, and what it must add before any kernel moves + +Running the suite with the JIT off proves nothing by itself: every model-backed cell self-skips +on `if (!jit_enabled())`, and `guard_interp_inference` (`dasllama_math.das:29-33`) turns anything +that tries to serve into a panic. The portable tier's real safety net is the synthetic, +direct-call unit layer, and that layer is complete exactly where the K-quant work happened +(`tests/test_kquant.das`: 13 formats x dots, gemv rows, repack grp, tile, groupn against fp64 +dequant and bit-exact disk references; its `test_kq_tile` header at `:1673-1676` claims validity +on reference bodies and stamped kernels alike) and thin to absent elsewhere. The additions, in +priority order: + +**(a) A `[tuned]` elementwise unit.** `dot` (`dasllama_math.das:48`), `axpy` (`:66`), +`dot_bf16` (`:81`), `add_inplace` (`:97`), `add_scale_inplace` (`:111`), `mul_inplace` (`:123`), +`scale_inplace` (`:137`), `copy_floats` (`:151`). All `[tuned]`, the highest-traffic kernels in the +module (`copy_floats` 62 engine sites, `add_inplace` 35, `scale_inplace` 33, `axpy` 26), zero +coverage. `dot` is pinned only through three hand-computed `matmul` cases. The unit sweeps +lengths with tails against fp64 accumulation references. + +**(b) `test_q8q8_family.das`.** The crown family (`q8q8_tile_gen` `dasllama_math_gen.das:1222`, +`q8q8_gemv_gen` `:107`, the mx4 and s16 companions `:124, :145, :1153, :1170`) has its +tile-vs-gemv-vs-reference parity only in hand-run probes (`harness/gen_parity_probe.das:110,131`, +`harness/gen_slot_parity_probe.das`). Promote them into the suite with the ladder the 13 kq +families already have, and cover what has no gate at all: the whole s16 (`wscale_f16`) arm +(`dot_q8q8_f16s` `dasllama_math_default.das:64`, `q8q8_rows_kernel_s16` `:320`, +`q8q8_groupn_kernel_s16` `:354`, the s16 overloads `dasllama_math.das:2096-2138`), +`matmul_q8q8_group3` (`:2081`), `matmul_q8` / `dot_q8` (`:906, :881`), and +`matmul_mx4q8_batch` / `_batch_groupn` (`:2194-2226`). + +**(c) A CPU rope-application unit.** `rope_scaled`, `rope_scaled_neox`, `rope_scaled_tab`, +`rope_scaled_neox_tab`, the `_part` twins and `rope_apply` (`dasllama_math.das:2992-3160`) are +today only the oracle of the Apple-only Metal rope tests; `test_rope.das` covers the table +builders and the legacy `rope()`. An in-test angle reference makes them a subject. + +**(d) The activation and norm cells.** `softmax_sink` (`:2569`, six gpt-oss sites, no test), +`swiglu_oai` / `swiglu_oai4` (`:2749, :2820`), `softcap` / `softcap4` (`:2932, :2947`), +`sigmoid_gate` / `sigmoid_gate4` (`:2884, :2893`), `silu_mul4` (`:2911`), `silu4_batch` +(`:2869`), `l2_norm_rows` (`:2677`), `gemm_f32` / `gemm_f32_jo` / `gemm_f32_uk_4x16` +(`:834, :858, :828`; 23 engine sites, the tower and TTS float GEMM). Alongside the existing +`test_silu.das` / `test_softmax.das` property tests. + +**(e) Prefill and KV cells.** `matmul_kq_batch` (`:1944`, no test; portable has no slot, so +the serial-GEMV fallback is also ungated), `matmul_kq_batch_groupn` (`:1918`, whose only gate +`test_kquant.das:2136` self-skips off-JIT at `:1962-1966`), `kv_row_to_f32` and +`cvt_q8kv_to_f32` (`dasllama_kv_codec.das:394-409, :140`), `requant_rows_q8` / +`requant_rows_q8k_bs` (`dasllama_convert.das:1490, :1509`), `repack_kq_weight` +(`dasllama_math.das:1974`). + +**(f) Suite membership and honesty.** `test_kv_codec.das` and `test_vad.das` (a frozen +onnxruntime oracle with weights in-tree, `tests/test_vad.das:75`) are absent from +`tests/run.das:75-171`'s `model-free` list; add them. `test_mxfp4.das:168-170` returns silently +instead of `t |> skip` (against `tests/CLAUDE.md:40-42`). `test_groupn.das:12-15` takes its +expected values from the per-region GEMV under the same backend pin, so a bug shared by +`q8q8_kernel` and `q8q8_groupn_kernel` passes; give it an independent reference. + +**(f2) The same shared-bug class in the two mx4 groupn cells.** `test_mx4q8_groupn` takes its +expected values from the per-region `matmul_mx4q8` under the same backend pin, and +`test_groupn_bias_fold` takes its q8 and mx4 expectations from the unbiased groupn kernel; the +fp64 mx4 reference already exists as `ref_dot_f64` in `test_mxfp4.das:35` (private to that +file). Lift it into a shared `_kernel_ref.das` beside the two files and close both cells the +way (f) closed the q8 one. Lint candidate for a folder-local `tests/REVIEW.das` gate: a +`t |> run` arm that returns before asserting registers a skip (the guard whitelist is +`model_available`, `arm_on`, `family_on`, `model_missing`, `gguf_missing`, +`facade_data_missing`); the `test_mxfp4.das` silent return was the one raw-capability return +that did neither. + +**(g) A way to point the parity rails at the portable backend.** `tests/test_parity.das` +(`:177`) and `harness/parity.das` cannot select a backend from the command line; the +documented `DASLLAMA_PIN_BACKEND` knob (`ENVIRONMENT.md:25`, `dasllama_env.das:50`) is read by +three benchmarks and never by the library. Wire it into the parity harness (or add +`--backend`), so the same frozen ids replay against the tier this arc ships. + +**(h) Tower, ASR and TTS.** Encoder-tower GEMM wrappers `mm_blob_b` / `mm_bf16_b` / +`mm_plane_b` (`dasllama_tower.das:253-277`) and `layernorm_batch` (`:182`) have no CPU unit; +the ASR conv stems and mels (`dasllama_qwen3a.das:514, :406`, `dasllama_canary.das:786-841, +:670`, `dasllama_gemma4a.das:778, :744`, `dasllama_parakeet.das:905, :987, :805`, the whisper +stem `dasllama_whisper.das:987`) are `private` and covered only by jit-gated, model-gated +transcript oracles in `test_whisper.das`; `fft_pow2_run` (`dasllama_audio.das:157`) has no +direct test. TTS: `bilstm`, `sine_source`, `magnitude_phase`, `istft_envelope_divide` +(`dasllama_tts_blocks.das:1633, :1920, :1973, :2037`) reach only `_tts_parity.das` (model + +oracle-dir gated); `resize_linear` / `resize_linear_torch` (`:1738, :1776`) have none. Since ASR +and TTS are the main targets, the private stems get exposed to a synthetic unit or a +weights-in-tree oracle the way VAD already has one. + +**(h) findings from writing the tower/ASR/TTS unit.** The four family mels and every TTS +kernel named above are already public; only the seven conv-stem helpers and `lstm_direction` +are private, and the whisper-class stem is not in the whisper file at all but lines 1300-1333 of +`audio_encode_blocks` (`dasllama_audio.das`), serving whisper, ultravox, voxtral, qwen2audio and +omni. Ranked exposure: (1) split that stem into one public kernel in `dasllama_audio.das` +(ARCHITECTURE_MEDIA sec.2.14 names the seam); (2)-(3) the four mels need no engine change, only +an fp64 STFT reference; (4) `bilstm` is public, gate `lstm_direction` through it and run one +shape twice for the scratch-global reset; (5) `magnitude_phase` / `istft_envelope_divide` are +pure and now covered; (6) drop `private` on the two parakeet convs; (7) `g4a_conv_stage` splits +into a public im2col and a LayerNorm+ReLU epilogue that has no twin anywhere; (8)-(10) the +qwen3a and canary stems converge on the parakeet shape (the three im2col walks are +near-identical copies); `cn_conv_pw` is a wrapper, not a kernel. `sine_source` cannot take an +fp64 reference (ARCHITECTURE_TTS sec.2.33 phase law): its gate is a golden vector. + +**(i) findings from writing the other units.** `test_kquant.das`'s synthetic plane builders are +private in a `[test]` root; lift them into a `_kq_fixtures.das` sibling (the +`_metal_kernel_common.das` precedent) so the kq gates stop hand-packing blocks. +`kq_batch_kernel_gen` / `kq_batch_groupn_gen` (`dasllama_math_gen.das:2185, :2233`) are private +unlike their groupn sibling, so their reference bodies are reachable only through a backend +pin; make them public. `KernelBackend` has no `kernel_backend_has_mx4_batch()` predicate, so a +sweep of `kernel_backend_names()` that touches `matmul_mx4q8_batch` panics on `arm64-sdot` +(`mx4q8_unset_batch`, `dasllama_math.das:1051, :1573`). An exact-equality float compare whose +fixture lattice no longer guarantees exactness has no lint; the two elementwise files carry +hand-derived lattice arguments, worth a fixture-intent annotation before any rule. + +Stage 0 also lands the four rule-document and gate fixes owed from the is_nan arc +(a `tests-cpp/REVIEW.das` gate that every `set_source_files_properties` path exists, the +`tests-cpp/REVIEW.md` rule 3 wording, the `include/daScript/simulate/REVIEW.md` hot-path rule +naming the shipped build as the judge, and the `writing_cpp_tests.md` line about CMake edits). + +**Exit:** every kernel family in the inventory has a synthetic gate that does not self-skip +without the JIT, or a written reason it cannot; the suite's `model-free` arm lists every such +file. + +--- + +## Stage 1 - the refactor, and the portable tier on a regular `-jit` build + +1. `daslib/f16_cvt.das`; 37 require spellings (landed). +2. The inert annotation module in daslib carrying `tune`, `tune_perm`, `tune_companion`, + `tune_scope`, `tune_policy` and `llvm_code` as shells: `require ?llvm/daslib/llvm_tune + llvm/daslib/llvm_tune`, then `static_if (typeinfo module_exists(llvm_tune))` in each + `apply` delegating to the framework, else return true and do nothing. The shell's own + path-guarded require is the witness (`module_exists`, not `builtin_module_exists` - see the + ruling under "Open questions"). Framework change: its annotation classes + stop self-registering under those names (two registrations of `[tune]` would resolve + ambiguously) and expose their `apply` as plain functions the shells forward to. The + `_variants()` registries are emitted only by the framework; the engine references none + outside the guarded generator tier. +3. `[tuned]` goes framework-optional: `dasllama_tune.das:14` takes the path guard, and + `resolve_perm` (`:330-346`) with no framework returns the declared `fallback=` or the default + perm and reports nothing - no sidecar opened, no `tune_status` line, no box profile. +4. `dasllama_exchange.das` becomes a guarded require from the umbrella and the server. +5. The NEON tier's require takes the path guard. +6. `tests/run.das` gets a no-tune arm (the framework's reference policy; `run.das:309` + hard-codes `-jit` with no knob today), and the suite plus the parity rails run under it on + the stage-0 gates. (Landed as `--no-tune`; the whole-suite run under it is 2.5 h and was + cancelled - the suite rework in flight elsewhere is the gate, this arc runs single files.) +7. Measure the portable floor on the reference small models (decode and prefill, ASR and + TTS cells), under the measurement discipline: it is the number every later tier is judged + against. +8. The `-jit -exe` rail under the reference policy (landed). A standalone exe targets the host + CPU only when the program carries `[llvm_code]` kernels (`llvm_jit_run.das`, + `has_generated_kernel`); with the framework short-circuited there are none, the exe is the + generic ARMv8.0 baseline, and the NEON tier's reference bodies still name-resolved + `aarch64_neon::sdot4` to SDOT - a fatal `Cannot select` at codegen. The SDOT and SMMLA + tables now gate on `g_target_arm64_dotprod` / `g_target_arm64_i8mm` (host rail: always / + detected; generic and cross rails: the force env only), and a force-env feature reaches the + generic machine's string, so the gates and the machine stay one truth + (`modules/dasLLVM/ARCHITECTURE.md#aarch64-feature-truth`). The generic exe runs the NEON + tier on the scalar `sdot4` fallback - slower than the portable backend's auto-vectorized + `dot_q8q8`; whether the `[init]` backend gates should also consult the exe's feature rail is a + stage-2 question, alongside the same shape under AOT. Gates: `tests/jit_tests/aarch64_neon.das` + (hardware vs reference per intrinsic), the baseline probe in `tests/jit_tests/exe_host_cpu.das` + (red on the pre-fix emitter), and `test_exe_smoke` under `DAS_TUNE_POLICY=reference`. + +**Exit:** a `-DDAS_LLVM_DISABLED=ON` tree compiles the engine and its suite; a `-jit` run with +the framework short-circuited is green on the stage-0 gates and parity; the floor is in +`PERF_LEDGER.md`. + +--- + +## Stage 2 - an AOT compilation target under `examples/dasLLAMA/` + +**Status: the target exists and the exit's parity half holds (2026-09-04).** `examples/dasLLAMA/CMakeLists.txt` +builds `dasllama_aot` (opt-in, `EXCLUDE_FROM_ALL`): the daslang host with every CPU engine +module's C++ stubs linked in (`DAS_AOT_LIB` over `dasllama/*.das` minus the Vulkan and Metal +tiers, the LLVM IR generators and the exchange server, which interpret as scaffolding) plus the +daslib modules the CPU path calls that `libDaScriptAot` does not carry. `bin/dasllama_aot -use-aot +examples/dasLLAMA/run.das -- SmolLM2-135M-Instruct-Q8_0.gguf` reproduces the `-jit` reference-policy +run's 64 greedy tokens exactly. The run links 4637 functions; the 1178 it does not are the excluded +Metal tier plus 38 generic instantiations attributed to `json` / `strings_boost` / `strings_convert` +(libDaScriptAot's stubs, minted from another root - harmless, they interpret). + +What it took, each a general fix rather than a dasLLAMA one: + +- `-use-aot` now does what its help text says: the host compiles the script with `policies.aot` + (and `tune_frozen`, the generator's policy) so the linked stubs bind; a function with no stub + interprets (`fail_on_no_aot` stays off in the host). Before, only dastest honored the flag. +- The guards read the AOT tier through `aot_kernels_linked()` - `is_aot_function` on one kernel + (`dasllama_math::silu`), a runtime probe - never `aot_enabled()`, which folds differently under + generation and the consuming run and would desync every caller's hash. `ARCHITECTURE_INVARIANTS.md` + sec.3 carries the adjusted rule. +- `das_accelerate` had no `aotRequire`, and a C++ module without one AOT-disables every das + module whose require closure reaches it - the whole engine above `dasllama_math_accelerate` + emitted empty TUs (`// AOT disabled due to module requirements`) with no error anywhere. The + module now ships `dasAccelerate.h`. Lint candidate: every in-tree C++ module class overrides + `aotRequire` (a tests-cpp or CMake-time gate). +- The emitter's topological structure sort read a function type's parameter types as by-value + dependencies, saw a false cycle (`BatchWorkspace` embeds `Session`; a `function<(...BatchWorkspace...)>` + typedef names it) and fell back to source order - C++ then embedded an incomplete type. + `collectStructDeps` skips function / lambda / block types, and a residual (container-mediated) + cycle keeps the sorted prefix. Gate: `tests/aot/test_struct_order.das`. +- `das_is_aot_function` / `das_is_jit_function` and the jobque affinity pair had no AOT header + declaration (`aot_builtin.h`, `aot_builtin_jobque.h`); the generated C++ named them undeclared. +- The AOT TUs include every closure module's C++ header, so the target compiles with dasAudio's, + dasMinfft's and dasVulkan's (volk) include dirs; the generator reads `aotRequire` out of the + loaded shared modules, so those are regeneration inputs too (`DAS_AOT_EXTRA_DEPENDS`). + +The exit's numbers half is open: on the M5 box, contended by a build, the AOT decode ran 54 t/s +against the JIT reference policy's 360 t/s on the same model (prefill 74 vs 617) - not a +measurement, a gap of the size the vectorization leg predicts (clang over the emitted loop +shapes versus the JIT's auto-vectorizer). Next: the same rail for the ASR and TTS examples, a +model-free gate for the AOT target (CI has no models; the build itself is the compile gate, and a +synthetic-weights run is the cheapest runtime one), Linux and Windows builds of the target, and the +stage-2 profile that decides which kernels the vectorization leg rewrites first. + +**Exit:** the example compiles and runs the small model with exact parity to the JIT run; its +decode/prefill numbers sit beside the stage-1 floor. + +--- + +## Stage 3 - WASM64 + +**Status: the AOT-through-emcc rail runs (2026-09-04).** The same `dasllama_aot` target builds in +the wasm64 web tree (`web/build64`, `DAS_WASM_MEMORY64` + pthreads): the repo's host daslang +generates the engine C++ (the host-generated set compiles for wasm64 unchanged - 8-byte pointers +on both sides, every layout `static_assert` holds), em++ compiles the ~100 TUs, and the engine's +das files embed into MEMFS beside daslib. `examples/dasLLAMA/wasm/run_node.js` mounts the repo +and the model's directory through NODEFS and calls main with `-use-aot`; on SmolLM2-135M-Q8 the +64 greedy tokens match the native JIT run exactly, decode 46 t/s under node 25 against 54 on the +native AOT host (same box, not a measurement). No dasLLVM is embedded, so `require ?llvm` is +false inside the artifact and no tune framework exists in it. Two link-time findings: the +Accelerate driver's TU stays out wherever `das_accelerate` is not built (its C++ calls the +binding by symbol), and the AOT type tables need a 512 MB initial heap on top of the web build's +embeds. The artifact is 60 MB - 35 MB of code, 24 MB of data (about 10 MB of embedded `.das`, +the rest the AOT type tables). It was 306 MB before the web build's embed step learned to +exclude build residue: `_aot_generated/` C++ and `.jitted_scripts/` caches sit inside the +embedded source trees in a working checkout and outweighed the sources twenty to one; the +playground's `daslang_static` carried the same 95 MB. Next: wasmtime (no JS host), the browser +page, ASR and TTS examples. + +**The JIT-cross rail runs too (2026-09-04 night).** `daspkg release wasm` on a package whose +main is `examples/dasLLAMA/speak.das` or `run.das` produces a 20 MB wasm64 app (compiled code +only, no compiler, no embeds beyond the modules' own) that runs under node. Two rail fixes +made it so, both general: `release_wasm_disable_module("dasvulkan")` (and dasmetal, +dasaccelerate) keeps host-only modules out of the cross-compile so their guarded requires +resolve as absent, and the tune sidecar's module-cache pin moved out of the manifest reader the +runtime shares (`ast_core`'s `add_module_cache_dependency` rode into every exe; the wasm runtime +archive has no thunk for it). The tune framework is inert on the cross target by the +cross-target rule, so the artifact carries reference bodies through LLVM's own vectorizer. + +Kitten nano, the same line (6.7 s of audio), four workers on wasm, seven native; one box, +contended, not measurements: + +| host | Kitten nano, x real time | stories15M Q8, decode t/s | +|---|---|---| +| native JIT, tuned | 29-37 | 4550 | +| native AOT host (`dasllama_aot`) | 21 | - | +| wasm64, AOT C++ through emcc (node) | 4.0 | 357 | +| wasm64, JIT cross (`daspkg release wasm`, node) | 4.4 | 376-389 | + +Findings on the way: (1) the JIT-cross Kitten artifact dies with SIGILL under node's default +memory64 trap handling and runs clean under `--no-wasm-memory64-trap-handling`; the AOT +artifact never trips it, and the story artifact runs either way - a V8 trap-handler edge or an +access pattern of the emitted code, open, and a browser risk for that rail. (2) emscripten +builds the C environment from `Module.ENV` during startup, so a node driver must set the +engine's knobs in `preRun`, not after the runtime initializes: the first drivers set +`DASLLAMA_IMAGE=0` too late, the wasm runs minted wasm-identity `.dlim` files beside the box's +models, and the image GC reaped the box's own q8 Kitten image (re-minted by the next native +run - the hazard the image rail documents; both drivers now hold the knob in `preRun`, and a +run with the fixed driver mints nothing). (3) Parity is +numeric, not token-exact, across tiers on stories15M: with `_jit_fast_math` off, the JIT's +reference and tuned policies agree with each other, and the native AOT host, the wasm AOT host +and the JIT-cross artifact each diverge from it around token 40 - every tier orders its +reductions differently, and a 15M model's flat logits flip on that. SmolLM-135M matched +token-exact across all three; the frozen parity gates stay per tier. + +**The browser demo runs (2026-09-05).** `examples/dasLLAMA/storyteller/` is the first card of +the examples page: stories15M-Q8 writes a children's story on screen two tokens a frame +(dasGlfw + dasOpenGL text), each finished sentence goes to a `new_thread` speech worker, and +Kitten nano reads it in the kid voice through dasAudio while the next one is written; Space +starts a new story. One `main.das` for both targets, the same shape as `examples/games` +(`glfw_live` + `live_host` + `eval_main_loop`); `daspkg release wasm` gives a 27 MB wasm64 +app, the JIT-cross rail. The package's own shell (`web_shell.html`) fetches the four model +files (113 MB: the two ggufs and the g2p / POS-tag packs) into MEMFS with one progress bar +and waits for a click - browsers start audio only from a gesture - so the release rail now +exports `FS` and `ENV` to the shell (`-sEXPORTED_RUNTIME_METHODS=FS,ENV`). In Chrome under a +cross-origin-isolated server: stories at typing speed, Kitten nano at 0.37-0.43x real time +(three sentences of audio per second of synthesis, four workers), the AudioContext running, +no SIGILL - finding (1) did not reproduce in Chrome's trap handling. Three lessons from the +native run, general: the jobque fork-context pool is armed PER CONTEXT, so a `new_thread` body +that runs inference calls `setup_dasllama_jobque()` itself or every parallel kernel clones the +whole program (Kitten at rtf 15 instead of 0.05 - the facade doc-comment says so now); a +`new_thread` lambda's captured `Stream?` / `Channel?` are released by the thread body or the +lambda finalizer panics and takes the process down; a fresh `Session` samples from a fixed seed +(`set_seed` from the clock, or every story is the same story). The TinyStories corpus separates +stories with BOS, so BOS is a stop token beside EOS, and it writes curly quotes the screen +font has no glyphs for. Serving: the models sit beside the page (`models/`, or `?models=`). + +**Images only across the wire (2026-09-05).** The page fetches two `.dlim` files and the two +English front-end packs, no gguf. The TTS carrier now serves without its gguf: the family's +driver data (Kitten's speed priors and aliases, Kokoro's symbol table) rides the image meta, +`load_tts_model` takes a `.dlim` path (the file names its lane), and `dasllama-convert` grew a +TTS arm (`-o` anywhere). An image is keyed by the build's identity, so the browser's images +are baked natively against the wasm build's configuration: a scratch script cross-compiled to +wasm64 prints `dlim_config_json` under node (identity +`v32p1|q8|portable|s16|q8 mr4 b0 g4|kq 4/../4|q51 mr1|nat 1101`), and `dasllama-convert +--config wasm.json -o serve/.dlim` bakes against it - the portable backend exists on +every host, so the fail-closed verify passes. Kitten's image is half its gguf (30 MB against +59). The story model's is not: 51 MB against a 26.7 MB gguf, because the image widens the +token-embedding table to f32 (9.2M of the 15.2M parameters) where the gguf holds it as Q8_0. +A q8 token-table plane in the image (the PLE rail already gathers rows from one) is the engine +item that would bring it to ~17 MB; until then the two-image download is 108 MB against the +gguf pair's 113. An OPFS reader stays open. + +- Route: the stage-2 example's AOT C++ through emcc against `web/output64` (memory64 + + pthreads), under node first (NODEFS mounts), the browser after. +- The GPU tiers already self-gate the way a wasm build needs (verified 2026-09-04 with + `--disable-module dasvulkan`: the umbrella loads, `builtin_module_exists(vulkan)` is false, + kernel units green interpreted and under the JIT). The Vulkan tier hangs off one guarded + require in `dasllama_transformer.das` (`require ?vulkan dasllama/dasllama_math_vulkan`), the + same shape as the five `?das_metal` lines beside it; every unguarded `require vulkan` sits + inside the tier's own files, which only load through that guard. Owed once the suite rework + lands: a model-free gate that spawns the umbrella under `--disable-module dasvulkan + --disable-module dasmetal` (the wasm and console shape) so the seam cannot regress silently. +- Repairs on the way, all small: the `wasm_cross` CI lane configures without + `-DDAS_WASM_MEMORY64=ON` (`wasm_build.yml:209`) so `web/CMakeLists.txt:374` registers no + examples and the lane is a green no-op; `DAS_MAX_HW_JOBS` is 4 on wasm (`platform.h:629`); + the two emsdk pins contradict (`wasm_build.yml:188` vs `pages.yml:73`). Done 2026-09-04: + `get_architecture_name()` answers `wasm64` on a memory64 build. +- **No tuning under a cross target, by construction (landed 2026-09-04).** Two folds answer for + the artifact a compile EMITS: `get_target_triple()` (the `--jit-target` triple, "" native) and + `get_target_architecture_name()` (the triple's arch, the host's with no triple). The tune + framework's inert gate (`tune_aot_gate`) fires on any cross target, so `[tune]`, `[tuned]` and + `[llvm_code]` emit nothing but reference bodies into a wasm or console exe; the engine's + kernel-tier `[init]` gates ask the target arch, so a wasm cross-compile from an arm64 box + registers no NEON tier. The `--jit-target` argument keys the module cache (it sits after the + `--` separator, which the key ignored, so a cached native compile served a cross one with the + host's folds). Gate: `tests/jit_tests/cross_target_folds.das`. +- Limits to design around: SIMD is 128-bit through the `-msse2` shim and that flag is + load-bearing (`web/CMakeLists.txt:34`; without it vecmath drops to scalar); a process-wide + malloc spinlock under shared memory (`src/hal/wasm_thread_malloc.cpp`); the main-thread join + is sliced at 500 ms; no dasHV on wasm, so no server surface. +- Model IO is the wall: `fmap` on MEMFS copies the whole file into the heap, there is no + OPFS or WASMFS anywhere, and no `-sMAXIMUM_MEMORY`. Small quantized models load; a + windowed OPFS reader is its own item if anything larger is ever wanted. + +**Exit:** the ASR and TTS examples run under wasmtime with parity; a small LLM decodes; the +browser demo is a stretch on the same artifact. + +--- + +## The vectorization leg (between stages 1 and 3, measured) + +**First rung landed (2026-09-05): the int8 dot on wasm.** The profile came from node's V8 tick +logger over the JIT-cross Kitten artifact (`--cpu-prof` hangs emscripten's pthread pool at +worker load; `--prof` with `DAS_JOBQUE_THREADS=2` does not): 77% of every tick in one lambda, +the portable `q8q8_batch_kernel`'s row loop, which is `dot_q8q8` inlined - the TTS rows GEMMs. +A one-lane micro-bench sized the alternatives - a scratch `q8bench` daspkg package (not checked in) +timing all three forms over one 4096-wide row in ONE node process, `release wasm` with +`--profiling-funcs`, GMAC/s; the native column is the same package under `-jit` on this M1 Max: + +| form | native arm64 JIT | wasm64, before | wasm64, after | +|---|---|---|---| +| `dot_q8q8` tuned template (auto-vectorized) | 36 | 4.8 | 4.8 | +| `dot_q8q8_idot4x4` (idot4 builtin, per-block hsum) | 52 | 1.9 | 9.1 | +| `dot_q8q8_idot4_ps` (idot4, vector epilogue, one hsum per row) | 46-54 | - | 13.8 | + +Two changes, both general: the JIT's signed `idot` family gets a wasm lowering - the exact +extmul + `extadd_pairwise` chain (the 9.1 row above); the relaxed 7-bit dot behind the 13.8 row was +withdrawn in review, because the sign trick that feeds it wraps at -128 in either operand and +`+relaxed-simd` re-means float-vector min/max/mad engine-wide - and the portable kernels pick +`dot_q8q8_idot4_ps` (and its s16 twin) on a wasm target at compile time and keep the template +everywhere else. Gates: `tests/jit_tests/wasm_idot_lowering.das` (the cross dump carries the +pairwise adds and no relaxed dot, the host dump no wasm intrinsic) and the new `test_q8q8_idot4_ps` cell in the +kernel-family test (fp64 bar, s16 twin bit-identical). End to end: Kitten nano under node on 4 +lanes 0.27x -> 0.14x real time; in Chrome 0.37-0.43x -> 0.23-0.26x (the generator stage 1.1 s +-> 0.57 s per sentence). Not measurements, one box. Next rungs by the same profile: the +remaining 23% is spread thin (Snake's `sin` rows at 2.6%, libm `sinf`, the q8 quantizer), so the +next win is structural - the two-token GEMM shape for the rows conv (`q8q8_batch_kernel` reads +each weight row once per token) - or the story model's decode profile, not yet taken. + +Under the JIT the reference loops are auto-vectorized, so stage 1 is the SIMD-achievable floor +for the default kernels. AOT and WASM have no vectorizer we control: the emitted C++ goes +through clang, gcc or emcc, whose auto-vectorization of daslang loop shapes is hit and miss, +and the interpreter never vectorizes a loop. Explicit `float4` / `int4` bodies are how those +tiers reach the floor structurally. Do it per hot kernel by the stage-2 profile, not up +front: the q8 dot and GEMV/GEMM, the k-quant dequant-dot, f16 widen, attention dot and axpy, +the KV codec, the TTS conv and GEMM. The crux is the int8 dot: das has no int8 vector type, +so it either widens through `int4` lanes or gets a name-recognized lattice intrinsic with a +fallback body (the `x64_avx.das` pattern), which on wasm would want SIMD128's +`i32x4.dot_i16x8_s`. That decision is taken with stage-2 numbers in hand. + +--- + +## Models + +ASR and TTS first: whisper tiny, the silero VAD (weights in-tree), Kitten nano, Kokoro. For the +LLM: gemma-3-270m, SmolLM2-135M, Qwen3-0.6B are all registered arches; the two under 300M are +the honest wasm candidates, and smaller is on the table. + +## Open questions + +- **RULED: "dasLLVM absent" is the build configuration's word.** The path guard + `?llvm/daslib/llvm_tune` satisfied itself off the filesystem, so a source tree configured + with `-DDAS_LLVM_DISABLED=ON` still loaded the framework. The two scenarios that matter - a + wasm build, where no tuning may exist, and a console build (PS5 in dagor) with the LLVM + sources on disk for the PC build but configured out - both need the configure to decide. The + `llvm` C++ witness module (see "Settled decisions") is that word; every guard is `?llvm`. +- **RULED: a second trait.** `builtin_module_exists` on a shared das module flipped under + tool-driven compiles (`Module::requireEx` scans the process's promoted-module list, which a + nested compile never populates), so lint, ast-verify and the MCP checks audited the + no-framework arm of every guarded file. The ruling keeps `builtin_module_exists` as it was + (the process registry) and adds `typeinfo module_exists(X)`: the compiling program's own + library, the same answer on both rails. Every path-guarded das target in the tree + (`llvm_tune`, `llvm_code`, `dasllama_exchange`, `dasllama_gemm_gen`) now asks `module_exists`; + C++-module guards keep `builtin_module_exists`. Gate: `tests/language/optional_require.das`. + +- Which small-LLM carrier gets the parity fixture for stages 1 and 2 (SmolLM2-135M already has + a cls_q8 parity cell, `test_parity.das:76`). +- Whether `test_groupn.das`'s independent reference is fp64 dequant (the kq ladder's choice) or + a second, scalar implementation. +- Whether the stage-2 example is its own daspkg app or a plain `examples/` program. diff --git a/modules/dasLLAMA/tests/CLAUDE.md b/modules/dasLLAMA/tests/CLAUDE.md index 96ed59a7e9..6151a82893 100644 --- a/modules/dasLLAMA/tests/CLAUDE.md +++ b/modules/dasLLAMA/tests/CLAUDE.md @@ -39,7 +39,14 @@ files and `test_model_image` - it omits `test_mtp_gemma_drafter.das`, `image-vul Every suite but `model-free` and `stocked` needs `--arm`. `--full` parses and is then refused, so `--arm` is the only way in. `--suite model-free` and `--suite stocked` take neither - each is a whole gate; `--exclude ` drops the files whose name contains a token and names each -on an `EXCLUDED` line, so a trimmed run cannot read as full. The runner redirects +on an `EXCLUDED` line, so a trimmed run cannot read as full. `--no-tune` composes with any suite: +every child runs with `DAS_TUNE_POLICY=reference`, so the `[tune]` families and the `[tuned]` +hints fall to their reference bodies - the portable tier's arm of the gate. That arm also runs +with `DASLLAMA_IMAGE=0`: the reference policy is a different box identity, and a `.dlim` minted +under it would GC-purge the box's tuned images, so the runner refuses `--no-tune` with the +`image` and `image-vulkan` suites, and the two image-reading cells outside them (`test_tts_kitten`'s +image cell, `test_audio_embedder`'s direct-route cell) skip on the knob and keep their coverage on +the tuned arm. The runner redirects the COMPLETE output to a log file, and prints that path on the DONE line. It owns the dastest timeout, and repeats a file only when `--nreps` is passed explicitly (default 1, never best-of-N). Every child runs `-jit -module-cache .jitted_scripts/module_cache/dastest.dascache`; @@ -338,7 +345,10 @@ generic loop drives every evidence-carrying spec of the model-set table through pinned arms (evidence is DATA on `ModelSpec.parity` - ids + arms, regenerated via `../harness/parity.sh`); hand-written arms remain only for the tied-cls bit-match and the gpt-oss shared-load double fixture. Every compare logs both decoded streams. Large carriers gate -on `DASLLAMA_PARITY_FULL=1` via `model_available`. +on `DASLLAMA_PARITY_FULL=1` via `model_available`. `DASLLAMA_PIN_BACKEND` points the compares +at one kernel backend: the generic loop and the gpt-oss arm pin it before their load, restore +it on every exit, and name the served backend in every assert label; a name the box has not +registered skips the cell naming it, never a silent run on the default backend. `test_parity_pregate.das` - stocked suite; model-gated: the board parity pregate (`lcpp_bench --parity`, via `parity_check` fed a controlled spec) on small carriers - the real fixture passes, a flipped id fails, an evidence-less spec is refused, the text-form prompt @@ -520,13 +530,18 @@ pack-derived rhotic rows and its keep-the-r fallback for a vowel with no row, an rewrite on the fixture's probe words with the reference's espeak readings logged beside ours, and the British lexicon tier itself - the bath-trap split, the LOT vowel and the non-rhotic vowels answered from the British tier rather than the rewrite, with the American reading of the same -line beside it. +line beside it. Then the AMERICAN-ONLY pack (`tts_g2p_en_us.bin`, skips when absent): phoneme-identical +with the full pack on the 200 sentences, British refused by name on it (the full pack reading +British is the control), and the directory rule of `g2p_pack_path` - the full pack wins where +both sit, the American twin serves alone, an empty directory panics naming it. `test_tts_kitten.das` - stocked suite; the symbol-map and token-rule cells run everywhere (the front end's inventory into espeak-style IPA against the reference rewrite over the corpus, the reference driver's re-spacing and wrapping), the model-gated cells (`kitten-.gguf` + `tts_oracle/kitten_/` under the models dir, both from `performance/build_tts_data.das`) run the parity rail of `_tts_parity.das` per size and a facade smoke cell that speaks one -sentence and checks the PCM is finite, non-silent, of speech length, and carries its timings. +sentence and checks the PCM is finite, non-silent, of speech length, and carries its timings; +the image cell (skips under `DASLLAMA_IMAGE=0`) loads kitten-nano through `load_styletts2` on a +`.dlim` - the speed priors and aliases ride the image meta, the q8 sibling bakes when absent. `test_tts_kokoro.das` - stocked suite; model-free cells: the symbol map over a synthetic phoneme string, the out-of-vocabulary drop, the style-row clamps, and the pack-name language rule (`_` for all nine codes, and every other shape reading "" whatever letter it opens @@ -572,6 +587,106 @@ F0 phase drift the file header explains). `test_tower_helpers.das` - model-free: the shared encoder-tower helpers in `dasllama/dasllama_tower` (clamp, row norms, f16-table GEGLU-quick, im2col, two-axis rope, avg-pool, `attention_bidir`), each against an in-test reference. +`test_tower_asr_kernels.das` - model-free: the public tower / audio / TTS kernels with no CPU +unit of their own, each against an in-test fp64 reference over a procedural seeded-LCG fixture +and each bar carrying its own must-EXCEED poison arm - the padded-width GEMM wrappers +(`mm_blob_b` on an owned blob and on a borrowed plane, `mm_bf16_b` on bf16-exact operands, +`mm_plane_b`'s per-tensor routing, and the sec.2.13 claim that a zero-padded served width is +bit-identical to the unpadded GEMM), `layernorm` and both `layernorm_batch` overloads at a dim +off every lane width, `add_bias_rows` and `add_inplace_rows` against the exact scalar loop at +lengths off the float4 block, `gelu_erf_batch` against an fp64 erf series and against its own +whole-image form, `rope_neox_tab_rows` with a per-row-table-index leg, `attention_bidir_windows` +against an oracle that attends strictly inside each ragged window with per-window content so a +leak is material, `interpolate_grid_bilinear_aa` down and up at a non-zero plane offset, the +audio front end's `build_fft_plan` / `fft_pow2_run` / `build_dft_twiddles` against an fp64 +O(n^2) DFT of the same forward convention (plus the plan's non-power-of-2 refusal), the TTS +`resize_linear` / `resize_linear_torch` against an fp64 reference of each interpolation +convention, and the STFT pair `magnitude_phase` (with the zero-imaginary branch planted at both +zero signs - a bare `atan2` reads -pi where torch.angle reads +pi) and `istft_envelope_divide` +(four overlap-add geometries plus the zero-window leg that must leave the wave untouched through +the negligible-envelope guard). +`test_math_elementwise.das` - model-free: the `[tuned]` elementwise kernels (`dot`, `dot_bf16`, +`axpy`, `add_inplace`, `add_scale_inplace`, `mul_inplace`, `scale_inplace`, `copy_floats`) over +a 27-length sweep that reaches every width-8/unroll-2 tail (1..9, 15/16/17, 23/24/25, 31/32/33, +63/64/65, 100, 127/128/129, 1023/1024/1025). Inputs come from a seeded LCG tainted through a +`[sideeffects]` seam so no fixture folds at compile time, and every reference is accumulated in +`double` in-test. The two dots run twice: once on a dyadic lattice chosen so the whole fp32 +accumulation is provably exact (products are multiples of 3/64 bounded well inside 2^24, so any +vectorized or reassociated reduction still bit-matches), and once on unit-range inputs against +a length-only bar `1e-7*n^2 + 1e-6` - the textbook sequential-FMA bound at |a|,|b| <= 1 - each +length shipping its must-EXCEED control, the expected result offset by an added 0.5. `dot_bf16` +builds its operands AS bf16 and widens them for the reference, so the compare never measures +input rounding. The six in-place kernels are exact-equality gates: their operand lattices make +the fp64 reference land on a representable float and differ from the input at every element by +construction (`copy_floats` writes over a 1e9 sentinel, `add_scale_inplace` uses positive +operands with a negative scale so the result cannot be the input, `mul_inplace`'s multiplier is +a multiple of 0.375 so it can never be one), and every cell asserts that count. A closing cell +pins a zero count as a no-op on all eight. +`test_math_activations.das` - model-free: the activation, norm and fp32 GEMM kernels against +closed forms and in-test fp64 references over the same tail sweep. `softmax_sink` is gated per +element plus the closed form its name carries - the row's sum plus the sink mass is one - at a +negligible sink (-1000, where the -80 clamp engages) and a dominant one, and by shift +invariance (adding 5 to the scores and to the sink leaves the distribution alone). +`softcap`/`softcap4` are gated as `cap*tanh(x/cap)` at caps 30 and 50 over inputs out to 200, +with the band assert `|out| <= cap`; `sigmoid_gate`/`sigmoid_gate4`, `swiglu_oai`/`swiglu_oai4` +and `silu_mul4` (writing over a sentinel destination) each get the scalar and vec4 forms against +the same fp64 reference, so the twins are held to an oracle rather than to each other. +`swiglu_oai` additionally carries the trained-clamp gate: a gate of 20 must read bit-identically +to a gate of 7, an up value of +-20 to +-7, with a gate pair inside the limit as the must-differ +control. `silu4_batch` is gated for split invariance - bit-identical to `silu4` over the whole +buffer inline and under a `with_job_que` scope, with the dispatch counter (`get_disp_count`) as +the anti-vacuous witness that the threaded leg really split - plus the two fallback shapes (row +length off the float4 grouping, single row) which must not dispatch. `l2_norm_rows` is gated +per element and by the unit-norm property over eight (rows, row_size) pairs, plus the eps floor +on a zero row and a 1e-9 row. `gemm_f32` and `gemm_f32_jo` run twelve shapes covering both +remainder strips and the pure 4x16 tile, each against an fp64 GEMM that starts from the +pre-initialized C (so the accumulate contract is part of the claim) and against each other +bit-for-bit. Every tolerance bar in the file ships its control in the same cell: the expected +value offset by an added 0.01, which must land outside the bar. +`test_q8q8_family.das` - model-free: the q8q8 kernel family end to end, promoted from the +hand-run `harness/gen_parity_probe.das` / `gen_slot_parity_probe.das`. Six widths with tails +(64, 96, 512, 1024, 1056, 3072) across five cells, each judged by an in-test fp64 dequant +reference (int8 products summed exactly, both block scales applied in double) whose bar is the +per-block envelope times the block count, and each bar carrying a poison leg - 0.25 ADDED to +one expected element - that must EXCEED it. The stamped family repacks through +`repack_q8q8_grp` at the layout/wbias/kgroup companions and walks `q8q8_tile_gen` at the +tokstep companion's stride with a `q8q8_gemv_gen` token tail, bit-exact against per-token +GEMVs. The dispatch wrappers run on every backend `kernel_backend_names()` lists - repack +backends through their own `repack_q8q8_weight`, never on row-major data - with +`matmul_q8q8_batch` bit-exact against ntok x `matmul_q8q8`. The wscale_f16 arm covers +`dot_q8q8_f16s`, the `_s16` rows and groupN kernels, the stamped s16 tile/GEMV twins and the +`matmul_q8q8` / `_batch` / `_groupn` s16 overloads, against their f32 twins over f16-exact +scales - bit-exact on the portable backend, within the fp64 bar on a generated one whose s16 +and f32 stamps fold differently. `matmul_q8q8_group3` (f32 and s16) runs against three +independent GEMVs on unequal regions 32/40/44 (the row tail); `matmul_q8` / `dot_q8` cover the fp32-activation rail; +the mx4 cell drives `matmul_mx4q8_batch` and `matmul_mx4q8_batch_groupn` against ntok +independent `matmul_mx4q8` GEMVs and the `dot_mx4q8_scalar` leaf, pinned portable and swept +over the repack backends that carry those slots. Nothing here self-skips off-JIT - every cell +holds on the reference bodies. +`test_rope_apply.das` - model-free: the CPU rope APPLICATION leaves (`dasllama_math`'s +`rope_scaled`, `rope_scaled_neox`, the `_tab` and `_part` twins and `rope_apply`) against an +in-test fp64 angle reference over the same theta schedule, on head sizes 40/64/96/256 with +tails, positions 0/1/7/37/512, two thetas, fscale and mscale arms and the `rope_freqs` divisor +- the bar is one rotation's f32 accumulation plus the f32 angle's own rounding, each with an +added-value poison that must exceed it. The `_tab` forms also ride a tight twin bar against the +un-tabled forms (loose only by the cross-compilation-unit cos/sin ulp drift), the `_part` forms +are bit-exact against a full apply over the gathered rotated prefix with the un-rotated dims +proven to pass through, and `rope_apply` is bit-exact against the leaf its `neox` flag names, +including the `use_ff = false` p-RoPE arm. These kernels were previously only the Apple-only +Metal rope tests' oracle; this file makes them a subject on every platform. +`test_prefill_cpu_kernels.das` - model-free: the prefill and KV CPU kernels no suite gated +before, on q4_K / q6_K / q4_0 synthetic disk planes built in-file. `matmul_kq_batch` and +`matmul_kq_batch_groupn` - the per-position and per-expert GEMV routes a tier with no kq batch +slot runs, bit-matched against per-(token,row) and per-(region,token) disk dots, with no skip on +any tier; where a kq-carrying backend can be pinned (restored on exit) the native batched +kernels additionally ride bit-for-bit against `matmul_kq_active` and the rows-core GEMVs. +`cvt_q8kv_to_f32` / `kv_row_to_f32` (all four overloads, at a non-zero `kv_head_off` base) +against `quantize_q8kv_row`'s stored blocks plus a half-step round-trip assert. +`requant_rows_q8` vs `quantize_q8_0` per row and `requant_rows_q8k_bs` vs an in-test Q8_K +reference, on both `par` arms. `repack_kq_weight`'s round trip: the pure grp repack read back +through `kq_grp_row_dot` on every tier, then the dispatch seam - byte-identical to that layout +where the tier carries kq slots, the registered identity where it does not. Every compare ships +an added-value poison control. `test_attn_span.das` - stocked suite; the non-causal image span (`eval_embd_` with `non_causal = true`): mask direction by perturbation (causal row 0 blind to the last row, span row 0 sees it), classic/blocked/flash agreement, and the flag-reset bit-exactness; plus the FUSED mid-turn span diff --git a/modules/dasLLAMA/tests/REVIEW.md b/modules/dasLLAMA/tests/REVIEW.md index 855be65458..ca053e605a 100644 --- a/modules/dasLLAMA/tests/REVIEW.md +++ b/modules/dasLLAMA/tests/REVIEW.md @@ -45,8 +45,8 @@ or skip condition.** A clause that only names the file (a brace list, a suite ro nothing to correct. **A diff that changes `run.das`'s flag surface - a flag, a suite name, an area name, or what a -flag runs - corrects `CLAUDE.md`'s "Run suites ONLY through the runner" block and -`../CLAUDE.md`'s "Test workflow" section in the same change.** Both restate the surface for an +flag runs - adds it to or corrects it in `CLAUDE.md`'s "Run suites ONLY through the runner" block +and `../CLAUDE.md`'s "Test workflow" section in the same change.** Both restate the surface for an agent that reads them cold; a copy the code has left behind sends that agent to a flag that no longer does what the text says. @@ -172,12 +172,12 @@ which can wrongly satisfy a tolerant compare. transcript against a reference leg, external dump or CPU control alike. **A test in this folder that loads a stocked artifact whose producer the test cannot name is a -defect.** Stocked artifacts are model files, mmprojs, image fixtures, and oracle dumps. Any -one of these names the producer: a row in `../performance/model_specs.das`; a row's +defect.** A stocked artifact is anything the test reads out of `models_dir()` that this repo's +build does not produce - model files, mmprojs, front-end packs, image fixtures, oracle dumps. +Any one of these names the producer: a row in `../performance/model_specs.das`; a row's `companions` list; a row in `asr_catalog` (`../performance/profile_common.das`); a convert -script beside `model_specs.das` in `../performance/`. For an oracle dump, the mint script -stocked beside the dumps under `models_dir()` also counts, and the test that loads the dump -names it. +script beside `model_specs.das` in `../performance/`; for an oracle dump, the mint script stocked +beside the dumps under `models_dir()`, named by the test that loads the dump. **A test that reads a vision encode oracle dump without naming the minting arm in its header - the backend, the flash-attention setting, and the mmproj precision the dump came from - is a @@ -213,10 +213,10 @@ one CPU inference reads) and the stages a decode override selects on that model' (`blob_twin(t, path, seq_cap)`, `test_metal_decode_parity.das`), in one session.** The planar model and its blob twin share one shape, so one session serves both. -**A diff that adds a model-loading block to a `run.das` MODEL suite (every suite the -`--family` filter reaches - not the model-free suite) tags it with its family.** The family -tag is the token passed to `family_on(t, name)` (`_model_tier.das`). An untagged block -silently joins every family's gate. +**A diff that adds a model-loading block to a `run.das` MODEL suite (`decode`, `mtp`, `prefill`, +`matrix`, `image`, `image-vulkan`, `coverage` - not `model-free` or `stocked`) tags it with its +family.** The family tag is the token passed to `family_on(t, name)` (`_model_tier.das`). An +untagged block silently joins every family's gate. **No CPU-control batch parity runs against `Llama-3.3-70B-Instruct-Q4_K_M.gguf`.** The batched code paths get their parity on small models, through pins. diff --git a/modules/dasLLAMA/tests/_metal_kernel_common.das b/modules/dasLLAMA/tests/_metal_kernel_common.das index b058f6de6a..c66a069e7a 100644 --- a/modules/dasLLAMA/tests/_metal_kernel_common.das +++ b/modules/dasLLAMA/tests/_metal_kernel_common.das @@ -9,7 +9,7 @@ module _metal_kernel_common shared public require dasllama/dasllama_lint public require math require dasllama/dasllama_convert // dequant_k{4,5,6}_plane_superblock — the kq reference dequant -require llvm/daslib/f16_cvt // f32_to_f16 — packing synthetic scale planes +require daslib/f16_cvt // f32_to_f16 — packing synthetic scale planes require ?das_metal metal/das_metal_boost // ===== device buffers ===== diff --git a/modules/dasLLAMA/tests/_vk_kq_fixtures.das b/modules/dasLLAMA/tests/_vk_kq_fixtures.das index e4313e6bc7..35078df9e0 100644 --- a/modules/dasLLAMA/tests/_vk_kq_fixtures.das +++ b/modules/dasLLAMA/tests/_vk_kq_fixtures.das @@ -1,7 +1,7 @@ options gen2 options _dasllama_internal = true -require llvm/daslib/f16_cvt +require daslib/f16_cvt require math // Synthetic K-quant expert stacks in the vulkan tier's DEVICE layout, shared by the tier suite diff --git a/modules/dasLLAMA/tests/run.das b/modules/dasLLAMA/tests/run.das index 75b3cb30f1..a47cc38a2c 100644 --- a/modules/dasLLAMA/tests/run.das +++ b/modules/dasLLAMA/tests/run.das @@ -52,6 +52,9 @@ struct RunArgs { @clarg_short = "n" @clarg_doc = "Repeat count — explicit only (default 1, never best-of-N)" nreps : int = 1 + @clarg_name = "no-tune" + @clarg_doc = "Short-circuit the [tune] framework in every child (DAS_TUNE_POLICY=reference): the reference bodies run — the portable tier's arm" + no_tune : bool @clarg_doc = "daslang binary (default: the binary running this script — a hardcoded ./bin/daslang picked up a STALE top-level exe on the MSVC layout)" das : string = "" @@ -117,6 +120,8 @@ def suite_files(name : string) : array { // nolint:STYLE038 - a flat s "modules/dasLLAMA/tests/test_kernel_backend.das", "modules/dasLLAMA/tests/test_kgroup_repack.das", "modules/dasLLAMA/tests/test_kqformat.das", + "modules/dasLLAMA/tests/test_math_activations.das", + "modules/dasLLAMA/tests/test_math_elementwise.das", "modules/dasLLAMA/tests/test_matmul.das", "modules/dasLLAMA/tests/test_matmul_batch.das", "modules/dasLLAMA/tests/test_metal_gemm.das", @@ -128,12 +133,15 @@ def suite_files(name : string) : array { // nolint:STYLE038 - a flat s "modules/dasLLAMA/tests/test_jobque_tripwire.das", "modules/dasLLAMA/tests/test_plane.das", "modules/dasLLAMA/tests/test_ple_check.das", + "modules/dasLLAMA/tests/test_prefill_cpu_kernels.das", "modules/dasLLAMA/tests/test_program_roots.das", + "modules/dasLLAMA/tests/test_q8q8_family.das", "modules/dasLLAMA/tests/test_quant.das", "modules/dasLLAMA/tests/test_repack.das", "modules/dasLLAMA/tests/test_repack_lane_context.das", "modules/dasLLAMA/tests/test_rmsnorm.das", "modules/dasLLAMA/tests/test_rope.das", + "modules/dasLLAMA/tests/test_rope_apply.das", "modules/dasLLAMA/tests/test_run_suites.das", "modules/dasLLAMA/tests/test_run_summary.das", "modules/dasLLAMA/tests/test_silu.das", @@ -145,6 +153,7 @@ def suite_files(name : string) : array { // nolint:STYLE038 - a flat s "modules/dasLLAMA/tests/test_tts_textnorm.das", "modules/dasLLAMA/tests/test_tts_blocks.das", "modules/dasLLAMA/tests/test_tool_formats.das", + "modules/dasLLAMA/tests/test_tower_asr_kernels.das", "modules/dasLLAMA/tests/test_tower_helpers.das", "modules/dasLLAMA/tests/test_tune_interrupt.das", "modules/dasLLAMA/tests/test_unicode.das", @@ -222,7 +231,7 @@ let TESTS_DIR = "modules/dasLLAMA/tests" def area_tests(area : string) : array { if (area == "audio") { return <- [ "test_asr_verbs.das", "test_audio.das", "test_audio_embedder.das", "test_dasllama_lint_contracts.das", - "test_tower_helpers.das", "test_vad.das", "test_whisper.das" ] + "test_tower_asr_kernels.das", "test_tower_helpers.das", "test_vad.das", "test_whisper.das" ] } elif (area == "vision") { return <- [ "test_attn_span.das", "test_gemma3v.das", "test_gemma4uv.das", "test_gemma4v.das", "test_qwen25v.das", "test_qwen3v.das", "test_tower_helpers.das", "test_vision.das", "test_vision_chat.das", @@ -243,12 +252,15 @@ def area_tests(area : string) : array { "test_flash.das", "test_forward.das", "test_fused_decode.das", "test_gguf_quant.das", "test_gguf_shards.das", "test_gpu_model_swap.das", "test_gpu_slot_swap.das", "test_gpu_tier.das", "test_groupn.das", "test_jobque_tripwire.das", "test_kernel_backend.das", "test_kgroup_repack.das", "test_kqformat.das", "test_kquant.das", - "test_kv_codec.das", "test_kv_paged.das", "test_kv_prefix.das", "test_matmul.das", "test_matmul_batch.das", + "test_kv_codec.das", "test_kv_paged.das", "test_kv_prefix.das", "test_math_activations.das", + "test_math_elementwise.das", "test_matmul.das", "test_matmul_batch.das", "test_metal_float_a_gate.das", "test_metal_gemm.das", "test_metal_role_infer.das", "test_mtp.das", "test_mtp_gemma_drafter.das", "test_mtp_sampled_walk.das", "test_mtp_snapshot.das", "test_mxfp4.das", "test_par_indexed.das", "test_parity.das", "test_parity_cache.das", "test_parity_pregate.das", - "test_plane.das", "test_ple_check.das", "test_ple_modes.das", "test_prefill.das", "test_quant.das", - "test_repack.das", "test_repack_lane_context.das", "test_rmsnorm.das", "test_rope.das", "test_sampling.das", + "test_plane.das", "test_ple_check.das", "test_ple_modes.das", "test_prefill.das", "test_prefill_cpu_kernels.das", + "test_q8q8_family.das", "test_quant.das", + "test_repack.das", "test_repack_lane_context.das", "test_rmsnorm.das", "test_rope.das", "test_rope_apply.das", + "test_sampling.das", "test_scheduler.das", "test_silu.das", "test_softmax.das", "test_think_split.das", "test_tokenizer.das", "test_tool_formats.das", "test_unicode.das", "test_vulkan_dec_tail.das", "test_vulkan_kernels.das", "test_vulkan_moe_cm2.das", "test_vulkan_tier.das" ] @@ -414,6 +426,10 @@ def private child_envs(cfg : RunArgs; win : bool; arm : string) : string { envs = "DASLLAMA_TEST_FAMILY='{cfg.family}' {envs}" } } + if (cfg.no_tune) { + envs = (win ? "{envs}set \"DAS_TUNE_POLICY=reference\"&& set \"DASLLAMA_IMAGE=0\"&& " + : "DAS_TUNE_POLICY=reference DASLLAMA_IMAGE=0 {envs}") + } if (cfg.suite == "image-vulkan") { // the offline-bake suite: arm the tier so the probed config carries a vulkan section — // the bake itself is DRY (no device calls), so this is safe on GPU-less boxes too @@ -425,8 +441,9 @@ def private child_envs(cfg : RunArgs; win : bool; arm : string) : string { // the perf rig's. Without this every child auto-mints its own sidecar tune, computes a // FOREIGN image identity, and its save GC-purges the rig's images as dead siblings // (observed 2026-07-30: fam-gemma4 freed 34 GB of bench flavors). A caller-set - // DAS_TUNE_MANIFEST inherits; a box without a minted manifest keeps sidecar behavior. - if (empty(g_env_core.tune_manifest)) { + // DAS_TUNE_MANIFEST inherits; a box without a minted manifest keeps sidecar behavior. The + // --no-tune arm pins nothing: an explicit manifest disables the policy rail it runs on. + if (empty(g_env_core.tune_manifest) && !cfg.no_tune) { let manifest = "modules/dasLLAMA/performance/{derive_box_name(g_env_prof.box)}.tune.json" var mst : FStat if (stat(manifest, mst)) { @@ -542,6 +559,10 @@ def select_areas(cfg : RunArgs) : tuple> { //! the suite forms' argument contract: 0 = run, else main's exit code def check_suite_args(cfg : RunArgs) : int { + if (cfg.no_tune && (cfg.suite == "image" || cfg.suite == "image-vulkan")) { //! the arm runs with the image rail off, and the image suites ARE the rail + to_log(LOG_ERROR, "--no-tune sets DASLLAMA_IMAGE=0 in every child, so it cannot drive the image suites - drop --no-tune or pick another suite\n") + return 2 + } if (cfg.suite == "model-free" || cfg.suite == "stocked") { if (cfg.full || !empty(cfg.arm)) { //! the per-PR gates have no arms to scope: a filter would silently shrink them to_log(LOG_ERROR, "--suite {cfg.suite} takes neither --arm nor --full: it is a whole per-PR gate (--exclude names what it drops)\n") diff --git a/modules/dasLLAMA/tests/test_audio_embedder.das b/modules/dasLLAMA/tests/test_audio_embedder.das index c55ac79e61..59684015d1 100644 --- a/modules/dasLLAMA/tests/test_audio_embedder.das +++ b/modules/dasLLAMA/tests/test_audio_embedder.das @@ -73,7 +73,9 @@ def test_audio_embedder_gemma4a_arm(t : T?) { // scan only skips under DASLLAMA_IMAGE=0 — a model gate, not an artifact gate if (empty(image_path) && starts_with(name, "{base_name(mm)}.0x") && ends_with(name, ".dlim")) { let cand = path_join(models_dir(), name) - if (image_family_tag(cand) |> starts_with("gemma4a-")) { + let tag = image_family_tag(cand) + //! only the path the rail mints for THIS box identity is loadable by the direct route - a foreign-identity bake sits beside it under its own hash + if (tag |> starts_with("gemma4a-") && cand == image_path_for(mm, tag)) { image_path = cand } } diff --git a/modules/dasLLAMA/tests/test_batch_decode.das b/modules/dasLLAMA/tests/test_batch_decode.das index 1f61e2535f..32c479c012 100644 --- a/modules/dasLLAMA/tests/test_batch_decode.das +++ b/modules/dasLLAMA/tests/test_batch_decode.das @@ -6,7 +6,7 @@ options _dasllama_internal = true require dastest/testing_boost public require dasllama/dasllama_transformer // umbrella fires each arch [init] registration require dasllama/dasllama_math // setup_dasllama_jobque_ + the runtime knob setters (engine-level, not re-exported) -require llvm/daslib/f16_cvt +require daslib/f16_cvt require daslib/jobque_boost require daslib/fio require math diff --git a/modules/dasLLAMA/tests/test_batch_grid.das b/modules/dasLLAMA/tests/test_batch_grid.das index dcce3e2fd7..975e3e7864 100644 --- a/modules/dasLLAMA/tests/test_batch_grid.das +++ b/modules/dasLLAMA/tests/test_batch_grid.das @@ -5,7 +5,7 @@ require dastest/testing_boost public require dasllama/dasllama_math require dasllama/dasllama_math_default // portable backend (knob-inert control) require dasllama/dasllama_math_aarch64_neon // arm64 backends (registered under arm64 + JIT only) -require ?llvm/daslib/llvm_tune dasllama/dasllama_math_gen // the gen batch walk (the knob's target) +require ?llvm dasllama/dasllama_math_gen // the gen batch walk (the knob's target) require dasllama/dasllama_convert require daslib/jobque_boost require math diff --git a/modules/dasLLAMA/tests/test_box_profile.das b/modules/dasLLAMA/tests/test_box_profile.das index 9b79af0d3f..4f8a1ce95d 100644 --- a/modules/dasLLAMA/tests/test_box_profile.das +++ b/modules/dasLLAMA/tests/test_box_profile.das @@ -7,7 +7,7 @@ require dasllama/dasllama_math // the knob setters/getters (incl. set/get_q8_to require daslib/fio require dasllama/dasllama_env require dasllama/dasllama_tune // box_profile_verdict_at -require llvm/daslib/llvm_tune // tune_box_identity, TuneSidecarReason +require llvm/daslib/llvm_tune // tune_box_identity // The tune sidecar's "runtime" section: apply_box_profile_runtime(path) sets the runtime knobs // and logs each applied entry. The explicit-path form keeps the test independent of the app's @@ -260,7 +260,7 @@ def test_stale_binary_sidecar_keeps_runtime(t : T?) { t |> success(fwrite(path, js), "profile written") var err : string t |> success(set_mtime(path, mktime(2020, 1, 1, 0, 0, 0), err), "mtime set into the past ({err})") - t |> equal(box_profile_verdict_at(path), TuneSidecarReason.stale_binary, "reads as binary-stale") + t |> equal(box_profile_verdict_at(path), "stale_binary", "reads as binary-stale") apply_box_profile_runtime_checked(path) t |> equal(get_q8_token_block(), 320l, "runtime knob applied off a binary-stale sidecar") t |> success(box_profile_runtime_applied(), "the applied flag is up") @@ -276,7 +276,7 @@ def test_foreign_sidecar_applies_nothing(t : T?) { let path = path_join(dir, "dasllama_test_box_profile_foreign.json") let js = "\{\"kernels\":\{\},\"runtime\":\{\"q8_token_block\":320\},\"provenance\":\{\"box\":\"dasllama-test-foreign-box\"\}\}" t |> success(fwrite(path, js), "profile written") - t |> equal(box_profile_verdict_at(path), TuneSidecarReason.foreign_box, "reads as another box's") + t |> equal(box_profile_verdict_at(path), "foreign_box", "reads as another box's") apply_box_profile_runtime_checked(path) t |> equal(get_q8_token_block(), 128l, "runtime knob NOT applied off another box's sidecar") t |> equal(box_profile_runtime_declined(), "foreign_box", "the decline names its reason") diff --git a/modules/dasLLAMA/tests/test_groupn.das b/modules/dasLLAMA/tests/test_groupn.das index d4a6a3888d..32c441ace0 100644 --- a/modules/dasLLAMA/tests/test_groupn.das +++ b/modules/dasLLAMA/tests/test_groupn.das @@ -9,9 +9,11 @@ require dasllama/dasllama_convert // quantize_q8_0 (deterministic require math // Region-list (groupN) GEMVs — the MoE decode dispatch fuse: N same-shape [d x n] projections in -// one dispatch, offs packing (weight, activation) element-offset pairs. Expected values come from -// the per-region single GEMV under the SAME backend pin (identical dots, only scheduling differs), -// so equality is exact per backend. Regions 0/1 share activation row 0; region 2 reads row 1. +// one dispatch, offs packing (weight, activation) element-offset pairs. Every q8 row carries TWO +// compares: the per-region single GEMV under the SAME backend pin (identical dots, only scheduling +// differs, so equality is exact per backend — it catches groupn-vs-gemv divergence and nothing a +// shared kernel bug would move), and the in-test fp64 dequant dot below, which shares no code with +// any kernel. Regions 0/1 share activation row 0; region 2 reads row 1. def private close(t : T?; got, want : float; what : string) { t |> success(abs(got - want) <= 1e-6 * max(abs(want), 1.0), "{what}: got {got} want {want}") @@ -20,6 +22,8 @@ def private close(t : T?; got, want : float; what : string) { let N = 64l let D = 4l // one interleaved group per region (laneq-compatible), no tail let NREG = 3l +let REF_ULP_PER_BLOCK = 2.0e-6 +let REF_POISON = 0.05 def private build_offs() : array { return <- [N * D * 0l, 0l, N * D * 1l, 0l, N * D * 2l, N] @@ -37,6 +41,22 @@ def private slice_f(src : array; off, n : int64) : array { return <- [for (i in range64(off, off + n)); src[i]] } +//! The independent fp64 truth for one q8 row; `env` is the same sum on block magnitudes, the envelope the bar derives from. Layout-free, so repacked and row-major backends judge alike. +def private ref_dot_q8_f64(wq : array; ws : array; woff : int64; + xq : array; xs : array; xoff : int64; n : int64) : tuple { + var res : tuple + for (bi in range64(n / 32l)) { + var iacc = 0 + for (k in range64(32l)) { + iacc += int(wq[woff + bi * 32l + k]) * int(xq[xoff + bi * 32l + k]) + } + let term = double(iacc) * double(ws[woff / 32l + bi]) * double(xs[xoff / 32l + bi]) + res.dot += term + res.env += abs(term) + } + return res +} + [test] def test_q8q8_groupn(t : T?) { t |> run("q8 groupN == per-region GEMV under every backend (repack where needed)") @(t : T?) { @@ -54,6 +74,18 @@ def test_q8q8_groupn(t : T?) { let wq0 <- quantize_q8_0(wf, NREG * D * N) let xq <- quantize_q8_0(xf, 2l * N) var offs <- build_offs() + //! reference and bar built once off the row-major quants - a repack only permutes the same weights, so both are backend-free + var refy : array + var refbar : array + refy |> resize(int(NREG * D)) + refbar |> resize(int(NREG * D)) + for (r in range64(NREG)) { + for (row in range64(D)) { + let rf = ref_dot_q8_f64(wq0.q, wq0.scales, r * D * N + row * N, xq.q, xq.scales, offs[r * 2l + 1l], N) + refy[r * D + row] = float(rf.dot) + refbar[r * D + row] = REF_ULP_PER_BLOCK * float(N / 32l) * max(float(rf.env), 1.0) + } + } let saved = active_kernel_backend() for (name in kernel_backend_names()) { pin_kernel_backend(name) @@ -86,7 +118,11 @@ def test_q8q8_groupn(t : T?) { matmul_q8q8_groupn(y, wq, ws, offs, NREG, xq.q, xq.scales, N, D) for (i in range(int(NREG * D))) { close(t, y[i], want[i], "q8 row {i} (backend {name})") + t |> success(abs(y[i] - refy[i]) <= refbar[i], + "q8 row {i} vs fp64 dequant dot (backend {name}): got {y[i]} want {refy[i]} bar {refbar[i]}") } + t |> success(abs(y[0] - (refy[0] + REF_POISON)) > refbar[0], + "poison control: row 0 expectation + {REF_POISON} must exceed bar {refbar[0]} (backend {name})") } clear_kernel_backend_pin() if (saved != "") { diff --git a/modules/dasLLAMA/tests/test_kernel_backend.das b/modules/dasLLAMA/tests/test_kernel_backend.das index da6453363b..65434586dd 100644 --- a/modules/dasLLAMA/tests/test_kernel_backend.das +++ b/modules/dasLLAMA/tests/test_kernel_backend.das @@ -5,7 +5,7 @@ require dastest/testing_boost public require dasllama/dasllama_math require dasllama/dasllama_math_default // registers the "portable" backend at [init] require dasllama/dasllama_math_aarch64_neon // registers "arm64-sdot" on arm64+jit -require ?llvm/daslib/llvm_tune dasllama/dasllama_math_gen +require ?llvm dasllama/dasllama_math_gen // The kernel-backend registry must self-populate at [init]: the portable Q8·Q8 backend is always // registered (the fallback everywhere), and one is auto-activated as the row-major default. A repack diff --git a/modules/dasLLAMA/tests/test_kquant.das b/modules/dasLLAMA/tests/test_kquant.das index 367da547a3..aa2c92f4f4 100644 --- a/modules/dasLLAMA/tests/test_kquant.das +++ b/modules/dasLLAMA/tests/test_kquant.das @@ -6,7 +6,7 @@ options _function_length = 300 // and each gate carries its whole fixture options _dasllama_internal = true require dastest/testing_boost public -require dasllama/dasllama_gemm_gen // vbmi_pack_word / vbmi_alphabet / pm1_of - the lattice's pure table builders +require dasllama/dasllama_gemm_schema // vbmi_pack_word / vbmi_alphabet / pm1_of - the lattice's pure table builders require dasllama/dasllama_layout // moe_gpu_gather_stack_kq - the device-plane gather oracle require _model_tier require dasllama/dasllama_gguf diff --git a/modules/dasLLAMA/tests/test_kv_codec.das b/modules/dasLLAMA/tests/test_kv_codec.das index a1a4ffb6ee..1c6299bf1c 100644 --- a/modules/dasLLAMA/tests/test_kv_codec.das +++ b/modules/dasLLAMA/tests/test_kv_codec.das @@ -8,7 +8,7 @@ require dasllama/dasllama_transformer // umbrella fires each arch [init] regis require dasllama/dasllama_math // setup_dasllama_jobque_ + target_chunk_work + fused knob require dasllama/dasllama_kv_codec // the codec kernel families under unit test require dasllama/dasllama_convert // cvt_f32_to_f16/cvt_f16_to_f32 — the generic f16 rows -require llvm/daslib/f16_cvt +require daslib/f16_cvt require daslib/jobque_boost require daslib/fio require math diff --git a/modules/dasLLAMA/tests/test_math_activations.das b/modules/dasLLAMA/tests/test_math_activations.das new file mode 100644 index 0000000000..ffe175056e --- /dev/null +++ b/modules/dasLLAMA/tests/test_math_activations.das @@ -0,0 +1,636 @@ +options gen2 +options _dasllama_internal = true + +require dastest/testing_boost public +require dasllama/dasllama_math +require dasllama/dasllama_par +require daslib/jobque_boost +require math + +//! The activation, norm and fp32 GEMM kernels - softmax_sink, swiglu_oai(4), softcap(4), +//! sigmoid_gate(4), silu_mul4, silu4_batch, l2_norm_rows and gemm_f32 / _jo / _uk_4x16 - against +//! closed forms and in-test fp64 references over shapes that reach every tail. + +var g_taint = 0u + +[init] +def init_activation_taint() { + g_taint = 0u +} + +[sideeffects] +def private opaque_seed(s : uint) : uint { + return s ^ g_taint +} + +let ACT_SWEEP <- [ 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65, + 100, 127, 128, 129, 1023, 1024, 1025 ] +let L2_SHAPES <- [ 1, 1, 1, 7, 3, 8, 4, 64, 2, 127, 5, 33, 3, 128, 7, 4 ] +let GEMM_SHAPES <- [ 1, 1, 1, 2, 3, 5, 3, 7, 16, 4, 5, 16, 4, 16, 16, 5, 9, 17, + 7, 4, 33, 8, 32, 48, 9, 13, 19, 4, 1, 16, 6, 2, 31, 16, 40, 64 ] + +let POISON = 0.01 +let XMAG = 4.0 +let ZMAG = 8.0 +let OAI_MAG = 56.0 +let SENTINEL = 1.0e9 +let L2_EPS = 1.0e-6 + +struct ActSweep { + bad : int + inert : int + cases : int + elems : int + worst : float + worst_bar : float + worst_n : int + poison_inside : int +} + +def private lcg_pm(n : int; seed : uint; lo, hi : float) : array { + var st = opaque_seed(seed) + var a : array + a |> resize(n) + for (v in a) { + st = st * 1664525u + 1013904223u + let u01 = float(st >> 8u) * (1.0 / 16777216.0) + let mag = lo + (hi - lo) * u01 + v = ((st >> 4u) & 1u) == 0u ? -mag : mag + } + return <- a +} + +def private filled(n : int; v : float) : array { + var a : array + a |> resize(n) + for (x in a) { + x = v + } + return <- a +} + +def private check(var r : ActSweep; n : int; bar : float; got, want : array) { + r.cases++ + for (g, w in got, want) { + r.elems++ + let diff = abs(g - w) + if (diff > bar) { + r.bad++ + } + if (abs(g - (w + POISON)) <= bar) { + r.poison_inside++ + } + if (diff >= r.worst) { + r.worst = diff + r.worst_bar = bar + r.worst_n = n + } + } +} + +def private count_inert(var r : ActSweep; got, before : array) { + for (g, b in got, before) { + if (g == b) { + r.inert++ + } + } +} + +def private report(t : T?; name : string; r : ActSweep) { + t |> success(r.cases > 0 && r.elems > 0, "{name}: the sweep must run at least one case") + t |> equal(r.bad, 0, "{name}: values past the bar, worst |diff| {r.worst} against bar {r.worst_bar} at n={r.worst_n}") + t |> equal(r.poison_inside, 0, "{name}: poison legs inside the bar - a bar nothing exceeds discriminates nothing") + feint("{name}: {r.elems} elements, worst |diff| {r.worst} against bar {r.worst_bar} at n={r.worst_n}\n") +} + +def private report_moved(t : T?; name : string; r : ActSweep) { + report(t, name, r) + t |> equal(r.inert, 0, "{name}: elements the kernel left exactly where it found them") +} + +def private softmax_bar(n : int) : float { + return 1.0e-6 + 4.0e-7 * float(n) +} + +def private softmax_sink_ref(x : array; sink : float) : tuple; float> { + var mx = double(sink) + for (v in x) { + mx = max(mx, double(v)) + } + var terms : array + terms |> resize(length(x)) + let sk = exp(max(double(sink) - mx, -80.0lf)) + var sum = sk + for (e, v in terms, x) { + e = exp(max(double(v) - mx, -80.0lf)) + sum += e + } + var out : array + out |> resize(length(x)) + for (o, e in out, terms) { + o = float(e / sum) + } + return out => float(sk / sum) +} + +def private sweep_softmax_sink(sink : float; seed : uint) : ActSweep { + var r = ActSweep() + for (n in ACT_SWEEP) { + let x0 <- lcg_pm(n, opaque_seed(seed) + uint(n), 0.25, 6.0) + let want <- softmax_sink_ref(x0, sink) + var x := x0 + softmax_sink(x, int64(n), sink) + check(r, n, softmax_bar(n), x, want._0) + var total = 0.0lf + for (v in x) { + total += double(v) + } + if (abs(total + double(want._1) - 1.0lf) > double(softmax_bar(n))) { + r.bad++ + } + } + return r +} + +def private sweep_softmax_sink_shift() : ActSweep { + var r = ActSweep() + for (n in ACT_SWEEP) { + let x0 <- lcg_pm(n, opaque_seed(0x71a5u) + uint(n), 0.25, 6.0) + let want <- softmax_sink_ref(x0, 1.5) + var plain := x0 + softmax_sink(plain, int64(n), 1.5) + check(r, n, softmax_bar(n), plain, want._0) + var shifted : array + shifted |> resize(n) + for (s, v in shifted, x0) { + s = v + 5.0 + } + softmax_sink(shifted, int64(n), 6.5) + check(r, n, softmax_bar(n), shifted, plain) + } + return r +} + +def private softcap_ref(x : array; cap : float) : array { + var out : array + out |> resize(length(x)) + for (o, v in out, x) { + o = float(double(cap) * tanh(double(v) / double(cap))) + } + return <- out +} + +def private softcap_bar(cap : float) : float { + return 1.0e-6 + 1.0e-6 * cap +} + +def private sweep_softcap(cap : float; vec : bool) : ActSweep { + var r = ActSweep() + for (n in ACT_SWEEP) { + let x0 <- lcg_pm(n, opaque_seed(vec ? 0x2c81u : 0x9b44u) + uint(n), 0.5, 200.0) + let want <- softcap_ref(x0, cap) + var x := x0 + if (vec) { + softcap4(x, int64(n), cap) + } else { + softcap(x, int64(n), cap) + } + check(r, n, softcap_bar(cap), x, want) + count_inert(r, x, x0) + for (v in x) { + if (abs(v) > cap) { + r.bad++ + } + } + } + return r +} + +def private sigmoid_gate_ref(x, g : array) : array { + var out : array + out |> resize(length(x)) + for (o, xv, gv in out, x, g) { + o = float(double(xv) / (1.0lf + exp(-double(gv)))) + } + return <- out +} + +def private sweep_sigmoid_gate(vec : bool) : ActSweep { + var r = ActSweep() + for (n in ACT_SWEEP) { + let x0 <- lcg_pm(n, opaque_seed(0x5ab3u) + uint(n), 0.25, XMAG) + let g <- lcg_pm(n, opaque_seed(0x9d10u) + uint(n), 0.25, ZMAG) + let want <- sigmoid_gate_ref(x0, g) + var x := x0 + if (vec) { + sigmoid_gate4(unsafe(addr(x[0])), unsafe(addr(g[0])), int64(n)) + } else { + sigmoid_gate(unsafe(addr(x[0])), unsafe(addr(g[0])), int64(n)) + } + check(r, n, 1.0e-6 + 1.0e-6 * XMAG, x, want) + count_inert(r, x, x0) + } + return r +} + +def private swiglu_oai_ref(g, u : array) : array { + var out : array + out |> resize(length(g)) + for (o, gv, uv in out, g, u) { + let xv = min(double(gv), 7.0lf) + let yv = clamp(double(uv), -7.0lf, 7.0lf) + o = float((xv / (1.0lf + exp(-1.702lf * xv))) * (yv + 1.0lf)) + } + return <- out +} + +def private sweep_swiglu_oai(vec : bool) : ActSweep { + var r = ActSweep() + for (n in ACT_SWEEP) { + let g0 <- lcg_pm(n, opaque_seed(0x4411u) + uint(n), 0.25, 12.0) + let u <- lcg_pm(n, opaque_seed(0x77c2u) + uint(n), 0.25, 12.0) + let want <- swiglu_oai_ref(g0, u) + var g := g0 + if (vec) { + swiglu_oai4(unsafe(addr(g[0])), unsafe(addr(u[0])), int64(n)) + } else { + swiglu_oai(unsafe(addr(g[0])), unsafe(addr(u[0])), int64(n)) + } + check(r, n, 1.0e-6 + 1.0e-6 * OAI_MAG, g, want) + } + return r +} + +struct ClampCase { + clamp_diffs : int + control_same : int +} + +def private oai_pair_diffs(ga, gb, ua, ub : array; vec : bool) : int { + var a := ga + var b := gb + let n = long_length(ga) + if (vec) { + swiglu_oai4(unsafe(addr(a[0])), unsafe(addr(ua[0])), n) + swiglu_oai4(unsafe(addr(b[0])), unsafe(addr(ub[0])), n) + } else { + swiglu_oai(unsafe(addr(a[0])), unsafe(addr(ua[0])), n) + swiglu_oai(unsafe(addr(b[0])), unsafe(addr(ub[0])), n) + } + var d = 0 + for (x, y in a, b) { + if (x != y) { + d++ + } + } + return d +} + +def private oai_clamp_case(vec : bool) : ClampCase { + let n = 18 + let u <- lcg_pm(n, opaque_seed(0x3f00u), 1.5, 5.0) + var res = ClampCase() + res.clamp_diffs += oai_pair_diffs(filled(n, 20.0), filled(n, 7.0), u, u, vec) + res.clamp_diffs += oai_pair_diffs(filled(n, 2.0), filled(n, 2.0), filled(n, 20.0), filled(n, 7.0), vec) + res.clamp_diffs += oai_pair_diffs(filled(n, 2.0), filled(n, 2.0), filled(n, -20.0), filled(n, -7.0), vec) + res.control_same = n - oai_pair_diffs(filled(n, 3.0), filled(n, 7.0), u, u, vec) + return res +} + +def private silu_mul_ref(a, z : array) : array { + var out : array + out |> resize(length(a)) + for (o, av, zv in out, a, z) { + let d = double(zv) + o = float(double(av) * (d / (1.0lf + exp(-d)))) + } + return <- out +} + +def private sweep_silu_mul4() : ActSweep { + var r = ActSweep() + for (n in ACT_SWEEP) { + let a <- lcg_pm(n, opaque_seed(0x6e21u) + uint(n), 0.25, XMAG) + let z <- lcg_pm(n, opaque_seed(0xf13cu) + uint(n), 0.25, ZMAG) + let want <- silu_mul_ref(a, z) + let before <- filled(n, SENTINEL) + var o := before + silu_mul4(unsafe(addr(o[0])), unsafe(addr(a[0])), unsafe(addr(z[0])), int64(n)) + check(r, n, 1.0e-6 + 1.0e-6 * XMAG * ZMAG, o, want) + count_inert(r, o, before) + } + return r +} + +def private silu_ref(x : array) : array { + var out : array + out |> resize(length(x)) + for (o, v in out, x) { + let d = double(v) + o = float(d / (1.0lf + exp(-d))) + } + return <- out +} + +struct BatchCase { + split_diffs : int + dispatches : int64 + elems : int +} + +def private silu4_batch_case(var r : ActSweep; d, npos : int) : BatchCase { + var res = BatchCase(elems = d * npos) + let x0 <- lcg_pm(d * npos, opaque_seed(0x84c1u) + uint(d * 131 + npos), 0.25, ZMAG) + var whole := x0 + silu4(whole, int64(d) * int64(npos)) + var batched := x0 + let before = get_disp_count() + silu4_batch(batched, int64(d), int64(npos)) + res.dispatches = get_disp_count() - before + let want <- silu_ref(x0) + check(r, d * npos, 1.0e-6 + 1.0e-6 * ZMAG, whole, want) + count_inert(r, batched, x0) + for (bv, wv in batched, whole) { + if (bv != wv) { + res.split_diffs++ + } + } + return res +} + +def private l2_ref(v : array; rows, row_size : int; eps : float) : array { + var out : array + out |> resize(rows * row_size) + for (r in range(rows)) { + let base = r * row_size + var sum = 0.0lf + for (i in range(row_size)) { + let x = v[base + i] + sum += double(x * x) + } + let scale = 1.0 / max(sqrt(float(sum)), eps) + for (i in range(row_size)) { + out[base + i] = v[base + i] * scale + } + } + return <- out +} + +def private l2_bar(row_size : int) : float { + return 1.0e-6 + 2.0e-7 * float(row_size) +} + +def private sweep_l2_norm_rows() : ActSweep { + var r = ActSweep() + for (s in range(length(L2_SHAPES) / 2)) { + let rows = L2_SHAPES[s * 2] + let row_size = L2_SHAPES[s * 2 + 1] + let v0 <- lcg_pm(rows * row_size, opaque_seed(0xa1b2u) + uint(s), 1.25, 3.0) + let want <- l2_ref(v0, rows, row_size, L2_EPS) + var v := v0 + l2_norm_rows(unsafe(addr(v[0])), int64(rows), int64(row_size), L2_EPS) + check(r, row_size, l2_bar(row_size), v, want) + count_inert(r, v, v0) + for (rw in range(rows)) { + var sum = 0.0lf + for (i in range(row_size)) { + sum += double(v[rw * row_size + i]) * double(v[rw * row_size + i]) + } + if (abs(sqrt(sum) - 1.0lf) > double(l2_bar(row_size))) { + r.bad++ + } + } + } + return r +} + +def private l2_eps_floor() : ActSweep { + var r = ActSweep() + let rows = 3 + let row_size = 8 + var v0 <- filled(rows * row_size, 0.0) + for (i in range(row_size)) { + v0[row_size + i] = 1.0e-9 + } + let want <- l2_ref(v0, rows, row_size, L2_EPS) + var v := v0 + l2_norm_rows(unsafe(addr(v[0])), int64(rows), int64(row_size), L2_EPS) + check(r, row_size, l2_bar(row_size), v, want) + return r +} + +struct GemmCase { + bad : int + jo_diffs : int + cases : int + elems : int + poison_inside : int + worst : float + worst_bar : float + worst_k : int +} + +def private gemm_ref(c0, a, b : array; M, K, N : int) : array { + var out : array + out |> resize(M * N) + for (i in range(M)) { + for (j in range(N)) { + var acc = double(c0[i * N + j]) + for (k in range(K)) { + acc += double(a[i * K + k]) * double(b[k * N + j]) + } + out[i * N + j] = float(acc) + } + } + return <- out +} + +def private gemm_bar(K : int) : float { + return 6.0e-8 * float(K) * (float(K) + 2.0) + 1.0e-6 +} + +def private sweep_gemm() : GemmCase { + var r = GemmCase() + for (s in range(length(GEMM_SHAPES) / 3)) { + let M = GEMM_SHAPES[s * 3] + let K = GEMM_SHAPES[s * 3 + 1] + let N = GEMM_SHAPES[s * 3 + 2] + let a <- lcg_pm(M * K, opaque_seed(0x1717u) + uint(s), 0.25, 1.0) + let b <- lcg_pm(K * N, opaque_seed(0x2828u) + uint(s), 0.25, 1.0) + let c0 <- lcg_pm(M * N, opaque_seed(0x3939u) + uint(s), 0.25, 1.0) + let want <- gemm_ref(c0, a, b, M, K, N) + var c1 := c0 + gemm_f32(unsafe(addr(c1[0])), unsafe(addr(a[0])), unsafe(addr(b[0])), int64(M), int64(K), int64(N)) + var c2 := c0 + gemm_f32_jo(unsafe(addr(c2[0])), unsafe(addr(a[0])), unsafe(addr(b[0])), int64(M), int64(K), int64(N)) + r.cases++ + tally_gemm(r, K, c1, c2, want) + } + return r +} + +def private tally_gemm(var r : GemmCase; K : int; c1, c2, want : array) { + let bar = gemm_bar(K) + for (g, j, w in c1, c2, want) { + r.elems++ + let diff = abs(g - w) + if (diff > bar || abs(j - w) > bar) { + r.bad++ + } + if (g != j) { + r.jo_diffs++ + } + if (abs(g - (w + POISON)) <= bar) { + r.poison_inside++ + } + if (diff >= r.worst) { + r.worst = diff + r.worst_bar = bar + r.worst_k = K + } + } +} + +def private zero_size_moves() : int { + let before <- lcg_pm(8, opaque_seed(0x0f01u), 0.25, 2.0) + let g <- lcg_pm(8, opaque_seed(0x0f02u), 0.25, 2.0) + var x := before + softmax_sink(x, 0l, 1.0) + softcap(x, 0l, 30.0) + softcap4(x, 0l, 30.0) + silu4_batch(x, 8l, 0l) + sigmoid_gate(unsafe(addr(x[0])), unsafe(addr(g[0])), 0l) + sigmoid_gate4(unsafe(addr(x[0])), unsafe(addr(g[0])), 0l) + swiglu_oai(unsafe(addr(x[0])), unsafe(addr(g[0])), 0l) + swiglu_oai4(unsafe(addr(x[0])), unsafe(addr(g[0])), 0l) + silu_mul4(unsafe(addr(x[0])), unsafe(addr(g[0])), unsafe(addr(g[0])), 0l) + l2_norm_rows(unsafe(addr(x[0])), 0l, 8l, L2_EPS) + var moved = 0 + for (v, b in x, before) { + if (v != b) { + moved++ + } + } + return moved +} + +[test] +def test_softmax_sink(t : T?) { + t |> run("softmax_sink against an fp64 reference, the row plus the sink mass summing to one") @(t : T?) { + report(t, "softmax_sink negligible sink", sweep_softmax_sink(-1000.0, 0x1001u)) + report(t, "softmax_sink dominant sink", sweep_softmax_sink(8.0, 0x1002u)) + } + t |> run("softmax_sink is invariant to a shift of the scores and the sink together") @(t : T?) { + report(t, "softmax_sink shift", sweep_softmax_sink_shift()) + } +} + +[test] +def test_softcap(t : T?) { + t |> run("softcap is cap*tanh(x/cap) and never leaves the band") @(t : T?) { + report_moved(t, "softcap cap=30", sweep_softcap(30.0, false)) + report_moved(t, "softcap cap=50", sweep_softcap(50.0, false)) + } + t |> run("softcap4 holds the same closed form through its tail") @(t : T?) { + report_moved(t, "softcap4 cap=30", sweep_softcap(30.0, true)) + report_moved(t, "softcap4 cap=50", sweep_softcap(50.0, true)) + } +} + +[test] +def test_sigmoid_gate(t : T?) { + t |> run("sigmoid_gate scales by the logistic of the gate row") @(t : T?) { + report_moved(t, "sigmoid_gate", sweep_sigmoid_gate(false)) + } + t |> run("sigmoid_gate4 holds the same form through its tail") @(t : T?) { + report_moved(t, "sigmoid_gate4", sweep_sigmoid_gate(true)) + } +} + +[test] +def test_swiglu_oai(t : T?) { + t |> run("swiglu_oai against an fp64 reference carrying the trained clamps") @(t : T?) { + report_moved(t, "swiglu_oai", sweep_swiglu_oai(false)) + report(t, "swiglu_oai4", sweep_swiglu_oai(true)) + } + t |> run("the gate limit and the up-branch band are the activation, not an optimization") @(t : T?) { + let scalar = oai_clamp_case(false) + t |> equal(scalar.clamp_diffs, 0, "swiglu_oai: a value past the limit must read as the limit") + t |> equal(scalar.control_same, 0, "swiglu_oai: gates inside the limit must not collapse together") + let vec = oai_clamp_case(true) + t |> equal(vec.clamp_diffs, 0, "swiglu_oai4: a value past the limit must read as the limit") + t |> equal(vec.control_same, 0, "swiglu_oai4: gates inside the limit must not collapse together") + } +} + +[test] +def test_silu_mul4(t : T?) { + t |> run("silu_mul4 writes a*silu(z) over a sentinel destination") @(t : T?) { + report_moved(t, "silu_mul4", sweep_silu_mul4()) + } +} + +[test] +def test_silu4_batch(t : T?) { + t |> run("silu4_batch splits rows without moving a bit, threaded and inline") @(t : T?) { + var r = ActSweep() + let inline_wide = silu4_batch_case(r, 256, 512) + var threaded = BatchCase() + with_job_que() { + setup_dasllama_jobque_() + threaded = silu4_batch_case(r, 256, 512) + } + report_moved(t, "silu4_batch", r) + t |> equal(inline_wide.split_diffs, 0, "silu4_batch inline: elements differing from silu4 over the whole buffer") + t |> equal(threaded.split_diffs, 0, "silu4_batch threaded: elements differing from silu4 over the whole buffer") + t |> success(!is_job_que_available(), "the inline leg needs a que-less process to be one lane") + t |> equal(inline_wide.dispatches, 0l, "a que-less process must run silu4_batch inline") + t |> success(threaded.dispatches > 0l, "the threaded leg must dispatch, else both legs are the same one lane") + } + t |> run("silu4_batch falls back whole where the row length or the row count forbids a split") @(t : T?) { + var r = ActSweep() + with_job_que() { + setup_dasllama_jobque_() + let odd = silu4_batch_case(r, 250, 64) + let single = silu4_batch_case(r, 64, 1) + let tail = silu4_batch_case(r, 4, 1025) + t |> equal(odd.split_diffs, 0, "a row length off the float4 grouping must run silu4 whole") + t |> equal(single.split_diffs, 0, "a single row must run silu4 whole") + t |> equal(tail.split_diffs, 0, "a narrow many-row image must still land the same bits") + t |> equal(odd.dispatches, 0l, "a row length off the float4 grouping must not dispatch") + t |> equal(single.dispatches, 0l, "a single row must not dispatch") + t |> success(tail.dispatches > 0l, "a narrow many-row image must still split across lanes") + } + report_moved(t, "silu4_batch fallbacks", r) + } +} + +[test] +def test_l2_norm_rows(t : T?) { + t |> run("l2_norm_rows drives every row to unit length") @(t : T?) { + report_moved(t, "l2_norm_rows", sweep_l2_norm_rows()) + } + t |> run("l2_norm_rows floors the divisor at eps") @(t : T?) { + report(t, "l2_norm_rows eps floor", l2_eps_floor()) + } +} + +[test] +def test_gemm_f32(t : T?) { + t |> run("gemm_f32 and gemm_f32_jo accumulate the same fp64 product over tiles and both strips") @(t : T?) { + let r = sweep_gemm() + t |> equal(r.cases, length(GEMM_SHAPES) / 3, "every declared shape must run") + t |> success(r.elems > 0, "the sweep must compare at least one element") + t |> equal(r.bad, 0, "gemm_f32: elements past the bar, worst |diff| {r.worst} against bar {r.worst_bar} at K={r.worst_k}") + t |> equal(r.jo_diffs, 0, "gemm_f32_jo must be bit-identical to gemm_f32") + t |> equal(r.poison_inside, 0, "gemm_f32: poison legs inside the bar - a bar nothing exceeds discriminates nothing") + feint("gemm_f32: {r.elems} elements, worst |diff| {r.worst} against bar {r.worst_bar} at K={r.worst_k}\n") + } +} + +[test] +def test_zero_size(t : T?) { + t |> run("a zero count is a no-op on every activation and norm kernel") @(t : T?) { + t |> equal(zero_size_moves(), 0, "a zero count must leave the buffer untouched") + } +} diff --git a/modules/dasLLAMA/tests/test_math_elementwise.das b/modules/dasLLAMA/tests/test_math_elementwise.das new file mode 100644 index 0000000000..fc86ba7e5b --- /dev/null +++ b/modules/dasLLAMA/tests/test_math_elementwise.das @@ -0,0 +1,395 @@ +options gen2 +options _dasllama_internal = true + +require dastest/testing_boost public +require dasllama/dasllama_math +require math + +//! The [tuned] elementwise kernels - dot, dot_bf16, axpy, add_inplace, add_scale_inplace, +//! mul_inplace, scale_inplace, copy_floats - swept over tail lengths against fp64 references. + +var g_taint = 0u + +[init] +def init_elementwise_taint() { + g_taint = 0u +} + +[sideeffects] +def private opaque_seed(s : uint) : uint { + return s ^ g_taint +} + +let SWEEP <- [ 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 23, 24, 25, 31, 32, 33, + 63, 64, 65, 100, 127, 128, 129, 1023, 1024, 1025 ] + +let TEST_SCALE = -0.75 +let SENTINEL = 1.0e9 +let POISON = 0.5 +let ROUND_U = 1.0e-7 +let BAR_FLOOR = 1.0e-6 + +struct KernelSweep { + bad : int + inert : int + cases : int + worst : float + worst_bar : float + worst_n : int + poison_inside : int +} + +def private note(var r : KernelSweep; n : int; diff, bar : float) { + if (diff >= r.worst) { + r.worst = diff + r.worst_bar = bar + r.worst_n = n + } +} + +def private dot_bar(n : int) : float { + return ROUND_U * float(n) * float(n) + BAR_FLOOR +} + +def private lcg_grid(n : int; seed : uint; step : float; signed_values : bool) : array { + var st = opaque_seed(seed) + var a : array + a |> resize(n) + for (v in a) { + st = st * 1664525u + 1013904223u + let k = int((st >> 9u) & 31u) + 1 + let neg = signed_values && ((st >> 20u) & 1u) == 0u + v = float(neg ? -k : k) * step + } + return <- a +} + +def private lcg_unit(n : int; seed : uint) : array { + var st = opaque_seed(seed) + var a : array + a |> resize(n) + for (v in a) { + st = st * 1664525u + 1013904223u + v = float(st >> 8u) * (2.0 / 16777216.0) - 1.0 + } + return <- a +} + +def private to_bf16(src : array) : array { + var b : array + b |> resize(length(src)) + for (w, v in b, src) { + w = uint16(unsafe(reinterpret(v)) >> 16u) + } + return <- b +} + +def private bf16_value(w : uint16) : float { + return unsafe(reinterpret(uint(w) << 16u)) +} + +def private tally(var r : KernelSweep; n : int; got, want, before : array) { + r.cases++ + for (g, w, b in got, want, before) { + if (g != w) { + r.bad++ + note(r, n, abs(g - w), 0.0) + } + if (g == b) { + r.inert++ + } + } +} + +def private report_cases(t : T?; name : string; r : KernelSweep) { + t |> equal(r.cases, length(SWEEP), "{name}: every swept length must run") +} + +def private report_exact(t : T?; name : string; r : KernelSweep) { + report_cases(t, name, r) + t |> equal(r.bad, 0, "{name}: values off the in-test fp64 reference, worst |diff| {r.worst} at n={r.worst_n}") +} + +def private report_inplace(t : T?; name : string; r : KernelSweep) { + report_exact(t, name, r) + t |> equal(r.inert, 0, "{name}: elements the kernel left exactly where it found them") +} + +def private report_bar(t : T?; name : string; r : KernelSweep) { + report_cases(t, name, r) + t |> equal(r.bad, 0, "{name}: lengths past the bar, worst |diff| {r.worst} against bar {r.worst_bar} at n={r.worst_n}") + t |> equal(r.poison_inside, 0, "{name}: poison legs inside the bar - a bar nothing exceeds discriminates nothing") + feint("{name}: worst |diff| {r.worst} against bar {r.worst_bar} at n={r.worst_n}\n") +} + +def private sweep_dot_exact() : KernelSweep { + var r = KernelSweep() + for (n in SWEEP) { + let a <- lcg_grid(n, opaque_seed(0x11a3u) + uint(n), 0.125, true) + let b <- lcg_grid(n, opaque_seed(0x9e37u) + uint(n), 0.375, true) + var want = 0.0lf + for (av, bv in a, b) { + want += double(av) * double(bv) + } + let got = dot(unsafe(addr(a[0])), unsafe(addr(b[0])), int64(n)) + r.cases++ + if (got != float(want)) { + r.bad++ + note(r, n, abs(got - float(want)), 0.0) + } + } + return r +} + +def private sweep_dot_bar() : KernelSweep { + var r = KernelSweep() + for (n in SWEEP) { + let a <- lcg_unit(n, opaque_seed(0x4d2bu) + uint(n)) + let b <- lcg_unit(n, opaque_seed(0xa51fu) + uint(n)) + var want = 0.0lf + for (av, bv in a, b) { + want += double(av) * double(bv) + } + let got = dot(unsafe(addr(a[0])), unsafe(addr(b[0])), int64(n)) + let bar = dot_bar(n) + let diff = abs(got - float(want)) + r.cases++ + if (diff > bar) { + r.bad++ + } + if (abs(got - (float(want) + POISON)) <= bar) { + r.poison_inside++ + } + note(r, n, diff, bar) + } + return r +} + +def private sweep_dot_bf16_exact() : KernelSweep { + var r = KernelSweep() + for (n in SWEEP) { + let a <- lcg_grid(n, opaque_seed(0x3c07u) + uint(n), 0.125, true) + let bf <- lcg_grid(n, opaque_seed(0x7b91u) + uint(n), 0.375, true) + let b <- to_bf16(bf) + var want = 0.0lf + for (av, bw in a, b) { + want += double(av) * double(bf16_value(bw)) + } + let got = dot_bf16(unsafe(addr(a[0])), unsafe(addr(b[0])), int64(n)) + r.cases++ + if (got != float(want)) { + r.bad++ + note(r, n, abs(got - float(want)), 0.0) + } + } + return r +} + +def private sweep_dot_bf16_bar() : KernelSweep { + var r = KernelSweep() + for (n in SWEEP) { + let a <- lcg_unit(n, opaque_seed(0x2f6eu) + uint(n)) + let bf <- lcg_unit(n, opaque_seed(0xc0deu) + uint(n)) + let b <- to_bf16(bf) + var want = 0.0lf + for (av, bw in a, b) { + want += double(av) * double(bf16_value(bw)) + } + let got = dot_bf16(unsafe(addr(a[0])), unsafe(addr(b[0])), int64(n)) + let bar = dot_bar(n) + let diff = abs(got - float(want)) + r.cases++ + if (diff > bar) { + r.bad++ + } + if (abs(got - (float(want) + POISON)) <= bar) { + r.poison_inside++ + } + note(r, n, diff, bar) + } + return r +} + +def private sweep_copy_floats() : KernelSweep { + var r = KernelSweep() + for (n in SWEEP) { + let s <- lcg_unit(n, opaque_seed(0x51c0u) + uint(n)) + var before : array + before |> resize(n) + for (v in before) { + v = SENTINEL + } + var d := before + copy_floats(unsafe(addr(d[0])), unsafe(addr(s[0])), int64(n)) + tally(r, n, d, s, before) + } + return r +} + +def private sweep_add_inplace() : KernelSweep { + var r = KernelSweep() + for (n in SWEEP) { + let before <- lcg_grid(n, opaque_seed(0x6a12u) + uint(n), 0.125, true) + let s <- lcg_grid(n, opaque_seed(0xb33fu) + uint(n), 0.375, true) + var want : array + want |> resize(n) + for (w, b, sv in want, before, s) { + w = float(double(b) + double(sv)) + } + var d := before + add_inplace(unsafe(addr(d[0])), unsafe(addr(s[0])), int64(n)) + tally(r, n, d, want, before) + } + return r +} + +def private sweep_add_scale_inplace() : KernelSweep { + var r = KernelSweep() + for (n in SWEEP) { + let before <- lcg_grid(n, opaque_seed(0x1de5u) + uint(n), 0.125, false) + let s <- lcg_grid(n, opaque_seed(0x8c44u) + uint(n), 0.375, false) + var want : array + want |> resize(n) + for (w, b, sv in want, before, s) { + w = float((double(b) + double(sv)) * double(TEST_SCALE)) + } + var d := before + add_scale_inplace(unsafe(addr(d[0])), unsafe(addr(s[0])), TEST_SCALE, int64(n)) + tally(r, n, d, want, before) + } + return r +} + +def private sweep_mul_inplace() : KernelSweep { + var r = KernelSweep() + for (n in SWEEP) { + let before <- lcg_grid(n, opaque_seed(0x9077u) + uint(n), 0.125, true) + let s <- lcg_grid(n, opaque_seed(0x2bb8u) + uint(n), 0.375, true) + var want : array + want |> resize(n) + for (w, b, sv in want, before, s) { + w = float(double(b) * double(sv)) + } + var d := before + mul_inplace(unsafe(addr(d[0])), unsafe(addr(s[0])), int64(n)) + tally(r, n, d, want, before) + } + return r +} + +def private sweep_scale_inplace() : KernelSweep { + var r = KernelSweep() + for (n in SWEEP) { + let before <- lcg_grid(n, opaque_seed(0x4e19u) + uint(n), 0.125, true) + var want : array + want |> resize(n) + for (w, b in want, before) { + w = float(double(b) * double(TEST_SCALE)) + } + var d := before + scale_inplace(unsafe(addr(d[0])), TEST_SCALE, int64(n)) + tally(r, n, d, want, before) + } + return r +} + +def private sweep_axpy() : KernelSweep { + var r = KernelSweep() + for (n in SWEEP) { + let before <- lcg_grid(n, opaque_seed(0x7231u) + uint(n), 0.125, true) + let s <- lcg_grid(n, opaque_seed(0xd4a0u) + uint(n), 0.375, true) + var want : array + want |> resize(n) + for (w, b, sv in want, before, s) { + w = float(double(b) + double(TEST_SCALE) * double(sv)) + } + var d := before + axpy(unsafe(addr(d[0])), unsafe(addr(s[0])), TEST_SCALE, int64(n)) + tally(r, n, d, want, before) + } + return r +} + +def private zero_length_moves() : int { + let before <- lcg_grid(4, opaque_seed(0x0e11u), 0.125, true) + let s <- lcg_grid(4, opaque_seed(0x0e12u), 0.375, true) + var d := before + unsafe { + copy_floats(addr(d[0]), addr(s[0]), 0l) + add_inplace(addr(d[0]), addr(s[0]), 0l) + add_scale_inplace(addr(d[0]), addr(s[0]), TEST_SCALE, 0l) + mul_inplace(addr(d[0]), addr(s[0]), 0l) + scale_inplace(addr(d[0]), TEST_SCALE, 0l) + axpy(addr(d[0]), addr(s[0]), TEST_SCALE, 0l) + } + var moved = 0 + for (x, b in d, before) { + if (x != b) { + moved++ + } + } + return moved +} + +def private zero_length_dots() : float { + let a <- lcg_grid(4, opaque_seed(0x0e13u), 0.125, true) + let bf <- lcg_grid(4, opaque_seed(0x0e14u), 0.375, true) + let b <- to_bf16(bf) + unsafe { + return abs(dot(addr(a[0]), addr(a[0]), 0l)) + abs(dot_bf16(addr(a[0]), addr(b[0]), 0l)) + } +} + +[test] +def test_dot(t : T?) { + t |> run("dot bit-matches an fp64 accumulation on inputs fp32 cannot round") @(t : T?) { + report_exact(t, "dot exact", sweep_dot_exact()) + } + t |> run("dot against an fp64 accumulation on unit-range inputs, with the bar's control") @(t : T?) { + report_bar(t, "dot bar", sweep_dot_bar()) + } +} + +[test] +def test_dot_bf16(t : T?) { + t |> run("dot_bf16 bit-matches an fp64 accumulation over exactly-bf16 operands") @(t : T?) { + report_exact(t, "dot_bf16 exact", sweep_dot_bf16_exact()) + } + t |> run("dot_bf16 against an fp64 accumulation over the widened operands, with the bar's control") @(t : T?) { + report_bar(t, "dot_bf16 bar", sweep_dot_bf16_bar()) + } +} + +[test] +def test_copy_and_add(t : T?) { + t |> run("copy_floats replaces every sentinel with the source bit for bit") @(t : T?) { + report_inplace(t, "copy_floats", sweep_copy_floats()) + } + t |> run("add_inplace is exact on dyadic operands") @(t : T?) { + report_inplace(t, "add_inplace", sweep_add_inplace()) + } + t |> run("add_scale_inplace is exact and its sign flip cannot return the input") @(t : T?) { + report_inplace(t, "add_scale_inplace", sweep_add_scale_inplace()) + } +} + +[test] +def test_mul_scale_axpy(t : T?) { + t |> run("mul_inplace is exact on operands no multiplier of which is one") @(t : T?) { + report_inplace(t, "mul_inplace", sweep_mul_inplace()) + } + t |> run("scale_inplace is exact at a dyadic scale") @(t : T?) { + report_inplace(t, "scale_inplace", sweep_scale_inplace()) + } + t |> run("axpy is exact on dyadic operands") @(t : T?) { + report_inplace(t, "axpy", sweep_axpy()) + } +} + +[test] +def test_zero_length(t : T?) { + t |> run("a zero count is a no-op on every elementwise kernel") @(t : T?) { + t |> equal(zero_length_moves(), 0, "a zero count must leave the destination untouched") + t |> equal(zero_length_dots(), 0.0, "a zero count must answer an empty sum") + } +} diff --git a/modules/dasLLAMA/tests/test_metal_decode_parity.das b/modules/dasLLAMA/tests/test_metal_decode_parity.das index 2f14d1fcb1..b04ec33458 100644 --- a/modules/dasLLAMA/tests/test_metal_decode_parity.das +++ b/modules/dasLLAMA/tests/test_metal_decode_parity.das @@ -12,7 +12,7 @@ require ?das_metal dasllama/dasllama_metal_decode // stats/mirror/shutdown rail require ?das_metal dasllama/dasllama_metal_common // the greedy chain's stats and mode (arm 15) require ?das_metal dasllama/dasllama_metal_prefill // the Metal-prefill→GPU-decode hybrid arm require ?das_metal metal/das_metal_boost // metal_live_object_count for the leak gate -require llvm/daslib/f16_cvt // f16 KV row decode in the tolerance gate (Apple half only) +require daslib/f16_cvt // f16 KV row decode in the tolerance gate (Apple half only) require daslib/jobque_boost require daslib/fio // nolint:STYLE030,LINT019 — path_join/stat, used only inside the das_metal static_if halves require math diff --git a/modules/dasLLAMA/tests/test_metal_misc_kernels.das b/modules/dasLLAMA/tests/test_metal_misc_kernels.das index 9256daadd5..c418c91e59 100644 --- a/modules/dasLLAMA/tests/test_metal_misc_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_misc_kernels.das @@ -13,7 +13,7 @@ require ?das_metal metal/das_metal_boost // device/pipeline/buffer plumbing + t require ?das_metal _metal_kernel_common // shared buf helpers + eyeball-dump compares require ?das_metal _mtl_toy // the [metal_dispatch] multi-kernel fixture require ?das_metal dasllama/dasllama_metal_lens // set_metal_requires_hook - the requires= tripwire observer -require llvm/daslib/f16_cvt // f32_to_f16 — packing the synthetic scale planes +require daslib/f16_cvt // f32_to_f16 — packing the synthetic scale planes require math require daslib/fio // the lens tgmem gate's spawned compiles require strings diff --git a/modules/dasLLAMA/tests/test_metal_prefill_kernels.das b/modules/dasLLAMA/tests/test_metal_prefill_kernels.das index dae9c85c5e..7325121fb5 100644 --- a/modules/dasLLAMA/tests/test_metal_prefill_kernels.das +++ b/modules/dasLLAMA/tests/test_metal_prefill_kernels.das @@ -12,7 +12,7 @@ require ?das_metal dasllama/dasllama_metal_prefill // the Phase-6 prefill kerne require ?das_metal dasllama/dasllama_metal_kernels // the Wave C MoE routing kernel set (Apple builds) require ?das_metal metal/das_metal_boost // device/pipeline/buffer plumbing + the leak gate require ?das_metal _metal_kernel_common // shared buf helpers + fixtures; only the tag-less mismatch compares stay local -require llvm/daslib/f16_cvt // f32_to_f16: the q51 gates hand-pack f16 d/m scale halfwords +require daslib/f16_cvt // f32_to_f16: the q51 gates hand-pack f16 d/m scale halfwords require math // The Phase-6 GPU prefill kernels vs their dasLLAMA CPU twins on synthetic data. rmsnorm / diff --git a/modules/dasLLAMA/tests/test_mxfp4.das b/modules/dasLLAMA/tests/test_mxfp4.das index 3853a29150..5228e611d8 100644 --- a/modules/dasLLAMA/tests/test_mxfp4.das +++ b/modules/dasLLAMA/tests/test_mxfp4.das @@ -5,7 +5,7 @@ require dastest/testing_boost public require dasllama/dasllama_math require dasllama/dasllama_math_default // dot_mx4q8_scalar + the "portable" backend require dasllama/dasllama_math_aarch64_neon // dot_mx4q8 (TBL+SDOT under arm64 JIT, scalar fallback elsewhere) -require ?llvm/daslib/llvm_tune dasllama/dasllama_math_gen // nolint:STYLE030,LINT019 — registers the "arm64-gen"/"x64-gen" repack tier when dasLLVM is mounted +require ?llvm dasllama/dasllama_math_gen // nolint:STYLE030,LINT019 — registers the "arm64-gen"/"x64-gen" repack tier when dasLLVM is mounted require math // MXFP4·Q8 kernels: the E8M0 scale decode, the dot leaves (scalar reference vs the NEON-intrinsic @@ -165,7 +165,8 @@ def test_matmul_mx4q8(t : T?) { } } t |> run("gen tier: repack + interleaved GEMV matches the scalar leaf (incl. tail rows)") @(t : T?) { - if (!has_backend("arm64-gen")) { // registered on arm64 + JIT (with dasLLVM) only + if (!has_backend("arm64-gen")) { + t |> skip("backend 'arm64-gen' is not registered here (arm64 + JIT with dasLLVM only)") return } // d = 6 = one 4-row group + 2 row-major tail rows diff --git a/modules/dasLLAMA/tests/test_parity.das b/modules/dasLLAMA/tests/test_parity.das index 77fd6b7536..bbc64eda69 100644 --- a/modules/dasLLAMA/tests/test_parity.das +++ b/modules/dasLLAMA/tests/test_parity.das @@ -4,7 +4,10 @@ options _dasllama_internal = true require dastest/testing_boost public require dasllama/dasllama // the public facade — public-path tests exercise the API through it (engine + arch registrations + chat) -require llvm/daslib/f16_cvt // f16_to_f32 — the tied-cls arm reads the live scale plane +require daslib/f16_cvt // f16_to_f32 — the tied-cls arm reads the live scale plane +require dasllama/dasllama_math // pin_kernel_backend / kernel_backend_names / active_kernel_backend — the facade wraps none of them +require dasllama/dasllama_env // g_env_engine.pin_backend +require daslib/defer // the backend pin is restored on every path out of a pinned cell require daslib/jobque_boost require daslib/fio require _model_tier // models_dir() + model_available() presence/size-tier gate (see the file's header) @@ -15,6 +18,28 @@ require ../performance/model_specs.das // the model-set table — parity evide // frozen under; regenerate with harness/parity.sh. The board's --parity pregate reads the same rows. // Models are gitignored multi-GB files: absent = clean skip (a real local regression, a no-op in CI). +def private with_parity_backend(t : T?; blk : block<() : void>) { + let name = g_env_engine.pin_backend + if (empty(name)) { + invoke(blk) + return + } + var names <- kernel_backend_names() + let known = find_index(names, name) >= 0 + delete names + if (!known) { + t |> skip("DASLLAMA_PIN_BACKEND='{name}' is not a registered kernel backend on this box") + return + } + let prev = active_kernel_backend() + defer() { + clear_kernel_backend_pin() + select_kernel_backend(prev) // "" is a no-op: nothing had activated on entry + } + pin_kernel_backend(name) + invoke(blk) +} + def private same(a, b : array) : bool { if (length(a) != length(b)) { return false @@ -184,27 +209,28 @@ def private parity_spec_arms(t : T?; spec : ModelSpec; path : string) { if (tr.config.seq_len > 2048l) { tr.config.seq_len = 2048l } + let be = " [backend {active_kernel_backend()}]" // read AFTER the load: that is where the pin resolves var prompt <- (empty(spec.parity.prompt_text) ? clone_to_move(spec.parity.prompt) : encode(tr, spec.parity.prompt_text, true, false)) if (spec.parity.require_dn_kq) { t |> success(dn_kq_engaged(tr.dnqkv_fmt) > 0l && dn_kq_engaged(tr.dngate_fmt) > 0l && dn_kq_engaged(tr.dnout_fmt) > 0l, "{spec.file} dn planes kq-native (not demoted)") } - t |> success(parity_gen_ok(tr, prompt, spec.parity.gen, [eyeball = true]), "{spec.file} token-for-token") + t |> success(parity_gen_ok(tr, prompt, spec.parity.gen, [eyeball = true]), "{spec.file} token-for-token{be}") if (spec.parity.dn_chunked) { t |> success(parity_gen_ok(tr, prompt, spec.parity.gen, [dn_chunked = true, eyeball = true]), - "{spec.file} chunked-prefill token-for-token") + "{spec.file} chunked-prefill token-for-token{be}") } if (spec.parity.moe_reference) { set_moe_grouped_prefill(false) t |> success(parity_gen_ok(tr, prompt, spec.parity.gen, [eyeball = true]), - "{spec.file} reference-prefill token-for-token") + "{spec.file} reference-prefill token-for-token{be}") set_moe_grouped_prefill(true) } if (spec.parity.moe_per_expert) { set_moe_fused_expert_gemms(false) t |> success(parity_gen_ok(tr, prompt, spec.parity.gen, [eyeball = true]), - "{spec.file} per-expert-prefill token-for-token") + "{spec.file} per-expert-prefill token-for-token{be}") set_moe_fused_expert_gemms(true) } delete prompt @@ -232,7 +258,9 @@ def test_parity_specs(t : T?) { var inscope spec <- spec_by_file(file) let path = path_join(models_dir(), spec.file) if (!model_available(t, path)) return - parity_spec_arms(t, spec, path) + with_parity_backend(t) { + parity_spec_arms(t, spec, path) + } } } delete files @@ -263,13 +291,16 @@ def test_parity_gptoss_20b(t : T?) { var inscope spec <- spec_by_file("gpt-oss-20b-mxfp4.gguf") let path = path_join(models_dir(), spec.file) if (!model_available(t, path)) return - var tr <- load_model_(path, QuantMode.q8) - if (tr.config.seq_len > 2048l) { - tr.config.seq_len = 2048l + with_parity_backend(t) { + var tr <- load_model_(path, QuantMode.q8) + if (tr.config.seq_len > 2048l) { + tr.config.seq_len = 2048l + } + let be = " [backend {active_kernel_backend()}]" + t |> success(parity_gen_ok(tr, spec.parity.prompt, spec.parity.gen, [eyeball = true]), "gpt-oss-20b token-for-token{be}") + let long_prompt <- encode(tr, counting_prompt(150), true, false) + t |> success(parity_gen_ok(tr, long_prompt, GPTOSS_LONG_GEN, [eyeball = true]), "gpt-oss-20b window-engaged token-for-token{be}") + delete tr } - t |> success(parity_gen_ok(tr, spec.parity.prompt, spec.parity.gen, [eyeball = true]), "gpt-oss-20b token-for-token") - let long_prompt <- encode(tr, counting_prompt(150), true, false) - t |> success(parity_gen_ok(tr, long_prompt, GPTOSS_LONG_GEN, [eyeball = true]), "gpt-oss-20b window-engaged token-for-token") - delete tr } } diff --git a/modules/dasLLAMA/tests/test_prefill_cpu_kernels.das b/modules/dasLLAMA/tests/test_prefill_cpu_kernels.das new file mode 100644 index 0000000000..057d934bce --- /dev/null +++ b/modules/dasLLAMA/tests/test_prefill_cpu_kernels.das @@ -0,0 +1,741 @@ +//! The prefill and KV CPU kernels no suite gated before - the batched K-quant GEMMs and the per-position GEMV route a tier without those slots runs instead, the q8_0 cache dequant seam, the batch activation requants, and repack_kq_weight's round trip: every cell asserts on whatever tier is active, and the native batched arms ride a restored kq-backend pin. +options gen2 +options stack = 524288 +options _dasllama_internal = true + +require dastest/testing_boost public +require dasllama/dasllama_math +require dasllama/dasllama_math_default +require dasllama/dasllama_math_gen +require dasllama/dasllama_convert +require dasllama/dasllama_kv_codec +require dasllama/dasllama_repack +require dasllama/dasllama_gemm_schema +require daslib/f16_cvt +require math + +var g_taint = 0.0 + +[sideeffects] +def private opaque(x : float) : float => x + g_taint + +def private poison_of(v : float) : float => max(0.5, abs(v) * 1.0e-3) + +def private cmp_exact(t : T?; got : array; want : array; n : int64; tag : string) { + var bad = 0 + var seen = 0 + let pi = n / 2l + for (i in range64(n)) { + if (got[i] != want[i]) { + bad++ + } + if (got[i] != (i == pi ? want[i] + poison_of(want[i]) : want[i])) { + seen++ + } + } + t |> success(bad == 0, "{tag}: {bad} of {int(n)} elements differ") + t |> success(seen > 0, "{tag}: the poisoned expected element must be seen, and was not") +} + +def private put_f16(var b : array; off : int64; v : float) { + let bits = f32_to_f16(v) + b[off] = uint8(bits & 0xFFu) + b[off + 1l] = uint8(bits >> 8u) +} + +def private build_q4k_disk() : array { + var b : array + b |> resize(144l) + put_f16(b, 0l, 0.375) + put_f16(b, 2l, 0.125) + for (i in range64(12l)) { + b[4l + i] = uint8((i * 37l + 13l) % 256l) + } + for (i in range64(128l)) { + b[16l + i] = uint8((i * 29l + 7l) % 256l) + } + return <- b +} + +def private build_q6k_disk() : array { + var b : array + b |> resize(210l) + for (i in range64(192l)) { + b[i] = uint8((i * 23l + 5l) % 256l) + } + for (i in range64(16l)) { + b[192l + i] = uint8((i * 41l + 11l) % 256l) + } + put_f16(b, 208l, 0.25) + return <- b +} + +def private build_q40_disk() : array { + var b : array + b |> resize(144l) + for (blk in range64(8l)) { + put_f16(b, blk * 18l, 0.0625 * float(blk + 1l)) + for (j in range64(16l)) { + b[blk * 18l + 2l + j] = uint8(((blk * 16l + j) * 19l + 3l) % 256l) + } + } + return <- b +} + +def private fmt_tag(fmt : int) : string => fmt == 40 ? "q40" : "k{fmt}" + +def private build_kq_region(fmt : int; var kq : array; var ks : array; rows, n : int64; seed : uint) { + let nsb = n / 256l + let qsb = kq_qsb(fmt) + let ssb = kq_ssb(fmt) + let blk <- fmt == 4 ? build_q4k_disk() : (fmt == 40 ? build_q40_disk() : build_q6k_disk()) + kq |> resize(rows * nsb * qsb) + ks |> resize(rows * nsb * ssb) + for (r in range64(rows)) { + for (s in range64(nsb)) { + if (fmt == 4) { + transcode_q4k_superblock(blk, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } elif (fmt == 40) { + transcode_q40_superblock(blk, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } else { + transcode_q6k_superblock(blk, 0l, kq, (r * nsb + s) * qsb, ks, (r * nsb + s) * ssb) + } + } + var st = seed + uint(r) * 0x85EBCA6Bu + for (i in range64(nsb * qsb)) { + st = st * 1664525u + 1013904223u + kq[r * nsb * qsb + i] = uint8(uint(kq[r * nsb * qsb + i]) ^ (st >> 16u)) + } + } +} + +def private build_acts(var src : array; var xq : array; var xs : array; var xbs : array; ntok, n : int64) { + src |> resize(ntok * n) + for (i in range64(ntok * n)) { + src[i] = opaque(0.9 * sin(float(i) * 0.27) - 0.5 * cos(float(i) * 0.061)) + } + xq |> resize(ntok * n) + xs |> resize(ntok * n / 256l) + xbs |> resize(ntok * n / 16l) + unsafe { + for (p in range64(ntok)) { + quantize_q8_k_into_ptr(addr(src[p * n]), n, addr(xq[0]), addr(xs[0]), addr(xbs[0]), + p * n, p * (n / 256l), p * (n / 16l)) + } + } +} + +def private kq_row_dot(fmt : int; kq : array; ks : array; row, nsb : int64; + xq : array; xs : array; xbs : array; + xoff, soff, boff, n : int64) : float { + let qsb = kq_qsb(fmt) + let ssb = kq_ssb(fmt) + unsafe { + let kqr = addr(kq[row * nsb * qsb]) + let ksr = addr(ks[row * nsb * ssb]) + let xqp = addr(xq[xoff]) + let xsp = addr(xs[soff]) + let xbp = addr(xbs[boff]) + if (fmt == 4) { + return dot_k4q8(kqr, ksr, xqp, xsp, xbp, n) + } elif (fmt == 40) { + return dot_q40q8(kqr, ksr, xqp, xsp, xbp, n) + } + return dot_k6q8(kqr, ksr, xqp, xsp, xbp, n) + } +} + +def private gemv_gen_rows(fmt : int; var yp : float?; kqp : uint8 const?; ksp : uint8 const?; + xqp : int8 const?; xsp : float const?; xbsp : int const?; n, rb, re : int64) { + if (fmt == 4) { + k4q8_gemv_gen(yp, kqp, ksp, xqp, xsp, xbsp, n, rb, re) + } elif (fmt == 40) { + q40q8_gemv_gen(yp, kqp, ksp, xqp, xsp, xbsp, n, rb, re) + } else { + k6q8_gemv_gen(yp, kqp, ksp, xqp, xsp, xbsp, n, rb, re) + } +} + +def private grp_row_dot(fmt : int; kqg : uint8 const?; ksg : uint8 const?; r, mr : int64; + xqp : int8 const?; xsp : float const?; xbsp : int const?; n : int64) : float { + if (fmt == 40) { + return q40_grp_row_dot(kqg, ksg, r, mr, xqp, xsp, xbsp, n) + } + return kq_grp_row_dot(int64(fmt), kqg, ksg, r, mr, xqp, xsp, xbsp, n) +} + +def private repack_grp(fmt : int; var kq : array; var ks : array; n, d, mr : int64) { + unsafe { + if (fmt == 4) { + repack_k4_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } elif (fmt == 40) { + repack_q40_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } else { + repack_k6_grp(addr(kq[0]), addr(ks[0]), n, d, mr) + } + } +} + +def private gen_backend_name() : string => get_architecture_name() == "arm64" ? "arm64-gen" : "x64-gen" + +def private quant_one(src : array; poff, n : int64; var q1 : array; var s1 : array; var b1 : array) { + unsafe { + quantize_q8_k_into_ptr(addr(src[poff]), n, addr(q1[0]), addr(s1[0]), addr(b1[0]), 0l, 0l, 0l) + } +} + +def private alloc_row_scratch(var q1 : array; var s1 : array; var b1 : array; n : int64) { + q1 |> resize(n) + s1 |> resize(n / 256l) + b1 |> resize(n / 16l) +} + +def private kq_batch_gate(t : T?; fmt : int; n, d, ntok : int64) { + let nsb = n / 256l + var kq : array + var ks : array + build_kq_region(fmt, kq, ks, d, n, 0x51ED270Bu) + var src : array + var xq : array + var xs : array + var xbs : array + build_acts(src, xq, xs, xbs, ntok, n) + var want : array + want |> resize(ntok * d) + for (p in range64(ntok)) { + for (r in range64(d)) { + want[p * d + r] = kq_row_dot(fmt, kq, ks, r, nsb, xq, xs, xbs, p * n, p * nsb, p * (n / 16l), n) + } + } + var q1 : array + var s1 : array + var b1 : array + alloc_row_scratch(q1, s1, b1, n) + var y : array + y |> resize(ntok * d) + for (p in range64(ntok)) { + quant_one(src, p * n, n, q1, s1, b1) + matmul_kq(fmt, y, kq, ks, 0l, q1, s1, b1, n, d, p * d) + } + cmp_exact(t, y, want, ntok * d, + "{fmt_tag(fmt)} serial per-position GEMV (n={int(n)} d={int(d)} ntok={int(ntok)}) vs per-(token,row) disk dots") + let prev = active_kernel_backend() + pin_kernel_backend(gen_backend_name()) + if (kernel_backend_has_kq() && kernel_backend_has_kq_batch()) { + repack_kq_weight(fmt, kq, ks, 0l, n, d) + var ybatch : array + ybatch |> resize(ntok * d) + matmul_kq_batch(fmt, ybatch, kq, ks, 0l, xq, xs, xbs, n, d, ntok) + var yser : array + yser |> resize(ntok * d) + for (p in range64(ntok)) { + quant_one(src, p * n, n, q1, s1, b1) + matmul_kq_active(fmt, yser, kq, ks, 0l, q1, s1, b1, n, d, p * d) + } + cmp_exact(t, ybatch, yser, ntok * d, + "{fmt_tag(fmt)} native kq batch is bit-for-bit ntok x matmul_kq_active (ntok={int(ntok)})") + delete ybatch + delete yser + } + clear_kernel_backend_pin() + select_kernel_backend(prev) + delete kq + delete ks + delete src + delete xq + delete xs + delete xbs + delete want + delete q1 + delete s1 + delete b1 + delete y +} + +[test] +def test_kq_batch(t : T?) { + for (fmt in [4, 6, 40]) { + for (ntok in [1l, 5l, 13l]) { + t |> run("{fmt_tag(fmt)} batched kq GEMM at ntok={int(ntok)}: the tier's route matches the per-token disk dots") @(t : T?) { + kq_batch_gate(t, fmt, 512l, 32l, ntok) + } + } + t |> run("{fmt_tag(fmt)} batched kq GEMM at the multi-superblock shape (n=1024 d=64 ntok=6)") @(t : T?) { + kq_batch_gate(t, fmt, 1024l, 64l, 6l) + } + } +} + +def private groupn_fallback(t : T?; fmt : int; kq : array; ks : array; src : array; + cnts : array; n, d : int64; want : array; nk : int64) { + var q1 : array + var s1 : array + var b1 : array + alloc_row_scratch(q1, s1, b1, n) + var y : array + y |> resize(nk * d) + var row0 = 0l + for (r in range64(long_length(cnts))) { + for (tk in range64(row0, row0 + cnts[r])) { + quant_one(src, tk * n, n, q1, s1, b1) + matmul_kq(fmt, y, kq, ks, r * d * n, q1, s1, b1, n, d, tk * d) + } + row0 += cnts[r] + } + cmp_exact(t, y, want, nk * d, + "{fmt_tag(fmt)} per-expert per-token GEMV route (what a tier with no fused kq slot runs) vs per-(region,token) disk dots") + delete q1 + delete s1 + delete b1 + delete y +} + +def private groupn_fused(t : T?; fmt : int; var kq : array; var ks : array; + offs : array; cnts : array; xq : array; xs : array; + xbs : array; n, d, nk : int64) { + let nsb = n / 256l + let qsb = kq_qsb(fmt) + let ssb = kq_ssb(fmt) + let nreg = long_length(cnts) + for (r in range64(nreg)) { + repack_kq_weight(fmt, kq, ks, r * d * n, n, d) + } + var yb : array + yb |> resize(nk * d) + matmul_kq_batch_groupn(fmt, yb, kq, ks, offs, nreg, xq, xs, xbs, n, d) + var wantg : array + wantg |> resize(nk * d) + var row0 = 0l + unsafe { + for (r in range64(nreg)) { + let kqr = addr(kq[r * d * nsb * qsb]) + let ksr = addr(ks[r * d * nsb * ssb]) + for (tk in range64(row0, row0 + cnts[r])) { + gemv_gen_rows(fmt, addr(wantg[tk * d]), kqr, ksr, addr(xq[tk * n]), + addr(xs[tk * nsb]), addr(xbs[tk * (n / 16l)]), n, 0l, d) + } + row0 += cnts[r] + } + } + cmp_exact(t, yb, wantg, nk * d, + "{fmt_tag(fmt)} fused kq batch groupn bit-matches per-(region, token) rows-core calls") + delete yb + delete wantg +} + +def private kq_batch_groupn_gate(t : T?; fmt : int; n, d : int64) { + var cnts <- [1l, 4l, 6l] + let nreg = long_length(cnts) + var nk = 0l + for (c in cnts) { + nk += c + } + let nsb = n / 256l + var kq : array + var ks : array + build_kq_region(fmt, kq, ks, nreg * d, n, 0x7F4A7C15u) + var src : array + var xq : array + var xs : array + var xbs : array + build_acts(src, xq, xs, xbs, nk, n) + var offs : array + offs |> reserve(nreg * 3l) + var want : array + want |> resize(nk * d) + var row0 = 0l + for (r in range64(nreg)) { + offs |> push(r * d * n) + offs |> push(row0) + offs |> push(cnts[r]) + for (tk in range64(row0, row0 + cnts[r])) { + for (row in range64(d)) { + want[tk * d + row] = kq_row_dot(fmt, kq, ks, r * d + row, nsb, xq, xs, xbs, + tk * n, tk * nsb, tk * (n / 16l), n) + } + } + row0 += cnts[r] + } + groupn_fallback(t, fmt, kq, ks, src, cnts, n, d, want, nk) + let prev = active_kernel_backend() + pin_kernel_backend(gen_backend_name()) + if (kernel_backend_has_kq() && kernel_backend_has_kq_batch_groupn()) { + groupn_fused(t, fmt, kq, ks, offs, cnts, xq, xs, xbs, n, d, nk) + } + clear_kernel_backend_pin() + select_kernel_backend(prev) + delete cnts + delete kq + delete ks + delete src + delete xq + delete xs + delete xbs + delete offs + delete want +} + +[test] +def test_kq_batch_groupn_portable(t : T?) { + for (fmt in [4, 6, 40]) { + t |> run("{fmt_tag(fmt)} fused MoE-prefill region walk (runs 1/4/6) on whatever backend is active") @(t : T?) { + kq_batch_groupn_gate(t, fmt, 512l, 32l) + } + } + t |> run("k4 fused region walk at the expert-slice shape (n=1024 d=128)") @(t : T?) { + kq_batch_groupn_gate(t, 4, 1024l, 128l) + } +} + +def private grp_rows_after_repack(fmt : int; kq : array; ks : array; nsb, d, mr : int64; + xq : array; xs : array; xbs : array; n : int64; + var got : array) { + let qsb = kq_qsb(fmt) + let ssb = kq_ssb(fmt) + got |> resize(d) + unsafe { + for (r in range64(d)) { + let g = r / mr + got[r] = grp_row_dot(fmt, addr(kq[g * mr * nsb * qsb]), + addr(ks[g * mr * nsb * ssb]), r % mr, mr, + addr(xq[0]), addr(xs[0]), addr(xbs[0]), n) + } + } +} + +def private byte_diff(a : array; b : array) : int { + var n = 0 + for (x, y in a, b) { + if (x != y) { + n++ + } + } + return n +} + +def private repack_seam(t : T?; fmt : int; kq : array; ks : array; + kqg : array; ksg : array; want : array; + nsb, n, d : int64; xq : array; xs : array; xbs : array) { + var kqs : array + kqs := kq + var kss : array + kss := ks + let prev = active_kernel_backend() + pin_kernel_backend(gen_backend_name()) + let has_kq = kernel_backend_has_kq() + repack_kq_weight(fmt, kqs, kss, 0l, n, d) + if (has_kq) { + let moved = byte_diff(kqs, kqg) + byte_diff(kss, ksg) + t |> success(moved == 0, "repack_kq_weight lands on the grp<{int(kq_layout_of(fmt))}> layout the readers expect ({moved} bytes differ)") + var got : array + grp_rows_after_repack(fmt, kqs, kss, nsb, d, kq_layout_of(fmt), xq, xs, xbs, n, got) + cmp_exact(t, got, want, d, "{fmt_tag(fmt)} row dots through repack_kq_weight's own planes") + delete got + } else { + let moved = byte_diff(kqs, kq) + byte_diff(kss, ks) + t |> success(moved == 0, "with no kq slot on the tier repack_kq_weight is the registered identity ({moved} bytes moved)") + var got : array + got |> resize(d) + for (r in range64(d)) { + got[r] = kq_row_dot(fmt, kqs, kss, r, nsb, xq, xs, xbs, 0l, 0l, 0l, n) + } + cmp_exact(t, got, want, d, "{fmt_tag(fmt)} disk row dots survive the identity repack") + } + clear_kernel_backend_pin() + select_kernel_backend(prev) + delete kqs + delete kss +} + +def private repack_gate(t : T?; fmt : int; n, d : int64) { + let nsb = n / 256l + var kq : array + var ks : array + build_kq_region(fmt, kq, ks, d, n, 0x9E3779B9u) + var src : array + var xq : array + var xs : array + var xbs : array + build_acts(src, xq, xs, xbs, 1l, n) + var want : array + want |> resize(d) + for (r in range64(d)) { + want[r] = kq_row_dot(fmt, kq, ks, r, nsb, xq, xs, xbs, 0l, 0l, 0l, n) + } + let mr = kq_layout_of(fmt) + var kqg : array + kqg := kq + var ksg : array + ksg := ks + repack_grp(fmt, kqg, ksg, n, d, mr) + var got : array + grp_rows_after_repack(fmt, kqg, ksg, nsb, d, mr, xq, xs, xbs, n, got) + cmp_exact(t, got, want, d, "{fmt_tag(fmt)} grp<{int(mr)}> row dots after the repack (n={int(n)} d={int(d)})") + repack_seam(t, fmt, kq, ks, kqg, ksg, want, nsb, n, d, xq, xs, xbs) + delete kq + delete ks + delete kqg + delete ksg + delete src + delete xq + delete xs + delete xbs + delete want + delete got +} + +[test] +def test_repack_kq_weight(t : T?) { + for (fmt in [4, 6, 40]) { + t |> run("{fmt_tag(fmt)} the grp repack and repack_kq_weight preserve every row's dot") @(t : T?) { + repack_gate(t, fmt, 512l, 32l) + repack_gate(t, fmt, 1024l, 64l) + } + } +} + +def private q8kv_block_ref(blob : array; base, n : int64; var want : array) { + want |> resize(n) + for (bi in range64(n / 32l)) { + let bb = base + bi * 34l + let d = f16_to_f32(uint(blob[bb]) | (uint(blob[bb + 1l]) << 8u)) + for (k in range64(32l)) { + want[bi * 32l + k] = float(int8(blob[bb + 2l + k])) * d + } + } +} + +def private kv_q8_gate(t : T?; n : int64) { + let nb = n / 32l + var src : array + src |> resize(n) + for (i in range64(n)) { + src[i] = opaque(2.5 * sin(float(i) * 0.37 + 0.11)) + } + var blob : array + blob |> resize(2l * nb * 34l) + unsafe { + quantize_q8kv_row(addr(blob[0]), addr(src[0]), n) + quantize_q8kv_row(addr(blob[nb * 34l]), addr(src[0]), n) + } + var want : array + q8kv_block_ref(blob, 0l, n, want) + var got : array + got |> resize(n) + unsafe { + cvt_q8kv_to_f32(addr(got[0]), addr(blob[0]), n) + } + cmp_exact(t, got, want, n, "cvt_q8kv_to_f32 vs the per-block scalar reference") + var got2 : array + got2 |> resize(n) + unsafe { + let bp = addr(blob[0]) + kv_row_to_f32(addr(got2[0]), bp, kv_head_off(bp, n), n) + } + cmp_exact(t, got2, want, n, "kv_row_to_f32(q8_0) at the second row's byte base") + var rt_bad = 0 + for (bi in range64(nb)) { + let d = f16_to_f32(uint(blob[bi * 34l]) | (uint(blob[bi * 34l + 1l]) << 8u)) + for (k in range64(32l)) { + if (abs(src[bi * 32l + k] - want[bi * 32l + k]) > d * 0.57 + 1.0e-7) { + rt_bad++ + } + } + } + t |> success(rt_bad == 0, "every dequantized element within half a step of the source ({rt_bad} outside)") + delete src + delete blob + delete want + delete got + delete got2 +} + +def private kv_wide_gate(t : T?; n : int64) { + var src : array + src |> resize(n) + for (i in range64(n)) { + src[i] = opaque(1.75 * sin(float(i) * 0.21 + 0.4)) + } + var wide : array + wide |> resize(2l * n) + for (i in range64(n)) { + wide[n + i] = src[i] + } + var got : array + got |> resize(n) + unsafe { + let wp = addr(wide[0]) + kv_row_to_f32(addr(got[0]), wp, kv_head_off(wp, n), n) + } + cmp_exact(t, got, src, n, "kv_row_to_f32(f32) is the plain row copy") + var half : array + half |> resize(2l * n) + unsafe { + cvt_f32_to_f16(addr(half[n]), addr(src[0]), n) + } + var want : array + want |> resize(n) + for (i in range64(n)) { + want[i] = f16_to_f32(uint(half[n + i])) + } + var got2 : array + got2 |> resize(n) + unsafe { + let hp = addr(half[0]) + kv_row_to_f32(addr(got2[0]), hp, kv_head_off(hp, n), n) + } + cmp_exact(t, got2, want, n, "kv_row_to_f32(f16) widens the row it was pointed at") + delete src + delete wide + delete got + delete half + delete want + delete got2 +} + +[test] +def test_kv_q8_row_to_f32(t : T?) { + t |> run("cvt_q8kv_to_f32 and kv_row_to_f32 invert quantize_q8kv_row's stored blocks") @(t : T?) { + kv_q8_gate(t, 256l) + kv_q8_gate(t, 64l) + } + t |> run("kv_row_to_f32's f32 and f16 overloads carry a row at a non-zero base") @(t : T?) { + kv_wide_gate(t, 128l) + } +} + +def private ref_q8k_row(src : array; base, n : int64; var q : array; var sc : array; var bs : array) { + let nsb = n / 256l + q |> resize(n) + sc |> resize(nsb) + bs |> resize(n / 16l) + for (s in range64(nsb)) { + var amax = 0.0 + for (i in range64(256l)) { + amax = max(amax, abs(src[base + s * 256l + i])) + } + let d = amax / 127.0 + let id = d != 0.0 ? 1.0 / d : 0.0 + sc[s] = d + for (i in range64(256l)) { + q[s * 256l + i] = int8(round(src[base + s * 256l + i] * id)) + } + for (h in range64(16l)) { + var acc = 0 + for (i in range64(16l)) { + acc += int(q[s * 256l + h * 16l + i]) + } + bs[s * 16l + h] = acc + } + } +} + +def private fill_acts_flat(var x : array; npos, n : int64) { + x |> resize(npos * n) + for (i in range64(npos * n)) { + x[i] = opaque(1.4 * sin(float(i) * 0.19 + 0.3) - 0.6 * cos(float(i) * 0.083)) + } +} + +def private requant_q8_gate(t : T?; par : bool; npos, n : int64) { + var x : array + fill_acts_flat(x, npos, n) + let nb = n / 32l + var xqb : array + var xsb : array + xqb |> resize(npos * n) + xsb |> resize(npos * nb) + requant_rows_q8(x, n, npos, xqb, xsb, par) + var bad = 0 + var seen = 0 + for (p in range64(npos)) { + var row : array + row |> resize(n) + for (i in range64(n)) { + row[i] = x[p * n + i] + } + var qt <- quantize_q8_0(row, n) + for (i in range64(n)) { + if (xqb[p * n + i] != qt.q[i]) { + bad++ + } + if (p == 0l && i == n / 2l && int(xqb[p * n + i]) != int(qt.q[i]) + 7) { + seen++ + } + } + for (b in range64(nb)) { + if (xsb[p * nb + b] != qt.scales[b]) { + bad++ + } + } + delete qt + delete row + } + t |> success(bad == 0, "requant_rows_q8 (par={par}) bit-matches quantize_q8_0 per row ({bad} differ)") + t |> success(seen > 0, "the +7 poison on one expected quant must be seen (par={par})") + delete x + delete xqb + delete xsb +} + +def private requant_q8k_gate(t : T?; par : bool; npos, n : int64) { + var x : array + fill_acts_flat(x, npos, n) + let nsb = n / 256l + var xqb : array + var xsb : array + var xbsb : array + xqb |> resize(npos * n) + xsb |> resize(npos * nsb) + xbsb |> resize(npos * n / 16l) + requant_rows_q8k_bs(x, n, npos, xqb, xsb, xbsb, par) + var bad = 0 + var seen = 0 + for (p in range64(npos)) { + var q : array + var sc : array + var bs : array + ref_q8k_row(x, p * n, n, q, sc, bs) + for (i in range64(n)) { + if (xqb[p * n + i] != q[i]) { + bad++ + } + } + for (s in range64(nsb)) { + if (xsb[p * nsb + s] != sc[s]) { + bad++ + } + } + for (h in range64(n / 16l)) { + if (xbsb[p * (n / 16l) + h] != bs[h]) { + bad++ + } + if (p == 0l && h == 0l && xbsb[p * (n / 16l) + h] != bs[h] + 3) { + seen++ + } + } + delete q + delete sc + delete bs + } + t |> success(bad == 0, "requant_rows_q8k_bs (par={par}) matches the in-test Q8_K reference ({bad} differ)") + t |> success(seen > 0, "the +3 poison on one expected per-16 sum must be seen (par={par})") + delete x + delete xqb + delete xsb + delete xbsb +} + +[test] +def test_requant_rows(t : T?) { + for (par in [false, true]) { + t |> run("requant_rows_q8 par={par}: every row is quantize_q8_0 of that row") @(t : T?) { + requant_q8_gate(t, par, 5l, 512l) + requant_q8_gate(t, par, 1l, 1024l) + } + t |> run("requant_rows_q8k_bs par={par}: per-256 scales and per-16 sums against an in-test reference") @(t : T?) { + requant_q8k_gate(t, par, 5l, 512l) + requant_q8k_gate(t, par, 1l, 1024l) + } + } +} diff --git a/modules/dasLLAMA/tests/test_program_roots.das b/modules/dasLLAMA/tests/test_program_roots.das index ba5f96f9cf..ec0a0e78ab 100644 --- a/modules/dasLLAMA/tests/test_program_roots.das +++ b/modules/dasLLAMA/tests/test_program_roots.das @@ -25,11 +25,18 @@ def private read_lines(path : string) : array { return <- split(replace(text, "\r\n", "\n"), "\n") } +//! every .das under a root dir, subdirectories included: a program root may sit in its own folder +//! (examples/dasLLAMA/storyteller/main.das) def private das_files_of(dir_path : string) : array { var files : array fio::dir("{get_das_root()}/{dir_path}") $(name) { + return if (name == "." || name == "..") + let rel = "{dir_path}/{name}" if (name |> ends_with(".das")) { - files |> push("{dir_path}/{name}") + files |> push(rel) + } elif (stat("{get_das_root()}/{rel}").is_dir) { + var sub <- das_files_of(rel) + files |> push_from(sub) } } return <- files @@ -57,6 +64,8 @@ def test_program_roots(t : T?) { requires_dasllama ||= line |> starts_with("require dasllama/") has_main ||= (line |> starts_with("def main")) || (line |> starts_with("def main(")) has_stack ||= line |> starts_with(STACK_LINE) + // the loaders whose model prefills (LLM, ASR decoder, audio tower); a TTS + // carrier runs no prefill, so a TTS-only root owes no intent loads_model ||= find(line, "load_model(") >= 0 || find(line, "load_asr_model(") >= 0 || find(line, "load_audio_tower(") >= 0 // CODE only: roots' allow_cpu_prefill() tail comments NAME set_metal_mode, so a // raw-line scan would let a comment satisfy the check diff --git a/modules/dasLLAMA/tests/test_q8q8_family.das b/modules/dasLLAMA/tests/test_q8q8_family.das new file mode 100644 index 0000000000..a18de7488f --- /dev/null +++ b/modules/dasLLAMA/tests/test_q8q8_family.das @@ -0,0 +1,939 @@ +options gen2 +options stack = 524288 +options _dasllama_internal = true + +require dastest/testing_boost public +require dasllama/dasllama_math +require dasllama/dasllama_math_default +require dasllama/dasllama_math_aarch64_neon +require dasllama/dasllama_math_gen +require dasllama/dasllama_repack +require dasllama/dasllama_convert +require daslib/f16_cvt +require daslib/jobque_boost +require math + +//! The q8q8 kernel family - stamped tile vs GEMV, the mm/batch/group3/groupN slots on every registered backend, the wscale_f16 twins, matmul_q8/dot_q8 and the mx4 batch slots - each against an in-test fp64 dequant reference carrying its own poison control. + +let WIDTHS <- [64l, 96l, 512l, 1024l, 1056l, 3072l] +let REF_ULP = 1.0e-6 +let REF_POISON = 0.25 +let MX4_E2M1 <- [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0] + +def private mix32(seed, idx : uint) : uint { + var s = seed + idx * 2654435761u + s = s ^ (s >> 15u) + s = s * 2246822519u + s = s ^ (s >> 13u) + s = s * 3266489917u + return s ^ (s >> 16u) +} + +def private unit_at(seed, idx : uint) : float { + return float(int(mix32(seed, idx) % 20001u) - 10000) * 1.0e-4 +} + +def private fill_unit(var a : array; cnt : int64; seed : uint) { + a |> resize(cnt) + for (i in range64(cnt)) { + a[i] = unit_at(seed, uint(int(i))) + } +} + +def private slice_i8(src : array; off, cnt : int64) : array { + return <- [for (i in range64(off, off + cnt)); src[i]] +} + +def private slice_f(src : array; off, cnt : int64) : array { + return <- [for (i in range64(off, off + cnt)); src[i]] +} + +def private pow2d(k : int) : double { + var v = 1.0lf + for (_i in range(abs(k))) { + v = k >= 0 ? v * 2.0lf : v * 0.5lf + } + return v +} + +def private ref_bar(env : double; nb : int64) : float { + return REF_ULP * float(nb) * max(float(env), 1.0) +} + +def private ref_dot_f64(wq : array; ws : array; woff : int64; + xq : array; xs : array; xoff, n : int64) : tuple { + var res : tuple + for (bi in range64(n / 32l)) { + var iacc = 0 + for (k in range64(32l)) { + iacc += int(wq[woff + bi * 32l + k]) * int(xq[xoff + bi * 32l + k]) + } + let term = double(iacc) * double(ws[woff / 32l + bi]) * double(xs[xoff / 32l + bi]) + res.dot += term + res.env += abs(term) + } + return res +} + +def private ref_dot_q8f_f64(wq : array; ws : array; woff : int64; + x : array; n : int64) : tuple { + var res : tuple + for (bi in range64(n / 32l)) { + var bacc : double + for (k in range64(32l)) { + bacc += double(int(wq[woff + bi * 32l + k])) * double(x[bi * 32l + k]) + } + let term = bacc * double(ws[woff / 32l + bi]) + res.dot += term + res.env += abs(term) + } + return res +} + +def private mx4_val(nib : int) : double { + let m = MX4_E2M1[nib & 7] + return double((nib & 8) != 0 ? -m : m) +} + +def private ref_dot_mx4_f64(wn : array; we : array; roff, eoff : int64; + xq : array; xs : array; xoff, n : int64) : tuple { + var res : tuple + for (bi in range64(n / 32l)) { + var bacc : double + for (j in range64(16l)) { + let b = int(wn[roff + bi * 16l + j]) + bacc += mx4_val(b & 15) * double(int(xq[xoff + bi * 32l + j])) + bacc += mx4_val(b >> 4) * double(int(xq[xoff + bi * 32l + j + 16l])) + } + let term = bacc * pow2d(int(we[eoff + bi]) - 127) * double(xs[xoff / 32l + bi]) + res.dot += term + res.env += abs(term) + } + return res +} + +def private identical(t : T?; got : array; want : array; what : string) { + var bad = 0 + for (gv, wv in got, want) { + if (gv != wv) { + bad++ + } + } + t |> success(bad == 0, "{what}: {bad} of {long_length(got)} elements differ") +} + +def private held(t : T?; got : array; ref : array; bar : array; what : string) { + var over = 0 + var worst = 0.0 + for (gv, rv, bv in got, ref, bar) { + let df = abs(gv - rv) + if (df > bv) { + over++ + } + worst = max(worst, df) + } + t |> success(over == 0, "{what}: {over} of {long_length(got)} past the fp64 bar, maxdiff {worst}, bar[0] {bar[0]}") +} + +def private poisoned(t : T?; got : array; ref : array; bar : array; what : string) { + let i = long_length(ref) / 2l + t |> success(abs(got[i] - (ref[i] + REF_POISON)) > bar[i], + "{what}: poison {REF_POISON} added at element {i} must exceed bar {bar[i]}") +} + +def private quantize_rows(src : array; ntok, n : int64; var xq : array; var xs : array) { + let nb = n / 32l + xq |> resize(ntok * n) + xs |> resize(ntok * nb) + var row : array + row |> resize(n) + for (p in range64(ntok)) { + for (i in range64(n)) { + row[i] = src[p * n + i] + } + quantize_q8_0_into(row, n, xq, xs, p * n, p * nb) + } +} + +def private s16_planes(src : array; cnt : int64; var s16 : array; var s32 : array) { + s16 |> resize(cnt) + s32 |> resize(cnt) + for (i in range64(cnt)) { + let h = f32_to_f16(src[i]) + s16[i] = uint16(h) + s32[i] = f16_to_f32(h) + } +} + +def private narrow_scales(src : array; cnt : int64; var s16 : array) { + s16 |> resize(cnt) + for (i in range64(cnt)) { + s16[i] = uint16(f32_to_f16(src[i])) + } +} + +def private activation_bsums(xq : array; ntok, n : int64; var xbs : array) { + let nb = n / 32l + xbs |> resize(ntok * nb) + for (p in range64(ntok)) { + for (bi in range64(nb)) { + var sm = 0 + for (j in range64(32l)) { + sm += int(xq[p * n + bi * 32l + j]) + } + xbs[p * nb + bi] = -128 * sm + } + } +} + +def private ref_image(wq : array; ws : array; xq : array; xs : array; + ntok, d, n : int64; var ref : array; var bar : array) { + let nb = n / 32l + ref |> resize(ntok * d) + bar |> resize(ntok * d) + for (p in range64(ntok)) { + for (r in range64(d)) { + let rf = ref_dot_f64(wq, ws, r * n, xq, xs, p * n, n) + ref[p * d + r] = float(rf.dot) + bar[p * d + r] = ref_bar(rf.env, nb) + } + } +} + +def private groupn_ref(wq : array; ws : array; xq : array; xs : array; + nreg, dreg, n : int64; var ref : array; var bar : array) { + let nb = n / 32l + ref |> resize(nreg * dreg) + bar |> resize(nreg * dreg) + for (r in range64(nreg)) { + for (row in range64(dreg)) { + let rf = ref_dot_f64(wq, ws, (r * dreg + row) * n, xq, xs, r * n, n) + ref[r * dreg + row] = float(rf.dot) + bar[r * dreg + row] = ref_bar(rf.env, nb) + } + } +} + +def private gemv_image_q8(var y : array; wq : array; ws : array; + xq : array; xs : array; ntok, d, n : int64) { + let nb = n / 32l + y |> resize(ntok * d) + var yrow : array + yrow |> resize(d) + for (p in range64(ntok)) { + let rxq <- slice_i8(xq, p * n, n) + let rxs <- slice_f(xs, p * nb, nb) + matmul_q8q8(yrow, wq, ws, rxq, rxs, n, d) + for (r in range64(d)) { + y[p * d + r] = yrow[r] + } + } +} + +def private gemv_image_s16(var y : array; wq : array; ws : array; + xq : array; xs : array; ntok, d, n : int64) { + let nb = n / 32l + y |> resize(ntok * d) + var yrow : array + yrow |> resize(d) + for (p in range64(ntok)) { + let rxq <- slice_i8(xq, p * n, n) + let rxs <- slice_f(xs, p * nb, nb) + matmul_q8q8(yrow, wq, ws, 0l, rxq, rxs, n, d) + for (r in range64(d)) { + y[p * d + r] = yrow[r] + } + } +} + +def private concat3(y0 : array; y1 : array; y2 : array; var fused : array) { + fused |> resize(long_length(y0) + long_length(y1) + long_length(y2)) + var w = 0l + for (v in y0) { + fused[w] = v + w++ + } + for (v in y1) { + fused[w] = v + w++ + } + for (v in y2) { + fused[w] = v + w++ + } +} + +def private restore_backend(saved : string) { + clear_kernel_backend_pin() + if (saved != "") { + select_kernel_backend(saved) + } +} + +def private q8q8_stamped_gate(t : T?; n : int64) { + let d = 64l + let nb = n / 32l + let mr = int64(q8q8_layout_gen()) + let ts = int64(q8q8_tokstep_gen()) + let gs = ts == 4l ? 1l : ts / 16l + let ng = d / mr + let ntok = 2l * ts + 3l + var wf : array + var xf : array + fill_unit(wf, d * n, 0x51D0Au) + fill_unit(xf, ntok * n, 0x51D0Bu) + let wq0 <- quantize_q8_0(wf, d * n) + var xq : array + var xs : array + quantize_rows(xf, ntok, n, xq, xs) + var ref : array + var bar : array + ref_image(wq0.q, wq0.scales, xq, xs, ntok, d, n, ref, bar) + var xbs : array + activation_bsums(xq, ntok, n, xbs) + var q : array + var s : array + q := wq0.q + s := wq0.scales + unsafe { + repack_q8q8_grp(addr(q[0]), addr(s[0]), n, d, mr, int64(q8q8_wbias_gen()), int64(q8q8_kgroup_gen())) + } + var ytile : array + var ygemv : array + ytile |> resize(ntok * d) + ygemv |> resize(ntok * d) + unsafe { + let wp = addr(q[0]) + let sp = addr(s[0]) + let xqp = addr(xq[0]) + let xsp = addr(xs[0]) + let xbsp = addr(xbs[0]) + var ytp = addr(ytile[0]) + var ygp = addr(ygemv[0]) + q8q8_amx_cfg_gen() + for (gu in range64(ng / gs)) { + let gg = gu * gs + var tk = 0l + while (tk + ts <= ntok) { + q8q8_tile_gen(ytp, wp + gg * mr * n, sp + gg * mr * nb, xqp, xsp, xbsp, n, d, gg, tk) + tk += ts + } + while (tk < ntok) { + q8q8_gemv_gen(ytp + tk * d, wp, sp, xqp + tk * n, xsp + tk * nb, n, gg * mr, (gg + gs) * mr) + tk++ + } + } + for (p in range64(ntok)) { + q8q8_gemv_gen(ygp + p * d, wp, sp, xqp + p * n, xsp + p * nb, n, 0l, ng * mr) + } + } + identical(t, ytile, ygemv, "q8q8 tile (mr={mr} tokstep={ts}) bit-matches the per-token GEMV, n={n}") + held(t, ygemv, ref, bar, "q8q8 stamped GEMV, n={n}") + held(t, ytile, ref, bar, "q8q8 stamped tile, n={n}") + poisoned(t, ygemv, ref, bar, "q8q8 stamped GEMV, n={n}") +} + +def private q8q8_wrapper_gate(t : T?; n : int64) { + let d = 64l + let ntok = 2l * int64(q8q8_tokstep_gen()) + 3l + var wf : array + var xf : array + fill_unit(wf, d * n, 0x51D1Au) + fill_unit(xf, ntok * n, 0x51D1Bu) + let wq0 <- quantize_q8_0(wf, d * n) + var xq : array + var xs : array + quantize_rows(xf, ntok, n, xq, xs) + var ref : array + var bar : array + ref_image(wq0.q, wq0.scales, xq, xs, ntok, d, n, ref, bar) + let saved = active_kernel_backend() + for (name in kernel_backend_names()) { + pin_kernel_backend(name) + var q : array + var s : array + q := wq0.q + s := wq0.scales + if (kernel_backend_needs_repack(name)) { + repack_q8q8_weight(q, s, 0l, n, d) + } + var ymm : array + gemv_image_q8(ymm, q, s, xq, xs, ntok, d, n) + var ybat : array + ybat |> resize(ntok * d) + matmul_q8q8_batch(ybat, q, s, xq, xs, n, d, ntok) + if (name == "portable") { + identical(t, ybat, ymm, "{name} matmul_q8q8_batch bit-matches ntok x matmul_q8q8, n={n}") + } else { + held(t, ybat, ymm, bar, "{name} matmul_q8q8_batch within the fp64 bar of ntok x matmul_q8q8, n={n}") + } + held(t, ymm, ref, bar, "{name} matmul_q8q8, n={n}") + held(t, ybat, ref, bar, "{name} matmul_q8q8_batch, n={n}") + poisoned(t, ymm, ref, bar, "{name} matmul_q8q8, n={n}") + } + restore_backend(saved) +} + +[test] +def test_q8q8_tile_gemv_reference(t : T?) { + with_job_que() { + setup_dasllama_jobque_() + for (n in WIDTHS) { + t |> run("q8q8 stamped tile vs GEMV vs fp64 dequant, n={n}") @(t : T?) { + q8q8_stamped_gate(t, n) + } + t |> run("q8q8 mm/batch vs fp64 dequant on every backend, n={n}") @(t : T?) { + q8q8_wrapper_gate(t, n) + } + } + } +} + +def private s16_kernel_gate(t : T?; n : int64) { + let d = 64l + let dreg = d / 2l + let nb = n / 32l + let ntok = 2l + var wf : array + var xf : array + fill_unit(wf, d * n, 0x5F16Au) + fill_unit(xf, ntok * n, 0x5F16Bu) + let wq0 <- quantize_q8_0(wf, d * n) + var ws16 : array + var ws32 : array + s16_planes(wq0.scales, d * nb, ws16, ws32) + var xq : array + var xs : array + quantize_rows(xf, ntok, n, xq, xs) + var ref : array + var bar : array + ref_image(wq0.q, ws32, xq, xs, 1l, d, n, ref, bar) + var gref : array + var gbar : array + groupn_ref(wq0.q, ws32, xq, xs, 2l, dreg, n, gref, gbar) + var ydot : array + var ydot16 : array + var yrows : array + var yrows16 : array + var ygn : array + var ygn16 : array + ydot |> resize(d) + ydot16 |> resize(d) + yrows |> resize(d) + yrows16 |> resize(d) + ygn |> resize(d) + ygn16 |> resize(d) + var offs <- [0l, 0l, dreg * n, n] + unsafe { + let wp = addr(wq0.q[0]) + let s32p = addr(ws32[0]) + let s16p = addr(ws16[0]) + let xqp = addr(xq[0]) + let xsp = addr(xs[0]) + let offp = addr(offs[0]) + for (r in range64(d)) { + ydot[r] = dot_q8q8(wp + r * n, s32p + r * nb, xqp, xsp, n) + ydot16[r] = dot_q8q8_f16s(wp + r * n, s16p + r * nb, xqp, xsp, n) + } + q8q8_rows_kernel(addr(yrows[0]), wp, s32p, xqp, xsp, n, 0l, d) + q8q8_rows_kernel_s16(addr(yrows16[0]), wp, s16p, xqp, xsp, n, 0l, d) + q8q8_groupn_kernel(addr(ygn[0]), wp, s32p, offp, 2l, xqp, xsp, null, null, n, dreg) + q8q8_groupn_kernel_s16(addr(ygn16[0]), wp, s16p, offp, 2l, xqp, xsp, null, null, n, dreg) + } + identical(t, ydot16, ydot, "dot_q8q8_f16s bit-matches dot_q8q8 over the widened plane, n={n}") + identical(t, yrows16, yrows, "q8q8_rows_kernel_s16 bit-matches its f32 twin, n={n}") + identical(t, yrows16, ydot16, "q8q8_rows_kernel_s16 bit-matches the dot it wraps, n={n}") + identical(t, ygn16, ygn, "q8q8_groupn_kernel_s16 bit-matches its f32 twin, n={n}") + held(t, ydot16, ref, bar, "dot_q8q8_f16s, n={n}") + held(t, yrows16, ref, bar, "q8q8_rows_kernel_s16, n={n}") + held(t, ygn16, gref, gbar, "q8q8_groupn_kernel_s16, n={n}") + poisoned(t, ydot16, ref, bar, "dot_q8q8_f16s, n={n}") + poisoned(t, ygn16, gref, gbar, "q8q8_groupn_kernel_s16, n={n}") +} + +def private s16_stamped_gate(t : T?; n : int64) { + let d = 64l + let nb = n / 32l + let mr = int64(q8q8_layout_gen()) + let ts = int64(q8q8_tokstep_gen()) + let gs = ts == 4l ? 1l : ts / 16l + let ng = d / mr + let ntok = 2l * ts + 3l + var wf : array + var xf : array + fill_unit(wf, d * n, 0x5F17Au) + fill_unit(xf, ntok * n, 0x5F17Bu) + let wq0 <- quantize_q8_0(wf, d * n) + var ws16m : array + var ws32m : array + s16_planes(wq0.scales, d * nb, ws16m, ws32m) + var xq : array + var xs : array + quantize_rows(xf, ntok, n, xq, xs) + var ref : array + var bar : array + ref_image(wq0.q, ws32m, xq, xs, ntok, d, n, ref, bar) + var xbs : array + activation_bsums(xq, ntok, n, xbs) + var q : array + var s : array + q := wq0.q + s := ws32m + unsafe { + repack_q8q8_grp(addr(q[0]), addr(s[0]), n, d, mr, int64(q8q8_wbias_gen()), int64(q8q8_kgroup_gen())) + } + var s16 : array + narrow_scales(s, d * nb, s16) + var ytile : array + var ygemv : array + ytile |> resize(ntok * d) + ygemv |> resize(ntok * d) + unsafe { + let wp = addr(q[0]) + let sp = addr(s16[0]) + let xqp = addr(xq[0]) + let xsp = addr(xs[0]) + let xbsp = addr(xbs[0]) + var ytp = addr(ytile[0]) + var ygp = addr(ygemv[0]) + q8q8_amx_cfg_gen() + for (gu in range64(ng / gs)) { + let gg = gu * gs + var tk = 0l + while (tk + ts <= ntok) { + q8q8_tile_s16_gen(ytp, wp + gg * mr * n, sp + gg * mr * nb, xqp, xsp, xbsp, n, d, gg, tk) + tk += ts + } + while (tk < ntok) { + q8q8_gemv_s16_gen(ytp + tk * d, wp, sp, xqp + tk * n, xsp + tk * nb, n, gg * mr, (gg + gs) * mr) + tk++ + } + } + for (p in range64(ntok)) { + q8q8_gemv_s16_gen(ygp + p * d, wp, sp, xqp + p * n, xsp + p * nb, n, 0l, ng * mr) + } + } + identical(t, ytile, ygemv, "q8q8 s16 tile (mr={mr} tokstep={ts}) bit-matches the per-token s16 GEMV, n={n}") + held(t, ygemv, ref, bar, "q8q8 stamped s16 GEMV, n={n}") + held(t, ytile, ref, bar, "q8q8 stamped s16 tile, n={n}") + poisoned(t, ytile, ref, bar, "q8q8 stamped s16 tile, n={n}") +} + +def private s16_wrapper_gate(t : T?; n : int64) { + let d = 64l + let dreg = d / 2l + let nb = n / 32l + let ntok = 2l * int64(q8q8_tokstep_gen()) + 3l + var wf : array + var xf : array + fill_unit(wf, d * n, 0x5F18Au) + fill_unit(xf, ntok * n, 0x5F18Bu) + let wq0 <- quantize_q8_0(wf, d * n) + var ws16m : array + var ws32m : array + s16_planes(wq0.scales, d * nb, ws16m, ws32m) + var xq : array + var xs : array + quantize_rows(xf, ntok, n, xq, xs) + var ref : array + var bar : array + ref_image(wq0.q, ws32m, xq, xs, ntok, d, n, ref, bar) + var gref : array + var gbar : array + groupn_ref(wq0.q, ws32m, xq, xs, 2l, dreg, n, gref, gbar) + var offs <- [0l, 0l, dreg * n, n] + let saved = active_kernel_backend() + for (name in kernel_backend_names()) { + pin_kernel_backend(name) + if (!kernel_backend_has_wscale16()) { + continue + } + var q : array + var s : array + q := wq0.q + s := ws32m + if (kernel_backend_needs_repack(name)) { + repack_q8q8_weight(q, s, 0l, n, d) + } + var s16 : array + narrow_scales(s, d * nb, s16) + var ymm : array + var ymm16 : array + gemv_image_q8(ymm, q, s, xq, xs, ntok, d, n) + gemv_image_s16(ymm16, q, s16, xq, xs, ntok, d, n) + var ybat16 : array + ybat16 |> resize(ntok * d) + matmul_q8q8_batch(ybat16, q, s16, 0l, xq, xs, n, d, ntok) + var ygn : array + var ygn16 : array + ygn |> resize(d) + ygn16 |> resize(d) + matmul_q8q8_groupn(ygn, q, s, offs, 2l, xq, xs, n, dreg) + matmul_q8q8_groupn(ygn16, q, s16, offs, 2l, xq, xs, n, dreg) + identical(t, ymm16, ymm, "{name} matmul_q8q8 s16 bit-matches the f32 plane, n={n}") + identical(t, ybat16, ymm16, "{name} matmul_q8q8_batch s16 bit-matches ntok x matmul_q8q8 s16, n={n}") + identical(t, ygn16, ygn, "{name} matmul_q8q8_groupn s16 bit-matches the f32 plane, n={n}") + held(t, ymm16, ref, bar, "{name} matmul_q8q8 s16, n={n}") + held(t, ybat16, ref, bar, "{name} matmul_q8q8_batch s16, n={n}") + held(t, ygn16, gref, gbar, "{name} matmul_q8q8_groupn s16, n={n}") + poisoned(t, ybat16, ref, bar, "{name} matmul_q8q8_batch s16, n={n}") + poisoned(t, ygn16, gref, gbar, "{name} matmul_q8q8_groupn s16, n={n}") + } + restore_backend(saved) +} + +[test] +def test_q8q8_wscale_f16(t : T?) { + with_job_que() { + setup_dasllama_jobque_() + for (n in WIDTHS) { + t |> run("wscale_f16 dot/rows/groupN kernels vs their f32 twins and fp64, n={n}") @(t : T?) { + s16_kernel_gate(t, n) + } + t |> run("wscale_f16 stamped tile vs GEMV vs fp64, n={n}") @(t : T?) { + s16_stamped_gate(t, n) + } + t |> run("wscale_f16 matmul_q8q8 overloads on every backend, n={n}") @(t : T?) { + s16_wrapper_gate(t, n) + } + } + } +} + +//! The portable kernels' wasm form, gated on every host: n=64 is the paired-block main loop alone, n=96 adds the single-block tail. +def private idot4_ps_gate(t : T?; n : int64) { + let d = 64l + let nb = n / 32l + let ntok = 3l + var wf : array + var xf : array + fill_unit(wf, d * n, 0x1D074u) + fill_unit(xf, ntok * n, 0x1D075u) + let wq0 <- quantize_q8_0(wf, d * n) + var ws16m : array + var ws32m : array + s16_planes(wq0.scales, d * nb, ws16m, ws32m) + var xq : array + var xs : array + quantize_rows(xf, ntok, n, xq, xs) + var ref : array + var bar : array + ref_image(wq0.q, ws32m, xq, xs, ntok, d, n, ref, bar) + var y : array + var y16 : array + y |> resize(ntok * d) + y16 |> resize(ntok * d) + unsafe { + let wp = addr(wq0.q[0]) + let sp = addr(ws32m[0]) + let sp16 = addr(ws16m[0]) + let xqp = addr(xq[0]) + let xsp = addr(xs[0]) + for (p in range64(ntok)) { + for (i in range64(d)) { + y[p * d + i] = dot_q8q8_idot4_ps(wp + i * n, sp + i * nb, xqp + p * n, xsp + p * nb, n) + y16[p * d + i] = dot_q8q8_idot4_ps_f16s(wp + i * n, sp16 + i * nb, xqp + p * n, xsp + p * nb, n) + } + } + } + held(t, y, ref, bar, "dot_q8q8_idot4_ps, n={n}") + identical(t, y16, y, "dot_q8q8_idot4_ps_f16s bit-matches its f32 twin over the widened plane, n={n}") + poisoned(t, y, ref, bar, "dot_q8q8_idot4_ps, n={n}") +} + +[test] +def test_q8q8_idot4_ps(t : T?) { + for (n in WIDTHS) { + t |> run("vector-epilogue idot4 dots vs fp64 and the s16 twin, n={n}") @(t : T?) { + idot4_ps_gate(t, n) + } + } +} + +def private group3_gate(t : T?; n : int64) { + let d0 = 32l + let d1 = 40l + let d2 = 44l + let dt = d0 + d1 + d2 + let nb = n / 32l + let woff0 = 0l + let woff1 = d0 * n + let woff2 = woff1 + d1 * n + var wf : array + var xf : array + fill_unit(wf, dt * n, 0x63A0Au) + fill_unit(xf, n, 0x63A0Bu) + let wq0 <- quantize_q8_0(wf, dt * n) + var ws16m : array + var ws32m : array + s16_planes(wq0.scales, dt * nb, ws16m, ws32m) + var xq : array + var xs : array + quantize_rows(xf, 1l, n, xq, xs) + var ref : array + var bar : array + ref_image(wq0.q, ws32m, xq, xs, 1l, dt, n, ref, bar) + let saved = active_kernel_backend() + for (name in kernel_backend_names()) { + pin_kernel_backend(name) + var q := wq0.q + var s := ws32m + if (kernel_backend_needs_repack(name)) { + repack_q8q8_weight(q, s, woff0, n, d0) + repack_q8q8_weight(q, s, woff1, n, d1) + repack_q8q8_weight(q, s, woff2, n, d2) + } + var s16 : array + narrow_scales(s, dt * nb, s16) + var y0 : array + var y1 : array + var y2 : array + y0 |> resize(d0) + y1 |> resize(d1) + y2 |> resize(d2) + matmul_q8q8_group3(y0, y1, y2, q, s, woff0, woff1, woff2, xq, xs, n, d0, d1, d2) + var g0 : array + var g1 : array + var g2 : array + g0 |> resize(d0) + g1 |> resize(d1) + g2 |> resize(d2) + matmul_q8q8(g0, q, s, woff0, xq, xs, n, d0) + matmul_q8q8(g1, q, s, woff1, xq, xs, n, d1) + matmul_q8q8(g2, q, s, woff2, xq, xs, n, d2) + identical(t, y0, g0, "{name} group3 region 0 bit-matches its own GEMV, n={n}") + identical(t, y1, g1, "{name} group3 region 1 bit-matches its own GEMV, n={n}") + identical(t, y2, g2, "{name} group3 region 2 (row tail) bit-matches its own GEMV, n={n}") + var fused : array + concat3(y0, y1, y2, fused) + held(t, fused, ref, bar, "{name} matmul_q8q8_group3, n={n}") + poisoned(t, fused, ref, bar, "{name} matmul_q8q8_group3, n={n}") + if (!kernel_backend_has_wscale16()) { + continue + } + var h0 : array + var h1 : array + var h2 : array + h0 |> resize(d0) + h1 |> resize(d1) + h2 |> resize(d2) + matmul_q8q8_group3(h0, h1, h2, q, s16, woff0, woff1, woff2, xq, xs, n, d0, d1, d2) + if (name == "portable") { + identical(t, h0, y0, "{name} group3 s16 region 0 bit-matches the f32 plane, n={n}") + identical(t, h1, y1, "{name} group3 s16 region 1 bit-matches the f32 plane, n={n}") + identical(t, h2, y2, "{name} group3 s16 region 2 bit-matches the f32 plane, n={n}") + } else { + var fused16 : array + concat3(h0, h1, h2, fused16) + held(t, fused16, ref, bar, "{name} matmul_q8q8_group3 s16 within the fp64 bar, n={n}") + } + } + restore_backend(saved) +} + +[test] +def test_q8q8_group3(t : T?) { + with_job_que() { + setup_dasllama_jobque_() + for (n in WIDTHS) { + t |> run("matmul_q8q8_group3 (f32 and s16) vs three independent GEMVs and fp64, n={n}") @(t : T?) { + group3_gate(t, n) + } + } + } +} + +def private q8_f32x_gate(t : T?; n : int64) { + let d = 64l + let nb = n / 32l + var wf : array + var xf : array + fill_unit(wf, d * n, 0x8F00Au) + fill_unit(xf, n, 0x8F00Bu) + let wq0 <- quantize_q8_0(wf, d * n) + var ref : array + var bar : array + ref |> resize(d) + bar |> resize(d) + for (r in range64(d)) { + let rf = ref_dot_q8f_f64(wq0.q, wq0.scales, r * n, xf, n) + ref[r] = float(rf.dot) + bar[r] = ref_bar(rf.env, nb) + } + var ymm : array + var yoff : array + var ydot : array + ymm |> resize(d) + yoff |> resize(d) + ydot |> resize(d) + matmul_q8(ymm, wq0.q, wq0.scales, xf, n, d) + matmul_q8(yoff, wq0.q, wq0.scales, 0l, xf, n, d) + unsafe { + let wp = addr(wq0.q[0]) + let sp = addr(wq0.scales[0]) + let xp = addr(xf[0]) + for (r in range64(d)) { + ydot[r] = dot_q8(wp + r * n, sp + r * nb, xp, n) + } + } + identical(t, ymm, ydot, "matmul_q8 bit-matches the per-row dot_q8, n={n}") + identical(t, yoff, ydot, "matmul_q8 blob+offset bit-matches the per-row dot_q8, n={n}") + held(t, ydot, ref, bar, "dot_q8, n={n}") + held(t, ymm, ref, bar, "matmul_q8, n={n}") + poisoned(t, ydot, ref, bar, "dot_q8, n={n}") +} + +[test] +def test_q8_f32_activation(t : T?) { + with_job_que() { + setup_dasllama_jobque_() + for (n in WIDTHS) { + t |> run("matmul_q8 / dot_q8 vs fp64 dequant, n={n}") @(t : T?) { + q8_f32x_gate(t, n) + } + } + } +} + +def private mx4_planes(var wn : array; var we : array; cnt, nblk : int64) { + wn |> resize(cnt / 2l) + we |> resize(nblk) + for (i in range64(cnt / 2l)) { + wn[i] = uint8(mix32(0x3A4Bu, uint(int(i))) >> 24u) + } + for (i in range64(nblk)) { + we[i] = uint8(119u + mix32(0x3A4Cu, uint(int(i))) % 9u) + } +} + +def private mx4_ref_region(wn : array; we : array; woff : int64; + xq : array; xs : array; t0, te, d, n : int64; + var ref : array; var bar : array) { + let nb = n / 32l + for (p in range64(t0, te)) { + for (row in range64(d)) { + let rf = ref_dot_mx4_f64(wn, we, (woff + row * n) / 2l, (woff + row * n) / 32l, xq, xs, p * n, n) + ref[p * d + row] = float(rf.dot) + bar[p * d + row] = ref_bar(rf.env, nb) + } + } +} + +def private mx4_leaf_gate(t : T?; wn0 : array; we0 : array; + xq : array; xs : array; d, n : int64) { + let nb = n / 32l + var yleaf : array + var ymm : array + yleaf |> resize(d) + ymm |> resize(d) + let rxq <- slice_i8(xq, 0l, n) + let rxs <- slice_f(xs, 0l, nb) + pin_kernel_backend("portable") + matmul_mx4q8(ymm, wn0, we0, 0l, rxq, rxs, n, d) + var lut : int8[16] + mxfp4_fill_lut(lut) + unsafe { + let lutp = addr(lut[0]) + let wnp = addr(wn0[0]) + let wep = addr(we0[0]) + let xqp = addr(xq[0]) + let xsp = addr(xs[0]) + for (r in range64(d)) { + yleaf[r] = dot_mx4q8_scalar(lutp, wnp + r * (n / 2l), wep + r * nb, xqp, xsp, n) + } + } + identical(t, ymm, yleaf, "portable matmul_mx4q8 bit-matches the dot_mx4q8_scalar leaf, n={n}") +} + +def private mx4_batch_gate(t : T?; n : int64) { + let d = 32l + let nb = n / 32l + let ntok = 9l + let nreg = 2l + let cnt0 = 4l + let woff1 = d * n + var wn0 : array + var we0 : array + mx4_planes(wn0, we0, nreg * d * n, nreg * d * nb) + var xf : array + fill_unit(xf, ntok * n, 0x3A4Du) + var xq : array + var xs : array + quantize_rows(xf, ntok, n, xq, xs) + var bref : array + var bbar : array + bref |> resize(ntok * d) + bbar |> resize(ntok * d) + mx4_ref_region(wn0, we0, 0l, xq, xs, 0l, ntok, d, n, bref, bbar) + var gref : array + var gbar : array + gref |> resize(ntok * d) + gbar |> resize(ntok * d) + mx4_ref_region(wn0, we0, 0l, xq, xs, 0l, cnt0, d, n, gref, gbar) + mx4_ref_region(wn0, we0, woff1, xq, xs, cnt0, ntok, d, n, gref, gbar) + var goffs <- [0l, 0l, cnt0, woff1, cnt0, ntok - cnt0] + let saved = active_kernel_backend() + mx4_leaf_gate(t, wn0, we0, xq, xs, d, n) + var names : array + names |> push("portable") + for (nm in kernel_backend_names()) { + if (kernel_backend_needs_repack(nm)) { + names |> push(nm) + } + } + for (nm in names) { + pin_kernel_backend(nm) + var wn : array + var we : array + wn := wn0 + we := we0 + if (kernel_backend_needs_repack(nm)) { + for (r in range64(nreg)) { + repack_mx4_weight(wn, we, r * d * n, n, d) + } + } + var ymm : array + var ygmm : array + var yrow : array + ymm |> resize(ntok * d) + ygmm |> resize(ntok * d) + yrow |> resize(d) + for (p in range64(ntok)) { + let rxq <- slice_i8(xq, p * n, n) + let rxs <- slice_f(xs, p * nb, nb) + matmul_mx4q8(yrow, wn, we, 0l, rxq, rxs, n, d) + for (row in range64(d)) { + ymm[p * d + row] = yrow[row] + } + matmul_mx4q8(yrow, wn, we, p < cnt0 ? 0l : woff1, rxq, rxs, n, d) + for (row in range64(d)) { + ygmm[p * d + row] = yrow[row] + } + } + var ybat : array + ybat |> resize(ntok * d) + matmul_mx4q8_batch(ybat, wn, we, 0l, xq, xs, n, d, ntok) + var ygb : array + ygb |> resize(ntok * d) + matmul_mx4q8_batch_groupn(ygb, wn, we, goffs, nreg, xq, xs, n, d) + identical(t, ybat, ymm, "{nm} matmul_mx4q8_batch bit-matches ntok x matmul_mx4q8, n={n}") + identical(t, ygb, ygmm, "{nm} matmul_mx4q8_batch_groupn bit-matches the per-region GEMVs, n={n}") + held(t, ybat, bref, bbar, "{nm} matmul_mx4q8_batch, n={n}") + held(t, ygb, gref, gbar, "{nm} matmul_mx4q8_batch_groupn, n={n}") + poisoned(t, ybat, bref, bbar, "{nm} matmul_mx4q8_batch, n={n}") + poisoned(t, ygb, gref, gbar, "{nm} matmul_mx4q8_batch_groupn, n={n}") + } + restore_backend(saved) +} + +[test] +def test_mx4q8_batch_family(t : T?) { + with_job_que() { + setup_dasllama_jobque_() + for (n in WIDTHS) { + t |> run("mx4 batch / batch groupN vs per-token GEMVs, the scalar leaf and fp64, n={n}") @(t : T?) { + mx4_batch_gate(t, n) + } + } + } +} diff --git a/modules/dasLLAMA/tests/test_repack.das b/modules/dasLLAMA/tests/test_repack.das index ee58c4d08f..f6c56a18fd 100644 --- a/modules/dasLLAMA/tests/test_repack.das +++ b/modules/dasLLAMA/tests/test_repack.das @@ -13,7 +13,7 @@ require dasllama/dasllama_repack require dasllama/dasllama_math_default // dequant_kq_row_grp — the independent grp reader require dasllama/dasllama_gguf // dequant_*_plane_superblock — the independent disk reader require dasllama/dasllama_gemm_schema // kq_qsb/kq_ssb — the fmt-id plane strides -require llvm/daslib/f16_cvt // plausible f16 scale halfwords +require daslib/f16_cvt // plausible f16 scale halfwords // deterministic byte source (LCG) — no Math.random in tests struct private Rng { diff --git a/modules/dasLLAMA/tests/test_rope_apply.das b/modules/dasLLAMA/tests/test_rope_apply.das new file mode 100644 index 0000000000..b608213b61 --- /dev/null +++ b/modules/dasLLAMA/tests/test_rope_apply.das @@ -0,0 +1,481 @@ +//! The CPU rope APPLICATION leaves (dasllama_math's rope_scaled_* and rope_apply) against an in-test fp64 angle reference - a subject on every platform, not just the Apple Metal rope oracle. +options gen2 +options stack = 524288 +options _dasllama_internal = true + +require dastest/testing_boost public +require dasllama/dasllama_math +require dasllama/dasllama_rope +require math + +var g_taint = 0.0 + +[sideeffects] +def private opaque(x : float) : float => x + g_taint + +let ROPE_BAR = 2.0e-3 +let ROPE_POISON = 0.05 +let TWIN_BAR = 1.0e-5 + +def private ff_none() : array { + var e : array + return <- e +} + +def private ff_ptr(ff : array) : float const? { + unsafe { + return !empty(ff) ? addr(ff[0]) : null + } +} + +def private fill_vec(var v : array; n : int64) { + v |> resize(n) + for (i in range64(n)) { + v[i] = opaque(1.3 * sin(float(i) * 0.31 + 0.2) - 0.7 * cos(float(i) * 0.077)) + } +} + +def private copy_vec(src : array; var dst : array; n : int64) { + dst |> resize(n) + for (i in range64(n)) { + dst[i] = src[i] + } +} + +def private ref_freq(hd, hs : int64; theta : float; ff : array; has_ff : bool; fj : int64) : double { + var f = 1.0lf / pow(double(theta), double(hd) / double(hs)) + if (has_ff) { + f /= double(ff[fj]) + } + return f +} + +def private ref_norm(v : array; var outv : array; pos, hs, n : int64; + theta, fscale, mscale : float; ff : array; has_ff : bool) { + copy_vec(v, outv, n) + let posf = double(pos) * double(fscale) + for (p in range64(n / 2l)) { + let i = p * 2l + let hd = i % hs + let a = posf * ref_freq(hd, hs, theta, ff, has_ff, hd / 2l) + let fcr = cos(a) * double(mscale) + let fci = sin(a) * double(mscale) + let v0 = double(v[i]) + let v1 = double(v[i + 1l]) + outv[i] = float(v0 * fcr - v1 * fci) + outv[i + 1l] = float(v0 * fci + v1 * fcr) + } +} + +def private ref_neox(v : array; var outv : array; pos, hs, rot, n : int64; + theta, fscale, mscale : float; ff : array; has_ff : bool) { + copy_vec(v, outv, n) + let half = rot / 2l + let posf = double(pos) * double(fscale) + for (h in range64(n / hs)) { + let base = h * hs + for (j in range64(half)) { + let a = posf * ref_freq(2l * j, rot, theta, ff, has_ff, j) + let fcr = cos(a) * double(mscale) + let fci = sin(a) * double(mscale) + let v0 = double(v[base + j]) + let v1 = double(v[base + j + half]) + outv[base + j] = float(v0 * fcr - v1 * fci) + outv[base + j + half] = float(v0 * fci + v1 * fcr) + } + } +} + +def private maxdiff(a : array; b : array; n : int64) : float { + var worst = 0.0 + for (i in range64(n)) { + worst = max(worst, abs(a[i] - b[i])) + } + return worst +} + +def private poisoned_maxdiff(a : array; b : array; n : int64) : float { + var worst = 0.0 + for (i in range64(n)) { + let w = i == n / 2l ? b[i] + ROPE_POISON : b[i] + worst = max(worst, abs(a[i] - w)) + } + return worst +} + +def private cmp_bar(t : T?; got : array; want : array; n : int64; bar : float; tag : string) { + let worst = maxdiff(got, want, n) + t |> success(worst <= bar, "{tag}: maxdiff {worst} exceeds the bar {bar}") + let pworst = poisoned_maxdiff(got, want, n) + t |> success(pworst > bar, "{tag}: a +{ROPE_POISON} poison on one expected element scored {pworst}, which must exceed {bar}") +} + +def private cmp_exact(t : T?; got : array; want : array; n : int64; tag : string) { + var bad = 0 + var poisoned = 0 + for (i in range64(n)) { + if (got[i] != want[i]) { + bad++ + } + if (got[i] != (i == n / 2l ? want[i] + ROPE_POISON : want[i])) { + poisoned++ + } + } + t |> success(bad == 0, "{tag}: {bad} of {int(n)} elements differ") + t |> success(poisoned > 0, "{tag}: a +{ROPE_POISON} poison on one expected element must be seen") +} + +def private norm_gate(t : T?; pos, hs, nh : int64; theta, fscale, mscale : float) { + let n = hs * nh + var v : array + fill_vec(v, n) + var want : array + var ff <- ff_none() + ref_norm(v, want, pos, hs, n, theta, fscale, mscale, ff, false) + var got : array + copy_vec(v, got, n) + rope_scaled(got, pos, hs, n, theta, fscale, ff, mscale) + cmp_bar(t, got, want, n, ROPE_BAR, "rope_scaled hs={int(hs)} nh={int(nh)} pos={int(pos)}") + delete v + delete got + delete want + delete ff +} + +def private neox_gate(t : T?; pos, hs, nh : int64; theta, fscale, mscale : float) { + let n = hs * nh + var v : array + fill_vec(v, n) + var want : array + var ff <- ff_none() + ref_neox(v, want, pos, hs, hs, n, theta, fscale, mscale, ff, false) + var got : array + copy_vec(v, got, n) + rope_scaled_neox(got, pos, hs, n, theta, fscale, ff, mscale) + cmp_bar(t, got, want, n, ROPE_BAR, "rope_scaled_neox hs={int(hs)} nh={int(nh)} pos={int(pos)}") + delete v + delete got + delete want + delete ff +} + +[test] +def test_rope_apply_norm(t : T?) { + for (pos in [0l, 1l, 7l, 37l, 512l]) { + t |> run("rope_scaled at pos {pos}: head 64x2, 96x1 and the 40-wide tail head vs the fp64 angles") @(t : T?) { + norm_gate(t, pos, 64l, 2l, 10000.0, 1.0, 1.0) + norm_gate(t, pos, 96l, 1l, 1000000.0, 1.0, 1.0) + norm_gate(t, pos, 40l, 3l, 10000.0, 0.5, 1.19) + } + } +} + +[test] +def test_rope_apply_neox(t : T?) { + for (pos in [0l, 1l, 7l, 37l, 512l]) { + t |> run("rope_scaled_neox at pos {pos}: head 64x2, 96x1 and the 40-wide tail head vs the fp64 angles") @(t : T?) { + neox_gate(t, pos, 64l, 2l, 10000.0, 1.0, 1.0) + neox_gate(t, pos, 96l, 1l, 1000000.0, 1.0, 1.0) + neox_gate(t, pos, 40l, 3l, 10000.0, 0.5, 1.19) + } + } +} + +def private tab_gate(t : T?; pos, hs, nh : int64; theta, fscale, mscale : float) { + let n = hs * nh + var v : array + fill_vec(v, n) + var ff <- ff_none() + var want : array + ref_norm(v, want, pos, hs, n, theta, fscale, mscale, ff, false) + var direct : array + copy_vec(v, direct, n) + rope_scaled(direct, pos, hs, n, theta, fscale, ff, mscale) + var ct : array + var st : array + build_rope_tabs(ct, st, theta, fscale, mscale, ff, true, pos, 1l, hs) + var got : array + copy_vec(v, got, n) + unsafe { + rope_scaled_tab(addr(got[0]), hs, n, addr(ct[0]), addr(st[0])) + } + cmp_bar(t, got, want, n, ROPE_BAR, "rope_scaled_tab hs={int(hs)} nh={int(nh)} pos={int(pos)}") + cmp_bar(t, got, direct, n, TWIN_BAR, "rope_scaled_tab twin of rope_scaled hs={int(hs)} pos={int(pos)}") + delete v + delete got + delete direct + delete want + delete ct + delete st + delete ff +} + +def private neox_tab_gate(t : T?; pos, hs, nh : int64; theta, fscale, mscale : float) { + let n = hs * nh + var v : array + fill_vec(v, n) + var ff <- ff_none() + var want : array + ref_neox(v, want, pos, hs, hs, n, theta, fscale, mscale, ff, false) + var direct : array + copy_vec(v, direct, n) + rope_scaled_neox(direct, pos, hs, n, theta, fscale, ff, mscale) + var ct : array + var st : array + build_rope_tabs(ct, st, theta, fscale, mscale, ff, true, pos, 1l, hs) + var got : array + copy_vec(v, got, n) + unsafe { + rope_scaled_neox_tab(addr(got[0]), hs, n, addr(ct[0]), addr(st[0])) + } + cmp_bar(t, got, want, n, ROPE_BAR, "rope_scaled_neox_tab hs={int(hs)} nh={int(nh)} pos={int(pos)}") + cmp_bar(t, got, direct, n, TWIN_BAR, "rope_scaled_neox_tab twin of rope_scaled_neox hs={int(hs)} pos={int(pos)}") + delete v + delete got + delete direct + delete want + delete ct + delete st + delete ff +} + +[test] +def test_rope_apply_tab(t : T?) { + for (pos in [0l, 1l, 37l, 512l]) { + t |> run("rope_scaled_tab at pos {pos}: the fp64 angles, and the un-tabled twin") @(t : T?) { + tab_gate(t, pos, 64l, 2l, 10000.0, 1.0, 1.0) + tab_gate(t, pos, 40l, 3l, 10000.0, 0.5, 1.19) + } + } +} + +[test] +def test_rope_apply_neox_tab(t : T?) { + for (pos in [0l, 1l, 37l, 512l]) { + t |> run("rope_scaled_neox_tab at pos {pos}: the fp64 angles, and the un-tabled twin") @(t : T?) { + neox_tab_gate(t, pos, 64l, 2l, 10000.0, 1.0, 1.0) + neox_tab_gate(t, pos, 40l, 3l, 10000.0, 0.5, 1.19) + } + } +} + +def private part_gate(t : T?; pos, hs, rot, nh : int64; theta, fscale, mscale : float) { + let n = hs * nh + var v : array + fill_vec(v, n) + var ff <- ff_none() + var want : array + ref_neox(v, want, pos, hs, rot, n, theta, fscale, mscale, ff, false) + var got : array + copy_vec(v, got, n) + unsafe { + rope_scaled_neox_part(addr(got[0]), pos, hs, rot, n, theta, fscale, false, null, mscale) + } + cmp_bar(t, got, want, n, ROPE_BAR, "rope_scaled_neox_part hs={int(hs)} rot={int(rot)} pos={int(pos)}") + var compact : array + compact |> resize(nh * rot) + for (h in range64(nh)) { + for (j in range64(rot)) { + compact[h * rot + j] = v[h * hs + j] + } + } + rope_scaled_neox(compact, pos, rot, nh * rot, theta, fscale, ff, mscale) + var full : array + copy_vec(v, full, n) + for (h in range64(nh)) { + for (j in range64(rot)) { + full[h * hs + j] = compact[h * rot + j] + } + } + cmp_exact(t, got, full, n, "rope_scaled_neox_part == the full leaf over the rotated prefix hs={int(hs)} rot={int(rot)}") + var tail_moved = 0 + for (h in range64(nh)) { + for (j in range64(rot, hs)) { + if (got[h * hs + j] != v[h * hs + j]) { + tail_moved++ + } + } + } + t |> success(tail_moved == 0, "the {int(hs - rot)} un-rotated dims of each head must pass through ({tail_moved} moved)") + delete v + delete got + delete full + delete want + delete compact + delete ff +} + +def private tab_part_gate(t : T?; pos, hs, rot, nh : int64; theta, fscale, mscale : float) { + let n = hs * nh + var v : array + fill_vec(v, n) + var ff <- ff_none() + var want : array + ref_neox(v, want, pos, hs, rot, n, theta, fscale, mscale, ff, false) + var direct : array + copy_vec(v, direct, n) + unsafe { + rope_scaled_neox_part(addr(direct[0]), pos, hs, rot, n, theta, fscale, false, null, mscale) + } + var ct : array + var st : array + build_rope_tabs(ct, st, theta, fscale, mscale, ff, true, pos, 1l, rot) + var got : array + copy_vec(v, got, n) + unsafe { + rope_scaled_neox_tab_part(addr(got[0]), hs, rot, n, addr(ct[0]), addr(st[0])) + } + cmp_bar(t, got, want, n, ROPE_BAR, "rope_scaled_neox_tab_part hs={int(hs)} rot={int(rot)} pos={int(pos)}") + cmp_bar(t, got, direct, n, TWIN_BAR, "rope_scaled_neox_tab_part twin of the direct part form hs={int(hs)} rot={int(rot)}") + delete v + delete got + delete direct + delete want + delete ct + delete st + delete ff +} + +[test] +def test_rope_apply_part(t : T?) { + for (pos in [0l, 1l, 37l, 512l]) { + t |> run("rope_scaled_neox_part at pos {pos}: 64 of 256 (qwen35), 32 of 96, 20 of 40") @(t : T?) { + part_gate(t, pos, 256l, 64l, 2l, 1000000.0, 1.0, 1.0) + part_gate(t, pos, 96l, 32l, 2l, 10000.0, 0.5, 1.19) + part_gate(t, pos, 40l, 20l, 3l, 10000.0, 1.0, 1.0) + } + t |> run("rope_scaled_neox_tab_part at pos {pos}: the fp64 angles, and the direct twin") @(t : T?) { + tab_part_gate(t, pos, 256l, 64l, 2l, 1000000.0, 1.0, 1.0) + tab_part_gate(t, pos, 40l, 20l, 3l, 10000.0, 1.0, 1.0) + } + } +} + +[test] +def test_rope_apply_dispatch(t : T?) { + t |> run("rope_apply lands on the leaf its `neox` flag names, pointer and array forms alike") @(t : T?) { + let hs = 64l + let n = 128l + let pos = 37l + var v : array + fill_vec(v, n) + var ff <- ff_none() + for (neox in [false, true]) { + var leaf : array + copy_vec(v, leaf, n) + if (neox) { + rope_scaled_neox(leaf, pos, hs, n, 10000.0, 1.0, ff, 1.19) + } else { + rope_scaled(leaf, pos, hs, n, 10000.0, 1.0, ff, 1.19) + } + var pform : array + copy_vec(v, pform, n) + unsafe { + rope_apply(addr(pform[0]), pos, hs, n, 10000.0, 1.0, false, null, neox, 1.19) + } + cmp_exact(t, pform, leaf, n, "rope_apply(ptr, neox={neox}) == the leaf") + var aform : array + copy_vec(v, aform, n) + rope_apply(aform, pos, hs, n, 10000.0, 1.0, ff, neox, 1.19, true) + cmp_exact(t, aform, leaf, n, "rope_apply(array, neox={neox}) == the leaf") + delete leaf + delete pform + delete aform + } + delete v + delete ff + } + t |> run("use_ff = false drops loaded freq-factors for this call (the gemma4 p-RoPE arm)") @(t : T?) { + let hs = 64l + let n = 128l + let pos = 37l + var v : array + fill_vec(v, n) + var ff : array + ff |> resize(hs / 2l) + for (j in range64(hs / 2l)) { + ff[j] = 1.0 + 0.25 * float(j) + } + var none <- ff_none() + for (neox in [false, true]) { + var off : array + copy_vec(v, off, n) + rope_apply(off, pos, hs, n, 10000.0, 1.0, ff, neox, 1.0, false) + var bare : array + copy_vec(v, bare, n) + rope_apply(bare, pos, hs, n, 10000.0, 1.0, none, neox, 1.0, true) + cmp_exact(t, off, bare, n, "use_ff=false (neox={neox}) == the no-factors apply") + var on : array + copy_vec(v, on, n) + rope_apply(on, pos, hs, n, 10000.0, 1.0, ff, neox, 1.0, true) + var moved = 0 + for (i in range64(n)) { + if (on[i] != off[i]) { + moved++ + } + } + t |> success(moved > 0, "the factors must actually move the rotation when use_ff is true (neox={neox})") + delete off + delete bare + delete on + } + delete v + delete ff + delete none + } +} + +def private ff_gate(t : T?; pos, hs, nh : int64; neox : bool) { + let n = hs * nh + var v : array + fill_vec(v, n) + var ff : array + ff |> resize(hs / 2l) + for (j in range64(hs / 2l)) { + ff[j] = 1.0 + 0.5 * float(j % 7l) + } + var want : array + if (neox) { + ref_neox(v, want, pos, hs, hs, n, 500000.0, 1.0, 1.0, ff, true) + } else { + ref_norm(v, want, pos, hs, n, 500000.0, 1.0, 1.0, ff, true) + } + var got : array + copy_vec(v, got, n) + unsafe { + rope_apply(addr(got[0]), pos, hs, n, 500000.0, 1.0, true, ff_ptr(ff), neox, 1.0) + } + cmp_bar(t, got, want, n, ROPE_BAR, "freq-factors (neox={neox}) hs={int(hs)} pos={int(pos)}") + var ones : array + ones |> resize(hs / 2l) + for (c in ones) { + c = 1.0 + } + var none <- ff_none() + var with_ones : array + copy_vec(v, with_ones, n) + rope_apply(with_ones, pos, hs, n, 500000.0, 1.0, ones, neox, 1.0, true) + var without : array + copy_vec(v, without, n) + rope_apply(without, pos, hs, n, 500000.0, 1.0, none, neox, 1.0, true) + cmp_exact(t, with_ones, without, n, "all-ones factors (neox={neox}) must divide away exactly") + delete v + delete got + delete want + delete ff + delete ones + delete none + delete with_ones + delete without +} + +[test] +def test_rope_apply_freq_factors(t : T?) { + for (pos in [1l, 37l, 512l]) { + t |> run("rope_freqs divide the per-pair frequency at pos {pos}, both conventions") @(t : T?) { + ff_gate(t, pos, 64l, 2l, false) + ff_gate(t, pos, 64l, 2l, true) + ff_gate(t, pos, 40l, 3l, true) + } + } +} diff --git a/modules/dasLLAMA/tests/test_tower_asr_kernels.das b/modules/dasLLAMA/tests/test_tower_asr_kernels.das new file mode 100644 index 0000000000..4d0f9c427e --- /dev/null +++ b/modules/dasLLAMA/tests/test_tower_asr_kernels.das @@ -0,0 +1,1276 @@ +options gen2 +options stack = 524288 +options _dasllama_internal = true + +require dastest/testing_boost public +require dasllama/dasllama_tower +require dasllama/dasllama_audio +require dasllama/dasllama_tts_blocks +require dasllama/dasllama_math +require dasllama/dasllama_plane +require daslib/jobque_boost +require math + +//! Model-free CPU units for the public tower, audio and TTS kernels the ASR and TTS paths run: every bar is an in-test fp64 reference over a seeded-LCG fixture, and every bar carries a must-EXCEED poison. +struct KCase { + name : string + max_abs : double + max_ratio : double + bad : int +} + +var private g_lcg = 0x2545F4914F6CDD1Dul + +[sideeffects] +def private lcg_seed(s : uint64) { + g_lcg = s * 2862933555777941757ul + 3037000493ul +} + +[sideeffects] +def private lcg_unit() : float { + g_lcg = g_lcg * 6364136223846793005ul + 1442695040888963407ul + return float(uint((g_lcg >> 40ul) & 0xFFFFFFul)) / 8388608.0 - 1.0 +} + +def private fill(var a : array; n : int64; scale : float) { + a |> resize(n) + for (v in a) { + v = lcg_unit() * scale + } +} + +def private score(name : string; got : array; want : array; env : array; + rel : double; floor : double) : KCase { + var res = KCase(name = name) + if (long_length(got) != long_length(want) || long_length(got) != long_length(env) || empty(got)) { + res.bad = 1000000 + return res + } + for (g, w, e in got, want, env) { + let d = abs(double(g) - w) + let bar = max(rel * e, floor) + res.max_abs = max(res.max_abs, d) + res.max_ratio = max(res.max_ratio, d / bar) + if (d > bar) { + res.bad++ + } + } + return res +} + +def private score_poison(name : string; got : array; want : array; env : array; + rel : double; floor : double; delta : double) : KCase { + var p := want + if (empty(p)) { + return KCase(name = name) + } + p[long_length(p) / 2l] += delta + return score(name, got, p, env, rel, floor) +} + +def private score_exact(name : string; got : array; want : array) : KCase { + var res = KCase(name = name) + if (long_length(got) != long_length(want) || empty(got)) { + res.bad = 1000000 + return res + } + for (g, w in got, want) { + res.max_abs = max(res.max_abs, abs(double(g) - double(w))) + if (g != w) { + res.bad++ + } + } + return res +} + +def private score_exact_poison(name : string; got : array; want : array; delta : float) : KCase { + var p := want + if (empty(p)) { + return KCase(name = name) + } + p[long_length(p) / 2l] += delta + return score_exact(name, got, p) +} + +def private report(t : T?; r : KCase) { + to_log(LOG_INFO, "tower kernel {r.name}: max |kernel - reference| {r.max_abs}, worst ratio to the bar {r.max_ratio}, past the bar {r.bad}\n") + t |> equal(r.bad, 0, "{r.name}: elements past the bar") +} + +def private report_poison(t : T?; r : KCase) { + to_log(LOG_INFO, "tower kernel POISON {r.name}: max diff {r.max_abs}, worst ratio to the bar {r.max_ratio}, past the bar {r.bad}\n") + t |> success(r.bad > 0, "{r.name}: the poisoned expectation must exceed the bar, {r.bad} elements did") +} + +def private plane_f(a : array) : PlaneF { + var p = PlaneF() + unsafe { + p.p = addr(a[0]) + } + p.n = long_length(a) + return p +} + +def private plane_u16(a : array) : PlaneU16 { + var p = PlaneU16() + unsafe { + p.p = addr(a[0]) + } + p.n = long_length(a) + return p +} + +def private to_bf16(x : float) : uint16 { + return uint16(unsafe(reinterpret(x)) >> 16u) +} + +def private from_bf16(h : uint16) : float { + return unsafe(reinterpret(uint(h) << 16u)) +} + +let MM_N = 128l +let MM_D = 96l +let MM_T = 40l +let MM_WOFF = 64l +let MM_NPAD = 192l +let MM_REL = 2.0e-5lf +let MM_FLOOR = 1.0e-6lf + +def private gemm_ref(w : array; woff : int64; x : array; n : int64; d : int64; npos : int64; + var want : array; var env : array) { + want |> resize(npos * d) + env |> resize(npos * d) + for (p in range64(npos)) { + for (j in range64(d)) { + var acc = 0.0lf + var mag = 0.0lf + for (i in range64(n)) { + let prod = double(w[woff + j * n + i]) * double(x[p * n + i]) + acc += prod + mag += abs(prod) + } + want[p * d + j] = acc + env[p * d + j] = mag + } + } +} + +def private mm_fixture(var blob : array; var x : array) { + lcg_seed(0xA5A5ul) + fill(blob, MM_WOFF + MM_N * MM_D, 1.0) + fill(x, MM_T * MM_N, 1.0) +} + +def private mm_bf16_fixture(var wq : array; var wf : array; var x : array) { + lcg_seed(0xB16Bul) + fill(wf, MM_WOFF + MM_N * MM_D, 1.0) + wq |> resize(long_length(wf)) + for (f, q in wf, wq) { + q = to_bf16(f) + f = from_bf16(q) + } + fill(x, MM_T * MM_N, 1.0) + for (v in x) { + v = from_bf16(to_bf16(v)) + } +} + +def private mm_blob_run(var y : array; blob : array; x : array) { + y |> resize(MM_T * MM_D) + mm_blob_b(y, blob, MM_WOFF, x, MM_N, MM_D, MM_T) +} + +def private mm_blob_case(poison : bool) : KCase { + var blob : array + var x : array + mm_fixture(blob, x) + var want : array + var env : array + gemm_ref(blob, MM_WOFF, x, MM_N, MM_D, MM_T, want, env) + var y : array + mm_blob_run(y, blob, x) + let nm = "mm_blob_b blob+woff" + if (poison) { + return score_poison(nm, y, want, env, MM_REL, MM_FLOOR, 0.05lf) + } + return score(nm, y, want, env, MM_REL, MM_FLOOR) +} + +def private mm_plane_case(poison : bool) : KCase { + var blob : array + var x : array + mm_fixture(blob, x) + var y_arr : array + var y_pl : array + mm_blob_run(y_arr, blob, x) + y_pl |> resize(MM_T * MM_D) + mm_blob_b(y_pl, plane_f(blob), MM_WOFF, x, MM_N, MM_D, MM_T) + let nm = "mm_blob_b borrowed plane == owned blob" + if (poison) { + return score_exact_poison(nm, y_pl, y_arr, 1.0e-5) + } + return score_exact(nm, y_pl, y_arr) +} + +def private mm_bf16_case(poison : bool) : KCase { + var wq : array + var wf : array + var x : array + mm_bf16_fixture(wq, wf, x) + var want : array + var env : array + gemm_ref(wf, MM_WOFF, x, MM_N, MM_D, MM_T, want, env) + var y : array + y |> resize(MM_T * MM_D) + mm_bf16_b(y, plane_u16(wq), MM_WOFF, x, MM_N, MM_D, MM_T) + let nm = "mm_bf16_b on bf16-exact operands" + if (poison) { + return score_poison(nm, y, want, env, MM_REL, MM_FLOOR, 0.05lf) + } + return score(nm, y, want, env, MM_REL, MM_FLOOR) +} + +def private mm_route_case(bf16 : bool; poison : bool) : KCase { + var wq : array + var wf : array + var x : array + mm_bf16_fixture(wq, wf, x) + var want : array + var got : array + want |> resize(MM_T * MM_D) + got |> resize(MM_T * MM_D) + if (bf16) { + mm_bf16_b(want, plane_u16(wq), MM_WOFF, x, MM_N, MM_D, MM_T) + } else { + mm_blob_b(want, plane_f(wf), MM_WOFF, x, MM_N, MM_D, MM_T) + } + mm_plane_b(got, bf16, plane_u16(wq), plane_f(wf), MM_WOFF, x, MM_N, MM_D, MM_T) + let nm = "mm_plane_b bf16={bf16} routes to its wrapper" + if (poison) { + return score_exact_poison(nm, got, want, 1.0e-5) + } + return score_exact(nm, got, want) +} + +def private mm_pad_case(poison : bool) : KCase { + lcg_seed(0x9AD0ul) + var w : array + var x : array + fill(w, MM_N * MM_D, 1.0) + fill(x, MM_T * MM_N, 1.0) + var wp : array + var xp : array + wp |> resize(MM_NPAD * MM_D) + xp |> resize(MM_T * MM_NPAD) + for (j in range64(MM_D)) { + for (i in range64(MM_N)) { + wp[j * MM_NPAD + i] = w[j * MM_N + i] + } + } + for (p in range64(MM_T)) { + for (i in range64(MM_N)) { + xp[p * MM_NPAD + i] = x[p * MM_N + i] + } + } + var y_tight : array + var y_pad : array + y_tight |> resize(MM_T * MM_D) + y_pad |> resize(MM_T * MM_D) + mm_blob_b(y_tight, w, 0l, x, MM_N, MM_D, MM_T) + mm_blob_b(y_pad, wp, 0l, xp, MM_NPAD, MM_D, MM_T) + let nm = "mm_blob_b zero-padded width {MM_NPAD} == unpadded {MM_N}" + if (poison) { + return score_exact_poison(nm, y_pad, y_tight, 1.0e-5) + } + return score_exact(nm, y_pad, y_tight) +} + +let LN_DIM = 197l +let LN_POS = 33l +let LN_OFF = 8l +let LN_EPS = 1.0e-5 +let LN_REL = 1.0e-5lf +let LN_FLOOR = 1.0e-6lf + +def private ln_fixture(var x : array; var wb : array) { + lcg_seed(0x1EAFul) + fill(x, LN_POS * LN_DIM, 3.0) + fill(wb, LN_OFF + 2l * LN_DIM, 1.0) +} + +def private ln_ref(x : array; wb : array; var want : array; var env : array) { + want |> resize(LN_POS * LN_DIM) + env |> resize(LN_POS * LN_DIM) + for (p in range64(LN_POS)) { + let off = p * LN_DIM + var mean = 0.0lf + for (i in range64(LN_DIM)) { + mean += double(x[off + i]) + } + mean /= double(LN_DIM) + var vr = 0.0lf + for (i in range64(LN_DIM)) { + let dv = double(x[off + i]) - mean + vr += dv * dv + } + vr /= double(LN_DIM) + let scale = 1.0lf / sqrt(vr + double(LN_EPS)) + for (i in range64(LN_DIM)) { + let nrm = (double(x[off + i]) - mean) * scale * double(wb[LN_OFF + i]) + let bias = double(wb[LN_OFF + LN_DIM + i]) + want[off + i] = nrm + bias + env[off + i] = abs(nrm) + abs(bias) + } + } +} + +def private ln_scalar_run(var out : array; x : array; wb : array) { + out |> resize(LN_POS * LN_DIM) + unsafe { + var op = addr(out[0]) + let ip = addr(x[0]) + let wp = addr(wb[LN_OFF]) + let bp = addr(wb[LN_OFF + LN_DIM]) + for (p in range64(LN_POS)) { + layernorm(op + p * LN_DIM, ip + p * LN_DIM, wp, bp, LN_DIM, LN_EPS) + } + } +} + +def private ln_scalar_case(poison : bool) : KCase { + var x : array + var wb : array + ln_fixture(x, wb) + var want : array + var env : array + ln_ref(x, wb, want, env) + var out : array + ln_scalar_run(out, x, wb) + let nm = "layernorm row form" + if (poison) { + return score_poison(nm, out, want, env, LN_REL, LN_FLOOR, 0.01lf) + } + return score(nm, out, want, env, LN_REL, LN_FLOOR) +} + +def private ln_batch_case(poison : bool) : KCase { + var x : array + var wb : array + ln_fixture(x, wb) + var want : array + var env : array + ln_ref(x, wb, want, env) + var out : array + out |> resize(LN_POS * LN_DIM) + layernorm_batch(out, x, wb, LN_OFF, LN_OFF + LN_DIM, LN_DIM, LN_POS, LN_EPS) + let nm = "layernorm_batch blob+offsets" + if (poison) { + return score_poison(nm, out, want, env, LN_REL, LN_FLOOR, 0.01lf) + } + return score(nm, out, want, env, LN_REL, LN_FLOOR) +} + +def private ln_plane_case(poison : bool) : KCase { + var x : array + var wb : array + ln_fixture(x, wb) + var y_arr : array + var y_pl : array + y_arr |> resize(LN_POS * LN_DIM) + y_pl |> resize(LN_POS * LN_DIM) + layernorm_batch(y_arr, x, wb, LN_OFF, LN_OFF + LN_DIM, LN_DIM, LN_POS, LN_EPS) + layernorm_batch(y_pl, x, plane_f(wb), LN_OFF, LN_OFF + LN_DIM, LN_DIM, LN_POS, LN_EPS) + let nm = "layernorm_batch borrowed plane == owned blob" + if (poison) { + return score_exact_poison(nm, y_pl, y_arr, 1.0e-5) + } + return score_exact(nm, y_pl, y_arr) +} + +let AB_D = 37l +let AB_POS = 41l +let AB_OFF = 5l +let AI_N = 4093l + +def private ab_fixture(var y : array; var blob : array) { + lcg_seed(0xB1A5ul) + fill(y, AB_POS * AB_D, 2.0) + fill(blob, AB_OFF + AB_D, 1.0) +} + +def private ab_ref(y : array; blob : array) : array { + var want : array + want |> resize(AB_POS * AB_D) + for (p in range64(AB_POS)) { + for (i in range64(AB_D)) { + want[p * AB_D + i] = y[p * AB_D + i] + blob[AB_OFF + i] + } + } + return <- want +} + +def private add_bias_case(as_plane : bool; poison : bool) : KCase { + var y : array + var blob : array + ab_fixture(y, blob) + var want <- ab_ref(y, blob) + if (as_plane) { + add_bias_rows(y, plane_f(blob), AB_OFF, AB_D, AB_POS) + } else { + add_bias_rows(y, blob, AB_OFF, AB_D, AB_POS) + } + let nm = "add_bias_rows plane={as_plane} d={AB_D} off the float4 block" + if (poison) { + return score_exact_poison(nm, y, want, 1.0e-5) + } + return score_exact(nm, y, want) +} + +def private add_inplace_case(poison : bool) : KCase { + lcg_seed(0xADD1ul) + var x : array + var r : array + fill(x, AI_N, 2.0) + fill(r, AI_N, 1.0) + var want : array + want |> resize(AI_N) + for (a, b, w in x, r, want) { + w = a + b + } + add_inplace_rows(x, r, AI_N) + let nm = "add_inplace_rows n={AI_N} off the float4 block" + if (poison) { + return score_exact_poison(nm, x, want, 1.0e-5) + } + return score_exact(nm, x, want) +} + +let GE_D = 64l +let GE_POS = 25l +let GE_REL = 1.0e-6lf +let GE_FLOOR = 1.0e-6lf + +def private erf_ref(z : double) : double { + var term = z + var sum = z + for (k in range(1, 60)) { + term *= -z * z / double(k) + sum += term / double(2 * k + 1) + } + return sum * 2.0lf / sqrt(DBL_PI) +} + +def private gelu_fixture(var x : array) { + lcg_seed(0x9E10ul) + fill(x, GE_D * GE_POS, 4.0) +} + +def private gelu_ref(x : array; var want : array; var env : array) { + want |> resize(long_length(x)) + env |> resize(long_length(x)) + for (v, w, e in x, want, env) { + let d = double(v) + w = 0.5lf * d * (1.0lf + erf_ref(d * 0.7071067811865476lf)) + e = abs(d) + } +} + +def private gelu_batch_case(poison : bool) : KCase { + var x : array + gelu_fixture(x) + var want : array + var env : array + gelu_ref(x, want, env) + gelu_erf_batch(x, GE_D, GE_POS) + let nm = "gelu_erf_batch vs the fp64 erf series" + if (poison) { + return score_poison(nm, x, want, env, GE_REL, GE_FLOOR, 0.001lf) + } + return score(nm, x, want, env, GE_REL, GE_FLOOR) +} + +def private gelu_shape_case(poison : bool) : KCase { + var x : array + gelu_fixture(x) + var flat : array + flat := x + gelu_erf_batch(x, GE_D, GE_POS) + gelu_erf(flat, GE_D * GE_POS) + let nm = "gelu_erf_batch == the whole-image gelu_erf" + if (poison) { + return score_exact_poison(nm, x, flat, 1.0e-5) + } + return score_exact(nm, x, flat) +} + +let RP_POS = 17l +let RP_HEADS = 5l +let RP_HS = 16l +let RP_REL = 2.0e-6lf +let RP_FLOOR = 1.0e-7lf + +def private rope_tabs(var ct : array; var st : array; live_row : int64) { + let half = RP_HS / 2l + ct |> resize(RP_POS * half) + st |> resize(RP_POS * half) + for (r in range64(RP_POS)) { + for (j in range64(half)) { + let live = live_row < 0l || r == live_row + let ang = live ? 0.37 * float(r + 1l) / pow(2.0, float(j)) : 0.0 + ct[r * half + j] = cos(ang) + st[r * half + j] = sin(ang) + } + } +} + +def private rope_ref(v : array; ct : array; st : array; + var want : array; var env : array) { + let d = RP_HEADS * RP_HS + let half = RP_HS / 2l + want |> resize(RP_POS * d) + env |> resize(RP_POS * d) + for (r in range64(RP_POS)) { + for (h in range64(RP_HEADS)) { + let base = r * d + h * RP_HS + for (j in range64(half)) { + let c = double(ct[r * half + j]) + let s = double(st[r * half + j]) + let v0 = double(v[base + j]) + let v1 = double(v[base + j + half]) + want[base + j] = v0 * c - v1 * s + want[base + j + half] = v0 * s + v1 * c + env[base + j] = abs(v0 * c) + abs(v1 * s) + env[base + j + half] = abs(v0 * s) + abs(v1 * c) + } + } + } +} + +def private rope_case(poison : bool) : KCase { + lcg_seed(0x40BEul) + var v : array + var ct : array + var st : array + fill(v, RP_POS * RP_HEADS * RP_HS, 2.0) + rope_tabs(ct, st, -1l) + var want : array + var env : array + rope_ref(v, ct, st, want, env) + rope_neox_tab_rows(v, RP_POS, RP_HEADS, RP_HS, ct, st) + let nm = "rope_neox_tab_rows one table row per token, shared by every head" + if (poison) { + return score_poison(nm, v, want, env, RP_REL, RP_FLOOR, 0.001lf) + } + return score(nm, v, want, env, RP_REL, RP_FLOOR) +} + +def private rope_row_index_case(live_row : int64; poison : bool) : KCase { + lcg_seed(0x41BEul) + var v : array + var ct : array + var st : array + fill(v, RP_POS * RP_HEADS * RP_HS, 2.0) + let before := v + rope_tabs(ct, st, live_row) + rope_neox_tab_rows(v, RP_POS, RP_HEADS, RP_HS, ct, st) + let d = RP_HEADS * RP_HS + var got : array + var want : array + got |> resize((RP_POS - 1l) * d) + want |> resize((RP_POS - 1l) * d) + var o = 0l + for (r in range64(RP_POS)) { + if (r == live_row) { + continue + } + for (i in range64(d)) { + got[o * d + i] = v[r * d + i] + want[o * d + i] = before[r * d + i] + } + o++ + } + let nm = "rope_neox_tab_rows: only token {live_row}'s table row rotates" + if (poison) { + return score_exact_poison(nm, got, want, 1.0e-5) + } + var moved = false + for (i in range64(d)) { + if (v[live_row * d + i] != before[live_row * d + i]) { + moved = true + } + } + var res = score_exact(nm, got, want) + if (!moved) { + res.bad += 1000000 + } + return res +} + +let AW_D = 32l +let AW_HEADS = 4l +let AW_SCALE = 0.35 +let AW_REL = 1.0e-5lf +let AW_FLOOR = 1.0e-6lf + +def private aw_win() : array { + return <- [0l, 7l, 23l, 26l, 47l, 48l] +} + +def private aw_fixture(win : array; var qq : array; var kk : array; var vv : array) { + lcg_seed(0x77A0ul) + let rows = win[long_length(win) - 1l] + fill(qq, rows * AW_D, 1.0) + fill(kk, rows * AW_D, 1.0) + fill(vv, rows * AW_D, 1.0) + for (wi in range64(long_length(win) - 1l)) { + let amp = 1.0 + 3.0 * float(wi) + for (idx in range64(win[wi] * AW_D, win[wi + 1l] * AW_D)) { + kk[idx] = kk[idx] * amp + float(wi) + vv[idx] = vv[idx] * amp - float(wi) + } + } +} + +def private aw_ref_head(qq : array; kk : array; vv : array; var sc : array; + var want : array; var env : array; + r0 : int64; wr : int64; hoff : int64; hs : int64) { + for (p in range64(wr)) { + var mx = -1.0e300lf + for (q in range64(wr)) { + var s = 0.0lf + for (j in range64(hs)) { + s += double(qq[(r0 + p) * AW_D + hoff + j]) * double(kk[(r0 + q) * AW_D + hoff + j]) + } + sc[q] = s * double(AW_SCALE) + mx = max(mx, sc[q]) + } + var sum = 0.0lf + for (q in range64(wr)) { + sc[q] = exp(sc[q] - mx) + sum += sc[q] + } + for (j in range64(hs)) { + var acc = 0.0lf + var mag = 0.0lf + for (q in range64(wr)) { + let wgt = sc[q] / sum + let vq = double(vv[(r0 + q) * AW_D + hoff + j]) + acc += wgt * vq + mag += wgt * abs(vq) + } + want[(r0 + p) * AW_D + hoff + j] = acc + env[(r0 + p) * AW_D + hoff + j] = mag + } + } +} + +def private aw_ref(qq : array; kk : array; vv : array; win : array; + var want : array; var env : array) { + let rows = win[long_length(win) - 1l] + let hs = AW_D / AW_HEADS + want |> resize(rows * AW_D) + env |> resize(rows * AW_D) + var sc : array + for (wi in range64(long_length(win) - 1l)) { + let r0 = win[wi] + let wr = win[wi + 1l] - r0 + sc |> resize(wr) + for (h in range64(AW_HEADS)) { + aw_ref_head(qq, kk, vv, sc, want, env, r0, wr, h * hs, hs) + } + } +} + +def private aw_case(poison : bool) : KCase { + var win <- aw_win() + var qq : array + var kk : array + var vv : array + aw_fixture(win, qq, kk, vv) + var want : array + var env : array + aw_ref(qq, kk, vv, win, want, env) + var out : array + var hp : array + var att : array + out |> resize(long_length(qq)) + attention_bidir_windows(out, qq, kk, vv, hp, att, win, AW_D, AW_HEADS, AW_SCALE) + let nm = "attention_bidir_windows vs an oracle attending strictly inside each ragged window" + if (poison) { + return score_poison(nm, out, want, env, AW_REL, AW_FLOOR, 0.05lf) + } + return score(nm, out, want, env, AW_REL, AW_FLOOR) +} + +let IG_REL = 5.0e-6lf +let IG_FLOOR = 1.0e-7lf + +def private interp_ref(src : array; src_off : int64; sw : int64; sh : int64; dw : int64; dh : int64; + d : int64; var want : array; var env : array) { + want |> resize(dw * dh * d) + env |> resize(dw * dh * d) + let sf0 = float(dw) / float(sw) + let sf1 = float(dh) / float(sh) + let sup0 = max(1.0, 1.0 / sf0) + let sup1 = max(1.0, 1.0 / sf1) + let iv0 = 1.0 / sup0 + let iv1 = 1.0 / sup1 + for (i1 in range64(dh)) { + let yc = (float(i1) + 0.5) / sf1 + let ylo = max(int64(yc - sup1 + 0.5), 0l) + let yhi = min(int64(yc + sup1 + 0.5), sh) + for (i0 in range64(dw)) { + let xc = (float(i0) + 0.5) / sf0 + let xlo = max(int64(xc - sup0 + 0.5), 0l) + let xhi = min(int64(xc + sup0 + 0.5), sw) + let obase = (i1 * dw + i0) * d + for (c in range64(d)) { + want[obase + c] = 0.0lf + env[obase + c] = 0.0lf + } + var tw = 0.0lf + for (sy in range64(ylo, yhi)) { + let wy = max(1.0 - abs((float(sy) - yc + 0.5) * iv1), 0.0) + for (sx in range64(xlo, xhi)) { + let wgt = max(1.0 - abs((float(sx) - xc + 0.5) * iv0), 0.0) * wy + if (wgt <= 0.0) { + continue + } + let sbase = src_off + (sy * sw + sx) * d + for (c in range64(d)) { + let term = double(src[sbase + c]) * double(wgt) + want[obase + c] += term + env[obase + c] += abs(term) + } + tw += double(wgt) + } + } + if (tw > 0.0lf) { + for (c in range64(d)) { + want[obase + c] /= tw + env[obase + c] /= tw + } + } + } + } +} + +def private interp_case(sw : int64; sh : int64; dw : int64; dh : int64; d : int64; src_off : int64; + poison : bool) : KCase { + lcg_seed(0x1E40ul + uint64(dw * 131l + sh)) + var src : array + fill(src, src_off + sw * sh * d, 2.0) + var want : array + var env : array + interp_ref(src, src_off, sw, sh, dw, dh, d, want, env) + var out : array + interpolate_grid_bilinear_aa(out, plane_f(src), src_off, sw, sh, dw, dh, d) + let nm = "interpolate_grid_bilinear_aa {sw}x{sh} -> {dw}x{dh} d={d} at plane offset {src_off}" + if (poison) { + return score_poison(nm, out, want, env, IG_REL, IG_FLOOR, 0.01lf) + } + return score(nm, out, want, env, IG_REL, IG_FLOOR) +} + +let FFT_REL = 2.0e-6lf +let FFT_FLOOR = 1.0e-6lf +let DFT_N = 400l + +def private dft_ref(reim : array; n : int64; var want : array; var env : array) { + want |> resize(2l * n) + env |> resize(2l * n) + var envsum = 0.0lf + for (j in range64(n)) { + envsum += abs(double(reim[2l * j])) + abs(double(reim[2l * j + 1l])) + } + for (k in range64(n)) { + var acc_re = 0.0lf + var acc_im = 0.0lf + for (j in range64(n)) { + let th = 2.0lf * DBL_PI * double((j * k) % n) / double(n) + let cs = cos(th) + let sn = sin(th) + acc_re += double(reim[2l * j]) * cs + double(reim[2l * j + 1l]) * sn + acc_im += double(reim[2l * j + 1l]) * cs - double(reim[2l * j]) * sn + } + want[2l * k] = acc_re + want[2l * k + 1l] = acc_im + env[2l * k] = envsum + env[2l * k + 1l] = envsum + } +} + +def private fft_case(n : int64; poison : bool) : KCase { + lcg_seed(0xFF70ul + uint64(n)) + var reim : array + fill(reim, 2l * n, 1.0) + var want : array + var env : array + dft_ref(reim, n, want, env) + var p <- build_fft_plan(n) + unsafe { + fft_pow2_run(n, addr(p.rev[0]), addr(p.tw[0]), addr(reim[0])) + } + let nm = "fft_pow2_run n={n} vs the fp64 O(n^2) DFT of the same forward convention" + if (poison) { + return score_poison(nm, reim, want, env, FFT_REL, FFT_FLOOR, 0.05lf) + } + return score(nm, reim, want, env, FFT_REL, FFT_FLOOR) +} + +def private fft_plan_tw_case(n : int64; poison : bool) : KCase { + let p <- build_fft_plan(n) + var want : array + var env : array + want |> resize(n) + env |> resize(n) + for (k in range64(n / 2l)) { + let th = 2.0lf * DBL_PI * double(k) / double(n) + want[2l * k] = cos(th) + want[2l * k + 1l] = -sin(th) + env[2l * k] = 1.0lf + env[2l * k + 1l] = 1.0lf + } + let nm = "build_fft_plan n={n} twiddles vs fp64 (cos, -sin)" + if (poison) { + return score_poison(nm, p.tw, want, env, FFT_REL, FFT_FLOOR, 0.001lf) + } + return score(nm, p.tw, want, env, FFT_REL, FFT_FLOOR) +} + +def private fft_plan_rev_bad(n : int64) : int { + let p <- build_fft_plan(n) + var seen : array + seen |> resize(n) + var bad = 0 + for (i in range64(n)) { + let r = int64(p.rev[i]) + if (r < 0l || r >= n || seen[r] || int64(p.rev[r]) != i) { + bad++ + } else { + seen[r] = true + } + } + if (p.rev[0] != 0 || int64(p.rev[1]) != n / 2l) { + bad++ + } + return bad +} + +def private fft_plan_refuses(n : int64) : bool { + var tripped = false + try { + let p <- build_fft_plan(n) + tripped = long_length(p.rev) < 0l + } recover { + tripped = true + } + return tripped +} + +def private dft_twiddle_case(poison : bool) : KCase { + let n_bins = DFT_N / 2l + 1l + let b <- build_dft_twiddles(DFT_N) + var want : array + var env : array + want |> resize(DFT_N * n_bins * 2l) + env |> resize(DFT_N * n_bins * 2l) + for (j in range64(DFT_N)) { + for (k in range64(n_bins)) { + let th = 2.0lf * DBL_PI * double((j * k) % DFT_N) / double(DFT_N) + let at = j * n_bins * 2l + k * 2l + want[at] = cos(th) + want[at + 1l] = -sin(th) + env[at] = 1.0lf + env[at + 1l] = 1.0lf + } + } + let nm = "build_dft_twiddles n={DFT_N} vs fp64 (cos, -sin) at (j*k) mod n" + if (poison) { + return score_poison(nm, b, want, env, FFT_REL, FFT_FLOOR, 0.001lf) + } + return score(nm, b, want, env, FFT_REL, FFT_FLOOR) +} + +let RS_C = 5l +let RS_T = 41l +let RS_REL = 2.0e-6lf +let RS_FLOOR = 1.0e-7lf + +def private rs_fixture(var x : array; scale : float) { + lcg_seed(0x2E51ul + uint64(int64(scale * 1024.0))) + fill(x, RS_C * RS_T, 2.0) +} + +def private rl_onnx_ref(x : array; scale : float; var want : array; var env : array) : int64 { + let t_out = int64(floor(float(RS_T) * scale)) + want |> resize(RS_C * t_out) + env |> resize(RS_C * t_out) + for (to in range64(t_out)) { + let in_x = clamp((float(to) + 0.5) / scale - 0.5, 0.0, float(RS_T - 1l)) + let i1 = min(int64(in_x), RS_T - 1l) + let i2 = min(i1 + 1l, RS_T - 1l) + var dx1 = double(abs(in_x - float(i1))) + var dx2 = double(abs(in_x - float(i2))) + if (i1 == i2) { + dx1 = 0.5lf + dx2 = 0.5lf + } + for (ci in range64(RS_C)) { + let a = double(x[ci * RS_T + i1]) + let b = double(x[ci * RS_T + i2]) + want[ci * t_out + to] = dx2 * a + dx1 * b + env[ci * t_out + to] = dx2 * abs(a) + dx1 * abs(b) + } + } + return t_out +} + +def private rl_torch_ref(x : array; scale : float; var want : array; var env : array) : int64 { + let t_out = int64(floor(float(RS_T) * scale)) + want |> resize(RS_C * t_out) + env |> resize(RS_C * t_out) + let inv = double(1.0 / scale) + for (to in range64(t_out)) { + let src = max(0.0, float(inv * (double(to) + 0.5lf) - 0.5lf)) + let i0 = min(int64(src), RS_T - 1l) + let i1 = min(i0 + 1l, RS_T - 1l) + let l1 = double(src - float(i0)) + let l0 = 1.0lf - l1 + for (ci in range64(RS_C)) { + let a = double(x[ci * RS_T + i0]) + let b = double(x[ci * RS_T + i1]) + want[ci * t_out + to] = l0 * a + l1 * b + env[ci * t_out + to] = l0 * abs(a) + l1 * abs(b) + } + } + return t_out +} + +def private resize_onnx_case(scale : float; poison : bool) : KCase { + var x : array + rs_fixture(x, scale) + var want : array + var env : array + let want_t = rl_onnx_ref(x, scale, want, env) + var y : array + let got_t = resize_linear(x, RS_C, RS_T, scale, y) + let nm = "resize_linear (ONNX half-pixel) scale={scale}" + if (got_t != want_t) { + return KCase(name = nm, bad = 1000000) + } + if (poison) { + return score_poison(nm, y, want, env, RS_REL, RS_FLOOR, 0.001lf) + } + return score(nm, y, want, env, RS_REL, RS_FLOOR) +} + +def private resize_torch_case(scale : float; poison : bool) : KCase { + var x : array + rs_fixture(x, scale) + var want : array + var env : array + let want_t = rl_torch_ref(x, scale, want, env) + var y : array + let got_t = resize_linear_torch(x, RS_C, RS_T, scale, y) + let nm = "resize_linear_torch (align_corners false) scale={scale}" + if (got_t != want_t) { + return KCase(name = nm, bad = 1000000) + } + if (poison) { + return score_poison(nm, y, want, env, RS_REL, RS_FLOOR, 0.001lf) + } + return score(nm, y, want, env, RS_REL, RS_FLOOR) +} + +let MP_BINS = 13l +let MP_FRAMES = 37l +let MP_EPS = 1.0e-9 +let MP_PHASE_ENV = 4.0lf +let MP_REL = 2.0e-6lf +let MP_FLOOR = 1.0e-6lf + +[sideeffects] +def private neg_zero() : float { + return unsafe(reinterpret(0x80000000u)) +} + +def private mp_fixture(var re : array; var im : array) { + lcg_seed(0x3A9Ful) + fill(re, MP_BINS * MP_FRAMES, 2.0) + fill(im, MP_BINS * MP_FRAMES, 2.0) + re[0] = -1.5 + im[0] = 0.0 + re[1] = -1.5 + im[1] = neg_zero() + re[2] = 2.5 + im[2] = 0.0 + re[3] = 2.5 + im[3] = neg_zero() + re[4] = neg_zero() + im[4] = neg_zero() +} + +def private mp_ref(re : array; im : array; var want : array; var env : array) { + let nb = MP_BINS * MP_FRAMES + want |> resize(2l * nb) + env |> resize(2l * nb) + for (i in range64(nb)) { + let r = double(re[i]) + let m = double(im[i]) + let mag = sqrt(r * r + m * m + double(MP_EPS)) + want[i] = mag + env[i] = mag + want[nb + i] = im[i] == 0.0 ? (re[i] < 0.0 ? DBL_PI : 0.0lf) : atan2(m, r) + env[nb + i] = MP_PHASE_ENV + } +} + +def private mp_case(poison : bool) : KCase { + var re : array + var im : array + mp_fixture(re, im) + var want : array + var env : array + mp_ref(re, im, want, env) + var out : array + magnitude_phase(re, im, MP_BINS, MP_FRAMES, MP_EPS, out) + let nm = "magnitude_phase {MP_BINS}x{MP_FRAMES} with the zero-imaginary branch planted" + if (poison) { + return score_poison(nm, out, want, env, MP_REL, MP_FLOOR, 0.01lf) + } + return score(nm, out, want, env, MP_REL, MP_FLOOR) +} + +let IS_REL = 4.0e-6lf +let IS_FLOOR = 1.0e-7lf + +def private is_window(var w : array; k : int64; zeroed : bool) { + w |> resize(k) + for (v in w) { + let u = 0.4 + 0.6 * abs(lcg_unit()) + v = zeroed ? 0.0 : u + } +} + +def private istft_ref(wave0 : array; n : int64; window : array; k : int64; hop : int64; + pad : int64; var want : array; var env : array) { + want |> resize(n) + env |> resize(n) + for (i in range64(n)) { + let m = i + pad + var acc = 0.0lf + var f = (m - k + hop) / hop + if (f < 0l) { + f = 0l + } + while (f * hop <= m) { + let j = m - f * hop + if (j < k) { + acc += double(window[j]) * double(window[j]) + } + f++ + } + let w0 = double(wave0[i]) + want[i] = acc > 1.0e-11lf ? w0 / acc : w0 + env[i] = abs(want[i]) + } +} + +def private istft_case(n : int64; k : int64; hop : int64; pad : int64; zeroed : bool; poison : bool) : KCase { + lcg_seed(0x157Ful + uint64(k * 97l + hop)) + var window : array + is_window(window, k, zeroed) + var wave : array + fill(wave, n, 2.0) + let wave0 := wave + var want : array + var env : array + istft_ref(wave0, n, window, k, hop, pad, want, env) + istft_envelope_divide(wave, n, window, k, hop, pad) + let nm = "istft_envelope_divide n={n} k={k} hop={hop} pad={pad} zero-window={zeroed}" + if (poison) { + return score_poison(nm, wave, want, env, IS_REL, IS_FLOOR, 0.01lf) + } + return score(nm, wave, want, env, IS_REL, IS_FLOOR) +} + +[test] +def test_tower_gemm_wrappers(t : T?) { + with_job_que() { + setup_dasllama_jobque_() + t |> run("the wrappers against an fp64 GEMM, and the plane, per-tensor-flag and zero-padded forms against them") @(t : T?) { + report(t, mm_blob_case(false)) + report(t, mm_bf16_case(false)) + report(t, mm_plane_case(false)) + report(t, mm_route_case(true, false)) + report(t, mm_route_case(false, false)) + report(t, mm_pad_case(false)) + } + t |> run("the GEMM bars' must-red controls") @(t : T?) { + report_poison(t, mm_blob_case(true)) + report_poison(t, mm_bf16_case(true)) + report_poison(t, mm_plane_case(true)) + report_poison(t, mm_route_case(true, true)) + report_poison(t, mm_route_case(false, true)) + report_poison(t, mm_pad_case(true)) + } + } +} + +[test] +def test_tower_layernorm(t : T?) { + with_job_que() { + setup_dasllama_jobque_() + t |> run("layernorm and layernorm_batch against the fp64 population-variance form") @(t : T?) { + report(t, ln_scalar_case(false)) + report(t, ln_batch_case(false)) + report(t, ln_plane_case(false)) + } + t |> run("the layernorm bars' must-red controls") @(t : T?) { + report_poison(t, ln_scalar_case(true)) + report_poison(t, ln_batch_case(true)) + report_poison(t, ln_plane_case(true)) + } + } +} + +[test] +def test_tower_row_adds(t : T?) { + with_job_que() { + setup_dasllama_jobque_() + t |> run("the bias and residual row adds against the scalar f32 loop") @(t : T?) { + report(t, add_bias_case(false, false)) + report(t, add_bias_case(true, false)) + report(t, add_inplace_case(false)) + } + t |> run("the row adds' must-red controls") @(t : T?) { + report_poison(t, add_bias_case(false, true)) + report_poison(t, add_bias_case(true, true)) + report_poison(t, add_inplace_case(true)) + } + } +} + +[test] +def test_tower_gelu_erf_batch(t : T?) { + with_job_que() { + setup_dasllama_jobque_() + t |> run("gelu_erf_batch against an fp64 erf series, and against its whole-image form") @(t : T?) { + report(t, gelu_batch_case(false)) + report(t, gelu_shape_case(false)) + } + t |> run("the gelu bars' must-red controls") @(t : T?) { + report_poison(t, gelu_batch_case(true)) + report_poison(t, gelu_shape_case(true)) + } + } +} + +[test] +def test_tower_rope_neox_tab_rows(t : T?) { + with_job_que() { + setup_dasllama_jobque_() + t |> run("the full-head table rope against the fp64 rotation, and its per-row table index") @(t : T?) { + report(t, rope_case(false)) + report(t, rope_row_index_case(0l, false)) + report(t, rope_row_index_case(11l, false)) + } + t |> run("the rope bars' must-red controls") @(t : T?) { + report_poison(t, rope_case(true)) + report_poison(t, rope_row_index_case(11l, true)) + } + } +} + +[test] +def test_tower_attention_windows(t : T?) { + with_job_que() { + setup_dasllama_jobque_() + t |> run("attention_bidir_windows against an oracle that attends strictly inside each window") @(t : T?) { + report(t, aw_case(false)) + } + t |> run("the window attention bar's must-red control") @(t : T?) { + report_poison(t, aw_case(true)) + } + } +} + +[test] +def test_tower_interpolate_grid_aa(t : T?) { + with_job_que() { + setup_dasllama_jobque_() + t |> run("the antialiased grid resample down and up, at a non-zero plane offset") @(t : T?) { + report(t, interp_case(48l, 48l, 14l, 14l, 8l, 0l, false)) + report(t, interp_case(48l, 48l, 27l, 13l, 4l, 37l, false)) + report(t, interp_case(4l, 4l, 9l, 9l, 3l, 11l, false)) + } + t |> run("the resample bars' must-red controls") @(t : T?) { + report_poison(t, interp_case(48l, 48l, 14l, 14l, 8l, 0l, true)) + report_poison(t, interp_case(4l, 4l, 9l, 9l, 3l, 11l, true)) + } + } +} + +[test] +def test_audio_fft_and_dft_tables(t : T?) { + t |> run("the radix-2 plan's bit-reversal permutation and its twiddles") @(t : T?) { + t |> equal(fft_plan_rev_bad(256l), 0, "build_fft_plan(256) rev is an involutive permutation with rev[1] = n/2") + t |> equal(fft_plan_rev_bad(64l), 0, "build_fft_plan(64) rev is an involutive permutation with rev[1] = n/2") + report(t, fft_plan_tw_case(256l, false)) + t |> success(fft_plan_refuses(6l), "build_fft_plan refuses a non-power-of-2 size") + t |> success(fft_plan_refuses(1l), "build_fft_plan refuses a size below 2") + } + t |> run("fft_pow2_run against an fp64 O(n^2) DFT of the same forward convention") @(t : T?) { + report(t, fft_case(64l, false)) + report(t, fft_case(256l, false)) + } + t |> run("the 400-point DFT twiddle matrix against fp64") @(t : T?) { + report(t, dft_twiddle_case(false)) + } + t |> run("the FFT and DFT bars' must-red controls") @(t : T?) { + report_poison(t, fft_plan_tw_case(256l, true)) + report_poison(t, fft_case(256l, true)) + report_poison(t, dft_twiddle_case(true)) + } +} + +[test] +def test_tts_linear_resamplers(t : T?) { + t |> run("both interpolation conventions against an fp64 two-tap reference") @(t : T?) { + report(t, resize_onnx_case(2.0, false)) + report(t, resize_onnx_case(1.5, false)) + report(t, resize_onnx_case(0.5, false)) + report(t, resize_torch_case(2.0, false)) + report(t, resize_torch_case(1.5, false)) + report(t, resize_torch_case(0.5, false)) + } + t |> run("the resampler bars' must-red controls") @(t : T?) { + report_poison(t, resize_onnx_case(1.5, true)) + report_poison(t, resize_torch_case(1.5, true)) + } +} + +[test] +def test_tts_magnitude_phase(t : T?) { + t |> run("magnitude_phase against an fp64 reference, the zero-imaginary branch planted at both zero signs") @(t : T?) { + report(t, mp_case(false)) + } + t |> run("the magnitude_phase bar's must-red control") @(t : T?) { + report_poison(t, mp_case(true)) + } +} + +[test] +def test_tts_istft_envelope(t : T?) { + t |> run("the istft window-envelope divide against an fp64 envelope, and its zero-envelope guard") @(t : T?) { + report(t, istft_case(200l, 32l, 8l, 16l, false, false)) + report(t, istft_case(137l, 24l, 7l, 9l, false, false)) + report(t, istft_case(96l, 32l, 8l, 0l, false, false)) + report(t, istft_case(200l, 32l, 8l, 16l, true, false)) + } + t |> run("the envelope bars' must-red controls") @(t : T?) { + report_poison(t, istft_case(200l, 32l, 8l, 16l, false, true)) + report_poison(t, istft_case(200l, 32l, 8l, 16l, true, true)) + } +} diff --git a/modules/dasLLAMA/tests/test_tower_helpers.das b/modules/dasLLAMA/tests/test_tower_helpers.das index 49acf1900b..cc33cc6f86 100644 --- a/modules/dasLLAMA/tests/test_tower_helpers.das +++ b/modules/dasLLAMA/tests/test_tower_helpers.das @@ -8,7 +8,7 @@ require dasllama/dasllama_math // setup_dasllama_jobque_ require dasllama/dasllama_rope // build_rope_tabs_rows — the 2-axis rope's table source require dasllama/dasllama_plane require daslib/jobque_boost -require llvm/daslib/f16_cvt // the binary16 round-trip the geglu table property checks +require daslib/f16_cvt // the binary16 round-trip the geglu table property checks require math // Model-free unit tests for the encoder-tower helpers both the audio (gemma4a) and vision diff --git a/modules/dasLLAMA/tests/test_tts_g2p.das b/modules/dasLLAMA/tests/test_tts_g2p.das index 9150152c89..3314c8d7da 100644 --- a/modules/dasLLAMA/tests/test_tts_g2p.das +++ b/modules/dasLLAMA/tests/test_tts_g2p.das @@ -4,6 +4,7 @@ options _dasllama_internal = true require dastest/testing_boost public require dasllama/dasllama_g2p require dasllama/dasllama_postag +require dasllama/dasllama_tts // g2p_pack_path - the directory rule the facade loads by require strings require daslib/strings_boost require daslib/fio @@ -11,6 +12,7 @@ require dasllama/dasllama_unicode // utf8_to_cpts: the inventory sweep reads c require _tts_corpus require _tts_corpus_gb require _model_tier // models_dir(): tts_g2p.bin and tts_postag.bin live in the model store +require daslib/rtti // this_context().last_exception: the American-only pack's British refusal is read as text // TTS grapheme-to-phoneme against the 200-sentence corpus: phoneme-identical with the reference // front end (arm E of the fidelity experiment, fed the same normalized text), the heteronym @@ -246,6 +248,67 @@ def private write_pack_header(path : string; version : int) { fclose(f) } +[test] +def test_us_only_pack(t : T?) { + t |> run("the American-only pack reads the corpus as the full pack does and refuses British") @(t : T?) { + return if (!data_present(t)) + let us_path = path_join(models_dir(), "tts_g2p_en_us.bin") + if (!stat(us_path).is_valid) { + t |> skip("tts_g2p_en_us.bin absent in {models_dir()}") + return + } + let full <- load_g2p(g2p_path()) + let us <- load_g2p(us_path) + let tg <- load_pos_tagger(tagger_path()) + t |> success(!full.us_only, "the full pack carries both tiers") + t |> success(us.us_only, "the American pack says so in its source line: {us.source}") + let rows <- load_tts_corpus() + var differs = 0 + for (r in rows) { + let a = g2p_phonemize(full, tg, r.norm) + let b = g2p_phonemize(us, tg, r.norm) + if (a != b) { + differs++ + to_log(LOG_INFO, "tts g2p us-only differs {r.id}: full {a}\n us {b}\n") + } + } + t |> equal(differs, 0, "American readings identical on {length(rows)} sentences") + var refused = "" + try { + let gb = g2p_phonemize(us, tg, "the water is warm", true) + to_log(LOG_INFO, "tts g2p us-only british read: {gb}\n") + } recover { + refused = this_context().last_exception + } + t |> success(find(refused, "American tier only") >= 0, "British on the American pack is refused by name: {refused}") + let gb_full = g2p_phonemize(full, tg, "the water is warm", true) + t |> success(!empty(gb_full), "the control: the full pack reads British") + } + t |> run("the directory rule: the full pack wins where both sit, the American twin serves alone, an empty directory panics") @(tt : T?) { + return if (!data_present(tt)) + let us_path = path_join(models_dir(), "tts_g2p_en_us.bin") + if (!stat(us_path).is_valid) { + tt |> skip("tts_g2p_en_us.bin absent in {models_dir()}") + return + } + tt |> equal(g2p_pack_path(models_dir()), g2p_path(), "both packs beside the models: the full pack wins") + let only_us = tmp_path("packs_us") + mkdir(only_us) + write_pack_header(path_join(only_us, "tts_g2p_en_us.bin"), 2) // the path rule stats, it never reads + tt |> equal(g2p_pack_path(only_us), path_join(only_us, "tts_g2p_en_us.bin"), "the American twin serves alone") + let none = tmp_path("packs_none") + mkdir(none) + var refused = "" + try { + let p : string = g2p_pack_path(none) + to_log(LOG_INFO, "tts g2p pack path on an empty dir answered {p}\n") + } recover { + refused = this_context().last_exception + } + tt |> success(find(refused, "no phoneme pack") >= 0, "an empty directory panics naming the packs it lacks: {refused}") + } +} + [test] def test_missing_pack(t : T?) { t |> run("a missing pack panics by name") @(t : T?) { diff --git a/modules/dasLLAMA/tests/test_tts_kitten.das b/modules/dasLLAMA/tests/test_tts_kitten.das index 37dd6ce43e..187f589d1b 100644 --- a/modules/dasLLAMA/tests/test_tts_kitten.das +++ b/modules/dasLLAMA/tests/test_tts_kitten.das @@ -10,6 +10,7 @@ require dasllama/dasllama_tts_types require dasllama/dasllama_tts_blocks require dasllama/dasllama_styletts2 require dasllama/dasllama_image // image_path_for - the two lanes' file identities +require dasllama/dasllama_env // g_env_engine.image: the by-path load needs an image on disk require dasllama/dasllama_math // setup_dasllama_jobque_ require daslib/jobque_boost require daslib/defer @@ -263,7 +264,7 @@ def test_kitten_generator_hook(t : T?) { } } -[test] +[test, arch(at = "../ARCHITECTURE_TTS.md#tts-image-rail")] def test_kitten_image(t : T?) { t |> run("nano: the staged read, the mapped image and the in-memory mint agree on both lanes") @(t : T?) { if (!stat(gguf_path("nano")).is_valid) { @@ -291,6 +292,41 @@ def test_kitten_image(t : T?) { let q8_img = image_path_for(gguf_path("nano"), ST2_IMAGE_TAG_Q8) t |> success(f32_img != q8_img, "each lane is its own prepared image:\n {f32_img}\n {q8_img}") } + t |> run("nano: the image loads by its own path, the family data riding its meta") @(t : T?) { + if (!stat(gguf_path("nano")).is_valid) { + t |> skip("kitten-nano.gguf not present") + return + } + if (!g_env_engine.image) { + t |> skip("DASLLAMA_IMAGE=0: no image on disk to load by path") + return + } + with_job_que() { + setup_dasllama_jobque_() + var inscope from_gguf <- load_styletts2(gguf_path("nano")) // bakes the q8 sibling when it is not there yet + let img = image_path_for(gguf_path("nano"), ST2_IMAGE_TAG_Q8) + t |> success(stat(img).is_valid, "the q8 lane's image is on disk: {img}") + set_styletts2_q8(false) + defer() { + reset_styletts2_q8() + } + var inscope m <- load_styletts2(img) + t |> success(m.q8, "the file's lane is the served lane") + t |> equal(length(m.voice_names), 8) + t |> equal(length(m.kitten.aliases), length(from_gguf.kitten.aliases), "the aliases rode the image ({length(m.kitten.aliases)})") + t |> equal(length(m.kitten.speed_priors), length(from_gguf.kitten.speed_priors), "the speed priors rode the image ({length(m.kitten.speed_priors)})") + t |> success(!empty(m.kitten.aliases) && !empty(m.kitten.speed_priors), "the gguf carries family data to compare") + for (an, av in keys(m.kitten.aliases), values(m.kitten.aliases)) { + t |> equal(from_gguf.kitten.aliases?[an] ?? "", av, "alias {an}") + } + if (stat(path_join(models_dir(), "tts_g2p.bin")).is_valid && stat(path_join(models_dir(), "tts_postag.bin")).is_valid) { + var inscope tm <- load_tts_model(img) + var inscope c <- caps(tm) + var inscope a <- synthesize(tm, "The image spoke.", c.voices[length(c.voices) - 1], 1.0) + t |> success(long_length(a.pcm) > 8000l, "the facade synthesized {long_length(a.pcm)} samples from the image") + } + } + } } // ARCHITECTURE_TTS.md sec.2.32: q8 is the served default, f32 the reference lane the parity diff --git a/modules/dasLLAMA/tests/test_tts_kokoro.das b/modules/dasLLAMA/tests/test_tts_kokoro.das index 22db4a7f8b..e09bce79f9 100644 --- a/modules/dasLLAMA/tests/test_tts_kokoro.das +++ b/modules/dasLLAMA/tests/test_tts_kokoro.das @@ -54,7 +54,7 @@ def private panic_text_of(blk : block<() : void>) : string { // its dialect pick taken out, so a synthesis can be held against the string it should have read. // Noise never moves the length, so the seed here need not be the facade's. def private samples_for(var m : TtsModel; ps : string; voice : string) : int64 { - var inscope ids <- kokoro_token_ids(m.kokoro, ps) + var inscope ids <- kokoro_token_ids(m.model.kokoro, ps) var inscope style : array styletts2_style(m.model, voice, kokoro_style_row(ps, m.model.style_rows), style) var tm = TtsTimings() @@ -131,7 +131,10 @@ def test_kokoro_oracle(t : T?) { return } delete g_family - g_family <- load_kokoro_family(gguf_path()) + { + var inscope staged <- load_styletts2(gguf_path()) + g_family := staged.kokoro + } t |> success(length(g_family.sym) > 100, "{length(g_family.sym)} symbols in the vocabulary") var inscope ids <- kokoro_token_ids(g_family, "hˈɛlO wˈɜɹld") // nolint:STYLE039 t |> equal(length(ids), 14, "two pads around twelve known symbols") @@ -210,8 +213,8 @@ def test_kokoro_dialects(t : T?) { // every British symbol has to be in the model's own vocabulary, or the token // builder drops it silently and the voice speaks a shortened line var inscope cps <- utf8_to_cpts(ps_gb) - var inscope ids_gb <- kokoro_token_ids(m.kokoro, ps_gb) - var inscope ids_us <- kokoro_token_ids(m.kokoro, ps_us) + var inscope ids_gb <- kokoro_token_ids(m.model.kokoro, ps_gb) + var inscope ids_us <- kokoro_token_ids(m.model.kokoro, ps_us) t |> equal(length(ids_gb), length(cps) + 2, "no British symbol falls out of the vocabulary") var same_tokens = length(ids_us) == length(ids_gb) for (a, b in ids_us, ids_gb) { diff --git a/modules/dasLLAMA/tests/test_vulkan_dec_tail.das b/modules/dasLLAMA/tests/test_vulkan_dec_tail.das index 84961aaee0..c1f43375d1 100644 --- a/modules/dasLLAMA/tests/test_vulkan_dec_tail.das +++ b/modules/dasLLAMA/tests/test_vulkan_dec_tail.das @@ -12,7 +12,7 @@ require dasllama/dasllama_common // MoeGate, select_decode_override require dasllama/dasllama_blocks // registers the vulkan_moe_span decode override at init require dasllama/dasllama_moe // moe_select_core: the host select the span's device top-k mirrors require ?vulkan dasllama/dasllama_math_vulkan // the decode attention block + FFN tail under test; the test installs the hooks explicitly (no env gate) -require llvm/daslib/f16_cvt +require daslib/f16_cvt require math require _vk_kq_fixtures diff --git a/modules/dasLLAMA/tests/test_vulkan_moe_cm2.das b/modules/dasLLAMA/tests/test_vulkan_moe_cm2.das index cd909992a1..e0791c1668 100644 --- a/modules/dasLLAMA/tests/test_vulkan_moe_cm2.das +++ b/modules/dasLLAMA/tests/test_vulkan_moe_cm2.das @@ -7,7 +7,7 @@ options _dasllama_internal = true require dastest/testing_boost public require dasllama/dasllama_math require ?vulkan dasllama/dasllama_math_vulkan // the MoE GPU tier under test; the test installs its hooks explicitly (no env gate) -require llvm/daslib/f16_cvt +require daslib/f16_cvt require math require _vk_kq_fixtures require dasllama/dasllama_vulkan_common // find_stack_idx: the fresh-base guard diff --git a/modules/dasLLAMA/tests/test_vulkan_tier.das b/modules/dasLLAMA/tests/test_vulkan_tier.das index 4546f9c714..dc4d3a9e50 100644 --- a/modules/dasLLAMA/tests/test_vulkan_tier.das +++ b/modules/dasLLAMA/tests/test_vulkan_tier.das @@ -7,7 +7,7 @@ require dastest/testing_boost public require dasllama/dasllama_math require ?vulkan dasllama/dasllama_math_vulkan // the MoE GPU tier under test; the test installs its hooks explicitly (no env gate) require ?vulkan dasllama/dasllama_vulkan_seams // the single-op seams (vk_add_rms / rope_kv / decode_attn) on the class rail -require llvm/daslib/f16_cvt +require daslib/f16_cvt require math require _vk_kq_fixtures // the synthetic kq stacks + pow2 scale sets (shared with test_vulkan_moe_cm2) diff --git a/modules/dasLLVM/.das_module b/modules/dasLLVM/.das_module index e2a22f0cad..d2e55ab784 100644 --- a/modules/dasLLVM/.das_module +++ b/modules/dasLLVM/.das_module @@ -10,9 +10,6 @@ def initialize(project_path : string) { "llvm_jit_intrin", "llvm_jit_common", "llvm_jit_lower", "llvm_dll_utils", "llvm_exe", "llvm_macro", "llvm_jit_cli", "llvm_jit_run", "llvm_aot", "llvm_env", // [EnvConfig] environment-knob registry (ENVIRONMENT.md generates from it) - "aarch64_neon", // public NEON-intrinsic header (portable fallbacks + name-based JIT recognition) - "x64_avx", // public x86-64-intrinsic header (same contract, x64 mirror) - "f16_cvt", // public f16<->f32 convert header (same contract, aarch64 + x64/F16C) "llvm_code", // [llvm_code] user-side annotation "llvm_jit_code", // [llvm_code] JIT-side generator registry "llvm_user_modules", // user-extensible require list for generator modules @@ -28,4 +25,10 @@ def initialize(project_path : string) { for (path in bindings_paths) { register_native_path("llvm", "bindings/{path}", "{project_path}/bindings/{path}.das") } + if (das_is_dll_build()) { + // the `llvm` witness module (src/dasLLVM.cpp) exists only in a build configured with + // dasLLVM; the 2-arg form skips silently when the .shared_module was never built, and + // that silence is what `require ?llvm` reads as "this build has no LLVM" + register_dynamic_module("{project_path}/dasModuleLLVM.shared_module", "Module_LLVM") + } } diff --git a/modules/dasLLVM/ARCHITECTURE.md b/modules/dasLLVM/ARCHITECTURE.md index 5b4c7594cd..492f3e42c7 100644 --- a/modules/dasLLVM/ARCHITECTURE.md +++ b/modules/dasLLVM/ARCHITECTURE.md @@ -135,10 +135,18 @@ An aarch64 host target reads its CPU features from two sources, because neither the features - and a part this LLVM cannot name maps to the generic CPU, where SDOT and SMMLA have no instruction to select and codegen aborts. `cpu_supports` reads the operating system instead (sysctl / `AT_HWCAP` / `IsProcessorFeaturePresent`), so it answers for silicon LLVM has -never heard of. Both the tier gates (`init_jit_target_flags`) and the target machine's feature -string (`create_default_target_machine`) therefore take the union of the two: an LLVM host-string -hit OR a `cpu_supports` hit (fullfp16 additionally reads darwin-arm64 as always-on - every -Apple Silicon part has it). A cross-compile triple takes neither - only the force env. +never heard of. The tier gates (`init_jit_target_flags` - `g_target_arm64_dotprod`, `_i8mm`, +`_fullfp16`) and the target machine's feature string (`create_default_target_machine`) therefore +take the union of the two: an LLVM host-string hit OR a `cpu_supports` hit (fullfp16 additionally +reads darwin-arm64 as always-on - every Apple Silicon part has it). One asymmetry: the host rail's +machine string carries `+dotprod` unconditionally (every part the JIT has run on has it), while the +DotProd GATE probes like its siblings - on an ARMv8.0 host the gate declines and the `sdot4` family +compiles its fallback, whatever the string says. A cross-compile triple takes neither - only the force env - and so +does a generic-CPU standalone exe (one carrying no `[llvm_code]` kernel): its machine is the +ARMv8.0 baseline, which cannot select SDOT or SMMLA, so the DotProd and i8mm gates +(`g_target_arm64_dotprod`, `g_target_arm64_i8mm`) stay off there and every `aarch64_neon` call +that needs either compiles its daslang fallback body. The gates and the machine string are one +truth on both rails: a force-env feature raises the gate AND is appended to the generic machine. The two ways a feature reaches the target machine's string license different things. A detection-derived append - `+dotprod` always, `+i8mm` when `cpu_supports` confirms it - is @@ -151,19 +159,23 @@ have, so the artifact is for another machine and executing it here traps. `[tuned]` and `[tune_policy]` stamp a function's hints at macro time out of the tune sidecar, and the module cache stores the stamped AST. A re-mint therefore has to invalidate the cached record, or a later run serves stamps minted against the old sidecar until some source file -changes. `read_manifest` (`daslib/llvm_tune.das`) registers the sidecar path with -`add_module_cache_dependency` on every read; the record carries the path with the file's byte -size and content hash, and the reader re-validates both before it trusts the payload. Content, -not mtime: an app that rewrites its sidecar byte-identically on exit must not churn the cache. +changes. The stamping paths (`tune_apply`, `tune_kernel_pick`) register the sidecar path with +`add_module_cache_dependency` through `pin_module_cache_dependency` before they read it; the +record carries the path with the file's byte size and content hash, and the reader re-validates +both before it trusts the payload. Content, not mtime: an app that rewrites its sidecar +byte-identically on exit must not churn the cache. The registration runs before the staleness gate, and for a path that does not exist yet, because the mints that matter most produce no successful read - the first mint has no sidecar, and a re-mint replaces one the gate rejected. An absent file registers as size -1 and hash 0, -which the next run's re-validation sees change. Registering is a no-op outside compilation, so -the manifest's runtime readers reach the same call unconditionally. +which the next run's re-validation sees change. The pin sits beside the read, not inside +`read_manifest`: the runtime shares that reader (the box-profile pin at load, `tune_status`), +and a standalone exe binds every extern its functions name at startup, so a reader carrying the +`ast_core` extern would drag the compiler module into every exe - and a wasm cross-link, which +sees only the compiler-free runtime archive, has nothing to bind it to. The shipped defaults profiles are the same kind of input: with no sidecar entry a kernel -stamps its class entry out of `/.tune-defaults.json`, so `locate_profile_doc` +stamps its class entry out of `/.tune-defaults.json`, so `pin_profile_chain` registers every candidate on the class ladder it tries, existing or not - a profile that appears, or is re-exported after a re-mint, must invalidate the stamps minted without it. The staleness gate itself compares the sidecar's mtime with the running binary's, which no content @@ -269,3 +281,20 @@ vecmath carries no vector sinh/cosh/tanh, so `SimPolicy` binds `vsinh`/`vcosh`/` exp polynomial's error rather than agreeing with them - the opposite trade from every other emitter on the rail, taken because the consumer (GELU over float4 rows) otherwise pays four libm calls per vector. `tests/llvm_vector_math.das` asserts the size of that divergence. + +## 9. The idot family's target lowerings {#idot-lowerings} + +The exact integer dots on the 8-bit lattice have three lowerings, picked by target: one +`@llvm.aarch64.neon.sdot` where the target has DotProd (`g_target_arm64_dotprod` - the host rail's +`+dotprod` append, or the force env on the generic rail), the SIMD128 form on a wasm target +(`idot_wasm_simd128`), generic widen-multiply IR everywhere else. The native arms exist because +neither backend produces them from the generic form: AArch64 expands it to zip/uzp/smull instead +of folding to SDOT, and the wasm backend runs it a fifth as fast. The wasm form is the ISA's two +halves of an int8 dot, `i16x8.extmul_{low,high}_i8x16_s` (what LLVM makes of `mul(sext, sext)`) and +`i32x4.extadd_pairwise_i16x8_s`; the pairwise sums land as byte pairs, and one even/odd shuffle-add +folds them into the quad lanes the generic form defines - exact for every int8 lane. The +relaxed-SIMD dot (`i32x4.relaxed_dot_i8x16_i7x16_add_s`) is NOT used: its second operand is 7-bit, +so the sign trick that would feed it (`dot(w, x) == dot(sign(x)*w, |x|)`) wraps at -128 in either +operand and answers the wrong sign there, and `+relaxed-simd` is a whole-module switch that also +turns float-vector `min`/`max` and `mad` into engine-defined instructions (NaN and signed-zero +answers, fusion) - the feature string stays `+simd128,+nontrapping-fptoint`, the runtime archive's. diff --git a/modules/dasLLVM/CMakeLists.txt b/modules/dasLLVM/CMakeLists.txt index 0f26296b24..aa2498fb45 100644 --- a/modules/dasLLVM/CMakeLists.txt +++ b/modules/dasLLVM/CMakeLists.txt @@ -7,6 +7,14 @@ IF ((NOT DAS_LLVM_INCLUDED) AND ((NOT ${DAS_LLVM_DISABLED}) OR (NOT DEFINED DAS_ SET(DAS_LLVM_DIR ${PROJECT_SOURCE_DIR}/modules/dasLLVM) + # the `llvm` witness module (src/dasLLVM.cpp): compiled in exactly when this package is, so + # `require ?llvm ` follows the configure - the das files below are on disk in every + # checkout and would otherwise satisfy a path guard in a build without LLVM + ADD_MODULE_CPP(LLVM) + ADD_MODULE_LIB(libDasModuleLLVM dasModuleLLVM ${DAS_LLVM_DIR}/src/dasLLVM.cpp) + SETUP_CPP11(libDasModuleLLVM) + SETUP_CPP11(dasModuleLLVM) + LIST(APPEND CMAKE_MODULE_PATH ${DAS_LLVM_DIR}) set(DAS_LLVM_LIB_OUTPUT "${PROJECT_SOURCE_DIR}/lib") diff --git a/modules/dasLLVM/REVIEW.das b/modules/dasLLVM/REVIEW.das index ee9ac057a1..47fe021b91 100644 --- a/modules/dasLLVM/REVIEW.das +++ b/modules/dasLLVM/REVIEW.das @@ -26,9 +26,10 @@ var private SINK_CALLS <- { "fwrite", "copy_file", "copy_file_result", "set_mtime", "set_mtime_result" } -// The require category every register_native_path row in the descriptor carries; each of its -// other identifier-shaped quoted tokens names a module file. -var private DESCRIPTOR_CATEGORY_TOKENS <- { "llvm" } +// The require category every register_native_path row in the descriptor carries, and the C++ +// witness module's registration name (register_dynamic_module - a .shared_module, no .das file); +// each of the descriptor's other identifier-shaped quoted tokens names a module file. +var private DESCRIPTOR_CATEGORY_TOKENS <- { "llvm", "Module_LLVM" } def private is_ident_byte(b : int) : bool { return is_alnum(b) || b == '_' diff --git a/modules/dasLLVM/REVIEW.md b/modules/dasLLVM/REVIEW.md index 46cc279952..1344bdc807 100644 --- a/modules/dasLLVM/REVIEW.md +++ b/modules/dasLLVM/REVIEW.md @@ -89,8 +89,9 @@ `ConstantExpr`, and the setter's cast writes through the wrong type into it (`ARCHITECTURE.md#gep-constant-fold`). -- **A feature name used in a `requires=` list or a `g_target_x64_*` gate has its cpuid line in - `das_cpu_supports` (`src/builtin/module_builtin_runtime.cpp`, repo root) in the same diff** +- **A feature name used in a `requires=` list or a `g_target_*` tier gate + (`daslib/llvm_jit_common.das`) has its cpuid line in `das_cpu_supports` + (`src/builtin/module_builtin_runtime.cpp`, repo root) in the same diff** (`ARCHITECTURE.md#x64-tier-gates`). A name the cpuid table does not know answers false on every box, so every perm that requires it silently declines to its fallback and no error names the cause. @@ -101,13 +102,22 @@ sign-alternating chain moves the last few bits of the result, and the interpreter and AOT answers do not move with it. -- **A diff that adds or changes a `build_vector_*` emitter also adds two cells to - `tests/llvm_vector_math.das` (beside this file): one comparing the emitted result lane for - lane with the interpreted result, at every vector width the emitter serves, and one asserting - both answer NaN in the same lanes.** A clamp or a conversion written with ordered compares - turns a NaN lane into a number, and an accuracy bound reads that as success. - -- **Weakening `REVIEW.das` (beside this file) is a defect:** dropping a check, dropping a - directory from its tracked-fixture list or removing the last tracked file under one (a guard - over nothing), or a finding text that no longer names what failed. What the gate enforces is - read from the gate itself. +- **A diff that adds or changes an intrinsic emitter whose daslang body is the reference + implementation - a `build_vector_*` emitter, an `idot` lowering - also adds a cell comparing + the emitted result with the interpreted result over the operand range the emitter serves + (every vector width for `build_vector_*`, the full int8 lattice for a dot), and for a float + emitter one asserting both answer NaN in the same lanes; a lowering only a cross target runs + states in the PR body the artifact that compared them.** A clamp or a conversion written with + ordered compares turns a NaN lane into a number, and an accuracy bound reads that as success; + an IR-shape test names the instruction and never a number. + +- **A change that makes `REVIEW.das` (beside this file) report fewer inputs is a defect:** + dropping a check, shrinking a scanned set or a tracked-fixture directory (a guard over nothing), + widening an exemption list without naming the exempted input's reason beside it, or a finding + text that no longer names what failed. What the gate enforces is read from the gate itself. + +- **A diff that builds an x64 or aarch64 machine's feature string appends the forced ones + (`x64_forced_plus_features` / `arm64_forced_plus_features`, `daslib/llvm_jit_common.das`) AFTER + the detected host features.** LLVM's `SubtargetFeatures` takes the last occurrence of a name, so + a forced feature placed first is silently overridden by detection. A wasm machine has no force + knob and no detected features, so the rule does not reach it. diff --git a/modules/dasLLVM/daslib/llvm_code.das b/modules/dasLLVM/daslib/llvm_code.das index 355b95df37..14387ff4cc 100644 --- a/modules/dasLLVM/daslib/llvm_code.das +++ b/modules/dasLLVM/daslib/llvm_code.das @@ -14,22 +14,21 @@ require daslib/ast_boost //! IR instead of the body. The body stays the reference implementation — the interpreter, AOT, //! and any target the generator declines compile it verbatim. Extra annotation arguments are //! the generator's parameters (and fold into the JIT DLL cache key). -[function_macro(name = "llvm_code")] -class private LlvmCodeAnnotation : AstFunctionAnnotation { - def override apply(var func : FunctionPtr; var group : ModuleGroup; - args : AnnotationArgumentList; var errors : das_string) : bool { - for (arg in args) { - if (arg.name == "name") { - if (arg.basicType != Type.tString || empty(arg.sValue)) { - errors := "[llvm_code] on {func.name}: `name` must be a non-empty string (the generator key)" - return false - } - // force side effects, or constant-argument calls const-fold in the front-end and never reach the generator - func.sideEffectFlags.userScenario = true - return true +//! The annotation itself is registered by `daslib/tune` (inert without dasLLVM); this is the +//! body its shell forwards to. +def llvm_code_apply(var func : FunctionPtr; var _group : ModuleGroup; + args : AnnotationArgumentList; var errors : das_string) : bool { + for (arg in args) { + if (arg.name == "name") { + if (arg.basicType != Type.tString || empty(arg.sValue)) { + errors := "[llvm_code] on {func.name}: `name` must be a non-empty string (the generator key)" + return false } + // force side effects, or constant-argument calls const-fold in the front-end and never reach the generator + func.sideEffectFlags.userScenario = true + return true } - errors := "[llvm_code] on {func.name}: missing name=\"generator_key\" argument" - return false } + errors := "[llvm_code] on {func.name}: missing name=\"generator_key\" argument" + return false } diff --git a/modules/dasLLVM/daslib/llvm_exe.das b/modules/dasLLVM/daslib/llvm_exe.das index 1dc887ec8e..7152a1c15b 100644 --- a/modules/dasLLVM/daslib/llvm_exe.das +++ b/modules/dasLLVM/daslib/llvm_exe.das @@ -301,8 +301,8 @@ class public CollectExternVisitor : AstVisitor { if (empty(cpp_name)) return let reg_fn_name = "jit_register_{cpp_name}" to_log(LOG_INFO, "LLVM EXE: NEED_MODULE({cpp_name}) for `{mod_name}`\n") - // ast_core / network_core live in the compiler lib, so their jit_register thunk forces a whole-compiler-lib link. - if (mod_name == "ast_core" || mod_name == "network_core") { + // ast_core / ast / network_core / network live in the compiler lib, so their jit_register thunk forces a whole-compiler-lib link. + if (mod_name == "ast_core" || mod_name == "ast" || mod_name == "network_core" || mod_name == "network") { needs_whole_lib = true } // One call per thunk process-wide (not idempotent); get-or-add avoids a silent rename to an undefined symbol. @@ -918,9 +918,9 @@ def private emit_module_registration(m : Module?; dynamic_modules : table key_exists(mod_name)) reg_visited[mod_name] = true - // dylibs register via the runtime load path; ast_core / network_core / jit are prophylactic-skipped here (a real caller still gets them via ensure_module + needs_whole_lib). + // dylibs register via the runtime load path; the compiler-lib modules (ast_core, ast, network_core, network, jit) are prophylactic-skipped here (a real caller still gets them via ensure_module + needs_whole_lib) - an engine's macro-time require of daslib/ast reaches them with no runtime caller, and the wasm runtime archive has no thunk for them. return if (dynamic_modules |> key_exists(mod_name) - || mod_name == "ast_core" || mod_name == "network_core" || mod_name == "jit") + || mod_name == "ast_core" || mod_name == "ast" || mod_name == "network_core" || mod_name == "network" || mod_name == "jit") module_for_each_dependency(m) $(var dep : Module?; var _pub : bool) { emit_module_registration(dep, dynamic_modules, emitted_reg, reg_visited, builder, g_mod, register_mod_type, has_cpp_modules) diff --git a/modules/dasLLVM/daslib/llvm_jit_code.das b/modules/dasLLVM/daslib/llvm_jit_code.das index 4cf992501d..e6772691e9 100644 --- a/modules/dasLLVM/daslib/llvm_jit_code.das +++ b/modules/dasLLVM/daslib/llvm_jit_code.das @@ -9,8 +9,8 @@ require llvm/bindings/llvm_struct require llvm/daslib/llvm_jit_intrin require daslib/ast_boost -//! JIT-side half of the `[llvm_code]` rail (the user-side annotation lives in -//! `llvm/daslib/llvm_code.das`): a name-keyed registry of das generator functions that emit a +//! JIT-side half of the `[llvm_code]` rail (the user-side annotation registers in +//! `daslib/tune.das`, forwarding to `llvm_code_apply` in `llvm/daslib/llvm_code.das`): a name-keyed registry of das generator functions that emit a //! function's LLVM IR wholesale. Generator modules compile into the JIT's context via the //! require list in `llvm/daslib/llvm_user_modules.das` and register themselves from `[init]`. diff --git a/modules/dasLLVM/daslib/llvm_jit_common.das b/modules/dasLLVM/daslib/llvm_jit_common.das index 908f849619..e21bb93a2e 100644 --- a/modules/dasLLVM/daslib/llvm_jit_common.das +++ b/modules/dasLLVM/daslib/llvm_jit_common.das @@ -444,6 +444,9 @@ var public g_target_x64_amx = false // targets. Gates emission that calls Linux-only externs (the amx witness's arch_prctl). var public g_target_os_linux = false +// ARMv8.2 DotProd (SDOT — the aarch64_neon sdot4 family, the idot ss form); rails: ARCHITECTURE.md#aarch64-feature-truth +var public g_target_arm64_dotprod = false + // ARMv8.6 i8mm (SMMLA — 2×2 s8 matrix-multiply-accumulate; Apple M2/A15+, Graviton3+; M1 has // DotProd only). Host truth is LLVMGetHostCPUFeatures "+i8mm" OR cpu_supports (ARCHITECTURE.md#aarch64-feature-truth). var public g_target_arm64_i8mm = false @@ -462,6 +465,11 @@ def private arm64_forced_feature(name : string) : bool { return false } +def public arm64_forced_plus_features() : string { + let plus <- [for (part in split(g_env_jit.jit_arm64_force_features, ",")); "+{part}"; where !empty(part)] + return join(plus, ",") +} + // "+name" present in LLVM's host-features string (comma-separated "+feat"/"-feat" entries; // exact match on the enabled spelling, so a "-i8mm" entry never reads as support) def private host_llvm_feature(name : string) : bool { @@ -512,6 +520,7 @@ var public g_target_wasm_threads = false // mismatch. Diagnoses the MSVC-host vs wasm/clang-target C++ ABI layout class. Opt-in, off by default. var public g_jit_check_handled_abi = false +[arch(at = "../ARCHITECTURE.md#idot-lowerings")] def private wasm_target_features() : string { return g_target_wasm_threads ? "+simd128,+nontrapping-fptoint,+atomics,+bulk-memory" : "+simd128,+nontrapping-fptoint" } @@ -634,6 +643,8 @@ def public init_jit_target_flags(target_triple : string = ""; host_features : bo g_target_os_linux = (empty(target_triple) ? get_platform_name() == "linux" : find(target_triple, "linux") >= 0) + g_target_arm64_dotprod = g_target_is_aarch64 && (arm64_forced_feature("dotprod") + || (empty(target_triple) && host_features && (host_llvm_feature("dotprod") || cpu_supports("dotprod")))) g_target_arm64_i8mm = g_target_is_aarch64 && (arm64_forced_feature("i8mm") || (empty(target_triple) && host_features && (host_llvm_feature("i8mm") || cpu_supports("i8mm")))) // aarch64 fullfp16 (see declaration comment): darwin-arm64 == Apple Silicon == always. @@ -922,20 +933,18 @@ def public create_default_target_machine(opt_level : uint; use_host_cpu : bool) feats = "{feats},+i8mm" } // a forced feature's artifact cache-hits on the target box without a linker there - let forced_arm <- [for (part in split(g_env_jit.jit_arm64_force_features, ",")); "+{part}"; where !empty(part)] + let forced_arm = arm64_forced_plus_features() if (!empty(forced_arm)) { - feats = "{feats},{join(forced_arm, ",")}" + feats = "{feats},{forced_arm}" } targetMachine = create_target_machine(triple_msg, cpu_msg, feats, opt_level) - } elif (use_host_cpu && g_target_is_x64 && !empty(x64_forced_plus_features())) { - // DAS_JIT_X64_FORCE_FEATURES emission-only override (see the g_target_x64_* declaration - // comment): append AFTER the host features — the last occurrence of a feature wins in - // LLVM's SubtargetFeatures, so "+avx512f" here overrides the host list's "-avx512f". - let forced = x64_forced_plus_features() - let feats = empty(features_msg) ? forced : "{features_msg},{forced}" - targetMachine = create_target_machine(triple_msg, cpu_msg, feats, opt_level) } else { - targetMachine = create_target_machine(triple_msg, cpu_msg, features_msg, opt_level) + let forced = g_target_is_x64 ? x64_forced_plus_features() : (g_target_is_aarch64 ? arm64_forced_plus_features() : "") + let feats = empty(forced) ? features_msg : (empty(features_msg) ? forced : "{features_msg},{forced}") + if (!empty(forced)) { + to_log(LOG_INFO, "LLVM JIT: {g_target_is_x64 ? "DAS_JIT_X64_FORCE_FEATURES" : "DAS_JIT_ARM64_FORCE_FEATURES"} appends {forced} to the generic machine {cpu_msg}\n") + } + targetMachine = create_target_machine(triple_msg, cpu_msg, feats, opt_level) } if (use_host_cpu) { diff --git a/modules/dasLLVM/daslib/llvm_jit_intrin.das b/modules/dasLLVM/daslib/llvm_jit_intrin.das index cc8b2df1bd..41c3d7b754 100644 --- a/modules/dasLLVM/daslib/llvm_jit_intrin.das +++ b/modules/dasLLVM/daslib/llvm_jit_intrin.das @@ -179,17 +179,21 @@ let g_intrin_lookup <- { "math::normalize" => @@intrinsic_math_normalize } -// AArch64 NEON intrinsics (the llvm/aarch64_neon header), consulted only when the JIT target is +// AArch64 NEON intrinsics (the daslib/aarch64_neon module), consulted only when the JIT target is // aarch64. On any other target has_intrinsic returns false for these keys and the call's own // daslang body — the portable fallback in the declaring module — compiles as a normal function. // That silent degrade-to-fallback contract holds for every table in this section. let g_aarch64_intrin_lookup <- { + "aarch64_neon::tbl16_lo" => @@intrinsic_tbl16, + "aarch64_neon::tbl16_hi" => @@intrinsic_tbl16 +} +let g_aarch64_dotprod_intrin_lookup <- { // ARMv8.2 DotProd "aarch64_neon::sdot4" => @@intrinsic_sdot4, "aarch64_neon::sdot4_laneq" => @@intrinsic_sdot4_laneq, "aarch64_neon::sdot4_w" => @@intrinsic_sdot4_w, - "aarch64_neon::sdot4_laneq_w" => @@intrinsic_sdot4_laneq_w, - "aarch64_neon::tbl16_lo" => @@intrinsic_tbl16, - "aarch64_neon::tbl16_hi" => @@intrinsic_tbl16, + "aarch64_neon::sdot4_laneq_w" => @@intrinsic_sdot4_laneq_w +} +let g_aarch64_i8mm_intrin_lookup <- { // ARMv8.6 i8mm "aarch64_neon::smmla" => @@intrinsic_smmla } @@ -223,7 +227,7 @@ let g_x64_avx512vnni_intrin_lookup <- { // zmm VPDPBUSD (the bw gate is a prer "x64_avx::dot64_acc16_vnni" => @@intrinsic_dot64_acc16_vnni_x64 } -// Half<->float converts (the llvm/daslib/f16_cvt module) — one TARGET-INDEPENDENT table, consulted +// Half<->float converts (the daslib/f16_cvt module) — one TARGET-INDEPENDENT table, consulted // when the target lowers fpext/fptrunc through half in hardware: any aarch64 (scalar fcvt is // ARMv8.0 baseline, no finer gate) or x86-64 with F16C (g_target_x64_f16c). Elsewhere the calls // compile their exact bit-twiddle daslang bodies — a half convert without hardware support would @@ -276,6 +280,16 @@ def public has_intrinsic(expr : ExprCallFunc?) { result = true } } + if (!result && g_target_arm64_dotprod) { + g_aarch64_dotprod_intrin_lookup |> get(call_name) $(_pfun) { + result = true + } + } + if (!result && g_target_arm64_i8mm) { + g_aarch64_i8mm_intrin_lookup |> get(call_name) $(_pfun) { + result = true + } + } if (!result && g_target_x64_avx2) { g_x64_intrin_lookup |> get(call_name) $(_pfun) { result = true @@ -320,6 +334,16 @@ def public lookup_intinsic(g_ctx : LLVMContextRef; g_builder : LLVMOpaqueBuilder result = pfun |> invoke(JitCtx(ctx = g_ctx, builder = g_builder, types = types), expr, arguments) } } + if (result == null && g_target_arm64_dotprod) { + g_aarch64_dotprod_intrin_lookup |> get(call_name) $(pfun) { + result = pfun |> invoke(JitCtx(ctx = g_ctx, builder = g_builder, types = types), expr, arguments) + } + } + if (result == null && g_target_arm64_i8mm) { + g_aarch64_i8mm_intrin_lookup |> get(call_name) $(pfun) { + result = pfun |> invoke(JitCtx(ctx = g_ctx, builder = g_builder, types = types), expr, arguments) + } + } if (result == null && g_target_x64_avx2) { g_x64_intrin_lookup |> get(call_name) $(pfun) { result = pfun |> invoke(JitCtx(ctx = g_ctx, builder = g_builder, types = types), expr, arguments) @@ -1513,10 +1537,6 @@ def intrinsic_math_sinh_cosh_tanh(var ctx : JitCtx; expr : ExprCallFunc?; argume } // ===== the idot family: exact integer dots on the 8-bit lattice ===== -// Two lowerings: the signed x signed form emits ONE @llvm.aarch64.neon.sdot on the aarch64 -// host-features rail (g_target_host_features = exactly when +dotprod is force-appended, see -// init_jit_target_flags); everything else gets generic widen-multiply IR. The AArch64 backend -// does NOT fold the generic form to SDOT (probe: zip/uzp/smull expansion), hence the native arm. // The unsigned x signed form needs USDOT (i8mm — absent on M1); it rides the i8mm un-gate item. def private idot_products(var ctx : JitCtx; expr : ExprCallFunc?; a, b : LLVMOpaqueValue?; aIdx : int) : LLVMOpaqueValue? { // nolint:LINT014 — shared by the idot intrinsics let v16i32 = LLVMVectorType(ctx.types.t_int32, 16u) @@ -1543,9 +1563,55 @@ def private idot_sdot_aarch64(var ctx : JitCtx; acc, a, b : LLVMOpaqueValue?) : return LLVMBuildCall2(ctx.builder, fnTy, decl, callArgs, "idot4") } +// The exact wasm SIMD128 int8 dot: extmul halves + pairwise widening adds + one even/odd fold. Returns +// null when this LLVM lacks the pairwise intrinsic, so callers fall through to the generic lowering. +[arch(at = "../ARCHITECTURE.md#idot-lowerings")] +def private idot_wasm_simd128(var ctx : JitCtx; acc, a, b : LLVMOpaqueValue?) : LLVMOpaqueValue? { + let v8i16 = LLVMVectorType(ctx.types.t_int16, 8u) + let v4i32 = ctx.types.LLVMInt4Type() + let id = LLVMLookupIntrinsicID("llvm.wasm.extadd.pairwise.signed") + var declTypes <- [v4i32] // overloaded on the widened result vector type + var decl = id != 0u ? LLVMGetIntrinsicDeclaration(g_mod, id, declTypes) : null + if (decl == null) { + return null + } + var fnArgTypes <- [v8i16] + var fnTy = LLVMFunctionType(v4i32, fnArgTypes) + let alo = LLVMBuildSExt(ctx.builder, LLVMBuildShuffleVector(ctx.builder, ctx.types, a, a, [0, 1, 2, 3, 4, 5, 6, 7], ""), v8i16, "") + let ahi = LLVMBuildSExt(ctx.builder, LLVMBuildShuffleVector(ctx.builder, ctx.types, a, a, [8, 9, 10, 11, 12, 13, 14, 15], ""), v8i16, "") + let blo = LLVMBuildSExt(ctx.builder, LLVMBuildShuffleVector(ctx.builder, ctx.types, b, b, [0, 1, 2, 3, 4, 5, 6, 7], ""), v8i16, "") + let bhi = LLVMBuildSExt(ctx.builder, LLVMBuildShuffleVector(ctx.builder, ctx.types, b, b, [8, 9, 10, 11, 12, 13, 14, 15], ""), v8i16, "") + var plo <- [LLVMBuildMul(ctx.builder, alo, blo, "idot_lo")] + var phi <- [LLVMBuildMul(ctx.builder, ahi, bhi, "idot_hi")] + let qlo = LLVMBuildCall2(ctx.builder, fnTy, decl, plo, "idot_pairs_lo") + let qhi = LLVMBuildCall2(ctx.builder, fnTy, decl, phi, "idot_pairs_hi") + let even = LLVMBuildShuffleVector(ctx.builder, ctx.types, qlo, qhi, [0, 2, 4, 6], "") + let odd = LLVMBuildShuffleVector(ctx.builder, ctx.types, qlo, qhi, [1, 3, 5, 7], "") + var s = LLVMBuildAdd(ctx.builder, even, odd, "idot4") + if (acc != null) { + s = LLVMBuildAdd(ctx.builder, acc, s, "idot4_acc") + } + return s +} + +[arch(at = "../ARCHITECTURE.md#idot-lowerings")] def intrinsic_lattice_idot(var ctx : JitCtx; expr : ExprCallFunc?; arguments : array) : LLVMOpaqueValue? { let aUnsigned = expr.arguments[0]._type.vectorBaseType == Type.tUInt8 - if (g_target_is_aarch64 && g_target_host_features && !aUnsigned) { + if (g_target_is_wasm && !aUnsigned) { + let v4i32 = ctx.types.LLVMInt4Type() + var s = idot_wasm_simd128(ctx, null, arguments[0], arguments[1]) + if (s != null) { + let rid = LLVMLookupIntrinsicID("llvm.vector.reduce.add") + var rtys <- [v4i32] + var rdecl = rid != 0u ? LLVMGetIntrinsicDeclaration(g_mod, rid, rtys) : null + if (rdecl != null) { + var rtyp = LLVMFunctionType(ctx.types.t_int32, [v4i32]) + var rargs <- [s] + return LLVMBuildCall2(ctx.builder, rtyp, rdecl, rargs, "idot") + } + } + } + if (g_target_arm64_dotprod && !aUnsigned) { let v4i32 = ctx.types.LLVMInt4Type() var s = idot_sdot_aarch64(ctx, LLVMConstNull(v4i32), arguments[0], arguments[1]) if (s != null) { @@ -1577,7 +1643,13 @@ def intrinsic_lattice_idot4(var ctx : JitCtx; expr : ExprCallFunc?; arguments : let hasAcc = length(arguments) == 3 let aIdx = hasAcc ? 1 : 0 let aUnsigned = expr.arguments[aIdx]._type.vectorBaseType == Type.tUInt8 - if (g_target_is_aarch64 && g_target_host_features && !aUnsigned) { + if (g_target_is_wasm && !aUnsigned) { + var s = idot_wasm_simd128(ctx, hasAcc ? arguments[0] : null, arguments[aIdx], arguments[aIdx + 1]) + if (s != null) { + return s + } + } + if (g_target_arm64_dotprod && !aUnsigned) { let v4i32 = ctx.types.LLVMInt4Type() var acc = hasAcc ? arguments[0] : LLVMConstNull(v4i32) var s = idot_sdot_aarch64(ctx, acc, arguments[aIdx], arguments[aIdx + 1]) diff --git a/modules/dasLLVM/daslib/llvm_jit_run.das b/modules/dasLLVM/daslib/llvm_jit_run.das index 0b066b6c25..8c0f1b5f2c 100644 --- a/modules/dasLLVM/daslib/llvm_jit_run.das +++ b/modules/dasLLVM/daslib/llvm_jit_run.das @@ -37,11 +37,11 @@ var LINK_WHOLE_LIB = false // when true, standalone exe links against the whole // invalidates cached DLLs (e.g. edits to llvm_jit.das, llvm_macro.das, llvm_jit_common.das, // runtime helper ABI, default target triple). Cache filenames fold this in, so a bump // makes every previously written DLL miss the cache on the next run and get GC'd. -let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x6dul // the inline polynomial rail carries NaN: tanh selects the operand back over its ordered clamp, and the sincos quadrant / tan octant convert through llvm.fptosi.sat instead of poisoning on NaN and out-of-range (0x6c: aarch64 vector tan/exp2/log2/log/pow join the inline polynomial rail bit-exactly with the interpreter, sinh/cosh/tanh ride the exp one; 0x6b: aarch64 vector sin/cos ride the inline polynomial; 0x6a: srem/urem for 32-bit %; 0x69: every string argument of an extern is substituted, not just the ones which asked) +let LLVM_JIT_CODEGEN_VERSION : uint64 = 0x70ul // the wasm feature string drops +relaxed-simd and the idot family keeps only the exact extmul + extadd_pairwise lowering on wasm SIMD128 (0x6f: the first wasm idot lowering; 0x6e: the aarch64 SDOT / SMMLA tables gate on DotProd / i8mm, not the arch alone, and the force env reaches the generic exe machine (0x6d: the inline polynomial rail carries NaN: tanh selects the operand back over its ordered clamp, and the sincos quadrant / tan octant convert through llvm.fptosi.sat instead of poisoning on NaN and out-of-range (0x6c: aarch64 vector tan/exp2/log2/log/pow join the inline polynomial rail bit-exactly with the interpreter, sinh/cosh/tanh ride the exp one; 0x6b: aarch64 vector sin/cos ride the inline polynomial; 0x6a: srem/urem for 32-bit %; 0x69: every string argument of an extern is substituted, not just the ones which asked) // Read by tests-cpp/small/test_jit_emitter_pin.cpp: FNV-1a64 of the emitter sources // (normalized to LF; file list in the test) -let LLVM_JIT_EMITTER_HASH : uint64 = 0x6330d3756a092eebul +let LLVM_JIT_EMITTER_HASH : uint64 = 0x6edb28d5b95231c2ul let JIT_FNV_PRIME : uint64 = 1099511628211ul @@ -785,6 +785,12 @@ def public run_jit(prog : Program?; var ctx : Context?) : bool { // nolint:STYL // script-level `options jit_target = "..."`. let opt_target = (prog._options |> find_arg("jit_target")) ?as tString ?? "" let target_triple = cli_opts.target |> unwrap_or(opt_target) + // the compile-time target folds (get_target_triple, get_target_architecture_name) read the + // command line, never this option: a script-level triple the command line does not carry + // would emit for one target with every tier gate folded for the host + if (!empty(target_triple) && get_target_triple() != target_triple) { + panic("LLVM JIT: the target triple `{target_triple}` is not on the command line - pass --jit-target={target_triple} after the `--` separator (the compile-time target folds read argv, so `options jit_target` alone folds the host's tiers into a cross artifact)") + } let gen_wasm = gen_exe && (target_triple |> starts_with("wasm")) // Optional CLI override for the wasm runtime archive location. Falls // back to `options jit_runtime_lib = "..."` then write_wasm's diff --git a/modules/dasLLVM/daslib/llvm_tune.das b/modules/dasLLVM/daslib/llvm_tune.das index d772885818..b3fa21409e 100644 --- a/modules/dasLLVM/daslib/llvm_tune.das +++ b/modules/dasLLVM/daslib/llvm_tune.das @@ -70,12 +70,13 @@ def jit_cli_opt_level() : int { return cli.opt_level |> unwrap_or(compiling_program().policies.jit_opt_level) } - //! True when this compile must stay stamp-free: `policies.tune_frozen` (the cross-box - //! artifact mark) or an AOT-consuming run without the JIT (a stamp changes the semantic - //! hash). Every tune annotation goes inert; only reference-row registries are emitted. + //! True when this compile must stay stamp-free - `policies.tune_frozen`, an AOT-consuming run + //! without the JIT (a stamp changes the semantic hash), or a `--jit-target` cross target (the + //! artifact runs on a box this sidecar never measured): every tune annotation goes inert. def tune_aot_gate() : bool { return (compiling_program().policies.tune_frozen - || (compiling_program().policies.aot && !compiling_program().policies.jit_enabled)) + || (compiling_program().policies.aot && !compiling_program().policies.jit_enabled) + || !empty(get_target_triple())) } //! True when this compile must retain reference bodies. Documentation roots @@ -351,18 +352,29 @@ def tune_sidecar_stale(path : string) : bool { return box_match_key(sidecar_box(path, "{sfs.mtime}")) != box_match_key(tune_box_identity()) } + //! The compile-time half of a sidecar or profile read: the module cache records the file, existing + //! or not, so a re-mint invalidates the stamps minted against it. Kept out of every reader the + //! runtime shares - a standalone exe binds each extern at startup (the cited section says why). +[arch(at="../ARCHITECTURE.md#tune-sidecar-cache-pin")] +def private pin_module_cache_dependency(path : string) { + add_module_cache_dependency(path) +} + +def private pin_profile_chain(defaultsDir : string) { + var chain <- tune_class_chain() + for (c in chain) { + pin_module_cache_dependency(path_join(defaultsDir, "{c}.tune-defaults.json")) + } + delete chain +} + // the sidecar's "kernels" section as a flat { function name : perm suffix } map; // found = the file existed, parsed to an object, and is NOT stale vs the running binary // (a "kernels"-less but fresh sidecar counts as found-empty: runtime-knob-only files) -[arch(at="../ARCHITECTURE.md#tune-sidecar-cache-pin")] def private read_manifest(path : string; var found : bool&) : table { found = false var tab : table - if (empty(path)) { - return <- tab - } - add_module_cache_dependency(path) - if (tune_sidecar_stale(path)) { + if (empty(path) || tune_sidecar_stale(path)) { return <- tab } let text = fread(path) @@ -398,15 +410,11 @@ def tune_manifest_get(fname : string) : string { return r } -[arch(at="../ARCHITECTURE.md#tune-sidecar-cache-pin")] def private locate_profile_doc(defaultsDir, verKey, verVal : string; compileTime : bool; var klass, ppath : string&) : JsonValue? { var chain <- tune_class_chain() var doc : JsonValue? for (c in chain) { let p = path_join(defaultsDir, "{c}.tune-defaults.json") - if (compileTime) { - add_module_cache_dependency(p) - } let text = fread(p) continue if (empty(text)) var err = "" @@ -469,6 +477,7 @@ def private profile_kernels(var klass, ppath : string&) : table } var k = "" var p = "" + pin_profile_chain(dir) var doc = locate_profile_doc(dir, verKey, verVal, true, k, p) continue if (doc == null) let ks = doc?["kernels"] @@ -494,6 +503,7 @@ def private profile_kernels(var klass, ppath : string&) : table def tune_kernel_pick(fname : string; var source : string&) : string { var found = false let mpath = tune_manifest_path() + pin_module_cache_dependency(mpath) var tab <- read_manifest(mpath, found) var r = tab?[fname] ?? "" delete tab @@ -907,28 +917,6 @@ def tune_status() : array { return clone_to_move(g_tune_status) } - //! Log the tune status table at `LOG_INFO` — the shared "am I tuned?" surface, prefixed by - //! `appName`. Prints `/ kernels tuned for this box`, one line per function, and a - //! `--tune` hint when any kernel sits on a fallback/reference tier. No-op when empty. -def log_tune_status(appName : string) { - var st <- tune_status() - if (empty(st)) { - delete st - return - } - var tuned = 0 - for (s in st) { - tuned += (s.source == "manifest" || s.source == "profile") ? 1 : 0 - } - to_log(LOG_INFO, "{appName}: tune status ({tuned}/{length(st)} kernels tuned for this box):\n") - for (s in st) { - to_log(LOG_INFO, " {s.fname}: {empty(s.suffix) ? "reference" : s.suffix} ({s.source})\n") - } - if (tuned < length(st)) { - to_log(LOG_INFO, "{appName}: run with --tune to tune the remaining kernels for this box\n") - } - delete st -} // ===== [tune_scope] data — library-owned tuner wiring ===== @@ -1555,7 +1543,7 @@ def private stamp_llvm_code(var fn : FunctionPtr; genKey : string; perm : TunePe aargs |> push((argname = "tune_suffix", argvalue = RttiValue(tString = perm.suffix))) aargs |> push((argname = "tune_from", argvalue = RttiValue(tString = from))) aargs |> push((argname = "tune_source", argvalue = RttiValue(tString = source))) - fn |> append_annotation("llvm_code", "llvm_code", aargs) + fn |> append_annotation("tune", "llvm_code", aargs) fn.sideEffectFlags.userScenario = true delete aargs } @@ -1634,246 +1622,238 @@ def private emit_registry(var mod : Module?; base : FunctionPtr; perms : array)]` — one grid row for the `tune(...)` that must follow it in //! the same bracket. Args pass to the generator verbatim and fold into the JIT DLL cache //! key; `suffix="..."` overrides the auto-derived variant name. -[function_macro(name = "tune_perm")] -class private TunePermAnnotation : AstFunctionAnnotation { - def override apply(var func : FunctionPtr; var group : ModuleGroup; - args : AnnotationArgumentList; var errors : das_string) : bool { - g_perm_rows[perm_key(func)] |> emplace(build_perm_row(args)) - return true - } +def tune_perm_apply(var func : FunctionPtr; var _group : ModuleGroup; + args : AnnotationArgumentList; var _errors : das_string) : bool { + g_perm_rows[perm_key(func)] |> emplace(build_perm_row(args)) + return true } //! `[tune_companion(fn = "sibling", gen = "generator_key")]` — stamps `sibling` with the SAME //! perm from the same sidecar entry, so a kernel and (say) its repack-layout query can never //! desync. List these after the [tune_perm] grid and before tune(...). See `skills/tune.md`. -[function_macro(name = "tune_companion")] -class private TuneCompanionAnnotation : AstFunctionAnnotation { - def override apply(var func : FunctionPtr; var group : ModuleGroup; - args : AnnotationArgumentList; var errors : das_string) : bool { +def tune_companion_apply(var func : FunctionPtr; var _group : ModuleGroup; + args : AnnotationArgumentList; var errors : das_string) : bool { // the bank doubles as the order guard: rows after tune(...) or before any perm are ignored - if (!(g_perm_rows |> key_exists(perm_key(func)))) { - errors := "[tune_companion] on {func.name}: list it AFTER the [tune_perm] rows and BEFORE tune(...) in the same bracket" - return false - } - var row : TuneCompanion - let fv = find_arg(args, "fn") - if (fv is tString) { row.fnname = fv as tString } - let gv = find_arg(args, "gen") - if (gv is tString) { row.genkey = gv as tString } - if (empty(row.fnname) || empty(row.genkey)) { - errors := "[tune_companion] on {func.name}: fn=\"\" and gen=\"\" are both required" - return false - } - g_companion_rows[perm_key(func)] |> emplace(row) - return true + if (!(g_perm_rows |> key_exists(perm_key(func)))) { + errors := "[tune_companion] on {func.name}: list it AFTER the [tune_perm] rows and BEFORE tune(...) in the same bracket" + return false } + var row : TuneCompanion + let fv = find_arg(args, "fn") + if (fv is tString) { row.fnname = fv as tString } + let gv = find_arg(args, "gen") + if (gv is tString) { row.genkey = gv as tString } + if (empty(row.fnname) || empty(row.genkey)) { + errors := "[tune_companion] on {func.name}: fn=\"\" and gen=\"\" are both required" + return false + } + g_companion_rows[perm_key(func)] |> emplace(row) + return true } //! `[tune(gen="key", fallback="suffix")]` — LAST in a bracket of `[tune_perm]` rows, which it //! consumes. Normal runs stamp one winner (sidecar > fallback > reference); tune/test runs //! stamp the whole grid plus a `_variants()` registry. Mode contract: module docstring. -[function_macro(name = "tune")] -class private TuneAnnotation : AstFunctionAnnotation { - def override apply(var func : FunctionPtr; var group : ModuleGroup; // nolint:STYLE037,STYLE038 — the [tune] resolution ladder: manifest/fallback/reference arms in precedence order - args : AnnotationArgumentList; var errors : das_string) : bool { - let gv = find_arg(args, "gen") - if (!(gv is tString) || empty(gv as tString)) { - errors := "[tune] on {func.name}: gen=\"<[llvm_code] generator key>\" is required" - return false - } - let genKey = gv as tString - var perms <- take_perms(func, errors) - if (empty(errors) && empty(perms)) { - errors := "[tune] on {func.name}: no [tune_perm(...)] rows - list them BEFORE tune(...) in the same annotation bracket" - } - var comps <- take_companions(func, errors) - if (!empty(errors)) { - delete perms - delete comps - return false - } - if (func.result.baseType == Type.autoinfer) { +def tune_apply(var func : FunctionPtr; var _group : ModuleGroup; // nolint:STYLE037,STYLE038 — the [tune] resolution ladder: manifest/fallback/reference arms in precedence order + args : AnnotationArgumentList; var errors : das_string) : bool { + let gv = find_arg(args, "gen") + if (!(gv is tString) || empty(gv as tString)) { + errors := "[tune] on {func.name}: gen=\"<[llvm_code] generator key>\" is required" + return false + } + let genKey = gv as tString + var perms <- take_perms(func, errors) + if (empty(errors) && empty(perms)) { + errors := "[tune] on {func.name}: no [tune_perm(...)] rows - list them BEFORE tune(...) in the same annotation bracket" + } + var comps <- take_companions(func, errors) + if (!empty(errors)) { + delete perms + delete comps + return false + } + if (func.result.baseType == Type.autoinfer) { // function_to_type runs pre-infer: an implicit result reaches the registry as `auto` - errors := "[tune] on {func.name}: an explicit return type is required (add ': void' or the real type)" - delete perms - delete comps - return false - } + errors := "[tune] on {func.name}: an explicit return type is required (add ': void' or the real type)" + delete perms + delete comps + return false + } // overrides grid modes too: a stale DAS_TUNE_MODE must not leak stubs into generated C++ - if (tune_stamp_free_gate()) { - let noPerms : array - var gok = emit_registry(compiling_module(), func, noPerms, errors) - for (c in comps) { - if (gok) { - gok = emit_registry(compiling_module(), c.fn, noPerms, errors) - } + if (tune_stamp_free_gate()) { + let noPerms : array + var gok = emit_registry(compiling_module(), func, noPerms, errors) + for (c in comps) { + if (gok) { + gok = emit_registry(compiling_module(), c.fn, noPerms, errors) } - delete perms - delete comps - return gok - } - let mode = tune_mode() - if (mode == "tune" || mode == "test") { - // visible generation cost: a silent permutation stamp reads as a hung compile from outside - let ctail = empty(comps) ? "" : " (+{length(comps)} companions each)" - print("llvm_tune: [tune] {func.name}: stamping {length(perms)} permutations{ctail}\n") - let ok = stamp_grid(func, genKey, perms, comps, errors) - delete perms - delete comps - return ok - } - if (mode != "normal") { - errors := "[tune] on {func.name}: unknown DAS_TUNE_MODE '{mode}' (normal | tune | test)" - delete perms - delete comps - return false } - if (g_env_tune.tune_policy == "reference") { - let noPerms : array - var rok = emit_registry(compiling_module(), func, noPerms, errors) - for (c in comps) { - break if (!rok) - rok = emit_registry(compiling_module(), c.fn, noPerms, errors) - } - delete perms - delete comps - return rok + delete perms + delete comps + return gok + } + let mode = tune_mode() + if (mode == "tune" || mode == "test") { + // visible generation cost: a silent permutation stamp reads as a hung compile from outside + let ctail = empty(comps) ? "" : " (+{length(comps)} companions each)" + print("llvm_tune: [tune] {func.name}: stamping {length(perms)} permutations{ctail}\n") + let ok = stamp_grid(func, genKey, perms, comps, errors) + delete perms + delete comps + return ok + } + if (mode != "normal") { + errors := "[tune] on {func.name}: unknown DAS_TUNE_MODE '{mode}' (normal | tune | test)" + delete perms + delete comps + return false + } + if (g_env_tune.tune_policy == "reference") { + let noPerms : array + var rok = emit_registry(compiling_module(), func, noPerms, errors) + for (c in comps) { + break if (!rok) + rok = emit_registry(compiling_module(), c.fn, noPerms, errors) } + delete perms + delete comps + return rok + } // no entry falls back rather than dropping to reference, so a sidecar written before a kernel family landed cannot untune it - let fname = "{func.name}" - var manifestFound = false - let mpath = tune_manifest_path() - var manifest <- read_manifest(mpath, manifestFound) - var suffix = "" - var tuneSource = "" // manifest | profile | fallback - the tune_source the stamp records - var tuneSourcePath = "" - if (manifestFound) { - suffix = manifest?[fname] ?? "" - if (!empty(suffix)) { - tuneSource = "manifest" - tuneSourcePath = mpath + let fname = "{func.name}" + var manifestFound = false + let mpath = tune_manifest_path() + pin_module_cache_dependency(mpath) + var manifest <- read_manifest(mpath, manifestFound) + var suffix = "" + var tuneSource = "" // manifest | profile | fallback + var tuneSourcePath = "" + if (manifestFound) { + suffix = manifest?[fname] ?? "" + if (!empty(suffix)) { + tuneSource = "manifest" + tuneSourcePath = mpath + } + } + var profileKlass = "" + var profilePath = "" + var profile <- empty(suffix) ? profile_kernels(profileKlass, profilePath) : no_kernels() + if (empty(suffix)) { + suffix = profile?[fname] ?? "" + if (!empty(suffix)) { + tuneSource = "profile" + tuneSourcePath = profilePath + } + } + // clear a winner this box can't run - ISA-gated (declines to reference) or absent from this build's grid (crash-loop) - so the fallback chain picks a legal perm + if (!empty(suffix) && suffix != "reference") { + var known = false + var eligible = true + for (p in perms) { + if (p.suffix == suffix) { + known = true + eligible = perm_requires_ok(p) + break } } - var profileKlass = "" - var profilePath = "" - var profile <- empty(suffix) ? profile_kernels(profileKlass, profilePath) : no_kernels() - if (empty(suffix)) { - suffix = profile?[fname] ?? "" - if (!empty(suffix)) { - tuneSource = "profile" - tuneSourcePath = profilePath + if (!known || !eligible) { + if (tune_verbosity() != "silent") { + let why = known ? "fails its requires= on this box" : "is not in this build's grid" + print("llvm_tune: {fname} {tuneSource} winner '{suffix}' {why} - using the fallback chain\n") } + suffix = "" + tuneSource = "" + tuneSourcePath = "" } - // clear an adopted winner this box can't run (ISA-gated → would decline to reference; or not in this build's grid → would crash-loop) so the fallback ;-chain picks a legal perm and it boots - if (!empty(suffix) && suffix != "reference") { - var known = false - var eligible = true + } + var companionOwnSuffix : array // a companion's own entry when it names an eligible perm of this grid; empty follows the main + companionOwnSuffix |> resize(length(comps)) + if (tuneSource == "manifest" || tuneSource == "profile") { + for (ci, c in count(), comps) { + let own = tuneSource == "manifest" ? (manifest?["{c.fnname}"] ?? "") : (profile?["{c.fnname}"] ?? "") for (p in perms) { - if (p.suffix == suffix) { - known = true - eligible = perm_requires_ok(p) - break - } - } - if (!known || !eligible) { - if (tune_verbosity() != "silent") { - let why = known ? "fails its requires= on this box" : "is not in this build's grid" - print("llvm_tune: {fname} {tuneSource} winner '{suffix}' {why} - using the fallback chain\n") + if (p.suffix == own && perm_requires_ok(p)) { + companionOwnSuffix[ci] = own } - suffix = "" - tuneSource = "" - tuneSourcePath = "" } } - var companionOwnSuffix : array // a companion's OWN entry (the harness's gemv seat) when it names an eligible perm of this grid; else it follows the main - companionOwnSuffix |> resize(length(comps)) - if (tuneSource == "manifest" || tuneSource == "profile") { - for (ci, c in count(), comps) { - let own = tuneSource == "manifest" ? (manifest?["{c.fnname}"] ?? "") : (profile?["{c.fnname}"] ?? "") - for (p in perms) { - if (p.suffix == own && perm_requires_ok(p)) { - companionOwnSuffix[ci] = own - } - } - } - } - let fromPick = !empty(suffix) - var ok = true - if (empty(suffix)) { - tuneSource = "fallback" + } + let fromPick = !empty(suffix) + var ok = true + if (empty(suffix)) { + tuneSource = "fallback" // ';'-chain in declaration order, first passing requires= wins; none eligible => reference - let fv = find_arg(args, "fallback") - if (fv is tString) { - for (cand in split(fv as tString, ";")) { - continue if (empty(cand)) - var found = false - for (p in perms) { - if (p.suffix == cand) { - found = true - if (perm_requires_ok(p)) { - suffix = cand - } - break + let fv = find_arg(args, "fallback") + if (fv is tString) { + for (cand in split(fv as tString, ";")) { + continue if (empty(cand)) + var found = false + for (p in perms) { + if (p.suffix == cand) { + found = true + if (perm_requires_ok(p)) { + suffix = cand } + break } - if (!found) { - errors := "[tune] on {func.name}: fallback perm '{cand}' is not in the [tune_perm] grid" - ok = false - } - break if (!ok || !empty(suffix)) } + if (!found) { + errors := "[tune] on {func.name}: fallback perm '{cand}' is not in the [tune_perm] grid" + ok = false + } + break if (!ok || !empty(suffix)) } } - delete manifest - delete profile - if (ok && !empty(suffix) && suffix != "reference") { - ok = false - for (p in perms) { - if (p.suffix == suffix) { - stamp_llvm_code(func, genKey, p, fromPick ? tuneSourcePath : "", tuneSource) - for (ci, c in count(), comps) { - var stampedOwn = false - if (!empty(companionOwnSuffix[ci]) && companionOwnSuffix[ci] != suffix) { - for (q in perms) { - if (q.suffix == companionOwnSuffix[ci]) { - stamp_llvm_code(c.fn, c.genkey, q, tuneSourcePath, tuneSource) - stampedOwn = true - } - } - if (stampedOwn && tune_verbosity() == "verbose") { - print("llvm_tune: {c.fnname} <- {companionOwnSuffix[ci]} (its own entry, {tuneSourcePath})\n") + } + delete manifest + delete profile + if (ok && !empty(suffix) && suffix != "reference") { + ok = false + for (p in perms) { + if (p.suffix == suffix) { + stamp_llvm_code(func, genKey, p, fromPick ? tuneSourcePath : "", tuneSource) + for (ci, c in count(), comps) { + var stampedOwn = false + if (!empty(companionOwnSuffix[ci]) && companionOwnSuffix[ci] != suffix) { + for (q in perms) { + if (q.suffix == companionOwnSuffix[ci]) { + stamp_llvm_code(c.fn, c.genkey, q, tuneSourcePath, tuneSource) + stampedOwn = true } } - if (!stampedOwn) { - stamp_llvm_code(c.fn, c.genkey, p, fromPick ? tuneSourcePath : "", tuneSource) + if (stampedOwn && tune_verbosity() == "verbose") { + print("llvm_tune: {c.fnname} <- {companionOwnSuffix[ci]} (its own entry, {tuneSourcePath})\n") } } - if (tune_verbosity() == "verbose") { - print("llvm_tune: {fname} <- {suffix} ({fromPick ? tuneSourcePath : "fallback"})\n") + if (!stampedOwn) { + stamp_llvm_code(c.fn, c.genkey, p, fromPick ? tuneSourcePath : "", tuneSource) } - ok = true - break } - } - if (!ok) { - errors := "[tune] on {func.name}: perm '{suffix}' is not in the [tune_perm] grid" + if (tune_verbosity() == "verbose") { + print("llvm_tune: {fname} <- {suffix} ({fromPick ? tuneSourcePath : "fallback"})\n") + } + ok = true + break } } + if (!ok) { + errors := "[tune] on {func.name}: perm '{suffix}' is not in the [tune_perm] grid" + } + } // reference-row-only registries, so harness code referencing _variants() still compiles - if (ok) { - let noPerms : array - ok = emit_registry(compiling_module(), func, noPerms, errors) - for (c in comps) { - if (ok) { - ok = emit_registry(compiling_module(), c.fn, noPerms, errors) - } + if (ok) { + let noPerms : array + ok = emit_registry(compiling_module(), func, noPerms, errors) + for (c in comps) { + if (ok) { + ok = emit_registry(compiling_module(), c.fn, noPerms, errors) } } - delete companionOwnSuffix - delete perms - delete comps - return ok } + delete companionOwnSuffix + delete perms + delete comps + return ok } // ===== [tune_scope] / [tune_policy] — the tuner-wiring UX rail ===== @@ -1896,64 +1876,61 @@ def private module_compiled(name : string) : bool { //! `[tune_scope(name = "mylib", tuner = "harness.das")]` on a dummy struct in the library that //! owns the `[tune]` functions, declared ABOVE them: names the tuner that regenerates this //! library's winners in the ONE per-app sidecar. `tuner=` resolves against the declaring file. -[structure_macro(name = "tune_scope")] -class private TuneScopeAnnotation : AstStructureAnnotation { - def override apply(var st : StructurePtr; var group : ModuleGroup; - args : AnnotationArgumentList; var errors : das_string) : bool { - if (tune_aot_gate()) return true - let nv = find_arg(args, "name") - if (!(nv is tString) || empty(nv as tString)) { - errors := "[tune_scope]: name=\"\" is required" +def tune_scope_apply(var st : StructurePtr; var _group : ModuleGroup; + args : AnnotationArgumentList; var errors : das_string) : bool { + if (tune_aot_gate()) return true + let nv = find_arg(args, "name") + if (!(nv is tString) || empty(nv as tString)) { + errors := "[tune_scope]: name=\"\" is required" + return false + } + let sname = nv as tString + var tuner = "" + let tv = find_arg(args, "tuner") + if (tv is tString) { + tuner = resolve_decl_path(string(st.at.fileInfo.name), tv as tString) + } + var covers = "" + let cv = find_arg(args, "covers") + if (cv is tString) { + covers = cv as tString + } + var versionKey = "" + var versionExpected = "" + let vv = find_arg(args, "version_of") + if (vv is tString) { + var keyOverride = "" + let kv = find_arg(args, "version_key") + if (kv is tString) { + keyOverride = kv as tString + } + var verr = "" + if (!resolve_version_of(get_ptr(compiling_program()), vv as tString, keyOverride, versionKey, versionExpected, verr)) { + errors := "[tune_scope]: {verr}" return false } - let sname = nv as tString - var tuner = "" - let tv = find_arg(args, "tuner") - if (tv is tString) { - tuner = resolve_decl_path(string(st.at.fileInfo.name), tv as tString) - } - var covers = "" - let cv = find_arg(args, "covers") - if (cv is tString) { - covers = cv as tString - } - var versionKey = "" - var versionExpected = "" - let vv = find_arg(args, "version_of") - if (vv is tString) { - var keyOverride = "" - let kv = find_arg(args, "version_key") - if (kv is tString) { - keyOverride = kv as tString - } - var verr = "" - if (!resolve_version_of(get_ptr(compiling_program()), vv as tString, keyOverride, versionKey, versionExpected, verr)) { - errors := "[tune_scope]: {verr}" + } + var defaults = "" + let dv = find_arg(args, "defaults") + if (dv is tString) { + defaults = resolve_decl_path(string(st.at.fileInfo.name), dv as tString) + } + if (!empty(defaults)) { + for (m in split(covers, ";")) { + continue if (empty(m)) + if (module_compiled(m)) { + errors := "[tune_scope] '{sname}': covers= module '{m}' compiled before this scope, so its kernels could not read the defaults profile - require this scope's module from '{m}'" return false } } - var defaults = "" - let dv = find_arg(args, "defaults") - if (dv is tString) { - defaults = resolve_decl_path(string(st.at.fileInfo.name), dv as tString) - } - if (!empty(defaults)) { - for (m in split(covers, ";")) { - continue if (empty(m)) - if (module_compiled(m)) { - errors := "[tune_scope] '{sname}': covers= module '{m}' compiled before this scope, so its kernels could not read the defaults profile - require this scope's module from '{m}'" - return false - } - } - } - // overwrite, not error: shared-module macro state persists across compiles in one process - g_scopes["{compiling_module().name}"] = TuneScope(scopeName = sname, - modName = "{compiling_module().name}", covers = covers, - manifestPath = tune_manifest_path(), tunerPath = tuner, - versionKey = versionKey, versionExpected = versionExpected, - defaultsPath = defaults) - return true } + // overwrite, not error: shared-module macro state persists across compiles in one process + g_scopes["{compiling_module().name}"] = TuneScope(scopeName = sname, + modName = "{compiling_module().name}", covers = covers, + manifestPath = tune_manifest_path(), tunerPath = tuner, + versionKey = versionKey, versionExpected = versionExpected, + defaultsPath = defaults) + return true } // The tune flags, parsed off the APP's command line — we share an argv we do not own, so this @@ -2657,6 +2634,7 @@ def private warn_manifest_untuned() { for (scope in values(g_scopes)) { let key = "{scope.scopeName}|{scope.manifestPath}" continue if (g_manifest_warned |> key_exists(key)) + pin_module_cache_dependency(scope.manifestPath) let v = tune_sidecar_verdict(scope.manifestPath, scope_fnames_joined(scope)) continue if (v.complete) g_manifest_warned |> insert(key) @@ -2740,6 +2718,7 @@ def private apply_tune_policy(var func : FunctionPtr; declared : string; var err seen |> insert(scope.scopeName) // per-scope KEY completeness, not file existence — the sidecar is shared let scopeFnames = scope_fnames_joined(scope) + pin_module_cache_dependency(scope.manifestPath) let v = tune_sidecar_verdict(scope.manifestPath, scopeFnames) continue if (!force && v.complete && policy != "auto") if (force || policy == "auto") { @@ -2841,32 +2820,29 @@ def private apply_tune_policy(var func : FunctionPtr; declared : string; var err //! `[tune_policy(missing = "...")]` on the app's `main`: what to do about scopes whose sidecar //! entries are missing or stale — `fallback` | `warn` | `error` | `auto` (THE DEFAULT, declared //! or not) | `restart`. Full semantics and the escapes: `skills/tune.md`. -[function_macro(name = "tune_policy")] -class private TunePolicyAnnotation : AstFunctionAnnotation { - def override apply(var func : FunctionPtr; var group : ModuleGroup; - args : AnnotationArgumentList; var errors : das_string) : bool { - // the manifest warning is the default pass's job — it fires for every root, this apply's gate included - if (tune_policy_gated() && !exe_status_report()) { - return true - } - var errs = "" - if (tune_exe_gate()) { - // standalone build: no policy, but the artifact self-reports its stamps - emit_status_init(errs) - } else { - var policy = "auto" - let mv = find_arg(args, "missing") - if (mv is tString) { - policy = mv as tString - } - apply_tune_policy(func, policy, errs) - } - if (!empty(errs)) { - errors := errs - return false - } +def tune_policy_apply(var func : FunctionPtr; var _group : ModuleGroup; + args : AnnotationArgumentList; var errors : das_string) : bool { + // the manifest warning is the default pass's job — it fires for every root, this apply's gate included + if (tune_policy_gated() && !exe_status_report()) { return true } + var errs = "" + if (tune_exe_gate()) { + // standalone build: no policy, but the artifact self-reports its stamps + emit_status_init(errs) + } else { + var policy = "auto" + let mv = find_arg(args, "missing") + if (mv is tString) { + policy = mv as tString + } + apply_tune_policy(func, policy, errs) + } + if (!empty(errs)) { + errors := errs + return false + } + return true } // inert once-only marker the TuneDefaultPolicy pass stamps onto main — appending an @@ -2883,36 +2859,31 @@ class private TunePolicyDefaultMarker : AstFunctionAnnotation { //! The "untuned does not start" default: a program root with a `main`, some `[tune_scope]` in //! play and no declared `[tune_policy]` behaves as `missing = "auto"`. Escapes and the //! once-only marker: `skills/tune.md`. A root without `main` never tunes (dastest files). -[infer_macro] -class private TuneDefaultPolicy : AstPassMacro { - def override apply(prog : ProgramPtr; _mod : Module?) : bool { - // fires only for roots that SEE llvm_tune, so a scope-owning library must re-export it public - return false if (is_in_completion() || !empty("{compiling_module().name}") || empty(g_scopes)) - warn_manifest_untuned() // self-gates on DAS_TUNE_MANIFEST; covers exe and declared-policy roots too - return false if (tune_policy_gated() && !exe_status_report()) - var main_fn = compiling_module() |> find_unique_function("main", false) - return false if (main_fn == null) - for (ann in main_fn.annotations) { - // declared policy wins; the marker means this pass already ran - return false if (ann.annotation.name == "tune_policy" - || ann.annotation.name == "tune_policy_default_applied") - } - main_fn |> append_annotation("llvm_tune", "tune_policy_default_applied") - var errs = "" - var ok = true - if (tune_exe_gate()) { - // standalone build: no policy, but the artifact self-reports its stamps - emit_status_init(errs) - } else { - // O0 cannot represent winners raced under O3 - default fallback (DAS_TUNE_POLICY wins) - let flavor = jit_cli_opt_level() == 0 ? "fallback" : "auto" - ok = apply_tune_policy(main_fn, flavor, errs) - } - if (!ok || !empty(errs)) { +def tune_default_policy_apply(_prog : ProgramPtr; _mod : Module?) : bool { + return false if (is_in_completion() || !empty("{compiling_module().name}") || empty(g_scopes)) + warn_manifest_untuned() // self-gates on DAS_TUNE_MANIFEST; covers exe and declared-policy roots too + return false if (tune_policy_gated() && !exe_status_report()) + var main_fn = compiling_module() |> find_unique_function("main", false) + return false if (main_fn == null) + for (ann in main_fn.annotations) { + // declared policy wins; the marker means this pass already ran + return false if (ann.annotation.name == "tune_policy" + || ann.annotation.name == "tune_policy_default_applied") + } + main_fn |> append_annotation("llvm_tune", "tune_policy_default_applied") + var errs = "" + var ok = true + if (tune_exe_gate()) { + // standalone build: no policy, but the artifact self-reports its stamps + emit_status_init(errs) + } else { + let flavor = jit_cli_opt_level() == 0 ? "fallback" : "auto" + ok = apply_tune_policy(main_fn, flavor, errs) + } + if (!ok || !empty(errs)) { // macro_error mid-infer-pass is dropped by the retry loop — print + panic actually stops it - print("llvm_tune: {errs}\n") - panic("llvm_tune default policy: {errs}") - } - return true // modified — re-run infer over the injected status/guard + print("llvm_tune: {errs}\n") + panic("llvm_tune default policy: {errs}") } + return true // modified — re-run infer over the injected status/guard } diff --git a/modules/dasLLVM/daslib/llvm_user_modules.das b/modules/dasLLVM/daslib/llvm_user_modules.das index 5a6ff6ec41..31a9e58986 100644 --- a/modules/dasLLVM/daslib/llvm_user_modules.das +++ b/modules/dasLLVM/daslib/llvm_user_modules.das @@ -6,7 +6,7 @@ options no_global_variables = false module llvm_user_modules shared private require llvm/daslib/llvm_code_selftest -require ?dasllama/dasllama_gemm_gen dasllama/dasllama_gemm_register // nolint:STYLE030 — used inside the static_if below; on builds without the package both the require and the branch drop (path guard: pure-das package, nothing C++ to guard on) +require ?dasllama/dasllama_gemm_gen dasllama/dasllama_gemm_register // nolint:STYLE030,LINT019 — used inside the static_if below; on builds without the package both the require and the branch drop (path guard: pure-das package, nothing C++ to guard on) //! User-extensible wiring for `[llvm_code]` generator modules. The JIT calls //! `register_user_llvm_code_generators` (lazily, in the context that reads the registry) @@ -14,7 +14,8 @@ require ?dasllama/dasllama_gemm_gen dasllama/dasllama_gemm_register // nolint:S //! `require ? ` line above — a PATH guard loads exactly when the //! contributor's package is mounted (the guard's own file resolves), so this //! module never hard-depends on the contributor. Pair each line with a `static_if -//! (typeinfo builtin_module_exists())` around the registration call below; registered +//! (typeinfo module_exists())` around the registration call below - the +//! TARGET the guarded require loaded, which is what is visible from here; registered //! generators are name-keyed and inert for programs whose `[llvm_code]` annotations never //! name them. @@ -27,7 +28,7 @@ def public register_user_llvm_code_generators { } g_registered = true llvm_code_selftest::register_selftest() // worked example + self-test generator (add_i32, add_plus_k) - static_if (typeinfo builtin_module_exists(dasllama_gemm_gen)) { + static_if (typeinfo module_exists(dasllama_gemm_register)) { dasllama_gemm_register::register_dasllama_gemm_code_generators() } } diff --git a/modules/dasLLVM/src/dasLLVM.cpp b/modules/dasLLVM/src/dasLLVM.cpp new file mode 100644 index 0000000000..10eb63b578 --- /dev/null +++ b/modules/dasLLVM/src/dasLLVM.cpp @@ -0,0 +1,27 @@ +// llvm - the witness that this build was CONFIGURED with dasLLVM. The package itself is pure +// das (dasbind over LLVM.dll), so nothing else in it can answer "is LLVM part of this build" - +// its files sit on disk in every checkout, whatever DAS_LLVM_DISABLED says. This module is +// compiled in only when the package is, so `require ?llvm ` and +// `builtin_module_exists(llvm)` follow the configure, never the filesystem: a wasm or console +// build without LLVM loads no tune framework, no sidecar, no IR generator. + +#include "daScript/daScript.h" + +namespace das { + +class Module_LLVM : public Module { +public: + Module_LLVM() : Module("llvm") { + ModuleLibrary lib(this); + lib.addBuiltInModule(); + } + virtual ModuleAotType aotRequire ( TextWriter & ) const override { + return ModuleAotType::cpp; + } +}; + +REGISTER_DYN_MODULE(Module_LLVM, Module_LLVM); + +} // namespace das + +REGISTER_MODULE_IN_NAMESPACE(Module_LLVM, das); diff --git a/modules/dasLLVM/tests/cant_tune_bad_grids.das b/modules/dasLLVM/tests/cant_tune_bad_grids.das index 69bb887b51..671f0b578a 100644 --- a/modules/dasLLVM/tests/cant_tune_bad_grids.das +++ b/modules/dasLLVM/tests/cant_tune_bad_grids.das @@ -4,6 +4,7 @@ options gen2 expect 20800:12 +require daslib/tune require llvm/daslib/llvm_tune // 1. missing gen= diff --git a/modules/dasLLVM/tests/cant_tune_bad_scope.das b/modules/dasLLVM/tests/cant_tune_bad_scope.das index 960952638a..d296f32345 100644 --- a/modules/dasLLVM/tests/cant_tune_bad_scope.das +++ b/modules/dasLLVM/tests/cant_tune_bad_scope.das @@ -3,6 +3,7 @@ options gen2 expect 20800:3 +require daslib/tune require llvm/daslib/llvm_tune // 1. spec is not module/CONST diff --git a/modules/dasLLVM/tests/cant_tune_scope_after_covered.das b/modules/dasLLVM/tests/cant_tune_scope_after_covered.das index 2138a2351b..699a9df786 100644 --- a/modules/dasLLVM/tests/cant_tune_scope_after_covered.das +++ b/modules/dasLLVM/tests/cant_tune_scope_after_covered.das @@ -4,6 +4,7 @@ options gen2 expect 20800:1 +require daslib/tune require llvm/daslib/llvm_tune require llvm_tune_scope_early diff --git a/modules/dasLLVM/tests/llvm_tune.das b/modules/dasLLVM/tests/llvm_tune.das index b54667961a..f657e41940 100644 --- a/modules/dasLLVM/tests/llvm_tune.das +++ b/modules/dasLLVM/tests/llvm_tune.das @@ -1,6 +1,7 @@ options gen2 require dastest/testing_boost +require daslib/tune require llvm/daslib/llvm_tune // [tune] framework (llvm/daslib/llvm_tune): normal mode stamps the fallback perm's [llvm_code] diff --git a/modules/dasLLVM/tests/llvm_tune_manifest_client.das b/modules/dasLLVM/tests/llvm_tune_manifest_client.das index 750320fab5..5c915925b8 100644 --- a/modules/dasLLVM/tests/llvm_tune_manifest_client.das +++ b/modules/dasLLVM/tests/llvm_tune_manifest_client.das @@ -1,5 +1,6 @@ options gen2 +require daslib/tune require llvm/daslib/llvm_tune // Template fixture for llvm_tune_manifest.das (no [test] on purpose — the suite just diff --git a/modules/dasLLVM/tests/llvm_tune_modes_client.das b/modules/dasLLVM/tests/llvm_tune_modes_client.das index 5c92c5d4eb..791d1d1b92 100644 --- a/modules/dasLLVM/tests/llvm_tune_modes_client.das +++ b/modules/dasLLVM/tests/llvm_tune_modes_client.das @@ -1,12 +1,14 @@ options gen2 require dastest/testing_boost -require llvm/daslib/llvm_tune +require daslib/tune // Doubles as the child fixture for llvm_tune_modes.das (spawned with DAS_TUNE_MODE=test, -jit: // main prints every grid variant's result) and as an ordinary suite test (normal mode: the // [test] below checks the fallback contract). The variants registry exists in every mode — -// reference-row-only under normal — so both faces compile from the same source. +// reference-row-only under normal — so both faces compile from the same source. The framework +// is reached through `daslib/tune` alone, so `module_exists(llvm_tune)` below is the witness +// answer: in a build without it the shells are inert, no registry exists, and main prints nothing. // companion sibling — grid modes clone it per perm too (modes_kv__k0 ... ), so the child run // proves per-perm args reach the companion generator and the declined row falls back with it @@ -23,17 +25,19 @@ def modes_add(a, b : int) : int { [export] def main { - var n = 2 // nolint:LINT003 — var defeats const-fold so the calls survive to codegen - var vs <- modes_add_variants() - for (v in vs) { - print("VARIANT {v._0} -> {invoke(v._1, n, 3)}\n") - } - delete vs - var kvs <- modes_kv_variants() - for (v in kvs) { - print("KVARIANT {v._0} -> {invoke(v._1)}\n") + static_if (typeinfo module_exists(llvm_tune)) { + var n = 2 // nolint:LINT003 — var defeats const-fold so the calls survive to codegen + var vs <- modes_add_variants() + for (v in vs) { + print("VARIANT {v._0} -> {invoke(v._1, n, 3)}\n") + } + delete vs + var kvs <- modes_kv_variants() + for (v in kvs) { + print("KVARIANT {v._0} -> {invoke(v._1)}\n") + } + delete kvs } - delete kvs } [test] diff --git a/modules/dasLLVM/tests/llvm_tune_profiles_lib.das b/modules/dasLLVM/tests/llvm_tune_profiles_lib.das index 102da66cd2..3ec6a7ff06 100644 --- a/modules/dasLLVM/tests/llvm_tune_profiles_lib.das +++ b/modules/dasLLVM/tests/llvm_tune_profiles_lib.das @@ -2,6 +2,7 @@ options gen2 module llvm_tune_profiles_lib public +require daslib/tune public require llvm/daslib/llvm_tune public // Library-side fixture for llvm_tune_profiles.das (no [test] on purpose — the suite just diff --git a/modules/dasLLVM/tests/llvm_tune_scope_client.das b/modules/dasLLVM/tests/llvm_tune_scope_client.das index a2e566a779..2b15ac1490 100644 --- a/modules/dasLLVM/tests/llvm_tune_scope_client.das +++ b/modules/dasLLVM/tests/llvm_tune_scope_client.das @@ -1,5 +1,6 @@ options gen2 +require daslib/tune require llvm_tune_scope_client_lib require llvm_tune_scope_client_lib2 diff --git a/modules/dasLLVM/tests/llvm_tune_scope_client_lib.das b/modules/dasLLVM/tests/llvm_tune_scope_client_lib.das index b60544c059..3058c1104c 100644 --- a/modules/dasLLVM/tests/llvm_tune_scope_client_lib.das +++ b/modules/dasLLVM/tests/llvm_tune_scope_client_lib.das @@ -2,6 +2,7 @@ options gen2 module llvm_tune_scope_client_lib public +require daslib/tune public require llvm/daslib/llvm_tune public // Library-side fixture for llvm_tune_scope.das (no [test] on purpose — the suite just diff --git a/modules/dasLLVM/tests/llvm_tune_scope_client_lib2.das b/modules/dasLLVM/tests/llvm_tune_scope_client_lib2.das index f791540648..bfec8e622c 100644 --- a/modules/dasLLVM/tests/llvm_tune_scope_client_lib2.das +++ b/modules/dasLLVM/tests/llvm_tune_scope_client_lib2.das @@ -2,6 +2,7 @@ options gen2 module llvm_tune_scope_client_lib2 public +require daslib/tune public require llvm/daslib/llvm_tune public // Second library fixture for llvm_tune_scope.das (no [test] on purpose): a SEPARATE scope diff --git a/modules/dasLLVM/tests/llvm_tune_scope_client_resolver.das b/modules/dasLLVM/tests/llvm_tune_scope_client_resolver.das index 65c237868b..26902396e3 100644 --- a/modules/dasLLVM/tests/llvm_tune_scope_client_resolver.das +++ b/modules/dasLLVM/tests/llvm_tune_scope_client_resolver.das @@ -1,5 +1,6 @@ options gen2 +require daslib/tune require llvm_tune_scope_client_lib // Resolver fixture for llvm_tune_scope.das (no [test] on purpose — the suite just compiles diff --git a/modules/dasLLVM/tests/llvm_tune_scope_client_ver.das b/modules/dasLLVM/tests/llvm_tune_scope_client_ver.das index f80222fd38..48f18f7472 100644 --- a/modules/dasLLVM/tests/llvm_tune_scope_client_ver.das +++ b/modules/dasLLVM/tests/llvm_tune_scope_client_ver.das @@ -1,5 +1,6 @@ options gen2 +require daslib/tune require llvm_tune_scope_client_verlib require llvm_tune_scope_client_verlib2 diff --git a/modules/dasLLVM/tests/llvm_tune_scope_client_verlib.das b/modules/dasLLVM/tests/llvm_tune_scope_client_verlib.das index 07335d681e..9f560623b0 100644 --- a/modules/dasLLVM/tests/llvm_tune_scope_client_verlib.das +++ b/modules/dasLLVM/tests/llvm_tune_scope_client_verlib.das @@ -2,6 +2,7 @@ options gen2 module llvm_tune_scope_client_verlib public +require daslib/tune public require llvm/daslib/llvm_tune public require llvm_tune_scope_verlib // nolint:STYLE030 — version_of= below names TESTVER by string diff --git a/modules/dasLLVM/tests/llvm_tune_scope_client_verlib2.das b/modules/dasLLVM/tests/llvm_tune_scope_client_verlib2.das index dbc704bab6..75abac9127 100644 --- a/modules/dasLLVM/tests/llvm_tune_scope_client_verlib2.das +++ b/modules/dasLLVM/tests/llvm_tune_scope_client_verlib2.das @@ -2,6 +2,7 @@ options gen2 module llvm_tune_scope_client_verlib2 public +require daslib/tune public require llvm/daslib/llvm_tune public require llvm_tune_scope_verlib // nolint:STYLE030 — version_of= below names TESTVER by string diff --git a/modules/dasLLVM/tests/llvm_tune_scope_early.das b/modules/dasLLVM/tests/llvm_tune_scope_early.das index 71e22f657c..d13df26d49 100644 --- a/modules/dasLLVM/tests/llvm_tune_scope_early.das +++ b/modules/dasLLVM/tests/llvm_tune_scope_early.das @@ -2,6 +2,7 @@ options gen2 module llvm_tune_scope_early public +require daslib/tune public require llvm/daslib/llvm_tune public // Helper for cant_tune_scope_after_covered.das: a kernel module with no scope of its own, so the diff --git a/modules/dasSpirv/spirv/spirv_builder.das b/modules/dasSpirv/spirv/spirv_builder.das index 9153a862fe..c3d83d7b22 100644 --- a/modules/dasSpirv/spirv/spirv_builder.das +++ b/modules/dasSpirv/spirv/spirv_builder.das @@ -714,7 +714,8 @@ def public const_double(var m : SpirvModule; v : double) : uint { // value), so this is type + result-id only. Keyed separately so it never collides with const_int 0/1. // f32 -> f16 bits, round-to-nearest-even — the das CPU reference semantics exactly (its // float16 converts are RNE too, so a folded CPU constant and this routine agree bit-for-bit). -// Pure das because the f16_cvt daslib lives in dasLLVM (not requireable from here). +// Its own body rather than daslib/f16_cvt's f32_to_f16: that one flattens every nan to sign|0x7E00, +// this one carries a payload bit through. def public float_to_half_bits(v : float) : uint { let bits = unsafe(reinterpret(v)) let s = (bits >> 16u) & 0x8000 diff --git a/modules/dasUnitTest/unit_test.das.inc b/modules/dasUnitTest/unit_test.das.inc index 9a909d6317..9db5163853 100644 --- a/modules/dasUnitTest/unit_test.das.inc +++ b/modules/dasUnitTest/unit_test.das.inc @@ -1,32 +1,32 @@ -static unsigned char unit_test_das[] = { -0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20, -0x67,0x65,0x6e,0x32,0x0a, -0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20, -0x72,0x65,0x6d,0x6f,0x76,0x65,0x5f,0x75, -0x6e,0x75,0x73,0x65,0x64,0x5f,0x73,0x79, -0x6d,0x62,0x6f,0x6c,0x73,0x20,0x3d,0x20, -0x66,0x61,0x6c,0x73,0x65,0x0a, -0x0a, -0x72,0x65,0x71,0x75,0x69,0x72,0x65,0x20, -0x55,0x6e,0x69,0x74,0x54,0x65,0x73,0x74, -0x0a, -0x0a, -0x6c,0x65,0x74,0x20,0x4f,0x54,0x48,0x45, -0x52,0x5f,0x55,0x4e,0x49,0x54,0x5f,0x54, -0x45,0x53,0x54,0x5f,0x43,0x4f,0x4e,0x53, -0x54,0x41,0x4e,0x54,0x20,0x3d,0x20,0x55, -0x4e,0x49,0x54,0x5f,0x54,0x45,0x53,0x54, -0x5f,0x43,0x4f,0x4e,0x53,0x54,0x41,0x4e, -0x54,0x20,0x2b,0x20,0x31,0x3b,0x0a, -0x0a, -0x64,0x65,0x66,0x20,0x73,0x74,0x61,0x72, -0x74,0x5f,0x65,0x66,0x66,0x65,0x63,0x74, -0x28,0x6e,0x61,0x6d,0x65,0x3b,0x20,0x74, -0x6d,0x29,0x20,0x7b,0x0a, -0x20,0x20,0x20,0x20,0x72,0x65,0x74,0x75, -0x72,0x6e,0x20,0x73,0x74,0x61,0x72,0x74, -0x5f,0x65,0x66,0x66,0x65,0x63,0x74,0x28, -0x6e,0x61,0x6d,0x65,0x2c,0x20,0x74,0x6d, -0x2c,0x20,0x31,0x2e,0x30,0x29,0x0a, -0x7d,0x0a, -}; +static unsigned char unit_test_das[] = { +0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20, +0x67,0x65,0x6e,0x32,0x0a, +0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20, +0x72,0x65,0x6d,0x6f,0x76,0x65,0x5f,0x75, +0x6e,0x75,0x73,0x65,0x64,0x5f,0x73,0x79, +0x6d,0x62,0x6f,0x6c,0x73,0x20,0x3d,0x20, +0x66,0x61,0x6c,0x73,0x65,0x0a, +0x0a, +0x72,0x65,0x71,0x75,0x69,0x72,0x65,0x20, +0x55,0x6e,0x69,0x74,0x54,0x65,0x73,0x74, +0x0a, +0x0a, +0x6c,0x65,0x74,0x20,0x4f,0x54,0x48,0x45, +0x52,0x5f,0x55,0x4e,0x49,0x54,0x5f,0x54, +0x45,0x53,0x54,0x5f,0x43,0x4f,0x4e,0x53, +0x54,0x41,0x4e,0x54,0x20,0x3d,0x20,0x55, +0x4e,0x49,0x54,0x5f,0x54,0x45,0x53,0x54, +0x5f,0x43,0x4f,0x4e,0x53,0x54,0x41,0x4e, +0x54,0x20,0x2b,0x20,0x31,0x3b,0x0a, +0x0a, +0x64,0x65,0x66,0x20,0x73,0x74,0x61,0x72, +0x74,0x5f,0x65,0x66,0x66,0x65,0x63,0x74, +0x28,0x6e,0x61,0x6d,0x65,0x3b,0x20,0x74, +0x6d,0x29,0x20,0x7b,0x0a, +0x20,0x20,0x20,0x20,0x72,0x65,0x74,0x75, +0x72,0x6e,0x20,0x73,0x74,0x61,0x72,0x74, +0x5f,0x65,0x66,0x66,0x65,0x63,0x74,0x28, +0x6e,0x61,0x6d,0x65,0x2c,0x20,0x74,0x6d, +0x2c,0x20,0x31,0x2e,0x30,0x29,0x0a, +0x7d,0x0a, +}; diff --git a/site-dasllama/README.md b/site-dasllama/README.md index 3c8a1991f8..37c357d7ef 100644 --- a/site-dasllama/README.md +++ b/site-dasllama/README.md @@ -8,6 +8,10 @@ project "daslang.io Forge") on the daslang.io Forge system. Arc plan + follow-up ## Layout - `index.html` - news-first home (identity masthead, feed, how-to-get, ladder teaser) +- `examples.html` - the browser examples: one card per example, each opening the example on its own + page under `examples//` (the storyteller's page is `examples/dasLLAMA/storyteller/web_shell.html` + in the repo, built by `daspkg release wasm`; its model set is the rolling `dasllama-web` GitHub + release the deploy downloads beside it); posters under `files/examples/` - `stories.html` - the stories index (title, date, tag, lede per story), generated from `_stories/` between the `stories:begin` markers by `build_news.py`; the index rows use `forge.css`'s shared blog-list language (`forge-blog-*`), so their styling is shared with @@ -50,6 +54,10 @@ ledgered follow-up. py -3 serve.py # http://127.0.0.1:8932/ - stages shared css like the deploy does +`serve.py` maps `/examples//` onto `../web/output64/examples//` and sends the two +cross-origin-isolation headers the Caddy vhost sends under `/examples/` (`utils/internal/dasllama-ladder/caddy.snippet`), +so a locally built example runs here as deployed once its model set sits in `/models/`. + `serve.py` also proxies `/api/*` to a locally running ladder service on 127.0.0.1:8201 (mirroring the Caddy vhost), so a preview against real data is: start `utils/internal/dasllama-ladder/main.das` with `--official-dir modules/dasLLAMA/performance/records`, diff --git a/site-dasllama/REVIEW.md b/site-dasllama/REVIEW.md index 79146fae7a..ede3cb522c 100644 --- a/site-dasllama/REVIEW.md +++ b/site-dasllama/REVIEW.md @@ -37,8 +37,9 @@ top-level `.html` page here, or in a script or stylesheet under `files/`, that n `site/files/dasllama-table.css` (repo root) defines and no `id="..."` in those files declares - is a defect.** -**A claim in any text this folder publishes or documents - a page, a `_news/*.md` or -`_stories/*.md` entry, or `README.md` - that is not true at publish time is a defect.** A dated +**A claim in any text under this folder a reader or an operator reads as fact - a page, a +`_news/*.md` or `_stories/*.md` entry, `README.md`, a tool's docstring - that is not true at +publish time is a defect.** A dated `_news` or `_stories` entry is read as a claim about its own date; standing page copy - a masthead, section prose, a meta tag - is read as a claim about now, and a diff that falsifies standing copy fixes it in the same change. Check a claim about what a shipped tool does against diff --git a/site-dasllama/_news/2026-09-05-storyteller-in-the-browser.md b/site-dasllama/_news/2026-09-05-storyteller-in-the-browser.md new file mode 100644 index 0000000000..c7d96538fa --- /dev/null +++ b/site-dasllama/_news/2026-09-05-storyteller-in-the-browser.md @@ -0,0 +1,12 @@ +--- +date: 2026-09-05 +tag: examples +title: dasLLAMA runs in the browser - the storyteller, compiled to wasm64, opens the new examples page. +--- + +The engine behind the ladder now also ships as a WebAssembly build. `daspkg release wasm` +compiles the storyteller - llama2.c's stories15M writing a children's tale while KittenTTS nano +reads it aloud - into one 26 MB wasm64 module, and the two models arrive as prepared `.dlim` +images, 76 MB with the two English front-end packs, the same format the native engine maps. The +speech thread and the parallel kernels run on Web Workers in the page; nothing is interpreted. +Chrome or Edge 133+, Firefox 134+ (memory64). [Try it](examples.html). diff --git a/site-dasllama/_stories/template.html b/site-dasllama/_stories/template.html index 4cc721eebb..a9e31556d4 100644 --- a/site-dasllama/_stories/template.html +++ b/site-dasllama/_stories/template.html @@ -24,7 +24,7 @@
dasllama.io -
homestoriesladder
+
homeexamplesstoriesladder
GitHub built on daslang.io ↗v0.6.4
@@ -46,7 +46,7 @@

{{title}}

dasllama.io

LLM + speech-to-text inference written in daslang.
© 2018–2026 Gaijin Entertainment · BSD 3-Clause

- + diff --git a/site-dasllama/build_news.py b/site-dasllama/build_news.py index 90aa1ba631..80fdd1b399 100644 --- a/site-dasllama/build_news.py +++ b/site-dasllama/build_news.py @@ -218,7 +218,7 @@ def write_sitemap(root: Path, entries: list[dict], stories: list[dict], site_url newest = entries[0]['date'] if entries else None newest_story = stories[0]['date'] if stories else None # ONE line: site-dasllama/REVIEW.das reads it as the top-level page census - urls = [('', newest), ('stories.html', newest_story), ('ladder.html', None), ('sidecars.html', None)] + urls = [('', newest), ('examples.html', None), ('stories.html', newest_story), ('ladder.html', None), ('sidecars.html', None)] urls += [(f'stories/{s["slug"]}.html', s['date']) for s in stories] body = '\n'.join( f'{site_url}/{page}' diff --git a/site-dasllama/examples.html b/site-dasllama/examples.html new file mode 100644 index 0000000000..e1656fd7e5 --- /dev/null +++ b/site-dasllama/examples.html @@ -0,0 +1,75 @@ + + + +examples — dasllama.io + + + + + + + + + + + + + + + + +
+ + + +
+
+

The engine, running.

+

Every example here is dasLLAMA itself - compiled to WebAssembly and running in your browser, or built for your machine.

+
+
+ +
+
+
+ + +
+ Storyteller screenshot + +
+
+
+
+

Storyteller

+ llm + tts · in the browser +
+

A small story model writes a children's tale one token at a time while KittenTTS reads it aloud. Both models arrive as prepared .dlim images; the whole engine runs in your browser, the speech thread and the parallel kernels on Web Workers.

+
llmttswasm64threads
+
space · next storyChrome or Edge 133+, Firefox 134+ (memory64)
+
+ + +
+

Source for every example lives under examples/dasLLAMA in the repository; the browser builds are made with daspkg release wasm from the same sources.

+
+
+ + + +
diff --git a/site-dasllama/feed.xml b/site-dasllama/feed.xml index ecb7f0071a..7a7a55acac 100644 --- a/site-dasllama/feed.xml +++ b/site-dasllama/feed.xml @@ -4,9 +4,21 @@ dasllama.io -2026-09-03T00:00:00Z +2026-09-05T00:00:00Z https://dasllama.io/ +dasLLAMA runs in the browser - the storyteller, compiled to wasm64, opens the new examples page. + +https://dasllama.io/#n-2026-09-05-storyteller-in-the-browser +2026-09-05T00:00:00Z +<p>The engine behind the ladder now also ships as a WebAssembly build. <code>daspkg release wasm</code> +compiles the storyteller - llama2.c's stories15M writing a children's tale while KittenTTS nano +reads it aloud - into one 26 MB wasm64 module, and the two models arrive as prepared <code>.dlim</code> +images, 76 MB with the two English front-end packs, the same format the native engine maps. The +speech thread and the parallel kernels run on Web Workers in the page; nothing is interpreted. +Chrome or Edge 133+, Firefox 134+ (memory64). <a href="examples.html">Try it</a>.</p> + + dasLLAMA speaks - KittenTTS nano and mini, Kokoro-82M, and a text front end that is nothing but data. https://dasllama.io/#n-2026-09-03-text-to-speech diff --git a/site-dasllama/files/examples/storyteller-poster.png b/site-dasllama/files/examples/storyteller-poster.png new file mode 100644 index 0000000000..c944c6ce60 Binary files /dev/null and b/site-dasllama/files/examples/storyteller-poster.png differ diff --git a/site-dasllama/index.html b/site-dasllama/index.html index 2780b653ad..eb12553974 100644 --- a/site-dasllama/index.html +++ b/site-dasllama/index.html @@ -24,7 +24,7 @@
dasllama.io -
homestoriesladder
+
homeexamplesstoriesladder
GitHub built on daslang.io ↗v0.6.4
@@ -62,8 +62,17 @@

Local inference, written in daslang.

-
-
2026-09-03engine

dasLLAMA speaks - KittenTTS nano and mini, Kokoro-82M, and a text front end that is nothing but data.latest

▾
+
+
2026-09-05examples

dasLLAMA runs in the browser - the storyteller, compiled to wasm64, opens the new examples page.latest

▾
+

The engine behind the ladder now also ships as a WebAssembly build. daspkg release wasm +compiles the storyteller - llama2.c's stories15M writing a children's tale while KittenTTS nano +reads it aloud - into one 26 MB wasm64 module, and the two models arrive as prepared .dlim +images, 76 MB with the two English front-end packs, the same format the native engine maps. The +speech thread and the parallel kernels run on Web Workers in the page; nothing is interpreted. +Chrome or Edge 133+, Firefox 134+ (memory64). Try it.

+
+
+
2026-09-03engine

dasLLAMA speaks - KittenTTS nano and mini, Kokoro-82M, and a text front end that is nothing but data.

▾

Three text-to-speech models serve from the same engine and the same tuned kernels as the language models, on the CPU, through /v1/audio/speech on dasllama-server and a speech studio on its control page. On an M1 Max the served 8-bit lane reads a real-time factor of @@ -145,7 +154,7 @@

Latest measurements.

dasllama.io

LLM + speech-to-text inference written in daslang.
© 2018–2026 Gaijin Entertainment · BSD 3-Clause

- +
diff --git a/site-dasllama/ladder.html b/site-dasllama/ladder.html index 81827de611..1b258bb599 100644 --- a/site-dasllama/ladder.html +++ b/site-dasllama/ladder.html @@ -24,7 +24,7 @@
dasllama.io -
homestoriesladder
+
homeexamplesstoriesladder
GitHub built on daslang.io ↗v0.6.4
@@ -66,7 +66,7 @@ dasllama.io

LLM + speech-to-text inference written in daslang.
© 2018–2026 Gaijin Entertainment · BSD 3-Clause

- + diff --git a/site-dasllama/serve.py b/site-dasllama/serve.py index 6c2192e8e5..b1ddfad89c 100644 --- a/site-dasllama/serve.py +++ b/site-dasllama/serve.py @@ -4,6 +4,12 @@ github-star.js come from ../site/files (single source of truth in the repo), everything else from here. +/examples// is served from ../web/output64/examples// - where `daspkg release wasm` +writes a browser example - with the two cross-origin-isolation headers the Caddy vhost sends +there (the -pthread wasm64 builds need SharedArrayBuffer). Build one, copy its model set into +/models/ (the deploy downloads the same set from the `dasllama-web` release into the staged +tree), and the card works here as deployed. + /api/* is proxied to a locally running ladder service (utils/internal/dasllama-ladder on :8201), mirroring the Caddy vhost — start one with real data to preview the live pages: @@ -19,6 +25,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) SITE_FILES = os.path.normpath(os.path.join(HERE, "..", "site", "files")) +EXAMPLES_OUT = os.path.realpath(os.path.join(HERE, "..", "web", "output64", "examples")) SHARED = {"forge.css", "nav-dropdown.css", "dasllama-table.css", "github-star.js"} PORT = 8932 LADDER = "http://127.0.0.1:8201" @@ -33,6 +40,13 @@ def do_GET(self): return self.proxy_api() return super().do_GET() + def end_headers(self): + # mirrors the `header /examples/*` block of utils/internal/dasllama-ladder/caddy.snippet + if self.path.startswith("/examples/"): + self.send_header("Cross-Origin-Opener-Policy", "same-origin") + self.send_header("Cross-Origin-Embedder-Policy", "credentialless") + super().end_headers() + def proxy_api(self): try: with urllib.request.urlopen(LADDER + self.path, timeout=30) as up: @@ -63,6 +77,14 @@ def translate_path(self, path): name = os.path.basename(clean) if name in SHARED: return os.path.join(SITE_FILES, name) + if clean.startswith("/examples/"): + rel = clean[len("/examples/"):] + # containment, not pattern checks: `..`, an absolute remainder (a doubled slash) and a + # drive-qualified one on Windows (`C:foo`) all resolve outside EXAMPLES_OUT and fall + # through to the site tree, where no such path exists + full = os.path.realpath(os.path.join(EXAMPLES_OUT, rel)) + if rel and full.startswith(EXAMPLES_OUT + os.sep): + return full return super().translate_path(path) diff --git a/site-dasllama/sidecars.html b/site-dasllama/sidecars.html index 615a098945..4221acaa1c 100644 --- a/site-dasllama/sidecars.html +++ b/site-dasllama/sidecars.html @@ -25,7 +25,7 @@
dasllama.io -
homestoriesladdersidecars
+
homeexamplesstoriesladdersidecars
GitHub built on daslang.io ↗v0.6.4
@@ -61,7 +61,7 @@

The sidecar exchange is closed.

dasllama.io

LLM + speech-to-text inference written in daslang.
© 2018–2026 Gaijin Entertainment · BSD 3-Clause

- + diff --git a/site-dasllama/sitemap.xml b/site-dasllama/sitemap.xml index b855b18620..0dfdd433f4 100644 --- a/site-dasllama/sitemap.xml +++ b/site-dasllama/sitemap.xml @@ -1,6 +1,7 @@ -https://dasllama.io/2026-09-03 +https://dasllama.io/2026-09-05 +https://dasllama.io/examples.html https://dasllama.io/stories.html2026-09-03 https://dasllama.io/ladder.html https://dasllama.io/sidecars.html diff --git a/site-dasllama/stories.html b/site-dasllama/stories.html index 9b337dafed..beeb50a2ff 100644 --- a/site-dasllama/stories.html +++ b/site-dasllama/stories.html @@ -24,7 +24,7 @@
dasllama.io -
homestoriesladder
+
homeexamplesstoriesladder
GitHub built on daslang.io ↗v0.6.4
@@ -56,7 +56,7 @@

Stories from the build.

dasllama.io

LLM + speech-to-text inference written in daslang.
© 2018–2026 Gaijin Entertainment · BSD 3-Clause

- + diff --git a/site-dasllama/stories/2026-08-30-apple-m5.html b/site-dasllama/stories/2026-08-30-apple-m5.html index 61d47e42fe..68ca0ea610 100644 --- a/site-dasllama/stories/2026-08-30-apple-m5.html +++ b/site-dasllama/stories/2026-08-30-apple-m5.html @@ -24,7 +24,7 @@
dasllama.io -
homestoriesladder
+
homeexamplesstoriesladder
GitHub built on daslang.io ↗v0.6.4
@@ -157,7 +157,7 @@

Sixty-two kernels walked into a new chip.

dasllama.io

LLM + speech-to-text inference written in daslang.
© 2018–2026 Gaijin Entertainment · BSD 3-Clause

- + diff --git a/site-dasllama/stories/2026-09-03-mtp-on-mac.html b/site-dasllama/stories/2026-09-03-mtp-on-mac.html index f6a8943c59..e6622083c1 100644 --- a/site-dasllama/stories/2026-09-03-mtp-on-mac.html +++ b/site-dasllama/stories/2026-09-03-mtp-on-mac.html @@ -24,7 +24,7 @@
dasllama.io -
homestoriesladder
+
homeexamplesstoriesladder
GitHub built on daslang.io ↗v0.6.4
@@ -341,7 +341,7 @@

What a draft is worth

dasllama.io

LLM + speech-to-text inference written in daslang.
© 2018–2026 Gaijin Entertainment · BSD 3-Clause

- + diff --git a/site-dasllama/stories/2026-09-03-text-to-speech.html b/site-dasllama/stories/2026-09-03-text-to-speech.html index 04510e4776..4128281d8f 100644 --- a/site-dasllama/stories/2026-09-03-text-to-speech.html +++ b/site-dasllama/stories/2026-09-03-text-to-speech.html @@ -24,7 +24,7 @@
dasllama.io -
homestoriesladder
+
homeexamplesstoriesladder
GitHub built on daslang.io ↗v0.6.4
@@ -304,7 +304,7 @@

Where it ends

dasllama.io

LLM + speech-to-text inference written in daslang.
© 2018–2026 Gaijin Entertainment · BSD 3-Clause

- + diff --git a/site-dasllama/test_metadata.py b/site-dasllama/test_metadata.py index 14842ea1bb..d93f082d95 100644 --- a/site-dasllama/test_metadata.py +++ b/site-dasllama/test_metadata.py @@ -11,6 +11,7 @@ # and REVIEW.das reads this list to hold each served .html to it PAGES = { "index.html": "https://dasllama.io/", + "examples.html": "https://dasllama.io/examples.html", "stories.html": "https://dasllama.io/stories.html", "ladder.html": "https://dasllama.io/ladder.html", "sidecars.html": "https://dasllama.io/sidecars.html", diff --git a/site/_news/2026-09-05-dasllama-in-the-browser.md b/site/_news/2026-09-05-dasllama-in-the-browser.md new file mode 100644 index 0000000000..7a4095d6f9 --- /dev/null +++ b/site/_news/2026-09-05-dasllama-in-the-browser.md @@ -0,0 +1,6 @@ +--- +date: 2026-09-05 +tag: dasllama +title: You can now run dasLLAMA in the web browser - the whole engine compiled to wasm64, a story model writing while KittenTTS reads it aloud. Try it out on dasllama.io. +link: https://dasllama.io/examples.html +--- diff --git a/site/files/examples.js b/site/files/examples.js index 00ba7b85f1..d1c4f8bab9 100644 --- a/site/files/examples.js +++ b/site/files/examples.js @@ -258,7 +258,7 @@ '' + '
Needs a memory64 browser
' + '

' + esc(ex.name) + ' bundles the Dear ImGui ' + - 'module compiled to wasm64 — it runs on Chrome, Edge or Firefox 133+ (engines with ' + + 'module compiled to wasm64 — it runs on Chrome or Edge 133+ and Firefox 134+ (engines with ' + 'WebAssembly memory64). The universal interpreter can’t bind a native module, so ' + 'this one has no interpreted fallback.

' + 'view source ↗' + diff --git a/skills/das_macros.md b/skills/das_macros.md index 791285870d..809942d18d 100644 --- a/skills/das_macros.md +++ b/skills/das_macros.md @@ -60,7 +60,8 @@ To contribute macro-time state into A's registry the contributor must compile in two pieces: an optional require (`require ? `; a plain `require` resolves before `static_if`, so `static_if` alone cannot gate it), plus `static_if (typeinfo builtin_module_exists())` around the registration call and any -dispatcher branch naming the contributor's symbols. Adapters may still *emit* code referencing +dispatcher branch naming the contributor's symbols - `typeinfo module_exists()` when the +target is a das module, so a tool's nested compile takes the same arm. Adapters may still *emit* code referencing those symbols by name - that resolves at the user's splice site and is unaffected. ## Pass macros - which hook sees what diff --git a/skills/daslang/references/generics.md b/skills/daslang/references/generics.md index dc2f546ae2..9c43745b4f 100644 --- a/skills/daslang/references/generics.md +++ b/skills/daslang/references/generics.md @@ -126,7 +126,9 @@ Annotation traits all carry the `struct_` prefix - plain `has_annotation` does n argument to already be a struct or handled type, `safe_has_field` does not. `` takes the bare field name, not a string: `typeinfo safe_has_field(obj)`. -**Existence** - `builtin_module_exists(name)` (how a binding is made optional), +**Existence** - `builtin_module_exists(name)` (a module registered in the process - how a +C++ binding is made optional), `module_exists(name)` (a module visible from the compiling module, +C++ or das - the check behind a guarded require's target, same answer inside a tool's nested compile), `builtin_annotation_exists(type)` (true for registered native types), `builtin_function_exists(@@ name)` (native functions only - a daslang-defined function is a compile error here). diff --git a/skills/daslang/references/modules-and-stdlib.md b/skills/daslang/references/modules-and-stdlib.md index 953394810f..3d1562f449 100644 --- a/skills/daslang/references/modules-and-stdlib.md +++ b/skills/daslang/references/modules-and-stdlib.md @@ -91,7 +91,9 @@ require ?pugixml pugixml/PUGIXML_boost // load only if module `pugixml` is avai - **`?guard` skips silently** when the guard module is unavailable - no dependency, no error, even if the target does not exist; with the guard present a missing target errors normally. A guard containing `/` is satisfied when that path resolves, a plain name when that module is registered. - Pair with `static_if (typeinfo builtin_module_exists(guard)) { ... }`. + Pair with `static_if (typeinfo builtin_module_exists(guard)) { ... }` for a C++ guard, and + with `static_if (typeinfo module_exists(target)) { ... }` around uses of the guarded target + (true when the target is visible from this module, the same inside a tool's nested compile). ## Qualified calls diff --git a/skills/daspkg.md b/skills/daspkg.md index ec03e6a77e..814770924b 100644 --- a/skills/daspkg.md +++ b/skills/daspkg.md @@ -338,13 +338,14 @@ def release() { `release_wasm_main` exists because the desktop entry often `require`s host-only stacks (live-reload, libhv HTTP, threaded jobque) that don't cross-compile. The web variant swaps those for a `live_stub` shim (plain GLFW window + frame clock + no-op `[live_command]`) and guards music off (threaded strudel needs the jobque worker; SFX is single-threaded and stays on). One `.das_package` drives both targets. -**Module-side hooks** (in a module's own `.das_package`, accumulated across all linked modules): +**Module-side hooks** (in a module's own `.das_package`, accumulated across all linked modules; an app's own `.das_package` may call `release_emcc_arg` too - its flags come last on the link line, so they win): | Hook | Effect | |---|---| | `release_emcc_arg("-sUSE_GLFW=3")` | Append an emcc link flag (dasGlfw declares this) | | `release_embed_file(src, dst)` | `--embed-file src@dst` into MEMFS (dasStbImage embeds its HUD font) | -| `release_web_shell("path.html")` | Override the default canvas shell (`web/templates/wasm_canvas_shell.html`) | +| `release_web_shell("path.html")` | Override the default canvas shell (`web/templates/wasm_canvas_shell.html`). The link exports `FS` and `ENV`, so a shell's `Module.preRun` can stage files into MEMFS (`Module.FS.writeFile`) and set environment knobs (`Module.ENV.X = ...` - the C environment is built from it at startup, later writes never reach `getenv`); hold the run with `addRunDependency` while fetching | +| `release_wasm_disable_module("dasvulkan")` | Keep a host module out of the cross-compile so its guarded `require ?name` resolves as absent, as the wasm build sees it (the rail disables dashv itself; an app reaching vulkan, das_metal or das_accelerate names those) | ### Game source contract (cross-compiles UNCHANGED) diff --git a/skills/internal/babysit.md b/skills/internal/babysit.md index 745a97c733..f729d24175 100644 --- a/skills/internal/babysit.md +++ b/skills/internal/babysit.md @@ -56,7 +56,7 @@ matrix between rounds. 6. Repeat until Section 0 holds; **only then** wait for CI green and merge. **Every push that changes a `.das` file since the last push runs the pre-push lint gate -first**: `daslang utils/internal/preflight/main.das -- --only lint,review-md --lint-skip-exe-rail` +first**: `daslang utils/internal/preflight/main.das -- --only lint,review-md` (rails and cost: `skills/internal/preflight.md`, the lint row). MCP `lint` on the file just edited is NOT this gate: this run lints every `.das` the branch changed against `origin/master`, exactly as CI does, so a clean single-file lint still pushes red. **A push diff --git a/skills/internal/build_and_debug.md b/skills/internal/build_and_debug.md index 6393f22bda..7abd7d5951 100644 --- a/skills/internal/build_and_debug.md +++ b/skills/internal/build_and_debug.md @@ -19,6 +19,7 @@ The repo builds on **Windows, Linux, macOS, iOS, Android, and WASM** (CI runs th - **Compile-only check:** ` -compile-only path/to/script.das` - compiles without simulation or execution, useful for syntax/type checking without needing a window or GL context. Use `-dry-run` to also simulate (but not execute). - **Run tests:** ` dastest/dastest.das -- --test path/to/test.das` - **AOT tests (full):** `cmake --build build --config Release --target test_aot` then ` -use-aot dastest/dastest.das -- --use-aot --test tests`. The full binary is `EXCLUDE_FROM_ALL` (~1080 AOT TUs); the default build only makes `test_aot_subset` (tests/language - the per-PR CI gate; `--target run_tests_aot_subset` sweeps it) +- **dasLLAMA on the AOT tier:** `cmake --build build --target dasllama_aot` (opt-in, `EXCLUDE_FROM_ALL`, ~90 engine TUs) then `bin/dasllama_aot -use-aot examples/dasLLAMA/run.das -- `. The host is `daslang` with the CPU engine's C++ stubs linked in; `-use-aot` compiles the script with `policies.aot` so they bind, and a function without a stub interprets. `options log_aot` in the script root lists every function as linked or `NOT FOUND` with its hash - the coverage probe for a stub set - **IMPORTANT:** When adding a new test directory under `tests/`, register it in `tests/aot/CMakeLists.txt` for AOT compilation. See `skills/internal/aot_testing.md` for the step-by-step pattern. The nightly CI + `preflight --full` run ALL tests with AOT enabled - unregistered test directories cause `error[50101]: AOT link failed` there (per-PR CI won't catch it) This skill uses `bin/Release/daslang.exe` in examples below (the dominant local-dev case); substitute the right path on other platforms. diff --git a/skills/internal/llvm_tune_internals.md b/skills/internal/llvm_tune_internals.md index 87e1824a1e..a4404ca590 100644 --- a/skills/internal/llvm_tune_internals.md +++ b/skills/internal/llvm_tune_internals.md @@ -4,6 +4,16 @@ Read before touching `modules/dasLLVM/daslib/llvm_tune.das` or its tests. The fr itself - every annotation, the policy rail, the sidecar contract - is `skills/tune.md`; this file is only what maintaining the implementation needs on top of it. +`llvm_tune.das` registers no annotation of its own. All six names plus the default-policy +infer pass are registered by `daslib/tune`, whose shells forward here - `tune_apply`, +`tune_perm_apply`, `tune_companion_apply`, `tune_scope_apply`, `tune_policy_apply`, +`tune_default_policy_apply`, and `llvm_code_apply` in `llvm_code.das` - under +`static_if (typeinfo module_exists(llvm_tune))`, and do nothing when the guard is +false. Adding an annotation means adding both halves. A shell may not stay behind in this +module: **a module's macro state is per-module**, so `g_scopes`, `g_perm_rows` and +`g_companion_rows` are only consistent while every writer and reader runs in one macro +context - which is why the infer pass moved to `daslib/tune` with the annotations. + ## Macro-context traps - **Cross-module restamp does not work.** Mutating a *required* library's already-compiled @@ -29,7 +39,9 @@ file is only what maintaining the implementation needs on top of it. `collect_status` reads the stamps back off the AST. - **`stamp_llvm_code` records `tune_suffix` / `tune_from` / `tune_source`** as extra `[llvm_code]` args (generators ignore unknown args by contract), and `tune_status` reads that - stamped truth off the AST - there is no macro-state bank to keep coherent. + stamped truth off the AST - there is no macro-state bank to keep coherent. It appends the + annotation from module `tune`, the module that registers `[llvm_code]`; readers match it by + name, so nothing downstream cares which module it came from. - **`--tune` is decided at runtime** by the guard (`tune_cli_force`, the same `apply_cli_tune_flags` parse over `get_user_args()`), because a warm module cache can serve a compile whose macros never saw this run's argv; the policy pass reads it at macro time only to @@ -37,9 +49,11 @@ file is only what maintaining the implementation needs on top of it. ## Where the pieces live -- The default policy is injected by the `TuneDefaultPolicy` infer pass. A root is detected by - the anonymous module name, and the `tune_policy` marker annotation the pass appends is its - once-only guard. +- The default policy is injected by the `TuneDefaultPolicyShell` infer pass in `daslib/tune`, + which forwards to `tune_default_policy_apply` here. A root is detected by the anonymous + module name, and the `tune_policy` marker annotation the pass appends is its once-only guard. + The pass reaches a root that sees `daslib/tune`, so a scope-owning library re-exports THAT + (`require daslib/tune public`). - `scope_fnames_joined` appends a `version_of` pin to the joined kernel names as one `@version:key=value` token, so every carrier of that string - the policy guards, the resolver contract, daspkg's status walk - enforces the pin with no signature change. diff --git a/skills/internal/preflight.md b/skills/internal/preflight.md index d4e996d40a..6d9cffeaac 100644 --- a/skills/internal/preflight.md +++ b/skills/internal/preflight.md @@ -13,8 +13,7 @@ sequence`, `--only dasllama-model-free` when the module is the work. A gate with a **reach set** skips, with the reason, when nothing under its paths or the core (`src/`, `include/`, `daslib/`, `dastest/`, `CMakeLists.txt`, `cmake/`) changed; `--only` runs a gate whatever changed. `--list-gates` prints tier, reach -and description; `--skip ` drops gates; `--lint-skip-exe-rail` trims the -lint gate to its interp rails. A gate whose host tool or module is missing +and description; `--skip ` drops gates. A gate whose host tool or module is missing reports `SKIP` with an install/rebuild hint. The budget the tiers serve: a full run fits 20 minutes on the M5 box, or the gate is not in preflight (`plans/ci_preflight_budget.md`). @@ -172,10 +171,10 @@ cmake -B build -DDAS_HV_DISABLED=OFF -DDAS_LLVM_DISABLED=OFF -DDAS_AUDIO_DISABLE | Run examples - **nightly** | `cmake --build build --config Release --target run_examples`; per PR, preflight's `compile-sweep` gate compiles every example root | the run form is 5-8 minutes a lane | | Utils tests | `cmake --build build --config Release --target run_utils_tests` - preflight's `utils-tests` lane | | | Tutorial dry-runs - **nightly** | `cmake --build build --config Release --target dry_run_tutorials`; per PR, preflight's `compile-sweep` gate compiles every tutorial root | compile rot in `tutorials/` after daslib API changes; the run form is 8-11 minutes a lane | -| Standalone exes | `cmake --build build --config Release --target all_utils_exe`, plus ` -exe -output bin/das-fmt utils/das-fmt/dasfmt.das` and `... bin/das-lint utils/lint/main.das` | `-exe` needs dasLLVM + lld-link on PATH | +| Standalone exes | `cmake --build build --config Release --target all_utils_exe`, plus ` -exe -output bin/das-fmt utils/das-fmt/dasfmt.das` and `... bin/das-lint utils/lint/main.das` | `-exe` needs dasLLVM + lld-link on PATH; das-lint is built, not run - an exe carries the host's module paths without the modules, so its lint world is not one in-tree files answer to; the shipped `lint.exe` is exercised by the bundle smoke test on daslib | | Sequence smoke | Windows: `pwsh examples/games/sequence/ci_smoke_test.ps1 "$(pwd)"`; linux/mac: `bash examples/games/sequence/ci_smoke_test.sh "$(pwd)"` | build the runtime modules first: `cmake --build build --config Release --target dasModuleGlfw dasModuleLiveHost dasModuleHV dasModuleAudio dasModulePUGIXML dasModuleStbImage`. **The only pre-merge lane compiling GLFW-gated `.das` like dasOpenGL** - run it for type-system / daslib-generics changes | | Formatter `--verify` | preflight's `format` gate runs it exactly (tracked files via `--files-from`); manual: ` utils/das-fmt/dasfmt.das -- --path ./ --verify --exclude-mask build/` | CI's second verify pass uses an `-exe`-compiled `bin/das-fmt.exe`; the mask skips generated `.das` under the build dir (nightly doc-verify extracts RST snippets there) | -| Lint changed `.das` | preflight's `lint` gate - THREE rails: host-flavor interp, the LINUX-lane mirror (`--disable-module dasMetal` - CI's verdict comes only from linux, where dasMetal's platform `static_if` halves compile out so its requires/args read unused there and nowhere else; dasVulkan is in-tree on linux, so NOT disabled), and the `-exe`-compiled `das-lint` (different module resolution). `--lint-skip-exe-rail` drops the exe rail (per-push cadence - `skills/internal/babysit.md`). Single rail: `git diff --name-only origin/master..HEAD -- '*.das' \| xargs utils/lint/main.das -- --quiet` | zero warnings on ALL rails; a mirror-only STYLE030/LINT012 takes the both-worlds `nolint:...,LINT019` spelling | +| Lint changed `.das` | preflight's `lint` gate - TWO rails: host-flavor interp, then the LINUX-lane mirror (`--disable-module dasMetal` - CI's verdict comes only from linux, where dasMetal's platform `static_if` halves compile out so its requires/args read unused there and nowhere else; dasVulkan is in-tree on linux, so NOT disabled). Single rail: `git diff --name-only origin/master..HEAD -- '*.das' \| xargs utils/lint/main.das -- --quiet` | zero warnings on BOTH rails; a mirror-only STYLE030/LINT012 takes the both-worlds `nolint:...,LINT019` spelling | | ast-verify changed `.das` | preflight's `ast-verify` gate - ` -dry-run --ast-verify-batch ` per changed `.das` plus the `tests/linq/test_linq_fold.das` qmacro canary, parallel, 300 s per-file timeout, skipping `cant_`/`failed_`/`invalid_` and `utils/internal/ast-fuzz/selftest/`. An `AST verify` line, crash or timeout fails; a compile error belongs to whoever owns the file; a file inside the verifier's own require closure (`daslib/ast*.das`, `daslib/rtti.das`, `daslib/strings_boost.das` - `error[20510]` under the force-include) is reported *not verifiable*, never clean | mirrors the workflow's "Run ast-verify on changed .das files". Batch mode is the ruled gate form (`skills/das_macros.md`); with no pre-infer walk, a tree a macro breaks mid-inference surfaces as a compiler crash instead of a located report - hence crash = red, and plain `--ast-verify` on that file locates it. Each item is a whole-engine compile (2-3x a plain one). Width is physical cores halved; `-j` only lowers it | | REVIEW.das gates | ` utils/internal/review-md/all.das` | every `REVIEW.das` in the tree, fail-fix; also run per-diff in the make_pr step-0a walk | | dastest own suite | ` dastest/dastest.das -- --failures-only --test dastest/tests` | framework suite + `review_gate` library tests; whole-directory, so a new file needs no CI row | diff --git a/skills/internal/writing_cpp_tests.md b/skills/internal/writing_cpp_tests.md index 608485d7cb..160b4014c8 100644 --- a/skills/internal/writing_cpp_tests.md +++ b/skills/internal/writing_cpp_tests.md @@ -38,7 +38,7 @@ TEST_CASE("my feature does X") { Optionally drop `tests-cpp/small/test_my_feature.das` next to it; load via `getDasRoot() + "/tests-cpp/small/test_my_feature.das"`. -Rebuild - `cmake --build build`. The new test appears in `ctest -N` automatically (no CMake edit needed). +Rebuild - `cmake --build build`. The new test appears in `ctest -N` automatically; the glob needs no CMake edit. A test that must be compiled differently from the rest of the exe - a per-source flag such as the fast-math one on `test_isnan_fastmath.cpp` - adds its `set_source_files_properties` line to `tests-cpp/CMakeLists.txt`, and its TU `#error`s when built without the flag, so a moved or renamed file cannot silently build the ordinary way and stay green. **Don't include doctest with `DOCTEST_CONFIG_IMPLEMENT`** - that's owned exclusively by `doctest_main.cpp`. Including the header without that macro is the right pattern for every other TU. diff --git a/skills/tune.md b/skills/tune.md index 90c550b70e..30d05886ba 100644 --- a/skills/tune.md +++ b/skills/tune.md @@ -1,11 +1,20 @@ # Kernel tuning - `[tune]` and the per-app tune sidecar -`llvm/daslib/llvm_tune` turns **one reference function into a tuned kernel -family**: a grid of code-generation permutations, a per-app record of which -one wins, and a small policy rail that keeps an application honest about -whether it is running tuned code. It sits on top of `[llvm_code]` (the -JIT-time external code generator), so the family only *generates* under the -LLVM JIT; on every other tier the reference body runs verbatim. +`daslib/tune` turns **one reference function into a tuned kernel family**: a +grid of code-generation permutations, a per-app record of which one wins, and +a small policy rail that keeps an application honest about whether it is +running tuned code. It sits on top of `[llvm_code]` (the JIT-time external +code generator), so the family only *generates* under the LLVM JIT; on every +other tier the reference body runs verbatim. + +`daslib/tune` registers all six annotations - `[tune]`, `[tune_perm]`, +`[tune_companion]`, `[tune_scope]`, `[tune_policy]`, `[llvm_code]` - and the +default-policy pass; the framework behind them is `llvm/daslib/llvm_tune`, +which `daslib/tune` reaches through the `?llvm` witness guard (present exactly when the build is configured with dasLLVM) and re-exports; a direct `require llvm/daslib/llvm_tune` marks a program as framework-only - without dasLLVM its files still compile, but the shells are inert and no `_variants()` registry exists. **Any program +that reaches the framework through `daslib/tune` alone can spell the annotations and compile whether or not dasLLVM is configured.** +Without dasLLVM every one of them does nothing at all - no sidecar read, no +stamp, no `_variants()` registry, no tuner spawn - and the reference +body IS the function. The design goal is that a shipped application reaches *its own box's* floor with **defaults that are data** - a small JSON sidecar - rather than a fork @@ -38,7 +47,7 @@ under `"kernels"`, library runtime knobs under `"runtime"`. ```das options gen2 -require llvm/daslib/llvm_tune +require daslib/tune // one reference function, a grid of [llvm_code] generator permutations, // and a per-ISA fallback for an app with no sidecar yet. @@ -169,10 +178,10 @@ CPU class's profile at compile time instead of racing. ```{warning} The default-`auto` policy fires only for app roots that **see** -`llvm_tune` - a library owning scopes must re-export it -(`require llvm/daslib/llvm_tune public`), or its apps silently get no -default policy. Re-export `llvm_tune` alone, not your module's whole -public surface (a blanket `public` on a module that also re-exports +`daslib/tune` - a library owning scopes must re-export it +(`require daslib/tune public`), or its apps silently get no default +policy. Re-export `daslib/tune` alone, not your module's whole public +surface (a blanket `public` on a module that also re-exports `jobque_boost` floods requirers with name ambiguities). ``` diff --git a/src/ast/ast_infer_type.cpp b/src/ast/ast_infer_type.cpp index 8cd047af78..13eabd93ae 100644 --- a/src/ast/ast_infer_type.cpp +++ b/src/ast/ast_infer_type.cpp @@ -2313,7 +2313,7 @@ namespace das { } // verify bool allowMissingTypeExpr = false; - if (expr->trait == "builtin_function_exists" || expr->trait == "builtin_module_exists") { + if (expr->trait == "builtin_function_exists" || expr->trait == "builtin_module_exists" || expr->trait == "module_exists") { allowMissingTypeExpr = true; } bool allowMissingType = false; @@ -2852,6 +2852,26 @@ namespace das { expr->at, CompilationError::invalid_typeinfo_module_subexpression); } } + } else if (expr->trait == "module_exists") { + if (!expr->subexpr) { + error("module_exists requires subexpression", "", "", + expr->at, CompilationError::missing_typeinfo_subexpression); + } else { + if (expr->subexpr->rtti_isVar()) { + auto evar = static_cast(expr->subexpr); + // visible from the compiling module - the module itself, one it requires, or one a + // require re-exports public - C++ or das, promoted or not: the same answer whether + // the program compiles as the running script or inside a tool's nested compile, + // and "in the program somewhere" is not enough for the call the taken arm makes + auto mod = program->library.findModule(evar->name); + bool visible = mod && program->thisModule->isVisibleDirectly(mod); + reportAstChanged(); + return new ExprConstBool(expr->at, visible); + } else { + error("unsupported module name subexpression ", expr->subexpr->__rtti, "", + expr->at, CompilationError::invalid_typeinfo_module_subexpression); + } + } } else if (expr->trait == "builtin_annotation_exists") { if (expr->typeexpr->isAlias()) { reportAstChanged(); diff --git a/src/ast/ast_program.cpp b/src/ast/ast_program.cpp index 6c59fde820..b49aaad2c6 100644 --- a/src/ast/ast_program.cpp +++ b/src/ast/ast_program.cpp @@ -511,6 +511,8 @@ namespace das { static void collectStructDeps ( const TypeDeclPtr & type, Structure * owner, das_hash_set & deps ) { if ( !type ) return; + // a function, lambda or block value is a handle: the types in its signature are not stored by value + if ( type->baseType == Type::tFunction || type->baseType == Type::tLambda || type->baseType == Type::tBlock ) return; if ( type->baseType == Type::tStructure && type->structType && type->structType != owner ) { if ( type->isPointer() ) return; // pointers don't need full definition if ( !deps.insert(type->structType).second ) return; // already visited @@ -569,7 +571,18 @@ namespace das { } } } - if ( sorted.size() != structs.size() ) return; // cycle - keep original order + // a by-value cycle cannot exist (the layout would be infinite); what remains unsorted is a + // container-mediated cycle's members plus every struct that depends on one of them. The + // members need no definition order (TArray holds no T), so they keep the original order + // after everything the sort did place; a by-value dependent of a cycle member still emits + // an incomplete type there, as it did before this fallback existed + if ( sorted.size() != structs.size() ) { + das_hash_set placed; + for ( auto s : sorted ) placed.insert(s); + for ( auto & sp : structs ) { + if ( !placed.count(sp) ) sorted.push_back(sp); + } + } // reorder structs to match sorted order das_hash_map byPtr; for ( auto & sp : structs ) byPtr[sp] = sp; diff --git a/src/builtin/ARCHITECTURE.md b/src/builtin/ARCHITECTURE.md index 29c383e29c..0e0ceb1250 100644 --- a/src/builtin/ARCHITECTURE.md +++ b/src/builtin/ARCHITECTURE.md @@ -44,8 +44,10 @@ current directory, and it sits beside the JIT DLL cache. `` is the script' without its extension. `` is the first 8 hex digits of a 64-bit hash over the normalized script path, the running executable's mtime and size (the host resolves its own path through the OS, so a launch by bare name through `PATH` keys the same file as a launch by path), the host's -own command line - every argument up to the first `--`, the arguments after it belong to the -script - and every `NAME=VALUE` environment pair whose name starts with `DAS`, sorted. +own command line - every argument up to the first `--`, plus a `--jit-target` after it (the one +script argument that changes what the host compiles: the target's folds); the other arguments after +`--` belong to the script - and every `NAME=VALUE` environment pair whose name starts with `DAS`, +sorted. The binary, the command line and the environment are in the key because each changes the compile without changing a source file: macros read the tune and JIT environment at compile diff --git a/src/builtin/REVIEW.md b/src/builtin/REVIEW.md index 7276861558..57258ef318 100644 --- a/src/builtin/REVIEW.md +++ b/src/builtin/REVIEW.md @@ -3,19 +3,15 @@ **Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture doc: `ARCHITECTURE.md`. -- **A diff that adds or changes an `addExtern...`/`addInterop` registration under this folder - uses, for a plain-value bind, `addExternInline` or `addExternInlineEx` when the name its - file passes to `Module(...)` is an Inline module, and an `addExtern...` entry point whose - name does not contain `Inline` in every other module.** A bind is a C++ function registered - into a module with an `addExtern...` or `addInterop` entry point. The Inline modules are `$` - (builtin), `math`, `strings` and `jit`. A plain-value bind returns nothing at all, or a value - that is neither a reference nor a result the callee writes into the caller's result slot; it - is not an interop bind (`addInterop`), and not a generic container or equality helper - (`das_vector_*`, `das_equ*`, `das_nequ*`, `das_handle_equ*`, `das_handle_nequ*`). - - **Weakening `review_nttp.das`'s bind-flavor scan, which `REVIEW.das` runs, is a defect** - fix a bind the scan reports by switching the bind, and never drop a module from - `review_nttp.das`'s `require` list, which sets the modules the scan covers. + `review_nttp.das`'s `require` list, which sets the modules the scan covers. The scan reads the + binds compiled into the RUNNING binary, so a diff that changes a bind runs the gate against a + binary built from the diff - a stale binary yields a false green. The Inline modules are `$` + (builtin), `math`, `strings` and `jit`. In those, a plain-value bind - one returning nothing, or + a value that is neither a reference nor written into the caller's result slot - registers + through `addExternInline` or `addExternInlineEx`; `REVIEW.das` holds this list and the scan's + `inlineOnlyModules` to the same names. - **A diff that adds a module under this folder adds it to `review_nttp.das`'s `require` list, in the same change** - a module off the list is a module the scan never sees. @@ -30,12 +26,15 @@ outside `DAS_MODULE_CACHE_POLICY_FIELDS` is a defect - put the field on the list instead** - the list drives both the record's policy stream and the compare that refuses a record written under other policies, so a field handled outside it is written without being compared, or compared - without being written. A diff that adds a field to `CodeOfPolicies` itself applies - `include/daScript/simulate/REVIEW.md`. + without being written. - **A diff that adds a diagnostic to `AstSerializer::serializeProgram` or `AstSerializer::serializeProgramImpl` in `module_builtin_ast_serialize.cpp`, or drops the `quietCache` gate from one already there, is a defect - gate every line those two functions print on the serializer's `quietCache`** - the default cache is on unasked for an ordinary run, - so an ungated line becomes output every user sees. A diff to `trySerializeProgramModule` - (`src/ast/ast_parse.cpp`) applies `src/ast/REVIEW.md` too. + so an ungated line becomes output every user sees. + +- **A diff that changes what `ModuleFileCache::defaultPath` folds into the module-cache key - + the binary, the command line, the environment names, or which script arguments count - updates + the cache-key paragraph of `ARCHITECTURE.md` in the same change.** The key is what stops a + native-compiled module serving a cross compile, so a wrong description of it gets trusted. diff --git a/src/builtin/module_builtin_runtime.cpp b/src/builtin/module_builtin_runtime.cpp index 43b439339b..91ede69f87 100644 --- a/src/builtin/module_builtin_runtime.cpp +++ b/src/builtin/module_builtin_runtime.cpp @@ -2042,32 +2042,36 @@ namespace das #endif } - // The cross-compilation TARGET platform, or "" on a normal (non-cross) run. - // Unlike get_platform_name() (the host, a compile-time #if), this reads the - // active jit cross-compile target from the command line (`--jit-target=` - // after `--`) -- already in g_CommandLineArguments at process start, so it is - // valid at .das_module-initialize time, before the jit codegen macro runs. - // Lets a .das_module register a target-native module only when cross-compiling - // for that target (e.g. dasOpenGL registers its wasm GLES3 module for emscripten, - // while a normal desktop run keeps the pure-das opengl.das). Only the wasm triple - // is mapped today (-> "emscripten"); other cross targets return "". - const char * das_get_cross_platform_name() { + // The --jit-target triple this compile emits for, "" on a native compile. Read from the + // command line (`--jit-target=` or `--jit-target ` after `--`), already in + // g_CommandLineArguments at process start, so it is valid at .das_module-initialize time, + // before the jit codegen macro runs. A property of the whole compile (the triple is fixed + // before the first module parses), so the folds below are compile-time constants: an artifact + // for another machine decides its tiers from the target. das_get_cross_platform_name maps the + // triple to a platform name - only the wasm triples are mapped today (-> "emscripten"), which + // lets a .das_module register a target-native module only when cross-compiling for that + // target (dasOpenGL registers its wasm GLES3 module for emscripten; a desktop run keeps the + // pure-das opengl.das). + const char * das_get_target_triple() { char ** argv = (char **) g_CommandLineArguments.data; uint64_t n = g_CommandLineArguments.size; + const char * found = ""; // the LAST occurrence wins, as every other argv reader here takes it for ( uint64_t i=0; i(*this, lib, "get_architecture_name", SideEffects::none, "das_get_architecture_name"); + // the two target-side folds (see the definitions): SideEffects::none like the pair above, + // so a kernel-tier [init] gate and a static_if see the artifact's target, not the host + addExternInline(*this, lib, "get_target_triple", + SideEffects::none, "das_get_target_triple"); + addExternInline(*this, lib, "get_target_architecture_name", + SideEffects::none, "das_get_target_architecture_name"); // accessExternal (NOT ::none) on purpose: CPU features are a property of the RUNNING box, // so this must never const-fold into AOT artifacts built on a different machine. addExternInline(*this, lib, "cpu_supports", diff --git a/tests-cpp/REVIEW.das b/tests-cpp/REVIEW.das index 214d882f53..8a6a0febe2 100644 --- a/tests-cpp/REVIEW.das +++ b/tests-cpp/REVIEW.das @@ -61,6 +61,21 @@ def private check_test_targets(f : string; text : string) { } } +//! every source a `set_source_files_properties` names exists beside its CMakeLists - a property naming a moved file applies to nothing and its test builds green without the flag +def private check_source_property_paths(f : string; text : string) { + let folder = dir_name(f) + for (blk in cmake_command_blocks(text, "set_source_files_properties")) { + var inscope args <- cmake_args(blk) + for (a in args) { + break if (a == "PROPERTIES" || a == "DIRECTORY" || a == "TARGET_DIRECTORY") + continue if (a |> starts_with("$")) + if (!fexist(path_join(folder, a))) { + gate_finding(f, "set_source_files_properties names {a}, which does not exist beside {f} - the property applies to nothing and its test builds without the flag") + } + } + } +} + // Every lane a file labels tests into is a lane that file also builds for. def private check_lane_wiring { let top_cmake = "tests-cpp/CMakeLists.txt" @@ -79,6 +94,7 @@ def private check_lane_wiring { gate_finding(f, "labels a test \"{lab}\" but wires nothing into test-{lab} — wire it with add_dependencies(test-{lab} ), or the lane runs a binary it never built") } check_test_targets(f, text) + check_source_property_paths(f, text) } } diff --git a/tests-cpp/REVIEW.md b/tests-cpp/REVIEW.md index af4b8d8543..1fee1d8e67 100644 --- a/tests-cpp/REVIEW.md +++ b/tests-cpp/REVIEW.md @@ -9,6 +9,6 @@ checklist as well as this one.** **A test that owns its own `CMakeLists.txt`, wherever the diff puts it, answers to the `big/` subfolder's checklist as well as this one.** -**A diff adding or changing a test that skips or reports pass without running, under any -condition not every lane meets - a missing file or binary, an unset environment variable - -says in the PR which lane runs it for real, naming the command.** +**A diff adding or changing a C++ test under this folder that cannot fail on some lane - it +skips there, or its subject compiles there to an implementation the test does not exercise - says +in the PR which lane runs it for real, naming the command.** diff --git a/tests/README.md b/tests/README.md index 875c1d6596..050b532062 100644 --- a/tests/README.md +++ b/tests/README.md @@ -44,6 +44,7 @@ Every `.das` file in this directory tree is listed below, grouped by subdirector | test_range64_ctor_args.das | AOT most-vexing-parse - a for-source of constructor-style casts only (`range64(int64(a), int64(b))`) must emit a variable, not a function declaration | | | test_op2_operand_order_emit.das | AOT binary-op operand sequencing - impure-operand ops wrap in `das_ordered2` (braced init = left-to-right), pure and short-circuit ops don't | | | _ordered_op2_fixture.das | *(helper)* the op2 shapes the sequencing test emits - impure policy/plain/xor, pure, `&&`/`\|\|` | | +| test_struct_order.das | AOT structure emission order - an outer struct embeds a later-declared inner whose function-typed field names the outer in its signature; a signature is not a by-value dependency, so the topological sort must still place the inner first | | ## apply/ diff --git a/tests/aot/CMakeLists.txt b/tests/aot/CMakeLists.txt index 9a7b21dd46..3e1a08d586 100644 --- a/tests/aot/CMakeLists.txt +++ b/tests/aot/CMakeLists.txt @@ -51,10 +51,10 @@ FILE(GLOB AOT_DASPEG_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "tes FILE(GLOB AOT_DASGLTF_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "tests/dasGLTF/*.das") -# dasLLAMA IS NEVER AOT'D. No dasLLAMA test — old or new — joins test_aot: the engine is -# -jit only (its AOT story ends at the generic compile of the tree), and its require graph -# reaches dasVulkan, which is in-tree but carries no AOT registration at all, so a dasLLAMA -# AOT TU would have no Vulkan half to link against. Do not re-register it here. +# No dasLLAMA test — old or new — joins test_aot: the suite is -jit (tests/run.das), and its +# require graph reaches dasVulkan, which is in-tree but carries no AOT registration at all, so +# a dasLLAMA test TU would have no Vulkan half to link against. Do not re-register it here. +# The engine's own AOT tier is the opt-in examples/dasLLAMA target (CPU modules only). FILE(GLOB AOT_DECS_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "tests/decs/*.das") # Exclude expect-error tests (they intentionally fail to compile) @@ -273,9 +273,9 @@ FILE(GLOB AOT_METAL_FILES RELATIVE ${PROJECT_SOURCE_DIR} CONFIGURE_DEPENDS "test list(FILTER AOT_METAL_FILES EXCLUDE REGEX "/_") SET(AOT_METAL_MODULE_FILES tests/metal/_metal_common.das - # f16_cvt is a generic dasLLVM daslib helper the tensor-ops test requires; it used to ride + # f16_cvt is a generic daslib helper the tensor-ops test requires; it used to ride # the (deleted) dasLLAMA module lib — dasLLAMA itself is NEVER AOT'd (see above) - modules/dasLLVM/daslib/f16_cvt.das + daslib/f16_cvt.das ) IF(APPLE) # das_metal_boost hard-requires the APPLE-gated das_metal C++ module; the GPU-behavioral diff --git a/tests/aot/test_struct_order.das b/tests/aot/test_struct_order.das new file mode 100644 index 0000000000..ba3b09e555 --- /dev/null +++ b/tests/aot/test_struct_order.das @@ -0,0 +1,54 @@ +options gen2 +options no_unused_function_arguments = false +require dastest/testing_boost public + +// AOT emits structures in by-value dependency order, not source order: Outer embeds Inner, +// which is declared later, and Inner's function-typed field names Outer in its signature. A +// sort that reads the signature as a by-value dependency sees a cycle and falls back to source +// order, where the generated C++ embeds an incomplete type. Holder and Inc are the other shape: +// a genuine container cycle (each holds an array of the other), which the sort cannot place and +// appends unsorted - legal C++, because TArray holds no T by value. The AOT build of this +// file is the gate; the assertions only keep the shapes alive. + +struct Outer { + inner : Inner + tag : int +} + +struct Inner { + on_outer : function<(o : Outer) : int> + weight : float +} + +struct Holder { + arr : array + n : int +} + +struct Inc { + back : array + w : float +} + +def outer_tag(o : Outer) : int { + return o.tag * 2 +} + +[test] +def test_struct_order(t : T?) { + t |> run("an outer struct embeds a later-declared inner whose signature names the outer") @(t : T?) { + var o = Outer(tag = 21, inner = Inner(on_outer = @@outer_tag, weight = 0.5)) + t |> equal(invoke(o.inner.on_outer, o), 42) + t |> equal(o.inner.weight, 0.5) + } + t |> run("two structs holding arrays of each other sort as a container cycle") @(t : T?) { + var h = Holder(n = 3) + var inc = Inc(w = 1.5) + h.arr |> emplace(inc) + var inner = Holder(n = 7) + h.arr[0].back |> emplace(inner) + t |> equal(h.arr[0].back[0].n, 7) + t |> equal(h.arr[0].w, 1.5) + delete h + } +} diff --git a/tests/daslib/test_tune_shells.das b/tests/daslib/test_tune_shells.das new file mode 100644 index 0000000000..a99bb0c9cb --- /dev/null +++ b/tests/daslib/test_tune_shells.das @@ -0,0 +1,43 @@ +options gen2 + +require dastest/testing_boost public +require daslib/tune + +// The daslib/tune shells: any program spells the tune annotations and compiles on every tier - +// with dasLLVM (stamped, or unstamped under DAS_TUNE_POLICY=reference) and without it, where +// every apply is a no-op and the reference body IS the function. The gate is that the answer +// never moves between those tiers, so the reference body and the generator agree by +// construction: `llvm_code_selftest::add_plus_k` emits `a + b + k` for an (int, int) : int +// stub, and the k7 perm this grid falls back to emits exactly what the body below computes. + +[tune_scope(name = "test_tune_shells")] +struct private TuneShellsScope {} + +def private ref_add(a, b : int) : int { + return a + b + 7 +} + +[tune_perm(k = 7), tune_perm(k = 0), + tune(gen = "llvm_code_selftest::add_plus_k", fallback = "k7")] +def shell_add(a, b : int) : int { + return a + b + 7 +} + +[tune_policy(missing = "fallback")] +def private shell_policy_probe(a : int) : int { + return a * 2 +} + +[test] +def test_tune_shell_result_matches_the_reference(t : T?) { + for (a in range(-3, 4)) { + for (b in range(-3, 4)) { + t |> equal(shell_add(a, b), ref_add(a, b), "shell_add({a}, {b})") + } + } +} + +[test] +def test_tune_policy_leaves_the_body_alone(t : T?) { + t |> equal(shell_policy_probe(21), 42) +} diff --git a/tests/dastest/test_preflight_config.das b/tests/dastest/test_preflight_config.das index e677bf3133..2ebe3815d9 100644 --- a/tests/dastest/test_preflight_config.das +++ b/tests/dastest/test_preflight_config.das @@ -1,7 +1,6 @@ options gen2 options no_aot -require strings require dastest/testing_boost require daslib/fio require ../../dastest/worker_policy.das @@ -52,22 +51,6 @@ def test_preflight_config(tt : T?) { t |> equal(0, length(blanks)) } - tt |> run("lint rail selection and its Pass detail are pinned") <| @(t : T?) { - t |> success(lint_runs_exe_rail(false)) - t |> success(!lint_runs_exe_rail(true)) - t |> equal("3 file(s) clean on all three rails (host, linux-mirror, exe)", lint_pass_detail(3, true)) - t |> equal("1 file(s) clean on interp rails (host, linux-mirror); exe rail skipped", lint_pass_detail(1, false)) - } - - tt |> run("--show-help lists the rail-skip flag the skills hardcode") <| @(t : T?) { - let exe = get_command_line_arguments()[0] - let tool = path_join(get_das_root(), "utils/internal/preflight/main.das") - var out : string - let rc = run_and_capture([exe, tool, "--", "--show-help"], out, 120.0) - t |> equal(0, rc) - t |> success(find(out, "--lint-skip-exe-rail") >= 0, "help lists --lint-skip-exe-rail") - } - tt |> run("invalid settings fail closed to defaults") <| @(t : T?) { let temp = create_temp_file_result("preflight_config_bad", ".toml") t |> success(temp is value, "temporary config created") diff --git a/tests/jit_tests/_cross_target_fixture.das b/tests/jit_tests/_cross_target_fixture.das new file mode 100644 index 0000000000..469328062b --- /dev/null +++ b/tests/jit_tests/_cross_target_fixture.das @@ -0,0 +1,23 @@ +options gen2 + +// child for cross_target_folds.das: prints the compile-time target folds and which arm a static_if +// over the target triple selects (the same test the tune framework's inert gate makes), so the +// parent can compare a native compile against --jit-target= + +// the module-level `let` gate shape the engine's tier files use (dasllama_math_default's wasm dot +// pick), folded through a different path than the static_if below +let TARGET_IS_WASM = get_target_architecture_name() == "wasm64" || get_target_architecture_name() == "wasm32" + +[export] +def main { + print("TRIPLE=[{get_target_triple()}]\n") + print("TARGET_ARCH=[{get_target_architecture_name()}]\n") + print("LET_WASM=[{TARGET_IS_WASM}]\n") + print("HOST_ARCH=[{get_architecture_name()}]\n") + print("CROSS_PLATFORM=[{get_cross_platform_name()}]\n") + static_if (!empty(get_target_triple())) { + print("CROSS=[yes]\n") + } else { + print("CROSS=[no]\n") + } +} diff --git a/tests/jit_tests/_exe_baseline_intrin.das b/tests/jit_tests/_exe_baseline_intrin.das new file mode 100644 index 0000000000..2b4c79c5cf --- /dev/null +++ b/tests/jit_tests/_exe_baseline_intrin.das @@ -0,0 +1,26 @@ +options gen2 + +// child for exe_host_cpu.das: a kernel-free program reaching every feature-gated intrinsic +// family through a plain call, built with -exe onto the generic baseline machine — the build +// must not emit an instruction that machine cannot select, and the artifact prints the +// fallback bodies' exact answers +require daslib/aarch64_neon +require daslib/x64_avx + +def lane_sum(v : int4) : int => v.x + v.y + v.z + v.w + +[export] +def main { + var w : array + w |> resize(16) + for (i in range(16)) { + w[i] = int8(i + 1) + } + unsafe { + let p = addr(w[0]) + let sdot = lane_sum(aarch64_neon::sdot4(int4(0), p, p)) + let mmla = lane_sum(aarch64_neon::smmla(int4(0), p, p)) + let dot = lane_sum(x64_avx::dot4(int4(0), p, p)) + print("INTRIN=[{sdot},{mmla},{dot}]\n") + } +} diff --git a/tests/jit_tests/_wasm_idot_fixture.das b/tests/jit_tests/_wasm_idot_fixture.das new file mode 100644 index 0000000000..89db0182bb --- /dev/null +++ b/tests/jit_tests/_wasm_idot_fixture.das @@ -0,0 +1,24 @@ +// The fixture wasm_idot_lowering.das compiles for the wasm64 cross target and for the host, dumping +// the IR: one signed idot4 over byte16 operands the emitter cannot fold, since the bytes arrive +// through the command line's length. +options gen2 + +def taint(n : int) : byte16 { + var b : byte16 + unsafe { + var p = addr(b) + for (i in range(16)) { + p[i] = int8(i * 3 - n) + } + } + return b +} + +[export] +def main() { + let args <- get_command_line_arguments() + let w = taint(length(args)) + let x = taint(length(args) + 5) + let d = idot4(w, x) + print("IDOT4=[{d.x} {d.y} {d.z} {d.w}]\n") +} diff --git a/tests/jit_tests/aarch64_neon.das b/tests/jit_tests/aarch64_neon.das new file mode 100644 index 0000000000..1757a9acca --- /dev/null +++ b/tests/jit_tests/aarch64_neon.das @@ -0,0 +1,254 @@ +options gen2 + +require dastest/testing_boost +require daslib/aarch64_neon + +// aarch64_neon intrinsic emission: under -jit on an aarch64 host the module's sdot4 family, the +// tbl16 pair and smmla lower to SDOT / TBL / SMMLA (smmla only where the host has i8mm) at the +// call sites inside the jitted run_* helpers, while the ref_* copies — the same arithmetic in a +// different module, so the name-keyed lookup never matches — compile as-is; every op is exact +// integer, so hardware-vs-reference isolates the emitter. Interp, foreign-arch and generic-exe +// lanes run the identical body on both sides (trivially green) — that IS the fallback contract +// the aarch64_neon header states. The helpers take no T? (dastest keeps such functions off the +// JIT), and each [test] witnesses that its helper is jitted before trusting the comparison. + +def sx8(v : int) : int => ((v & 0xff) ^ 0x80) - 0x80 + +def packed_byte_sx(lane : int; e : int) : int => sx8(lane >> (e * 8)) + +def ref_sdot4(acc : int4; w : int8 const?; x : int8 const?) : int4 { + var r = acc + unsafe { + for (j in range(4)) { + var s = 0 + for (e in range(4)) { + s += int(w[4 * j + e]) * int(x[4 * j + e]) + } + r[j] += s + } + } + return r +} + +def ref_sdot4_w(acc : int4; w : int4; x : int8 const?) : int4 { + var r = acc + unsafe { + for (j in range(4)) { + var s = 0 + for (e in range(4)) { + s += packed_byte_sx(w[j], e) * int(x[4 * j + e]) + } + r[j] += s + } + } + return r +} + +def ref_sdot4_laneq(acc : int4; w : int8 const?; x : int8 const?; lane : int) : int4 { + var r = acc + unsafe { + for (j in range(4)) { + var s = 0 + for (e in range(4)) { + s += int(w[4 * j + e]) * int(x[4 * lane + e]) + } + r[j] += s + } + } + return r +} + +def ref_sdot4_laneq_w(acc : int4; w : int4; x : int8 const?; lane : int) : int4 { + var r = acc + unsafe { + for (j in range(4)) { + var s = 0 + for (e in range(4)) { + s += packed_byte_sx(w[j], e) * int(x[4 * lane + e]) + } + r[j] += s + } + } + return r +} + +def ref_tbl16(lut : int8 const?; nib : uint8 const?; high : bool) : int4 { + var r = int4(0) + unsafe { + for (j in range(4)) { + for (e in range(4)) { + let n = int(nib[4 * j + e]) + let idx = high ? (n >> 4) : (n & 0x0f) + r[j] |= (int(lut[idx]) & 0xff) << (e * 8) + } + } + } + return r +} + +def ref_smmla(acc : int4; a : int8 const?; b : int8 const?) : int4 { + var r = acc + unsafe { + for (i in range(2)) { + for (j in range(2)) { + var s = 0 + for (k in range(8)) { + s += int(a[8 * i + k]) * int(b[8 * j + k]) + } + r[2 * i + j] += s + } + } + } + return r +} + +struct Lcg { + state : uint +} + +def next_byte(var g : Lcg) : int8 { + g.state = g.state * 1664525u + 1013904223u + return int8(int(g.state >> 24u) - 128) +} + +def fill_bytes(var g : Lcg; var dst : array) { + for (v in dst) { + v = next_byte(g) + } +} + +def fill_nibbles(var g : Lcg; var dst : array) { + for (v in dst) { + v = uint8(next_byte(g)) + } +} + +let TRIALS = 256 + +//! each [test] bumps this before calling a helper: a pure call with constant arguments folds away at compile time, before the JIT ever sees its call sites +var g_salt = 0u + +// mismatch count over TRIALS random int8 vectors; the lane is a compile-time constant of the +// intrinsic, so the four lane forms are spelled out +def run_sdot4_family(salt : uint) : int { + var g = Lcg(state = 0x9e3779b9u ^ salt) + var w : array + var x : array + w |> resize(16) + x |> resize(16) + var bad = 0 + for (trial in range(TRIALS)) { + fill_bytes(g, w) + fill_bytes(g, x) + let acc = int4(trial, -trial, trial * 7, 3 - trial) + unsafe { + let wp = addr(w[0]) + let xp = addr(x[0]) + if (sdot4(acc, wp, xp) != ref_sdot4(acc, wp, xp)) { + bad++ + } + let packed = ref_tbl16(wp, reinterpret(xp), false) + if (sdot4_w(acc, packed, xp) != ref_sdot4_w(acc, packed, xp)) { + bad++ + } + if (sdot4_laneq(acc, wp, xp, 0) != ref_sdot4_laneq(acc, wp, xp, 0)) { + bad++ + } + if (sdot4_laneq(acc, wp, xp, 1) != ref_sdot4_laneq(acc, wp, xp, 1)) { + bad++ + } + if (sdot4_laneq(acc, wp, xp, 2) != ref_sdot4_laneq(acc, wp, xp, 2)) { + bad++ + } + if (sdot4_laneq(acc, wp, xp, 3) != ref_sdot4_laneq(acc, wp, xp, 3)) { + bad++ + } + if (sdot4_laneq_w(acc, packed, xp, 0) != ref_sdot4_laneq_w(acc, packed, xp, 0)) { + bad++ + } + if (sdot4_laneq_w(acc, packed, xp, 1) != ref_sdot4_laneq_w(acc, packed, xp, 1)) { + bad++ + } + if (sdot4_laneq_w(acc, packed, xp, 2) != ref_sdot4_laneq_w(acc, packed, xp, 2)) { + bad++ + } + if (sdot4_laneq_w(acc, packed, xp, 3) != ref_sdot4_laneq_w(acc, packed, xp, 3)) { + bad++ + } + } + } + return bad +} + +def run_tbl16(salt : uint) : int { + var g = Lcg(state = 0x2545f491u ^ salt) + var lut : array + var nib : array + lut |> resize(16) + nib |> resize(16) + var bad = 0 + for (_trial in range(TRIALS)) { + fill_bytes(g, lut) + fill_nibbles(g, nib) + unsafe { + let lp = addr(lut[0]) + let np = addr(nib[0]) + if (tbl16_lo(lp, np) != ref_tbl16(lp, np, false)) { + bad++ + } + if (tbl16_hi(lp, np) != ref_tbl16(lp, np, true)) { + bad++ + } + } + } + return bad +} + +def run_smmla(salt : uint) : int { + var g = Lcg(state = 0x7f4a7c15u ^ salt) + var a : array + var b : array + a |> resize(16) + b |> resize(16) + var bad = 0 + for (trial in range(TRIALS)) { + fill_bytes(g, a) + fill_bytes(g, b) + let acc = int4(-trial, trial, 11 - trial, trial * 5) + unsafe { + let ap = addr(a[0]) + let bp = addr(b[0]) + if (smmla(acc, ap, bp) != ref_smmla(acc, ap, bp)) { + bad++ + } + } + } + return bad +} + +[test] +def test_sdot4_family(t : T?) { + g_salt++ + t |> run("sdot4 / sdot4_w / sdot4_laneq / sdot4_laneq_w: hardware == reference") @(t : T?) { + t |> success(!jit_enabled() || is_jit_function(@@run_sdot4_family), "the helper is jitted, so its call sites reach the emitter") + t |> equal(run_sdot4_family(g_salt), 0, "every sdot4-family lane matches the reference over {TRIALS} random int8 vectors") + } +} + +[test] +def test_tbl16(t : T?) { + g_salt++ + t |> run("tbl16_lo / tbl16_hi: hardware == reference") @(t : T?) { + t |> success(!jit_enabled() || is_jit_function(@@run_tbl16), "the helper is jitted, so its call sites reach the emitter") + t |> equal(run_tbl16(g_salt), 0, "every tbl16 lookup matches the reference over {TRIALS} random tables") + } +} + +[test] +def test_smmla(t : T?) { + g_salt++ + t |> run("smmla: hardware == reference") @(t : T?) { + t |> success(!jit_enabled() || is_jit_function(@@run_smmla), "the helper is jitted, so its call sites reach the emitter") + t |> equal(run_smmla(g_salt), 0, "every smmla 2x2 product matches the reference over {TRIALS} random int8 matrices") + } +} diff --git a/tests/jit_tests/cross_target_folds.das b/tests/jit_tests/cross_target_folds.das new file mode 100644 index 0000000000..4889680486 --- /dev/null +++ b/tests/jit_tests/cross_target_folds.das @@ -0,0 +1,83 @@ +// The target-side folds: get_target_triple / get_target_architecture_name answer for the artifact +// a compile EMITS (the --jit-target triple), the host's values with no triple, and a static_if +// over the triple selects the cross arm (the tune framework's inert gate makes the same test), so +// a wasm or console artifact built on a tuned box carries reference bodies and no sidecar stamps. +// Compile-only keeps the cross run on this host: the folds are decided from argv before the +// first module parses, and main runs interpreted. The children run with the default module +// cache on purpose: a --jit-target after the separator keys the cache, or a cached native +// compile serves the cross one with the host's folds. +options gen2 +options no_aot + +require dastest/testing_boost +require daslib/fio +require daslib/strings_boost +require strings + +let FIXTURE = "{get_das_root()}/tests/jit_tests/_cross_target_fixture.das" + +def private spawn_child(cmd : string; var lines : array) : int { + var rc : int + unsafe { + rc = popen_timeout("{cmd} 2>&1", 300.0) $(f) { + if (f == null) { + return + } + while (!feof(f)) { + let ln = strip(fgets(f)) + if (!empty(ln)) { + lines |> push("{ln}") + } + } + } + } + return rc +} + +def private has(lines : array; needle : string) : bool { + for (ln in lines) { + if (find(ln, needle) >= 0) return true + } + return false +} + +[test] +def test_cross_target_folds(t : T?) { + if (!jit_enabled()) { + t |> success(true) + return + } + let args <- get_command_line_arguments() // locked view — do not delete + let bin = args[0] + t |> run("a native compile answers the host and keeps the framework live") @(t : T?) { + var ln : array + spawn_child("\"{bin}\" -jit \"{FIXTURE}\"", ln) + t |> success(ln |> has("TRIPLE=[]"), "no triple on a native compile: {join(ln, " | ")}") + t |> success(ln |> has("TARGET_ARCH=[{get_architecture_name()}]"), "the target arch is the host's") + t |> success(ln |> has("CROSS_PLATFORM=[]"), "no cross platform") + t |> success(ln |> has("CROSS=[no]"), "a native compile takes the native arm") + t |> success(ln |> has("LET_WASM=[false]"), "a module-level let over the target folds false natively") + } + t |> run("a wasm64 cross target answers wasm64 and turns the framework inert") @(t : T?) { + var ln : array + spawn_child("\"{bin}\" -jit \"{FIXTURE}\" -- --jit-target=wasm64-unknown-emscripten --jit-compile-only", ln) + t |> success(ln |> has("TRIPLE=[wasm64-unknown-emscripten]"), "the triple folds: {join(ln, " | ")}") + t |> success(ln |> has("TARGET_ARCH=[wasm64]"), "the target arch is the triple's") + t |> success(ln |> has("HOST_ARCH=[{get_architecture_name()}]"), "the host arch stays the host's") + t |> success(ln |> has("CROSS_PLATFORM=[emscripten]"), "the cross platform is emscripten") + t |> success(ln |> has("CROSS=[yes]"), "a cross compile takes the cross arm") + t |> success(ln |> has("LET_WASM=[true]"), "a module-level let over the target folds true for wasm64") + } + t |> run("the last --jit-target on the command line wins, as every other argv reader takes it") @(t : T?) { + var ln : array + spawn_child("\"{bin}\" -jit \"{FIXTURE}\" -- --jit-target=x86_64-unknown-linux-gnu --jit-target=wasm64-unknown-emscripten --jit-compile-only", ln) + t |> success(ln |> has("TRIPLE=[wasm64-unknown-emscripten]"), "the last triple folds: {join(ln, " | ")}") + } + t |> run("an x86_64 linux cross target answers x86_64") @(t : T?) { + var ln : array + spawn_child("\"{bin}\" -jit \"{FIXTURE}\" -- --jit-target=x86_64-unknown-linux-gnu --jit-compile-only", ln) + t |> success(ln |> has("TARGET_ARCH=[x86_64]"), "the target arch is the triple's: {join(ln, " | ")}") + t |> success(ln |> has("CROSS_PLATFORM=[]"), "linux is not a cross platform name") + t |> success(ln |> has("CROSS=[yes]"), "any cross target takes the cross arm") + } +} diff --git a/tests/jit_tests/exe_host_cpu.das b/tests/jit_tests/exe_host_cpu.das index 1021f05e22..796598263a 100644 --- a/tests/jit_tests/exe_host_cpu.das +++ b/tests/jit_tests/exe_host_cpu.das @@ -19,7 +19,10 @@ let KERNEL_SCRIPT = "{OUTPUT_DIR}/exe_host_cpu_kernel.das" let KERNEL_EXE = "{OUTPUT_DIR}/exe_host_cpu_kernel" let PLAIN_SCRIPT = "{OUTPUT_DIR}/exe_host_cpu_plain.das" let PLAIN_EXE = "{OUTPUT_DIR}/exe_host_cpu_plain" +let BASELINE_SCRIPT = "{get_das_root()}/tests/jit_tests/_exe_baseline_intrin.das" +let BASELINE_EXE = "{OUTPUT_DIR}/exe_baseline_intrin" let HOST_CPU_LINE = "targeting the current box" +let BASELINE_ANSWER = "INTRIN=[1496,2480,1496]" //! the fallback bodies over w = x = 1..16: sum of squares per lane group, the 2x2 block products, the same dot again def private write_probe_script(path : string; with_kernel : bool) : bool { var ok = false @@ -28,7 +31,7 @@ def private write_probe_script(path : string; with_kernel : bool) : bool { if (f != null) { fprint(f, "options gen2\n") if (with_kernel) { - fprint(f, "require llvm/daslib/llvm_code\n") + fprint(f, "require daslib/tune\n") fprint(f, "[llvm_code(name = \"llvm_code_selftest::host_features\")]\n") fprint(f, "def host_rail() : int \{ return -1 \}\n") fprint(f, "[export] def main() \{ print(\"HOSTF=[\{host_rail()}]\\n\"); \}\n") @@ -93,11 +96,23 @@ def test_exe_host_cpu(t : T?) { spawn_child("{neutral_env()}\"{bin}\" -exe -output \"{PLAIN_EXE}\" \"{PLAIN_SCRIPT}\"", lp) t |> success(!(lp |> has(HOST_CPU_LINE)), "kernel-free exe build must stay generic") - // run the built exe where the platform can load the shared lib (same gates as jit_exe.das: + //! an intrinsic call alone must not promote the exe off the generic baseline - the gates decline to the fallback bodies, so the build emits nothing that machine cannot select + var lb : array + let rcb = spawn_child("{neutral_env()}\"{bin}\" -exe -output \"{BASELINE_EXE}\" \"{BASELINE_SCRIPT}\"", lb) + t |> equal(0, rcb, "the intrinsic-calling kernel-free exe must build on the generic baseline: {join(lb, "\n")}") + t |> success(!(lb |> has(HOST_CPU_LINE)), "an intrinsic call alone never promotes the exe to the host cpu") + + // run the built exes where the platform can load the shared lib (same gates as jit_exe.das: // Windows lacks RPATH; static builds ship no runtime lib for the exe to call into) - if (get_platform_name() != "windows" && das_is_dll_build() && fexist("{KERNEL_EXE}.exe")) { + let can_run = get_platform_name() != "windows" && das_is_dll_build() + if (can_run && fexist("{KERNEL_EXE}.exe")) { var lr : array spawn_child("\"{KERNEL_EXE}.exe\"", lr) t |> success(lr |> has("HOSTF=[1]"), "the exe must bake the HOST feature rail, not generic (0) or reference (-1)") } + if (can_run && fexist("{BASELINE_EXE}.exe")) { + var lr : array + spawn_child("\"{BASELINE_EXE}.exe\"", lr) + t |> success(lr |> has(BASELINE_ANSWER), "the baseline exe computes the fallback bodies' answers: {join(lr, "\n")}") + } } diff --git a/tests/jit_tests/f16_cvt.das b/tests/jit_tests/f16_cvt.das index 4f6cb589b4..cd022e6ec7 100644 --- a/tests/jit_tests/f16_cvt.das +++ b/tests/jit_tests/f16_cvt.das @@ -1,7 +1,7 @@ options gen2 require dastest/testing_boost -require llvm/daslib/f16_cvt +require daslib/f16_cvt // f16_cvt intrinsic emission: under -jit on aarch64 / x64+F16C the module's f16_to_f32 / // f32_to_f16 lower to hardware half converts (fpext/fptrunc through half), while the ref_* diff --git a/tests/jit_tests/llvm_code.das b/tests/jit_tests/llvm_code.das index e690eb6df3..29408e4d75 100644 --- a/tests/jit_tests/llvm_code.das +++ b/tests/jit_tests/llvm_code.das @@ -1,7 +1,7 @@ options gen2 require dastest/testing_boost -require llvm/daslib/llvm_code +require daslib/tune // [llvm_code] external-codegen rail: the selftest generator (llvm/daslib/llvm_code_selftest, // wired via llvm/daslib/llvm_user_modules) emits `a + b` for (int, int) : int stubs and diff --git a/tests/jit_tests/wasm_idot_lowering.das b/tests/jit_tests/wasm_idot_lowering.das new file mode 100644 index 0000000000..bab3f1a312 --- /dev/null +++ b/tests/jit_tests/wasm_idot_lowering.das @@ -0,0 +1,70 @@ +// The signed idot family on the wasm64 cross target lowers to the ISA's exact int8 dot halves - +// the i16x8 extmul products summed by i32x4.extadd_pairwise_i16x8_s and folded even/odd - and never +// to the generic widen-multiply chain that runs a fifth as fast there; the relaxed 7-bit dot is +// refused by design (it wraps at -128, and +relaxed-simd re-means min/max/mad), so the dump carries +// no relaxed intrinsic either. A host compile of the same fixture emits no wasm intrinsic. +// Compile-only keeps both compiles on this host and never touches the artifact cache, so --jit-dump +// prints every time. +options gen2 +options no_aot + +require dastest/testing_boost +require daslib/fio +require daslib/strings_boost +require strings + +let FIXTURE = "{get_das_root()}/tests/jit_tests/_wasm_idot_fixture.das" +let PAIRWISE_ADD = "llvm.wasm.extadd.pairwise.signed" +let RELAXED_DOT = "llvm.wasm.relaxed.dot" + +def private spawn_child(cmd : string; var lines : array) : int { + var rc : int + unsafe { + rc = popen_timeout("{cmd} 2>&1", 300.0) $(f) { + if (f == null) { + return + } + while (!feof(f)) { + let ln = strip(fgets(f)) + if (!empty(ln)) { + lines |> push("{ln}") + } + } + } + } + return rc +} + +def private count_with(lines : array; needle : string) : int { + var n = 0 + for (ln in lines) { + if (find(ln, needle) >= 0) { + n++ + } + } + return n +} + +[test] +def test_wasm_idot_lowering(t : T?) { + if (!jit_enabled()) { + t |> success(true) + return + } + let args <- get_command_line_arguments() // locked view — do not delete + let bin = args[0] + t |> run("the wasm64 cross compile lowers idot4 to the exact pairwise int8 dot") @(t : T?) { + var ln : array + spawn_child("\"{bin}\" -jit \"{FIXTURE}\" -- --jit-target=wasm64-unknown-emscripten --jit-compile-only --jit-dump", ln) + let pairs = count_with(ln, PAIRWISE_ADD) + t |> success(pairs >= 3, "the declaration and the two half calls of {PAIRWISE_ADD} are in the dump ({pairs} lines, {length(ln)} lines total)") + t |> success(count_with(ln, RELAXED_DOT) == 0, "the relaxed 7-bit dot is never emitted") + t |> success(count_with(ln, "idot_prod") == 0, "no generic widen-multiply chain remains") + } + t |> run("a host compile emits no wasm intrinsic") @(t : T?) { + var ln : array + spawn_child("\"{bin}\" -jit \"{FIXTURE}\" -- --jit-compile-only --jit-dump", ln) + t |> success(length(ln) > 20, "the host dump printed ({length(ln)} lines)") + t |> success(count_with(ln, "llvm.wasm.") == 0, "no wasm intrinsic on the host") + } +} diff --git a/tests/language/optional_require.das b/tests/language/optional_require.das index 8776a22342..b566a841a3 100644 --- a/tests/language/optional_require.das +++ b/tests/language/optional_require.das @@ -22,6 +22,7 @@ require ?daslib/no_such_file_xyz optional_require_fixture_skipme // path guard // scan identically to ast_requireModule (it once stopped at '%', mangling guard AND target) require ?%/tests/language/optional_require_fixture.das optional_require_fixture_pctguard // nolint:STYLE030 — %/ guard resolves -> target loaded; referenced only under static_if require ?%/tests/language/no_such_file_xyz.das optional_require_fixture_pctskip // %/ guard unresolvable -> SKIPPED, target file resolves +require optional_require_fixture_private // requires optional_require_fixture_hidden PRIVATELY: the module_exists visibility pin below [test] def test_optional_require_guard(t : T?) { @@ -43,6 +44,26 @@ def test_optional_require_path_guard(t : T?) { // path guard resolved (daslib/fio is always mounted): the target loaded and its symbols work t |> equal(optional_require_fixture_pathguard::pathguard_marker(), 43) t |> success(typeinfo builtin_module_exists(optional_require_fixture_pathguard)) + //! module_exists is scoped to what THIS module can see, not the process registry - a loaded das target and a required builtin answer true, a skipped target and an unrequired name answer false + t |> success(typeinfo module_exists(optional_require_fixture_pathguard)) + t |> success(typeinfo module_exists(math)) + t |> success(!typeinfo module_exists(optional_require_fixture_skipme)) + t |> success(!typeinfo module_exists(no_such_guard_xyz)) + static_if (typeinfo module_exists(optional_require_fixture_pathguard)) { + t |> equal(optional_require_fixture_pathguard::pathguard_marker(), 43) + } else { + this_symbol_must_never_resolve_module_exists_xyz() + } + //! the half that separates module_exists from builtin_module_exists: a module another module + //! requires privately is IN the program (the registry says so) and NOT visible from here + t |> equal(optional_require_fixture_private::private_marker(), 46) + t |> success(typeinfo builtin_module_exists(optional_require_fixture_hidden), "a privately required module is in the process registry") + t |> success(!typeinfo module_exists(optional_require_fixture_hidden), "and module_exists answers for THIS module's view, where it is not visible") + static_if (typeinfo module_exists(optional_require_fixture_hidden)) { + this_symbol_must_never_resolve_hidden_xyz() + } else { + t |> success(true) + } // path guard unresolvable: the require was SKIPPED even though the target file exists on disk — // the pin for path-guard semantics (a plain-name guard would have loaded it via the fallback) t |> success(!typeinfo builtin_module_exists(optional_require_fixture_skipme)) diff --git a/tests/language/optional_require_fixture_hidden.das b/tests/language/optional_require_fixture_hidden.das new file mode 100644 index 0000000000..dc3aa0b889 --- /dev/null +++ b/tests/language/optional_require_fixture_hidden.das @@ -0,0 +1,8 @@ +options gen2 +// the module optional_require_fixture_private requires PRIVATELY: loaded into the program, so the +// process registry (builtin_module_exists) knows it, yet not visible from the program root +module optional_require_fixture_hidden shared public + +def hidden_marker() : int { + return 45 +} diff --git a/tests/language/optional_require_fixture_private.das b/tests/language/optional_require_fixture_private.das new file mode 100644 index 0000000000..6e163770b1 --- /dev/null +++ b/tests/language/optional_require_fixture_private.das @@ -0,0 +1,8 @@ +options gen2 +module optional_require_fixture_private shared public + +require optional_require_fixture_hidden // private on purpose: the root sees this module, not that one + +def private_marker() : int { + return hidden_marker() + 1 +} diff --git a/tests/metal/REVIEW.md b/tests/metal/REVIEW.md index a23d3e3326..4d4648338e 100644 --- a/tests/metal/REVIEW.md +++ b/tests/metal/REVIEW.md @@ -9,5 +9,5 @@ produces its expected values by running the kernel method on the CPU, in a drive sets `gl_GlobalInvocationID` once per thread id. Kernel members are per-thread state, so a reused instance carries the previous thread's values into the next. -**A file in this folder that creates any Metal object asserts `metal_live_object_count() == 0` -before it returns.** +**A diff that adds or changes a `[test]` in this folder that creates any Metal object asserts +`metal_live_object_count() == 0` before that test returns.** diff --git a/tests/metal/test_metal_tensor_ops.das b/tests/metal/test_metal_tensor_ops.das index 22849ca33b..b280e793b8 100644 --- a/tests/metal/test_metal_tensor_ops.das +++ b/tests/metal/test_metal_tensor_ops.das @@ -11,7 +11,7 @@ require dastest/testing_boost public require strings require metal/msl_shader require daslib/math_bits // float_bits_to_uint — bf16 halfword construction -require llvm/daslib/f16_cvt // f32_to_f16 — the f16s scale plane +require daslib/f16_cvt // f32_to_f16 — the f16s scale plane require _metal_common // nolint:STYLE030,LINT019 — used only inside the Apple static_if half; off-Apple lint sees the branch compiled out let M = 8 diff --git a/tutorials/dasLLAMA/16_text_to_speech.das b/tutorials/dasLLAMA/16_text_to_speech.das index 6d8ead3cbc..55cfa1ea11 100644 --- a/tutorials/dasLLAMA/16_text_to_speech.das +++ b/tutorials/dasLLAMA/16_text_to_speech.das @@ -5,6 +5,7 @@ require dasllama/dasllama require daslib/jobque_boost require daslib/defer require daslib/strings_boost +require daslib/fio require strings // Tutorial dasLLAMA-16: Text to Speech @@ -127,6 +128,11 @@ def main { // nolint:STYLE038 - one linear tutorial walkthrough // letters and two different sounds. synthesize runs both passes for you; run them alone and // you see what the model is asked to say. // + // g2p_pack_path names the phoneme pack the loader took from the model's directory: the full + // tts_g2p.bin when it is there, else the American-only tts_g2p_en_us.bin - the smaller pack a + // browser build ships, which reads no British voice. + print("phoneme pack: {base_name(g2p_pack_path(dir_name(model_path)))}\n") + // // A synthesis phonemizes in the DIALECT ITS VOICE WAS TRAINED ON, so this pass has to be // asked in the same one, or you are reading a document about a different sound. Every voice // carries its language: c.voice_langs[i] beside c.voices[i], and tts_voice_lang answers for diff --git a/tutorials/integration/cpp/class_adapters_module.das.inc b/tutorials/integration/cpp/class_adapters_module.das.inc index b79fa630fd..908c829486 100644 --- a/tutorials/integration/cpp/class_adapters_module.das.inc +++ b/tutorials/integration/cpp/class_adapters_module.das.inc @@ -1,53 +1,53 @@ -static unsigned char class_adapters_module_das[] = { -0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20, -0x67,0x65,0x6e,0x32,0x0a, -0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20, -0x72,0x65,0x6d,0x6f,0x76,0x65,0x5f,0x75, -0x6e,0x75,0x73,0x65,0x64,0x5f,0x73,0x79, -0x6d,0x62,0x6f,0x6c,0x73,0x20,0x3d,0x20, -0x66,0x61,0x6c,0x73,0x65,0x0a, -0x0a, -0x2f,0x2f,0x20,0x41,0x62,0x73,0x74,0x72, -0x61,0x63,0x74,0x20,0x62,0x61,0x73,0x65, -0x20,0x63,0x6c,0x61,0x73,0x73,0x20,0xe2, -0x80,0x94,0x20,0x64,0x61,0x73,0x6c,0x61, -0x6e,0x67,0x20,0x73,0x69,0x64,0x65,0x2e, -0x0a, -0x2f,0x2f,0x20,0x43,0x2b,0x2b,0x20,0x77, -0x69,0x6c,0x6c,0x20,0x69,0x6e,0x63,0x6c, -0x75,0x64,0x65,0x20,0x74,0x68,0x69,0x73, -0x20,0x76,0x69,0x61,0x20,0x63,0x6f,0x6d, -0x70,0x69,0x6c,0x65,0x42,0x75,0x69,0x6c, -0x74,0x69,0x6e,0x4d,0x6f,0x64,0x75,0x6c, -0x65,0x20,0x61,0x6e,0x64,0x20,0x74,0x68, -0x65,0x20,0x58,0x44,0x44,0x20,0x2e,0x64, -0x61,0x73,0x2e,0x69,0x6e,0x63,0x20,0x66, -0x69,0x6c,0x65,0x2e,0x0a, -0x2f,0x2f,0x20,0x64,0x61,0x73,0x6c,0x61, -0x6e,0x67,0x20,0x63,0x6c,0x61,0x73,0x73, -0x65,0x73,0x20,0x63,0x61,0x6e,0x20,0x64, -0x65,0x72,0x69,0x76,0x65,0x20,0x66,0x72, -0x6f,0x6d,0x20,0x74,0x68,0x69,0x73,0x20, -0x61,0x6e,0x64,0x20,0x6f,0x76,0x65,0x72, -0x72,0x69,0x64,0x65,0x20,0x74,0x68,0x65, -0x20,0x76,0x69,0x72,0x74,0x75,0x61,0x6c, -0x20,0x6d,0x65,0x74,0x68,0x6f,0x64,0x73, -0x2e,0x0a, -0x0a, -0x63,0x6c,0x61,0x73,0x73,0x20,0x54,0x75, -0x74,0x6f,0x72,0x69,0x61,0x6c,0x42,0x61, -0x73,0x65,0x43,0x6c,0x61,0x73,0x73,0x20, -0x7b,0x0a, -0x20,0x20,0x20,0x20,0x64,0x65,0x66,0x20, -0x61,0x62,0x73,0x74,0x72,0x61,0x63,0x74, -0x20,0x75,0x70,0x64,0x61,0x74,0x65,0x28, -0x64,0x74,0x20,0x3a,0x20,0x66,0x6c,0x6f, -0x61,0x74,0x29,0x20,0x3a,0x20,0x76,0x6f, -0x69,0x64,0x0a, -0x20,0x20,0x20,0x20,0x64,0x65,0x66,0x20, -0x61,0x62,0x73,0x74,0x72,0x61,0x63,0x74, -0x20,0x67,0x65,0x74,0x5f,0x70,0x6f,0x73, -0x69,0x74,0x69,0x6f,0x6e,0x20,0x3a,0x20, -0x66,0x6c,0x6f,0x61,0x74,0x33,0x0a, -0x7d,0x0a, -}; +static unsigned char class_adapters_module_das[] = { +0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20, +0x67,0x65,0x6e,0x32,0x0a, +0x6f,0x70,0x74,0x69,0x6f,0x6e,0x73,0x20, +0x72,0x65,0x6d,0x6f,0x76,0x65,0x5f,0x75, +0x6e,0x75,0x73,0x65,0x64,0x5f,0x73,0x79, +0x6d,0x62,0x6f,0x6c,0x73,0x20,0x3d,0x20, +0x66,0x61,0x6c,0x73,0x65,0x0a, +0x0a, +0x2f,0x2f,0x20,0x41,0x62,0x73,0x74,0x72, +0x61,0x63,0x74,0x20,0x62,0x61,0x73,0x65, +0x20,0x63,0x6c,0x61,0x73,0x73,0x20,0xe2, +0x80,0x94,0x20,0x64,0x61,0x73,0x6c,0x61, +0x6e,0x67,0x20,0x73,0x69,0x64,0x65,0x2e, +0x0a, +0x2f,0x2f,0x20,0x43,0x2b,0x2b,0x20,0x77, +0x69,0x6c,0x6c,0x20,0x69,0x6e,0x63,0x6c, +0x75,0x64,0x65,0x20,0x74,0x68,0x69,0x73, +0x20,0x76,0x69,0x61,0x20,0x63,0x6f,0x6d, +0x70,0x69,0x6c,0x65,0x42,0x75,0x69,0x6c, +0x74,0x69,0x6e,0x4d,0x6f,0x64,0x75,0x6c, +0x65,0x20,0x61,0x6e,0x64,0x20,0x74,0x68, +0x65,0x20,0x58,0x44,0x44,0x20,0x2e,0x64, +0x61,0x73,0x2e,0x69,0x6e,0x63,0x20,0x66, +0x69,0x6c,0x65,0x2e,0x0a, +0x2f,0x2f,0x20,0x64,0x61,0x73,0x6c,0x61, +0x6e,0x67,0x20,0x63,0x6c,0x61,0x73,0x73, +0x65,0x73,0x20,0x63,0x61,0x6e,0x20,0x64, +0x65,0x72,0x69,0x76,0x65,0x20,0x66,0x72, +0x6f,0x6d,0x20,0x74,0x68,0x69,0x73,0x20, +0x61,0x6e,0x64,0x20,0x6f,0x76,0x65,0x72, +0x72,0x69,0x64,0x65,0x20,0x74,0x68,0x65, +0x20,0x76,0x69,0x72,0x74,0x75,0x61,0x6c, +0x20,0x6d,0x65,0x74,0x68,0x6f,0x64,0x73, +0x2e,0x0a, +0x0a, +0x63,0x6c,0x61,0x73,0x73,0x20,0x54,0x75, +0x74,0x6f,0x72,0x69,0x61,0x6c,0x42,0x61, +0x73,0x65,0x43,0x6c,0x61,0x73,0x73,0x20, +0x7b,0x0a, +0x20,0x20,0x20,0x20,0x64,0x65,0x66,0x20, +0x61,0x62,0x73,0x74,0x72,0x61,0x63,0x74, +0x20,0x75,0x70,0x64,0x61,0x74,0x65,0x28, +0x64,0x74,0x20,0x3a,0x20,0x66,0x6c,0x6f, +0x61,0x74,0x29,0x20,0x3a,0x20,0x76,0x6f, +0x69,0x64,0x0a, +0x20,0x20,0x20,0x20,0x64,0x65,0x66,0x20, +0x61,0x62,0x73,0x74,0x72,0x61,0x63,0x74, +0x20,0x67,0x65,0x74,0x5f,0x70,0x6f,0x73, +0x69,0x74,0x69,0x6f,0x6e,0x20,0x3a,0x20, +0x66,0x6c,0x6f,0x61,0x74,0x33,0x0a, +0x7d,0x0a, +}; diff --git a/utils/daslang/main.cpp b/utils/daslang/main.cpp index a41e01d831..7ab98fd4ea 100644 --- a/utils/daslang/main.cpp +++ b/utils/daslang/main.cpp @@ -469,12 +469,17 @@ int compile_and_run ( const string & fn, const string & mainFnName, bool outputP // (--ast-verify-batch keeps only the post-infer one, read from argv by the module). access->addExtraModule("ast_verify", getDasRoot() + "/daslib/ast_verify.das"); } + // -use-aot links the stubs compiled into THIS binary against the script it runs; a function + // with no stub interprets (never fail_on_no_aot here - the host may be dastest, whose driver + // has no stubs even where its testing.das does, and which sets aot / fail_on_no_aot itself + // for the test files it compiles) + policies.aot = useAot; + policies.fail_on_no_aot = false; if ( useAot ) { - // don't set policies.aot here - the host program (e.g. dastest) doesn't need AOT linking - // the --use-aot flag (after --) tells dastest to enable AOT for test files it compiles - policies.fail_on_no_aot = false; - } else { - policies.fail_on_no_aot = false; + // the stubs were generated tune-frozen (a cross-box artifact carries no per-box [tune] + // stamps); a stamped compile here would hash every tune-family kernel differently and + // link none of them + policies.tune_frozen = true; } policies.fail_on_lack_of_aot_export = false; policies.aot_macros = aotMacros; // -aot-macros: force quote lowering (daslib/quote) in a normal run @@ -799,6 +804,19 @@ int MAIN_FUNC_NAME ( int argc, char * argv[] ) { hostOptions += argv[i]; hostOptions += '\n'; } + // a --jit-target after the separator is a compile input too: get_target_triple and every fold + // over it bake the target into each module, so a cached native compile must not serve a cross one + for ( int i=1, sep=0; i < argc; ++i ) { + if ( !sep ) { sep = strcmp(argv[i],"--")==0; continue; } + if ( strcmp(argv[i],"--jit-target")==0 || strncmp(argv[i],"--jit-target=",13)==0 ) { + hostOptions += argv[i]; + hostOptions += '\n'; + if ( strcmp(argv[i],"--jit-target")==0 && i+1 files; string mainName = "main"; bool scriptArgs = false; diff --git a/utils/dasllama-convert/main.das b/utils/dasllama-convert/main.das index 32c6400f95..290b055d47 100644 --- a/utils/dasllama-convert/main.das +++ b/utils/dasllama-convert/main.das @@ -30,6 +30,7 @@ require dasllama/dasllama_layout // convert_model_to_metal_blob — bake- require dasllama/dasllama_gguf // GGUFMeta — bake-side, deliberately not facade require dasllama/dasllama_math // pin_backend_for_bake — bake-side, deliberately not facade require dasllama/dasllama_gpu_tier // moe_gpu_bake_tag — bake-side, deliberately not facade +require dasllama/dasllama_styletts2 // bake_styletts2_image — the TTS arm, bake-side, deliberately not facade require daslib/clargs require daslib/jobque_boost require daslib/fio @@ -309,6 +310,88 @@ def private convert_asr(cfg : ConvertArgs; kind : string; qm : QuantMode) : int return 0 } +// "" = not a TTS carrier; else the StyleTTS2-lineage family the GGUF's architecture names +// (kitten-tts | kokoro) - a `styletts2.sample_rate` key is what makes it one +def private sniff_tts_family(path : string) : string { + var st : FStat + if (!stat(path, st) || st.is_dir || int64(st.size) < 8l) { + return "" + } + var family = "" + let f = fopen(path, "rb") + if (f == null) { + return "" + } + fmap(f) $(var bytes : array#) { + if (rd_u32(bytes, 0l) == 0x46554747u) { + var inscope m <- parse_gguf_meta(bytes) + if (gguf_has(m, "styletts2.sample_rate")) { + family = gguf_str(m, bytes, "general.architecture") + } + } + } + fclose(f) + return family +} + +// TTS carriers bake through the family's own staging on the lane -q names (q8 = the served +// default, fp32 = the reference lane); -o lands the image anywhere - a serving directory holds +// two .dlim files and the two front-end packs, no gguf. +def private convert_tts(cfg : ConvertArgs; family : string; qm : QuantMode) : int { + if (family != "kitten-tts" && family != "kokoro") { // a StyleTTS2-lineage file of a family the assembly has no data reader for would bake empty tables + to_log(LOG_ERROR, "dasllama-convert: '{cfg.model}' declares TTS architecture '{family}', which dasLLAMA does not serve (kitten-tts | kokoro)\n") + return 2 + } + if (cfg.flavor != Flavor.planar) { + to_log(LOG_ERROR, "dasllama-convert: TTS sources have a single (planar) image flavor\n") + return 2 + } + if (cfg.trim || cfg.stream) { + to_log(LOG_ERROR, "dasllama-convert: --trim/--stream apply to decoder GGUFs, not TTS sources\n") + return 2 + } + let q8 = qm == QuantMode.q8 + if (!q8 && qm != QuantMode.fp32) { + to_log(LOG_ERROR, "dasllama-convert: TTS sources bake q8 or fp32\n") + return 2 + } + let tag = q8 ? ST2_IMAGE_TAG_Q8 : ST2_IMAGE_TAG + let out = cfg.out != "" ? cfg.out : image_path_for(cfg.model, tag) + to_log(LOG_INFO, "dasllama-convert: {cfg.model} ({family})\n") + to_log(LOG_INFO, "dasllama-convert: -> {out}\n") + to_log(LOG_INFO, "dasllama-convert: identity {image_identity(tag)}\n") + if (!cfg.keep_stale_tmp) { + sweep_stale_tmp(cfg.model) + } + if (stat(out).is_valid) { + if (!cfg.force) { + to_log(LOG_INFO, "dasllama-convert: image already present — nothing to do (use --force to rebuild)\n") + return 0 + } + if (!remove(out)) { + to_log(LOG_ERROR, "dasllama-convert: cannot remove '{out}' for the --force rebuild\n") + return 1 + } + } + let ts = ref_time_ticks() + var baked = false + with_job_que() { + setup_dasllama_jobque() + baked = bake_styletts2_image(cfg.model, out, q8) + } + var st : FStat + if (!baked || !stat(out, st)) { + to_log(LOG_ERROR, "dasllama-convert: '{out}' did not land — see the log above\n") + return 1 + } + let total_ms = int64(get_time_usec(ts)) / 1000l + to_log(LOG_INFO, "dasllama-convert: wrote {int64(st.size) >> 20l} MB in {total_ms} ms total\n") + if (cfg.out == "") { + dlim_gc_stale(cfg.model, tag, q8 ? "q8" : "fp32") // the fresh bake beside the source proves ITS LANE's siblings dead + } + return 0 +} + def private flavor_tag(f : Flavor) : string { if (f == Flavor.metal) { return METAL_IMAGE_TAG @@ -429,6 +512,10 @@ def main() : int { if (asr_kind != "") { return convert_asr(cfg, asr_kind, qm) } + let tts_family = sniff_tts_family(cfg.model) + if (tts_family != "") { + return convert_tts(cfg, tts_family, qm) + } if (cfg.trim && cfg.flavor != Flavor.vulkan) { to_log(LOG_ERROR, "dasllama-convert: --trim is a vulkan-flavor property (add -f vulkan)\n") return 2 diff --git a/utils/dasllama-server/README.md b/utils/dasllama-server/README.md index fcb37d3844..0c149396b1 100644 --- a/utils/dasllama-server/README.md +++ b/utils/dasllama-server/README.md @@ -235,6 +235,8 @@ Config precedence: `defaults < config TOML < explicit CLI flags` - unless the TO `authoritative = true` (what the control page saves), which flips the top: `defaults < CLI < authoritative TOML`. +### The sidecar exchange {#exchange-policy} + The sidecar exchange rides three config-only keys (no CLI flags - one code path). **The exchange is closed: both policies default to off, and only an explicit key turns them on** - a box its shipped class profile covers stamps its kernels at compile time and never needed the @@ -245,6 +247,11 @@ unverified NEVER auto-applies), `exchange_submit = ask | always | never` (defaul `always` shares it automatically), and `exchange_url` (baked default `https://dasllama.io`). `DASLLAMA_EXCHANGE_URL` / `DASLLAMA_EXCHANGE_ACCEPT` env override for tests and one-shot watchdog relaunches. Lookup failure is never fatal - the boot falls through to the local tuner. +Every `/exchange*` route resolves its policy from `g_config_save_path`, the path a `POST /config` +writes. That path covers every start shape - `--config`, auto-discovered, and config-less, where +the file may not exist yet and the baked defaults stand. +The exchange client rides the tune framework, so a build without dasLLVM carries none of it: the +boot resolver and submit check are never registered and every `/exchange*` route answers 404. **First-contact consent (GDPR):** nothing is sent to the exchange until a choice is expressed. Setting any `exchange_*` key (TOML or env) IS that choice; on the zero-config diff --git a/utils/dasllama-server/REVIEW.md b/utils/dasllama-server/REVIEW.md index 53af926b18..c567907240 100644 --- a/utils/dasllama-server/REVIEW.md +++ b/utils/dasllama-server/REVIEW.md @@ -26,7 +26,7 @@ before - a new route requires all of its fields - adds that field to every `cont request to that route, in the same change.** **A diff that changes what a route answers - an item, a field, or a value, including one a -required `dasllama/*` module supplies - re-captures every fixture under `tests/fixtures/` that +`dasllama/*` module supplies - re-captures every fixture under `tests/fixtures/` that records that route, in the same change.** The fixtures are the recorded response shape. **A diff that makes `control.html` or a Playwright `.spec.js` under `tests/` (beside this file) @@ -35,3 +35,8 @@ fixture for the route that answers with that key first.** **A diff that adds a key to what a route answers lists it in that route's `README.md` row, in the same change.** The row is where a consumer learns the key exists. + +**A reference to a `dasllama_exchange` or `llvm_tune` symbol in this folder outside a +`static_if (typeinfo module_exists())` arm is a defect - a signature cannot carry that +guard, so it never names one of their types.** Both modules are optional (`require ?llvm`) and do +not exist in a build without dasLLVM; the unguarded reference fails that build's compile. diff --git a/utils/dasllama-server/ask.das b/utils/dasllama-server/ask.das index cbbfc9da44..980285a718 100644 --- a/utils/dasllama-server/ask.das +++ b/utils/dasllama-server/ask.das @@ -12,7 +12,7 @@ options stack = 524288 // every dasLLAMA program root takes this budget (optio // the three tools), and every sibling is tuned. require dasllama/dasllama // load_model / generate / the shared [tune_scope] -require llvm/daslib/llvm_tune // [tune_policy] + tune_status +require daslib/tune // [tune_policy] + log_tune_status - the shells; the framework rides in when the build has dasLLVM require daslib/jobque_boost require daslib/clargs require daslib/fio diff --git a/utils/dasllama-server/main.das b/utils/dasllama-server/main.das index f1fd564608..dbff780cf7 100644 --- a/utils/dasllama-server/main.das +++ b/utils/dasllama-server/main.das @@ -25,8 +25,9 @@ options stack = 524288 // every dasLLAMA program root takes this budget (optio require openai_server // the server + route handlers (this directory) require model_catalog // catalog_models_dir — the models-dir resolution require dasllama/dasllama // load_model / QuantMode / load_asr_model -require llvm/daslib/llvm_tune // [tune_policy] + tune_status (the injected guard resolves here) -require dasllama/dasllama_exchange // the sidecar exchange: boot resolver + submit check +require daslib/tune // [tune_policy] + log_tune_status (the injected guard resolves here) +require ?llvm llvm/daslib/llvm_tune // nolint:STYLE030,LINT019 — OPTIONAL: tune_suppress_mint, reached under module_exists(llvm_tune) +require ?llvm dasllama/dasllama_exchange // nolint:STYLE030,LINT019 — OPTIONAL: the sidecar exchange (boot resolver + submit check), which rides the tune framework require jobque // set_jobque_threads_cap / set_jobque_affinity require daslib/clargs require daslib/toml @@ -177,7 +178,9 @@ struct ModelEntry { // untuned boot consults the exchange only when the config opts in (the exchange is closed; off by default) [init] def register_exchange_resolver { - exchange_register_resolver("dasllama-server.toml") + static_if (typeinfo module_exists(dasllama_exchange)) { + exchange_register_resolver("dasllama-server.toml") + } } // A clearly model-free boot (no --model/-m, no --config/-c, no discoverable toml) heads for @@ -194,9 +197,11 @@ def suppress_tune_for_setup { } } delete args - if (!configured && !stat("dasllama-server.toml").is_valid - && !stat(path_join(get_this_module_dir(), "dasllama-server.toml")).is_valid) { - tune_suppress_mint("no model configured - setup mode boots on fallback kernels") + static_if (typeinfo module_exists(llvm_tune)) { + if (!configured && !stat("dasllama-server.toml").is_valid + && !stat(path_join(get_this_module_dir(), "dasllama-server.toml")).is_valid) { + tune_suppress_mint("no model configured - setup mode boots on fallback kernels") + } } } @@ -851,7 +856,9 @@ def init() { // nolint:STYLE037,STYLE038 — boot sequence: the knobs are orde log_tune_status("dasllama-server") // a fresh local tune that has not been shared: submit now (always) or surface the offer // (ask — the watchdog balloon + the control page's exchange card); quiet otherwise - exchange_boot_submit_check(exchange_policy_from_config(g_cfg.config)) + static_if (typeinfo module_exists(dasllama_exchange)) { + exchange_boot_submit_check(exchange_policy_from_config(g_cfg.config)) + } // GPU backend lands BEFORE the load: the metal mode pins the row-major backend the GPU // kernels read, and the vulkan tier want must be armed before the load walk uploads stacks let gpu = effective_gpu(g_cfg) diff --git a/utils/dasllama-server/openai_server.das b/utils/dasllama-server/openai_server.das index 427ad579fc..b4593b4a10 100644 --- a/utils/dasllama-server/openai_server.das +++ b/utils/dasllama-server/openai_server.das @@ -9,8 +9,8 @@ require dasllama/dasllama_scheduler // continuous-batching scheduler (layer 3 require dashv/dashv_boost public require daslib/json_boost require daslib/toml // write_toml: POST /config persists the authoritative config -require dasllama/dasllama_exchange // the /exchange surface + submit rails -require llvm/daslib/llvm_tune // tune_manifest_path — the /exchange apply target +require ?llvm dasllama/dasllama_exchange // nolint:STYLE030,LINT019 — OPTIONAL: the /exchange surface + submit rails, which ride the tune framework +require ?llvm llvm/daslib/llvm_tune // nolint:STYLE030,LINT019 — OPTIONAL: tune_manifest_path, the /exchange apply target, reached only inside the exchange arms require daslib/defer // defer() { delete js } — free the parsed request JSON on every exit path require model_catalog // the curated list + the /catalog download rail require daslib/jobque_boost @@ -3560,10 +3560,8 @@ def private handle_config_save(var req : HttpRequest?; var resp : HttpResponse?) // ===== the exchange surface (GET /exchange + the apply/submit/retune levers) ===== -def private exchange_policy_now : ExchangePolicy { - // the save path covers every start shape: --config, auto-discovered, and config-less - // (where the file may not exist yet and the baked defaults stand) - return exchange_policy_from_config(g_config_save_path) +def private exchange_absent(var resp : HttpResponse?) : http_status { // nolint:LINT022 — reached only from the no-framework static_if arm, which a build with dasLLVM drops + return resp |> JSON(error_body("the sidecar exchange is not in this build (no tune framework)", "not_found"), http_status.NOT_FOUND) } // The exchange control routes are an operator surface reached through the loopback control @@ -3614,12 +3612,17 @@ def private deny_operator(var resp : HttpResponse?) : http_status { return resp |> JSON(error_body("loopback-only (operator surface)", "forbidden"), http_status.FORBIDDEN) } +[arch(at = "README.md#exchange-policy")] def private handle_exchange_get(var req : HttpRequest?; var resp : HttpResponse?) : http_status { if (!is_loopback_req(req)) { // the status carries the local sidecar path (username in it) — operator-only, like the rest return resp |> JSON(error_body("loopback-only (operator surface)", "forbidden"), http_status.FORBIDDEN) } - return resp |> JSON(exchange_status_json(exchange_policy_now())) + static_if (typeinfo module_exists(dasllama_exchange)) { + return resp |> JSON(exchange_status_json(exchange_policy_from_config(g_config_save_path))) + } else { + return exchange_absent(resp) + } } def private handle_exchange_matches(var req : HttpRequest?; var resp : HttpResponse?) : http_status { @@ -3627,99 +3630,119 @@ def private handle_exchange_matches(var req : HttpRequest?; var resp : HttpRespo return resp |> JSON(error_body("loopback-only (operator surface)", "forbidden"), http_status.FORBIDDEN) } // a live exchange lookup — seconds on the tick thread; the page requests it explicitly - return resp |> JSON(exchange_matches_json(exchange_policy_now())) + static_if (typeinfo module_exists(dasllama_exchange)) { + return resp |> JSON(exchange_matches_json(exchange_policy_from_config(g_config_save_path))) + } else { + return exchange_absent(resp) + } } def private handle_exchange_apply(var req : HttpRequest?; var resp : HttpResponse?) : http_status { if (!is_loopback_req(req)) { return resp |> JSON(error_body("loopback-only (operator surface)", "forbidden"), http_status.FORBIDDEN) } - var jerr = "" - var js = read_json(string(req.body), jerr) - defer() { - unsafe { - delete js + static_if (typeinfo module_exists(dasllama_exchange)) { + var jerr = "" + var js = read_json(string(req.body), jerr) + defer() { + unsafe { + delete js + } } + let sha : string = js?["sha"] ?? "" + if (empty(sha)) { + return resp |> JSON(error_body("body must carry the sidecar sha", "invalid_request_error"), http_status.BAD_REQUEST) + } + // the listing box the page picked: passed through so the downloaded sidecar is verified to + // match what the exchange listed — the same lying-exchange gate the boot resolver applies + let box : string = js?["box"] ?? "" + let err = exchange_fetch_and_apply(exchange_policy_from_config(g_config_save_path).url, sha, tune_manifest_path(), box) + if (!empty(err)) { + return resp |> JSON(error_body(err, "invalid_request_error"), http_status.BAD_REQUEST) + } + // explicitly fetching a sidecar IS engaging the exchange - record the consent it expresses + exchange_record_consent(true) + to_log(LOG_INFO, "dasllama-server: exchange sidecar {sha} applied via the control page; draining for restart\n") + g_shutdown_requested = true + g_restart_requested = true + return resp |> JSON("\{\"ok\":true,\"restarting\":true}", http_status.OK) + } else { + return exchange_absent(resp) } - let sha : string = js?["sha"] ?? "" - if (empty(sha)) { - return resp |> JSON(error_body("body must carry the sidecar sha", "invalid_request_error"), http_status.BAD_REQUEST) - } - // the listing box the page picked: passed through so the downloaded sidecar is verified to - // match what the exchange listed — the same lying-exchange gate the boot resolver applies - let box : string = js?["box"] ?? "" - let err = exchange_fetch_and_apply(exchange_policy_now().url, sha, tune_manifest_path(), box) - if (!empty(err)) { - return resp |> JSON(error_body(err, "invalid_request_error"), http_status.BAD_REQUEST) - } - // explicitly fetching a sidecar IS engaging the exchange - record the consent it expresses - exchange_record_consent(true) - to_log(LOG_INFO, "dasllama-server: exchange sidecar {sha} applied via the control page; draining for restart\n") - g_shutdown_requested = true - g_restart_requested = true - return resp |> JSON("\{\"ok\":true,\"restarting\":true}", http_status.OK) } def private handle_exchange_submit(var req : HttpRequest?; var resp : HttpResponse?) : http_status { if (!is_loopback_req(req)) { return resp |> JSON(error_body("loopback-only (operator surface)", "forbidden"), http_status.FORBIDDEN) } - let err = exchange_submit_current(exchange_policy_now()) - if (!empty(err)) { - return resp |> JSON(error_body(err, "invalid_request_error"), http_status.BAD_REQUEST) + static_if (typeinfo module_exists(dasllama_exchange)) { + let err = exchange_submit_current(exchange_policy_from_config(g_config_save_path)) + if (!empty(err)) { + return resp |> JSON(error_body(err, "invalid_request_error"), http_status.BAD_REQUEST) + } + to_log(LOG_INFO, "dasllama-server: this box's tune submitted to the exchange\n") + return resp |> JSON("\{\"ok\":true}", http_status.OK) + } else { + return exchange_absent(resp) } - to_log(LOG_INFO, "dasllama-server: this box's tune submitted to the exchange\n") - return resp |> JSON("\{\"ok\":true}", http_status.OK) } def private handle_exchange_consent(var req : HttpRequest?; var resp : HttpResponse?) : http_status { if (!is_loopback_req(req)) { return resp |> JSON(error_body("loopback-only (operator surface)", "forbidden"), http_status.FORBIDDEN) } - var jerr = "" - var js = read_json(string(req.body), jerr) - defer() { - unsafe { - delete js + static_if (typeinfo module_exists(dasllama_exchange)) { + var jerr = "" + var js = read_json(string(req.body), jerr) + defer() { + unsafe { + delete js + } } + if (js == null) { + return resp |> JSON(error_body("invalid JSON: {jerr}", "invalid_request_error"), http_status.BAD_REQUEST) + } + let av = js?["accept"] + if (av == null || !(av.value is _bool)) { + return resp |> JSON(error_body("body must carry accept:true|false", "invalid_request_error"), http_status.BAD_REQUEST) + } + let accepted = av.value as _bool + if (!exchange_record_consent(accepted)) { + return resp |> JSON(error_body("could not write the consent file", "server_error"), http_status.INTERNAL_SERVER_ERROR) + } + let manifest = tune_manifest_path() + if (accepted && (!stat(manifest).is_valid || tune_sidecar_stale(manifest))) { + // untuned (or stale) box just enabled the exchange — restart so the boot resolver + // runs the lookup; a tuned box records the choice and keeps serving + to_log(LOG_INFO, "dasllama-server: exchange enabled via the control page; draining for restart to run the lookup\n") + g_shutdown_requested = true + g_restart_requested = true + return resp |> JSON("\{\"ok\":true,\"accepted\":true,\"restarting\":true}", http_status.OK) + } + to_log(LOG_INFO, "dasllama-server: exchange consent recorded ({accepted ? "accepted" : "declined"}) via the control page\n") + return resp |> JSON("\{\"ok\":true,\"accepted\":{accepted}}", http_status.OK) + } else { + return exchange_absent(resp) } - if (js == null) { - return resp |> JSON(error_body("invalid JSON: {jerr}", "invalid_request_error"), http_status.BAD_REQUEST) - } - let av = js?["accept"] - if (av == null || !(av.value is _bool)) { - return resp |> JSON(error_body("body must carry accept:true|false", "invalid_request_error"), http_status.BAD_REQUEST) - } - let accepted = av.value as _bool - if (!exchange_record_consent(accepted)) { - return resp |> JSON(error_body("could not write the consent file", "server_error"), http_status.INTERNAL_SERVER_ERROR) - } - let manifest = tune_manifest_path() - if (accepted && (!stat(manifest).is_valid || tune_sidecar_stale(manifest))) { - // untuned (or stale) box just enabled the exchange — restart so the boot resolver - // runs the lookup; a tuned box records the choice and keeps serving - to_log(LOG_INFO, "dasllama-server: exchange enabled via the control page; draining for restart to run the lookup\n") - g_shutdown_requested = true - g_restart_requested = true - return resp |> JSON("\{\"ok\":true,\"accepted\":true,\"restarting\":true}", http_status.OK) - } - to_log(LOG_INFO, "dasllama-server: exchange consent recorded ({accepted ? "accepted" : "declined"}) via the control page\n") - return resp |> JSON("\{\"ok\":true,\"accepted\":{accepted}}", http_status.OK) } def private handle_exchange_retune(var req : HttpRequest?; var resp : HttpResponse?) : http_status { if (!is_loopback_req(req)) { return resp |> JSON(error_body("loopback-only (operator surface)", "forbidden"), http_status.FORBIDDEN) } - let err = exchange_request_retune() - if (!empty(err)) { - // the marker write failed and the sidecar is untouched — do NOT drain/restart - return resp |> JSON(error_body(err, "server_error"), http_status.INTERNAL_SERVER_ERROR) + static_if (typeinfo module_exists(dasllama_exchange)) { + let err = exchange_request_retune() + if (!empty(err)) { + // the marker write failed and the sidecar is untouched — do NOT drain/restart + return resp |> JSON(error_body(err, "server_error"), http_status.INTERNAL_SERVER_ERROR) + } + to_log(LOG_INFO, "dasllama-server: re-tune armed (sidecar removed, exchange skipped once); draining for restart\n") + g_shutdown_requested = true + g_restart_requested = true + return resp |> JSON("\{\"ok\":true,\"restarting\":true,\"tune\":true}", http_status.OK) + } else { + return exchange_absent(resp) } - to_log(LOG_INFO, "dasllama-server: re-tune armed (sidecar removed, exchange skipped once); draining for restart\n") - g_shutdown_requested = true - g_restart_requested = true - return resp |> JSON("\{\"ok\":true,\"restarting\":true,\"tune\":true}", http_status.OK) } def private emit_asr_event(events : Channel?; event : AsrEvent) { diff --git a/utils/dasllama-server/test_model_catalog.das b/utils/dasllama-server/test_model_catalog.das index 0260210c7f..71a51c0fdb 100644 --- a/utils/dasllama-server/test_model_catalog.das +++ b/utils/dasllama-server/test_model_catalog.das @@ -92,9 +92,9 @@ def test_catalog_table(t : T?) { packs++ } } - t |> equal(length(set), 5, "the set is three models and two packs") - t |> equal(packs, 2, "two front-end packs") - for (want in ["kitten-nano.gguf", "kitten-mini.gguf", "kokoro-82m.gguf", "tts_g2p.bin", "tts_postag.bin"]) { + t |> equal(length(set), 6, "the set is three models and three packs") + t |> equal(packs, 3, "three front-end packs: the full phoneme pack, its American-only twin, the tagger") + for (want in ["kitten-nano.gguf", "kitten-mini.gguf", "kokoro-82m.gguf", "tts_g2p.bin", "tts_g2p_en_us.bin", "tts_postag.bin"]) { t |> success(key_exists(names, want), "the set carries {want}") } } diff --git a/utils/dasllama-server/txt2wav.das b/utils/dasllama-server/txt2wav.das index 9eb5a6b474..2d675194a5 100644 --- a/utils/dasllama-server/txt2wav.das +++ b/utils/dasllama-server/txt2wav.das @@ -8,7 +8,7 @@ options stack = 524288 // every dasLLAMA program root takes this budget (optio // bin/daslang -jit utils/dasllama-server/txt2wav.das -- --tts --text "Hello." --out hello.wav [--voice expr-voice-2-f] [--speed 1.0] require dasllama/dasllama // load_tts_model / synthesize / write_wav_pcm16 / the shared [tune_scope] -require llvm/daslib/llvm_tune // [tune_policy] + log_tune_status +require daslib/tune // [tune_policy] + log_tune_status - the shells; the framework rides in when the build has dasLLVM require daslib/jobque_boost require daslib/clargs require daslib/defer // defer() { reset_tts_q8() } — the weight-lane pin never outlives the run diff --git a/utils/dasllama-server/wav2txt.das b/utils/dasllama-server/wav2txt.das index cb1bc2a20b..c47422ea8f 100644 --- a/utils/dasllama-server/wav2txt.das +++ b/utils/dasllama-server/wav2txt.das @@ -10,7 +10,7 @@ options stack = 524288 // every dasLLAMA program root takes this budget (optio // bin/daslang -jit utils/dasllama-server/wav2txt.das -- --asr --file [--out t.txt] require dasllama/dasllama // load_asr_model / transcribe / the shared [tune_scope] -require llvm/daslib/llvm_tune // [tune_policy] + log_tune_status +require daslib/tune // [tune_policy] + log_tune_status - the shells; the framework rides in when the build has dasLLVM require daslib/jobque_boost require daslib/clargs require daslib/fio diff --git a/utils/daspkg/REVIEW.md b/utils/daspkg/REVIEW.md index b9f04facd1..b0a6525d07 100644 --- a/utils/daspkg/REVIEW.md +++ b/utils/daspkg/REVIEW.md @@ -20,10 +20,6 @@ vs flat directory). **A diff that adds a command or a flag also adds its `print_usage` line and its `README.md` table row, in the same change.** -**A diff that removes `daspkg` from `DAS_UTILS_SHIPPED_EXES` in `utils/CMakeLists.txt` is a -defect.** The removal drops the prebuilt daspkg exe from the release bundle, and the gate -cannot see a removed entry. - **A `cmd_release` bundle that ships an exe without a tune sidecar beside it is a defect** - the tune sidecar is the `.tune.json` file of measured kernel choices the exe reads at run time. @@ -39,12 +35,18 @@ missing a scope key, that is, an entry of the `tune_scopes` list in the deps JSO one the package's `release()` declares that way: a starter file deployed once, then owned by the user. -**A release path that finishes a bundle without writing `.daspkg_release.manifest` is a -defect.** +**A diff that removes `daspkg` from `DAS_UTILS_SHIPPED_EXES` in `utils/CMakeLists.txt` is a +defect.** The removal drops the prebuilt daspkg exe from the release bundle, and the parent's +gate cannot see a removed entry - it reads this sentence to know the decision stands. + +**A `cmd_release` bundle that finishes without writing `.daspkg_release.manifest` is a +defect.** `release wasm` is out: it wipes its output directory first, so it has no stale file to +account for. **A test in `test_daspkg.das` that reaches the network is a defect** - network coverage belongs in `test_daspkg_git.das`. -**A shell command built from a package, bundle, or app name outside `commands.das`, or without -an `is_safe_pkg_name` check on that name first, is a defect** - `is_safe_pkg_name` is private to -`commands.das`. +**A shell command built from any string a `.das_package` supplies - a package, bundle, app or +module name - outside `commands.das`, or without an `is_safe_pkg_name` check on that string +first, is a defect** - `is_safe_pkg_name` is private to `commands.das`. The emcc flags of +`release_emcc_arg` are the carve-out: free-form shell text by design, appended verbatim. diff --git a/utils/daspkg/commands.das b/utils/daspkg/commands.das index 8815952fa1..544dee14f7 100644 --- a/utils/daspkg/commands.das +++ b/utils/daspkg/commands.das @@ -1299,9 +1299,15 @@ def cmd_check(root : string; json : bool = false; is_global : bool = false) { return 0 } +//! The one gate a `.das_package` string passes before it reaches a path or a shell command line: +//! letters, digits, `_`, `-` and `.`, and not `.` or `..` - an allowlist, because the strings +//! are spliced unquoted into `run_cmd` lines, where a space splits argv and `;` or `$(` run. def private is_safe_pkg_name(name : string) : bool { if (empty(name) || name == "." || name == "..") return false - return !(find(name, "/") >= 0 || find(name, "\\") >= 0) + for (c in name) { + if (!(is_alpha(c) || is_number(c) || c == '_' || c == '-' || c == '.')) return false + } + return true } def cmd_cleanup(root : string; force : bool; is_global : bool = false) : int { // nolint:STYLE038 — flat removal checklist (modules/, lock, caches), one arm per artifact @@ -2150,7 +2156,7 @@ def cmd_release(root : string; out_dir : string; paranoid = false; quick = false // separators / `..` so a hostile or typo'd name can't escape out_dir // when path_join + force_rmdir run below. if (!is_safe_pkg_name(bundle_name)) { - to_log(LOG_ERROR, "Error: unsafe bundle name `{bundle_name}` (must not contain path separators or be `.` / `..`)\n") + to_log(LOG_ERROR, "Error: unsafe bundle name `{bundle_name}` (letters, digits, `_`, `-` and `.` only; not `.` or `..`)\n") return 1 } // macOS: emit a clickable .app bundle. The exe + dylibs + modules go into @@ -2665,11 +2671,17 @@ def private ensure_external_wasm_archives(pkg_dir, wasm_lib_dir : string; var bu // Build one app: cross-compile → discover modules → emcc-link → stage assets. def private release_one_wasm_app(root, out_dir, app_name, main_script : string; // nolint:STYLE037,STYLE038 — flat per-app wasm release steps + their error arms wasm_lib_dir, runtime_archive, shell_path : string; - include_globs, exclude_globs : array) : int { + include_globs, exclude_globs, disabled_modules, app_emcc_args : array) : int { if (!is_safe_pkg_name(app_name)) { - to_log(LOG_ERROR, "release wasm: unsafe app name `{app_name}` (no path separators / `.` / `..`)\n") + to_log(LOG_ERROR, "release wasm: unsafe app name `{app_name}` (letters, digits, `_`, `-` and `.` only; not `.` or `..`)\n") return 1 } + for (m in disabled_modules) { // a .das_package string that reaches the cross-compile command line + if (!is_safe_pkg_name(m)) { + to_log(LOG_ERROR, "release wasm: unsafe module name `{m}` in release_wasm_disable_module (letters, digits, `_`, `-` and `.` only; not `.` or `..`)\n") + return 1 + } + } let main_path = path_join(root, main_script) if (!fexist(main_path)) { to_log(LOG_ERROR, "release wasm: app `{app_name}` main script not found: {main_path}\n") @@ -2700,7 +2712,13 @@ def private release_one_wasm_app(root, out_dir, app_name, main_script : string; // cross-compile scans the script's folder and misses the package's modules/. // --jit-threads: emit +atomics,+bulk-memory so the object's ABI matches the -pthread runtime // archive (the wasm build is threaded — see cmd_build_wasm). The link below adds -pthread. - let xc_cmd = "\"{daslang}\" -exe -output \"{obj_path}\" --list-shared-modules \"{deps_file}\" -project-root \"{root}\" --disable-module dashv \"{main_path}\" -- --jit-target=wasm64-unknown-emscripten --jit-emit-object --jit-threads --jit-runtime-lib=\"{runtime_archive}\"" + let disable_args = build_string() $(w) { + write(w, "--disable-module dashv") + for (m in disabled_modules) { + write(w, " --disable-module {m}") + } + } + let xc_cmd = "\"{daslang}\" -exe -output \"{obj_path}\" --list-shared-modules \"{deps_file}\" -project-root \"{root}\" {disable_args} \"{main_path}\" -- --jit-target=wasm64-unknown-emscripten --jit-emit-object --jit-threads --jit-runtime-lib=\"{runtime_archive}\"" var xc_out : string let xc_rc = run_cmd(xc_cmd, xc_out) if (xc_rc != 0 || !fexist(obj_path)) { @@ -2777,6 +2795,9 @@ def private release_one_wasm_app(root, out_dir, app_name, main_script : string; } } + // the app's own release_emcc_arg() lines come last, after every module's: an app-level flag wins + emcc_args |> push_from(app_emcc_args) + // 4. emcc link → .{html,js,wasm}. run_cmd handles the Windows // cmd.exe outer-quote wrapping, so quoted paths survive on every host. let html_out = path_join(app_out_dir, "{app_name}.html") @@ -2800,7 +2821,9 @@ def private release_one_wasm_app(root, out_dir, app_name, main_script : string; // context/jobque mutexes). __wrap_emscripten_futex_wait (src/hal/wasm_thread_malloc.cpp) // busy-polls on the worklet instead, delegating all other threads to the real impl. No // emscripten fork needed — the wrap + shim live in libDaScript_runtime.a (like the malloc lock). - w |> write(" -sMEMORY64=1 -sFULL_ES3 -sMAX_WEBGL_VERSION=2 -sGL_ENABLE_GET_PROC_ADDRESS=1 -sALLOW_MEMORY_GROWTH=1 -sSTACK_SIZE=4MB -fwasm-exceptions -sWASM_LEGACY_EXCEPTIONS=0 -sEXIT_RUNTIME=0 -sFORCE_FILESYSTEM=1 -pthread -sPTHREAD_POOL_SIZE=16 -sAUDIO_WORKLET=1 -sWASM_WORKERS=1 -Wl,--wrap=emscripten_futex_wait") + // EXPORTED_RUNTIME_METHODS=FS,ENV: a custom shell stages files into MEMFS and sets the + // program's environment knobs in Module.preRun (the C environment is built from Module.ENV at startup). + w |> write(" -sMEMORY64=1 -sFULL_ES3 -sMAX_WEBGL_VERSION=2 -sGL_ENABLE_GET_PROC_ADDRESS=1 -sALLOW_MEMORY_GROWTH=1 -sSTACK_SIZE=4MB -fwasm-exceptions -sWASM_LEGACY_EXCEPTIONS=0 -sEXIT_RUNTIME=0 -sFORCE_FILESYSTEM=1 -sEXPORTED_RUNTIME_METHODS=FS,ENV -pthread -sPTHREAD_POOL_SIZE=16 -sAUDIO_WORKLET=1 -sWASM_WORKERS=1 -Wl,--wrap=emscripten_futex_wait") for (a in emcc_args) { w |> write(" {a}") } @@ -2892,5 +2915,5 @@ def cmd_release_wasm(root : string; out_dir : string; wasm_lib_dir_override : st } mkdir_rec(out_dir) - return release_one_wasm_app(root, out_dir, bundle_name, main_script, wasm_lib_dir, runtime_archive, shell_path, spec.include_globs, spec.exclude_globs) + return release_one_wasm_app(root, out_dir, bundle_name, main_script, wasm_lib_dir, runtime_archive, shell_path, spec.include_globs, spec.exclude_globs, spec.wasm_disabled_modules, spec.emcc_args) } diff --git a/utils/daspkg/fixtures/test_release.das_package b/utils/daspkg/fixtures/test_release.das_package index 2b390a33f0..0c43c50fb1 100644 --- a/utils/daspkg/fixtures/test_release.das_package +++ b/utils/daspkg/fixtures/test_release.das_package @@ -20,4 +20,6 @@ def release() { release_wasm_build("emcmake cmake -S . -B _wasm && cmake --build _wasm") release_wasm_archive("_wasm/liblibDasModuleTest.a") release_wasm_archive("_wasm/liblibTestApp.a") + release_wasm_disable_module("dasvulkan") + release_emcc_arg("--profiling-funcs") } diff --git a/utils/daspkg/package_runner.das b/utils/daspkg/package_runner.das index 132a38bd11..3071052394 100644 --- a/utils/daspkg/package_runner.das +++ b/utils/daspkg/package_runner.das @@ -43,6 +43,7 @@ struct PackageReleaseInfo { emcc_args : array embed_paths : array web_shell : string + wasm_disabled_modules : array wasm_build_command : string wasm_archives : array requires_jit : bool @@ -146,6 +147,9 @@ def run_das_package_release(das_package_path : string; var info : PackageRelease info.embed_paths |> push(clone_string(e)) } info.web_shell = clone_string(src.web_shell) + for (m in src.wasm_disabled_modules) { + info.wasm_disabled_modules |> push(clone_string(m)) + } info.wasm_build_command = clone_string(src.wasm_build_command) for (a in src.wasm_archives) { info.wasm_archives |> push(clone_string(a)) diff --git a/utils/daspkg/test_daspkg.das b/utils/daspkg/test_daspkg.das index 03a4afefa9..8f717e1144 100644 --- a/utils/daspkg/test_daspkg.das +++ b/utils/daspkg/test_daspkg.das @@ -1781,6 +1781,12 @@ def test_run_das_package_release(t : T?) { tt |> equal(2, length(info.wasm_archives)) tt |> equal("_wasm/liblibDasModuleTest.a", info.wasm_archives[0]) tt |> equal("_wasm/liblibTestApp.a", info.wasm_archives[1]) + // the two wasm-only knobs cross the same hand-written marshaller: a dropped copy would + // cross-compile with the module mounted and the flag missing, silently + tt |> equal(1, length(info.wasm_disabled_modules)) + tt |> equal("dasvulkan", info.wasm_disabled_modules[0]) + tt |> equal(1, length(info.emcc_args)) + tt |> equal("--profiling-funcs", info.emcc_args[0]) } t |> run("returns true with empty spec when release() is missing") @(tt : T?) { var info : PackageReleaseInfo @@ -1865,6 +1871,34 @@ def test_cmd_release_pure_daslang(t : T?) { } } +[test] +def test_cmd_release_refuses_unsafe_bundle_name(t : T?) { + // the bundle name is a .das_package string that reaches path_join, force_rmdir and shell + // command lines; the guard admits letters, digits, `_`, `-` and `.` only - a space or a + // shell metacharacter is refused before anything is built or removed + for (bad in ["bad name", "bad;name", "bad$(name)", "bad\"name"]) { + t |> run("refuses bundle name `{bad}`") @(tt : T?) { + let tmp_root = "{get_das_root()}/_test_release_badname" + let out_dir = "{get_das_root()}/_test_release_badname_out" + force_rmdir(tmp_root) + force_rmdir(out_dir) + mkdir(tmp_root) + fopen("{tmp_root}/main.das", "wb") $(f) { + fwrite(f, "options gen2\n[export]\ndef main() \{\n print(\"hello\\n\")\n\}\n") + } + let escaped = replace(replace(bad, "\\", "\\\\"), "\"", "\\\"") + fopen("{tmp_root}/.das_package", "wb") $(f) { + fwrite(f, "options gen2\nrequire daslib/daspkg\n[export]\ndef package() \{\n package_name(\"{escaped}\")\n\}\n[export]\ndef release() \{\n release_main(\"main.das\")\n\}\n") + } + let rc = cmd_release(tmp_root, out_dir) + tt |> equal(1, rc, "cmd_release refuses the name") + tt |> success(!fexist(out_dir), "nothing was built or removed under {out_dir}") + force_rmdir(tmp_root) + force_rmdir(out_dir) + } + } +} + [test] def test_cmd_release_ships_minted_scope_manifest(t : T?) { t |> run("a tune scope's RESOLVED manifest ships beside the exe") @(tt : T?) { @@ -1881,7 +1915,7 @@ def test_cmd_release_ships_minted_scope_manifest(t : T?) { fwrite(f, payload) } fopen("{tmp_root}/main.das", "wb") $(f) { - fwrite(f, "options gen2\nrequire llvm/daslib/llvm_tune\n[tune_scope(name=\"stubscope\", tuner=\"stub_tuner.das\")]\nstruct StubScope \{\n unused : int\n\}\n[export]\ndef main() \{\n print(\"hi\\n\")\n\}\n") + fwrite(f, "options gen2\nrequire daslib/tune\nrequire llvm/daslib/llvm_tune\n[tune_scope(name=\"stubscope\", tuner=\"stub_tuner.das\")]\nstruct StubScope \{\n unused : int\n\}\n[export]\ndef main() \{\n print(\"hi\\n\")\n\}\n") } fopen("{tmp_root}/stub_tuner.das", "wb") $(f) { fwrite(f, "options gen2\nrequire daslib/fio\n[export]\ndef main() \{\n fopen(get_env_variable(\"DAS_TUNE_MANIFEST\"), \"wb\") $(mf) \{\n fwrite(mf, fread(\"{tmp_root}/stub_payload.json\"))\n \}\n\}\n") diff --git a/utils/internal/dasllama-ladder/caddy.snippet b/utils/internal/dasllama-ladder/caddy.snippet index d3780086b7..cb7bbe9e95 100644 --- a/utils/internal/dasllama-ladder/caddy.snippet +++ b/utils/internal/dasllama-ladder/caddy.snippet @@ -59,6 +59,15 @@ handle /api/sidecar/* { reverse_proxy 127.0.0.1:8201 } +# The browser examples (/examples//) are -pthread wasm64 builds: SharedArrayBuffer needs +# a cross-origin-isolated page, so the two isolation headers ride every response under +# /examples/. `credentialless` (not require-corp) keeps the analytics script loading; every +# memory64-capable browser supports it, and the model files are same-origin. +header /examples/* { + Cross-Origin-Opener-Policy same-origin + Cross-Origin-Embedder-Policy credentialless +} + # /admin/* and /shutdown are deliberately absent: they are operator surfaces. # The service also refuses them from a non-loopback peer, so adding a route # here does not merely expose them — it breaks them. diff --git a/utils/internal/dasllama-ladder/dasllama-deploy.sh b/utils/internal/dasllama-ladder/dasllama-deploy.sh index a04dd739fe..06a3012215 100644 --- a/utils/internal/dasllama-ladder/dasllama-deploy.sh +++ b/utils/internal/dasllama-ladder/dasllama-deploy.sh @@ -97,12 +97,46 @@ EOF echo "provisioned: user=$SVCUSER data=$DATA unit=$UNIT; restic snapshot line added" } +vhost_has() { + # true when the dasllama.io block of the shared Caddyfile carries $1 - the other vhosts + # (daslang.io) may spell the same directive, so a whole-file grep is not the question + awk '/^dasllama\.io \{/ { b = 1 } b { print } b && /^\}/ { b = 0 }' "$CADDYFILE" | grep -q "$1" +} + caddy_apply() { # Splice caddy.snippet (from the installed release) into the dasllama.io vhost, ahead of # root/file_server. Idempotent, validated before reload, with a timestamped Caddyfile backup. snippet="$APP/current/caddy.snippet" [ -f "$snippet" ] || { echo "no $snippet - install a release first"; exit 1; } - if grep -q "reverse_proxy 127.0.0.1:$PORT" "$CADDYFILE"; then + if vhost_has "reverse_proxy 127.0.0.1:$PORT"; then + # the routes are in; a later snippet may still carry a block the vhost lacks - today the + # `header /examples/*` isolation block - so splice each such block on its own + if grep -q "header /examples/\*" "$snippet" && ! vhost_has "header /examples/\*"; then + ts=$(date +%Y%m%d-%H%M%S) + cp "$CADDYFILE" "$CADDYFILE.bak-$ts" + awk -v snip="$snippet" ' + /^dasllama\.io \{/ && !done { + print + inblock = 0 + while ((getline line < snip) > 0) { + if (line ~ /^header \/examples\/\*/) inblock = 1 + if (inblock) print "\t" line + if (inblock && line ~ /^\}/) inblock = 0 + } + close(snip); done=1; next + } + { print } + ' "$CADDYFILE.bak-$ts" > "$CADDYFILE" + if ! vhost_has "header /examples/\*"; then + echo "caddy: header splice inserted nothing (dasllama.io vhost not matched) - restoring $CADDYFILE.bak-$ts"; cp "$CADDYFILE.bak-$ts" "$CADDYFILE"; exit 1 + fi + if ! caddy validate --config "$CADDYFILE" --adapter caddyfile >/dev/null 2>&1; then + echo "caddy validate FAILED - restoring $CADDYFILE.bak-$ts"; cp "$CADDYFILE.bak-$ts" "$CADDYFILE"; exit 1 + fi + systemctl reload caddy + echo "caddy: /examples/* isolation headers spliced + reloaded (backup $CADDYFILE.bak-$ts)" + return 0 + fi echo "caddy: /api already spliced - nothing to do"; return 0 fi ts=$(date +%Y%m%d-%H%M%S) @@ -118,7 +152,7 @@ caddy_apply() { # The awk match is exact (`dasllama.io {`); if the vhost is ever reformatted (shared address, # renamed) it matches nothing, validate still passes on the unchanged file, and we would # wrongly report success. Confirm the proxy line actually landed before reloading. - if ! grep -q "reverse_proxy 127.0.0.1:$PORT" "$CADDYFILE"; then + if ! vhost_has "reverse_proxy 127.0.0.1:$PORT"; then echo "caddy: splice inserted nothing (dasllama.io vhost not matched) - restoring $CADDYFILE.bak-$ts"; cp "$CADDYFILE.bak-$ts" "$CADDYFILE"; exit 1 fi if ! caddy validate --config "$CADDYFILE" --adapter caddyfile >/dev/null 2>&1; then diff --git a/utils/internal/preflight/README.md b/utils/internal/preflight/README.md index 59604852dc..a6e1eec6db 100644 --- a/utils/internal/preflight/README.md +++ b/utils/internal/preflight/README.md @@ -27,9 +27,8 @@ daslang utils/internal/preflight/main.das -- --list-gates daslang utils/internal/preflight/main.das -- --only docs,ci-das daslang utils/internal/preflight/main.das -- --skip tests-aot --full -# pre-push lint gate: the changed-set lint without the -exe rail (rails and -# cost: skills/internal/preflight.md, the lint row) -daslang utils/internal/preflight/main.das -- --only lint --lint-skip-exe-rail +# pre-push lint gate: the changed-set lint (rails and cost: skills/internal/preflight.md, the lint row) +daslang utils/internal/preflight/main.das -- --only lint ``` A gate with a reach set (`dasgen`, `docs`, `utils-tests`, `ci-matrix`, the module gates) skips diff --git a/utils/internal/preflight/config.das b/utils/internal/preflight/config.das index 170991d6ed..e7bc42dbd0 100644 --- a/utils/internal/preflight/config.das +++ b/utils/internal/preflight/config.das @@ -184,20 +184,6 @@ def public valid_jit_max_file_time_override(value : float) : bool { return value == -1.0 || value >= 0.0 } -//! The pure half of the lint gate's rail selection: `--lint-skip-exe-rail` -//! keeps the host + linux-mirror interp rails and drops the -exe compiled one. -def public lint_runs_exe_rail(skip_exe_rail : bool) : bool { - return !skip_exe_rail -} - -//! The lint gate's Pass detail — the only observable that distinguishes a -//! full three-rail pass from a rail-skipping one (both return GateStatus.Pass). -def public lint_pass_detail(n_files : int; ran_exe_rail : bool) : string { - return (ran_exe_rail - ? "{n_files} file(s) clean on all three rails (host, linux-mirror, exe)" - : "{n_files} file(s) clean on interp rails (host, linux-mirror); exe rail skipped") -} - //! Resolve the read-only per-user config. ``DAS_PREFLIGHT_CONFIG`` overrides //! the platform default for hermetic tests and unusual installations. def public default_preflight_config_path() : string { diff --git a/utils/internal/preflight/main.das b/utils/internal/preflight/main.das index e553bf0b9b..564a3ab8d9 100644 --- a/utils/internal/preflight/main.das +++ b/utils/internal/preflight/main.das @@ -74,9 +74,6 @@ struct Config { @clarg_doc = "Stop at the first failing gate" fail_fast : bool - @clarg_doc = "Lint gate: skip the -exe compiled rail, keeping the host + linux-mirror interp rails — run before every push; PR-open preflight runs all three" - lint_skip_exe_rail : bool - @clarg_doc = "List gates with tier and description, then exit" list_gates : bool @@ -121,7 +118,6 @@ struct PreflightCtx { jit_jobs : int jit_max_file_time : float verbose : bool - lint_skip_exe_rail : bool } // ===== subprocess + small helpers ===== @@ -140,17 +136,6 @@ def run_argv(args : array) : tuple { return run_argv(args, 0.0) } -// A dynamically-linked artifact spawned outside the daslang bin dir (the exe rail's -// temp-path das-lint) resolves libDaScriptDyn only through the loader's environment — -// prepend the bin dir so spawned children inherit it. -def prepend_loader_search_path(bin_dir : string) { - let platform = get_platform_name() - let vname = platform == "windows" ? "PATH" : (platform == "darwin" ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH") - let sep = platform == "windows" ? ";" : ":" - let cur = get_env_variable(vname) - set_env_variable(vname, empty(cur) ? bin_dir : "{bin_dir}{sep}{cur}") -} - def tool_available(exe, probe_flag : string) : bool { let r = run_argv([exe, probe_flag], 30.0) return r.rc == 0 @@ -583,11 +568,9 @@ def gate_format(ctx : PreflightCtx) : GateResult { // CI's lint verdict is rendered ONLY by the linux lane (extended_checks gates the step on // matrix.target == 'linux'), in the linux MODULE WORLD (no dasMetal — platform static_if // halves compile out, so their requires/args read unused there and nowhere else; dasVulkan -// is in-tree and present on linux, so it is NOT disabled), -// and TWICE (interp, then the -exe compiled das-lint, which resolves modules differently). -// A host-flavor pass alone is structurally blind to both — so this gate runs three rails -// (--lint-skip-exe-rail keeps just the interp two). -// LINT019 is OFF on all three (mirrors CI): a changed-set ledger cannot see consumers +// is in-tree and present on linux, so it is NOT disabled). A host-flavor pass alone is +// structurally blind to that — so this gate runs two rails, host then linux-mirror. +// LINT019 is OFF on both (mirrors CI): a changed-set ledger cannot see consumers // outside the PR — staleness belongs to the nightly whole-tree -j 1 pass. def gate_lint(ctx : PreflightCtx) : GateResult { let t0 = ref_time_ticks() @@ -617,38 +600,8 @@ def gate_lint(ctx : PreflightCtx) : GateResult { return GateResult(name = "lint", status = GateStatus.Fail, seconds = seconds_since(t0), detail = "LINUX-lane lint warnings (host pass was clean — a platform static_if half hides the symbol here; nolint:...,LINT019 both-worlds spelling, see skills/internal/preflight.md)", output = mirror.out) } - if (!lint_runs_exe_rail(ctx.lint_skip_exe_rail)) { - return GateResult(name = "lint", status = GateStatus.Pass, seconds = seconds_since(t0), - detail = lint_pass_detail(length(files), false)) - } - // the exe rail: CI compiles utils/lint into a standalone das-lint and lints AGAIN — the - // -exe world resolves modules differently (dasAudio et al are not baked in). `-exe` - // appends .exe on every platform (CI's LINUX lane invokes $BIN/das-lint.exe verbatim) - let lint_exe_base = unique_temp_path("preflight_das_lint", "") - let build = run_argv([ctx.daslang, "-exe", "-output", lint_exe_base, "utils/lint/main.das"]) - if (build.rc != 0) { - remove("{lint_exe_base}.exe") - remove("{lint_exe_base}.o") - return GateResult(name = "lint", status = GateStatus.Fail, seconds = seconds_since(t0), - detail = "das-lint -exe build failed — the CI exe rail would fail the same way", output = build.out) - } - // the -exe artifact is dynamically linked (libDaScriptDyn); from the temp path the - // loader resolves it only via the environment — point it at the daslang bin dir - prepend_loader_search_path(dir_name(ctx.daslang)) - var exe_args <- ["{lint_exe_base}.exe"] - exe_args |> push_from(files) - exe_args |> push("--quiet") - exe_args |> push("--disable") - exe_args |> push("LINT019") - let exe = run_argv(exe_args) - remove("{lint_exe_base}.exe") - remove("{lint_exe_base}.o") - if (exe.rc != 0) { - return GateResult(name = "lint", status = GateStatus.Fail, seconds = seconds_since(t0), - detail = "das-lint.exe rail warnings (interp rails were clean — the -exe module world differs)", output = exe.out) - } return GateResult(name = "lint", status = GateStatus.Pass, seconds = seconds_since(t0), - detail = lint_pass_detail(length(files), true)) + detail = "{length(files)} file(s) clean on both rails (host, linux-mirror)") } // ===== ast-verify on changed .das (mirrors extended_checks "Run ast-verify on changed .das files") ===== @@ -1606,7 +1559,7 @@ def gate_table() : array { return <- [ GateInfo(name = "untracked", tier = "fast", doc = "no untracked files at PR time — commit, delete, or ignore each"), GateInfo(name = "format", tier = "fast", doc = "formatter --verify on tracked .das (mirrors CI)"), - GateInfo(name = "lint", tier = "fast", doc = "lint changed .das on three rails (host, linux-mirror, -exe), zero warnings; --lint-skip-exe-rail drops the exe rail"), + GateInfo(name = "lint", tier = "fast", doc = "lint changed .das on two rails (host, linux-mirror), zero warnings"), GateInfo(name = "hash-refs", tier = "fast", doc = "no bare #N in branch commit messages that GitHub would mislink - ledger cites spell out or backtick"), GateInfo(name = "review-md", tier = "fast", doc = "REVIEW.das gates of every folder the diff touches (utils/internal/review-md; CI's extended_checks step runs every gate)"), GateInfo(name = "md-ascii", tier = "fast", doc = "ci/fix_md_ascii.py --check when the diff touches .md (mirrors CI's Markdown ASCII gate in extended_checks)"), @@ -1779,8 +1732,7 @@ def main() : int { // nolint:STYLE037,STYLE038 — the gate loop + CLI surface; var ctx = PreflightCtx(base = cfg.base, jobs = cfg.jobs, jit_jobs = jit_jobs, jit_max_file_time = jit_max_file_time, - verbose = cfg.verbose, - lint_skip_exe_rail = cfg.lint_skip_exe_rail) + verbose = cfg.verbose) ctx.daslang = find_daslang(cfg.daslang_bin) if (empty(ctx.daslang)) { to_log(LOG_ERROR, "daslang binary not found — build it (cmake --build build --target daslang) or pass --daslang / set DASLANG\n") diff --git a/web/CMakeLists.txt b/web/CMakeLists.txt index b6b36b51de..a5002205e0 100644 --- a/web/CMakeLists.txt +++ b/web/CMakeLists.txt @@ -120,6 +120,13 @@ add_link_options("SHELL:--embed-file ${CMAKE_CURRENT_SOURCE_DIR}/../daslib@dasli # itself with "module not found". add_link_options("SHELL:--exclude-file */daslib/spoof.das") add_link_options("SHELL:--exclude-file */daslib/linked_list.das") +# Build residue inside the embedded source trees is not source: the AOT emitter's C++ under +# _aot_generated/ and the JIT's .jitted_scripts/ caches land beside the .das files in a +# working checkout and outweigh them twenty to one (daslib: 3.6 MB of .das against 50 MB of +# generated C++ and 45 MB of cache). Every embedded tree in this scope carries the exclusion. +add_link_options("SHELL:--exclude-file */_aot_generated/*") +add_link_options("SHELL:--exclude-file */.jitted_scripts/*") +add_link_options("SHELL:--exclude-file */_llvm_aot_generated/*") # embed dastest runner so the playground's Test button can invoke # /dastest/dastest.das against user-authored [test] functions. add_link_options("SHELL:--embed-file ${CMAKE_CURRENT_SOURCE_DIR}/../dastest@dastest") diff --git a/web/REVIEW.md b/web/REVIEW.md index e6a0b3876a..8a5268e218 100644 --- a/web/REVIEW.md +++ b/web/REVIEW.md @@ -1,6 +1,7 @@ # web (the WASM build and its shells) Code Review Checklist -**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** A file +**Read `REVIEW_COMMON.md` (repo root) first - its contract binds this checklist.** Architecture +doc: `README.md`. A file under this folder is served when the deploy (`.github/workflows/pages.yml`, repo root), `daspkg release wasm`, or a build step that feeds either copies it into a page a visitor loads.