From b38bea3d3a8c52af539ce99a1b35b19d41956d36 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Sat, 8 Aug 2026 03:31:09 +0000 Subject: [PATCH 1/4] llm.c: carry forward linux-gfx90a to de1dd62 (binary-equiv, no source/behavior change) --- projects/llm.c/notes.md | 60 ++++++++++++++++++++++++++++++++++++++ projects/llm.c/stats.jsonl | 2 ++ projects/llm.c/status.json | 18 +++++------- 3 files changed, 70 insertions(+), 10 deletions(-) diff --git a/projects/llm.c/notes.md b/projects/llm.c/notes.md index 8a9bb96f..bfc0bcf8 100644 --- a/projects/llm.c/notes.md +++ b/projects/llm.c/notes.md @@ -632,3 +632,63 @@ proven by it. Same determination on gfx1101/gfx1201 as gfx1151. State: windows-gfx1101 + windows-gfx1201 port-ready -> completed (validated_sha d19a322, fork unchanged). All five platforms terminal -> PR-ready. + +## Validation 2026-08-08 (linux-gfx90a, revalidate) -- carry-forward, no GPU re-run + +GPU: AMD Instinct MI250X (gfx90a), HIP_VISIBLE_DEVICES=2 (confirmed via `rocm-smi +--showproductname`: GPU index 2 is gfx90a). ROCm 7.2.1. + +Prior validated_sha: d19a3220d7b61af86c4d7a8a978a76f94dbbaa71. New head_sha: +de1dd620184356193b9864118dd4162b11938dd7 (two commits: `e33cc58` README AMD quick +start, `de1dd62` HIP build fix for partial-GCC installs / off-PATH amdgpu-arch). +`utils/moatlib.py classify` returned `unknown` (classification failed), so per +validator.md the carry-forward shortcut requires the binary-equivalence check +rather than the source classifier alone. + +### Delta + +`git diff d19a322 de1dd62 --stat`: Makefile (+25/-3), README.md (+11). README is +doc-only. The Makefile hunk is the same build-detection fallback already carried +forward for linux-gfx1100/windows-gfx1101/windows-gfx1201/windows-gfx1151 at this +exact head_sha: (1) locate `amdgpu-arch` via `hipconfig --rocmpath` when it is not +on PATH, (2) probe hipcc's libstdc++-header visibility and pin +`--gcc-install-dir` to the newest GCC with matching `/usr/include/c++/` +headers when the probe fails. Both are Makefile-shell probes guarded by whether +the tool/headers are already resolvable -- no effect when they already are, which +is the case in every prior validated build on this host (hipcc already finds +libstdc++ headers, amdgpu-arch was never relied upon since AMDGPU_TARGETS is +always pinned explicitly in our build commands). + +### Binary-equivalence check (utils/codeobj_diff.py) + +Built both shas into separate clones with the identical validated recipe: + +``` +export HIP_VISIBLE_DEVICES=2 +make USE_HIP=1 AMDGPU_TARGETS=gfx90a NO_MULTI_GPU=1 NO_USE_MPI=1 -j16 \ + test_gpt2fp32cu test_gpt2cu train_gpt2cu +``` + +Both builds: hipcc exit 0, only the pre-existing benign cudaGetLastError/ +cudaFreeHost/cudaGetDeviceProperties/cudaMemcpy nodiscard warnings (unchanged +from every prior gfx90a build log). + +``` +python3 utils/codeobj_diff.py /test_gpt2fp32cu /test_gpt2fp32cu +python3 utils/codeobj_diff.py /test_gpt2cu /test_gpt2cu +python3 utils/codeobj_diff.py /train_gpt2cu /train_gpt2cu +``` + +All three: `verdict=identical` (exported symbols + device ISA byte-identical). +Note: `codeobj_diff.py`'s directory walker only recognizes `.so`/lib-prefixed +binaries, so plain executables must be passed as individual file paths rather +than the containing directory (directory-mode returned `verdict=indeterminate, +no binaries found`) -- worth knowing for any other Makefile-built (non-CMake/ +non-.so) project using this tool. + +### Verdict + +Compiled program provably unchanged on gfx90a -> carried forward, no GPU test +re-run needed. Recorded via +`moatlib.py carry-forward llm.c linux-gfx90a de1dd62... binary-equiv "..."`. +State: revalidate -> completed. validated_sha: de1dd620184356193b9864118dd4162b11938dd7. diff --git a/projects/llm.c/stats.jsonl b/projects/llm.c/stats.jsonl index 68d321cf..cb92e187 100644 --- a/projects/llm.c/stats.jsonl +++ b/projects/llm.c/stats.jsonl @@ -12,3 +12,5 @@ {"kind":"phase","ts":"2026-06-01T19:32:49Z","phase":"test","seconds":1.815,"exit":0,"cmd":"bash -c cd /var/lib/jenkins/moat/projects/llm.c/src && HIP_VISIBLE_DEVICES=0 ./test_gpt2fp32cu"} {"kind":"phase","ts":"2026-06-01T19:33:01Z","phase":"test","seconds":5.236,"exit":1,"cmd":"bash -c cd /var/lib/jenkins/moat/projects/llm.c/src && HIP_VISIBLE_DEVICES=0 ./test_gpt2cu"} {"kind":"phase","ts":"2026-06-01T19:33:22Z","phase":"test","seconds":10.053,"exit":0,"cmd":"bash -c cd /var/lib/jenkins/moat/projects/llm.c/src && HIP_VISIBLE_DEVICES=0 OMP_NUM_THREADS=8 ./train_gpt2cu 2>&1 | tail -30"} +{"kind":"phase","ts":"2026-08-08T03:29:57Z","phase":"compile","seconds":5.683,"exit":0,"cmd":"bash -c cd ../llmc_old && make USE_HIP=1 AMDGPU_TARGETS=gfx90a NO_MULTI_GPU=1 NO_USE_MPI=1 -j16 test_gpt2fp32cu test_gpt2cu train_gpt2cu"} +{"kind":"phase","ts":"2026-08-08T03:30:10Z","phase":"compile","seconds":5.905,"exit":0,"cmd":"bash -c cd ../llmc_new && make USE_HIP=1 AMDGPU_TARGETS=gfx90a NO_MULTI_GPU=1 NO_USE_MPI=1 -j16 test_gpt2fp32cu test_gpt2cu train_gpt2cu"} diff --git a/projects/llm.c/status.json b/projects/llm.c/status.json index f70713ff..3918eeb3 100644 --- a/projects/llm.c/status.json +++ b/projects/llm.c/status.json @@ -7,7 +7,7 @@ "priority": 6.667, "ext_type": "makefile", "adopted_at": "2026-05-29T23:30:02Z", - "updated_at": "2026-08-07T07:05:15Z", + "updated_at": "2026-08-08T03:30:38Z", "head_sha": "de1dd620184356193b9864118dd4162b11938dd7", "depends_on": [], "pr_url": "https://github.com/karpathy/llm.c/pull/854", @@ -24,10 +24,10 @@ "state": "completed", "blocked": false, "blocked_reason": null, - "validated_sha": "d19a3220d7b61af86c4d7a8a978a76f94dbbaa71", + "validated_sha": "de1dd620184356193b9864118dd4162b11938dd7", "started_at": "2026-05-31T00:24:03Z", - "completed_at": "2026-06-04T06:37:40Z", - "updated_at": "2026-06-05T19:24:41Z", + "completed_at": "2026-08-08T03:30:38Z", + "updated_at": "2026-08-08T03:30:38Z", "stats": { "tokens_total": 0, "tokens_approx": true, @@ -43,12 +43,10 @@ }, "last_agent": "validator", "carry_forward": { - "from": "3282a74bc323a737fd55d53c2a88b1d8fe651619", - "to": "d19a3220d7b61af86c4d7a8a978a76f94dbbaa71", - "method": "source-class", - "class": "doc-only", - "detail": "no changes", - "at": "2026-06-04T06:59:06Z" + "to": "de1dd620184356193b9864118dd4162b11938dd7", + "method": "binary-equiv", + "detail": "Makefile build-detection fallback (gcc-install-dir pin, hipconfig amdgpu-arch lookup) + README doc addition; codeobj_diff.py confirms test_gpt2fp32cu/test_gpt2cu/train_gpt2cu byte-identical device ISA", + "at": "2026-08-08T03:30:38Z" } }, "linux-gfx1100": { From 5430e38e4472028115bfd16d51895e8e105103aa Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Sat, 8 Aug 2026 03:31:45 +0000 Subject: [PATCH 2/4] Regenerate README table --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c7b8d03a..76b5953e 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ The project name links upstream. | [bellhopcuda](https://github.com/A-New-BellHope/bellhopcuda) ([fork](https://github.com/AMD-Ecosystem/bellhopcuda/tree/moat-port)) | ✅ | ✅ | ✅ | 🟢 [#65](https://github.com/A-New-BellHope/bellhopcuda/pull/65) | | [brian2cuda](https://github.com/brian-team/brian2cuda) ([fork](https://github.com/AMD-Ecosystem/brian2cuda/tree/moat-port)) | ✅ | ✅ | 🔄 | 🟢 [#327](https://github.com/brian-team/brian2cuda/pull/327) | | [catboost](https://github.com/catboost/catboost) ([fork](https://github.com/AMD-Ecosystem/catboost-moat/tree/moat-port)) | 🔄 | ✅ | ✅ | 🟢 [#3111](https://github.com/catboost/catboost/pull/3111) | -| [colmap](https://github.com/colmap/colmap) ([fork](https://github.com/AMD-Ecosystem/colmap/tree/moat-port)) | ⬜ | ⬜ | ⬜ | — | +| [colmap](https://github.com/colmap/colmap) ([fork](https://github.com/AMD-Ecosystem/colmap/tree/moat-port)) | 🔧 | ⬜ | ⬜ | — | | [CPM.cu](https://github.com/OpenBMB/CPM.cu) | 🚫 | — | — | ⚪ not-portable | | [CubbyFlow](https://github.com/utilForever/CubbyFlow) ([fork](https://github.com/AMD-Ecosystem/CubbyFlow/tree/moat-port)) | ✅ | ✅ | ✅ | 🟢 [#145](https://github.com/utilForever/CubbyFlow/pull/145) | | [cuBQL](https://github.com/NVIDIA/cuBQL) ([fork](https://github.com/AMD-Ecosystem/cuBQL/tree/moat-port)) | ✅ | ✅ | ✅ | 🟣 [#35](https://github.com/NVIDIA/cuBQL/pull/35) | @@ -81,7 +81,7 @@ The project name links upstream. | [CudaSift](https://github.com/Celebrandil/CudaSift) ([fork](https://github.com/AMD-Ecosystem/CudaSift/tree/moat-port)) | ✅ | ✅ | ✅ | 🟢 [#97](https://github.com/Celebrandil/CudaSift/pull/97) | | [CuMesh](https://github.com/JeffreyXiang/CuMesh) ([fork](https://github.com/AMD-Ecosystem/CuMesh/tree/moat-port)) | 🔄 | ✅ | ✅ | 🟢 [#36](https://github.com/JeffreyXiang/CuMesh/pull/36) | | [cuPDLP-C](https://github.com/COPT-Public/cuPDLP-C) ([fork](https://github.com/AMD-Ecosystem/cuPDLP-C/tree/moat-port)) | ✅ | ✅ | ✅ | 🟢 [#41](https://github.com/COPT-Public/cuPDLP-C/pull/41) | -| [cuPDLPx](https://github.com/MIT-Lu-Lab/cuPDLPx) ([fork](https://github.com/AMD-Ecosystem/cuPDLPx/tree/moat-port)) | ✅ | 🔄 | 🔄 | 🟣 [#94](https://github.com/MIT-Lu-Lab/cuPDLPx/pull/94) | +| [cuPDLPx](https://github.com/MIT-Lu-Lab/cuPDLPx) ([fork](https://github.com/AMD-Ecosystem/cuPDLPx/tree/moat-port)) | ✅ | ✅ | 🔄 | 🟣 [#94](https://github.com/MIT-Lu-Lab/cuPDLPx/pull/94) | | [cupoch](https://github.com/neka-nat/cupoch) ([fork](https://github.com/AMD-Ecosystem/cupoch/tree/moat-port)) | ✅ | ✅ | ✅ | 🟣 [#143](https://github.com/neka-nat/cupoch/pull/143) | | [CuRast](https://github.com/m-schuetz/CuRast) ([fork](https://github.com/AMD-Ecosystem/CuRast/tree/moat-port)) | ✅ | ✅ | ✅ | 🟢 [#2](https://github.com/m-schuetz/CuRast/pull/2) | | [cuSZ](https://github.com/szcompressor/cuSZ) ([fork](https://github.com/AMD-Ecosystem/cuSZ/tree/moat-port)) | ✅ | ✅ | ✅ | 🟢 [#91](https://github.com/szcompressor/cuSZ/pull/91) | @@ -130,13 +130,13 @@ The project name links upstream. | [kaldifeat](https://github.com/csukuangfj/kaldifeat) ([fork](https://github.com/AMD-Ecosystem/kaldifeat/tree/moat-port)) | ✅ | ✅ | ✅ | — | | [KittenGpuLBVH](https://github.com/jerry060599/KittenGpuLBVH) ([fork](https://github.com/AMD-Ecosystem/KittenGpuLBVH/tree/moat-port)) | ✅ | ✅ | ✅ | 🟢 [#5](https://github.com/jerry060599/KittenGpuLBVH/pull/5) | | [LC-framework](https://github.com/burtscher/LC-framework) ([fork](https://github.com/AMD-Ecosystem/LC-framework/tree/moat-port)) | ✅ | ✅ | ⬜ | — | -| [lc0](https://github.com/LeelaChessZero/lc0) ([fork](https://github.com/AMD-Ecosystem/lc0/tree/moat-port)) | 🔄 | 🔄 | 🚫 | 🟢 [#2420](https://github.com/LeelaChessZero/lc0/pull/2420) | +| [lc0](https://github.com/LeelaChessZero/lc0) ([fork](https://github.com/AMD-Ecosystem/lc0/tree/moat-port)) | 🔄 | ✅ | 🚫 | 🟢 [#2420](https://github.com/LeelaChessZero/lc0/pull/2420) | | [LEAP](https://github.com/llnl/LEAP) ([fork](https://github.com/AMD-Ecosystem/LEAP/tree/moat-port)) | ✅ | ✅ | ✅ | — | | [libSGM](https://github.com/fixstars/libSGM) ([fork](https://github.com/AMD-Ecosystem/libSGM/tree/moat-port)) | ✅ | ✅ | ✅ | 🟢 [#89](https://github.com/fixstars/libSGM/pull/89) | | [LichtFeld-Studio](https://github.com/MrNeRF/LichtFeld-Studio) ([fork](https://github.com/AMD-Ecosystem/LichtFeld-Studio/tree/moat-port)) | ✅ | ✅ | 🚫 | — | | [LiteGS](https://github.com/MooreThreads/LiteGS) ([fork](https://github.com/AMD-Ecosystem/LiteGS/tree/moat-port)) | ✅ | ✅ | ✅ | — | | [llm-awq](https://github.com/mit-han-lab/llm-awq) ([fork](https://github.com/AMD-Ecosystem/llm-awq/tree/moat-port)) | ✅ | ✅ | ✅ | — | -| [llm.c](https://github.com/karpathy/llm.c) ([fork](https://github.com/AMD-Ecosystem/llm.c/tree/moat-port)) | 🔄 | ✅ | ✅ | 🟢 [#854](https://github.com/karpathy/llm.c/pull/854) | +| [llm.c](https://github.com/karpathy/llm.c) ([fork](https://github.com/AMD-Ecosystem/llm.c/tree/moat-port)) | ✅ | ✅ | ✅ | 🟢 [#854](https://github.com/karpathy/llm.c/pull/854) | | [llmq](https://github.com/IST-DASLab/llmq) | 🚫 | — | — | ⚪ not-portable | | [mahout](https://github.com/apache/mahout) ([fork](https://github.com/AMD-Ecosystem/mahout/tree/moat-port)) | ✅ | ✅ | 🔄 | 🟢 [#1399](https://github.com/apache/mahout/pull/1399) | | [marian-dev](https://github.com/marian-nmt/marian-dev) ([fork](https://github.com/AMD-Ecosystem/marian-dev/tree/moat-port)) | ✅ | ✅ | ✅ | — | From 2a52335bbf903aac8def004072031017b08be2c9 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Sat, 8 Aug 2026 17:38:04 +0000 Subject: [PATCH 3/4] llm.c: record the validator's token cost on this project's own branch `record-tokens` writes into the checkout it runs in, and it was run from another project's branch, so this landed in a phantom folder there. --- projects/llm.c/stats.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/llm.c/stats.jsonl b/projects/llm.c/stats.jsonl index cb92e187..126b2b28 100644 --- a/projects/llm.c/stats.jsonl +++ b/projects/llm.c/stats.jsonl @@ -14,3 +14,4 @@ {"kind":"phase","ts":"2026-06-01T19:33:22Z","phase":"test","seconds":10.053,"exit":0,"cmd":"bash -c cd /var/lib/jenkins/moat/projects/llm.c/src && HIP_VISIBLE_DEVICES=0 OMP_NUM_THREADS=8 ./train_gpt2cu 2>&1 | tail -30"} {"kind":"phase","ts":"2026-08-08T03:29:57Z","phase":"compile","seconds":5.683,"exit":0,"cmd":"bash -c cd ../llmc_old && make USE_HIP=1 AMDGPU_TARGETS=gfx90a NO_MULTI_GPU=1 NO_USE_MPI=1 -j16 test_gpt2fp32cu test_gpt2cu train_gpt2cu"} {"kind":"phase","ts":"2026-08-08T03:30:10Z","phase":"compile","seconds":5.905,"exit":0,"cmd":"bash -c cd ../llmc_new && make USE_HIP=1 AMDGPU_TARGETS=gfx90a NO_MULTI_GPU=1 NO_USE_MPI=1 -j16 test_gpt2fp32cu test_gpt2cu train_gpt2cu"} +{"kind": "tokens", "ts": "2026-08-08T03:32:29Z", "tokens": 78768, "source": "validator"} From 70059e03b2bc718f644cb785f688ac97f754da92 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Wed, 12 Aug 2026 22:11:17 +0000 Subject: [PATCH 4/4] llm.c: published_sha backfilled -- the open PR shows de1dd6201843 (verified against the live PR head) --- projects/llm.c/status.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/llm.c/status.json b/projects/llm.c/status.json index 3918eeb3..20037dfe 100644 --- a/projects/llm.c/status.json +++ b/projects/llm.c/status.json @@ -7,7 +7,7 @@ "priority": 6.667, "ext_type": "makefile", "adopted_at": "2026-05-29T23:30:02Z", - "updated_at": "2026-08-08T03:30:38Z", + "updated_at": "2026-08-12T22:11:17Z", "head_sha": "de1dd620184356193b9864118dd4162b11938dd7", "depends_on": [], "pr_url": "https://github.com/karpathy/llm.c/pull/854", @@ -165,5 +165,6 @@ "at": "2026-06-24T00:02:17Z" } } - } + }, + "published_sha": "de1dd620184356193b9864118dd4162b11938dd7" }