diff --git a/assets/embedding-demo.tape b/assets/embedding-demo.tape new file mode 100644 index 0000000..cf67537 Binary files /dev/null and b/assets/embedding-demo.tape differ diff --git a/cmd/toktape/card.go b/cmd/toktape/card.go index d2c14ab..3aa1dce 100644 --- a/cmd/toktape/card.go +++ b/cmd/toktape/card.go @@ -65,6 +65,12 @@ func runCard(c *cli, args []string) int { return c.usagef("toktape: %v", err) } + // An embedding tape has its own view, over its records (2026-10-15). + if tp.Summary.IsEmbedding() && format != outputPNG { + fmt.Fprint(c.stdout, card.EmbeddingText(tp)) + return exitOK + } + if *f.explain { explainCard(c.stderr, &tp.Summary) } diff --git a/cmd/toktape/compare.go b/cmd/toktape/compare.go index 5dcefff..f1862e8 100644 --- a/cmd/toktape/compare.go +++ b/cmd/toktape/compare.go @@ -3,6 +3,7 @@ package main import ( "fmt" + "github.com/midagedev/toktape/internal/card" "github.com/midagedev/toktape/internal/compare" "github.com/midagedev/toktape/internal/tape" ) @@ -25,6 +26,14 @@ func runCompare(c *cli, args []string) int { if err != nil { return c.usagef("toktape: %v", err) } + // Two embedding tapes compare vector by vector (2026-10-15). + if ea, eb := a.Summary.IsEmbedding(), b.Summary.IsEmbedding(); ea || eb { + if !ea || !eb { + return c.usagef("toktape compare: an embedding tape compares only with another embedding tape") + } + fmt.Fprint(c.stdout, card.EmbeddingCompare(a, b)) + return exitOK + } r, err := compare.Tapes(a, b) if err != nil { return c.usagef("toktape compare: %v", err) diff --git a/cmd/toktape/embed.go b/cmd/toktape/embed.go new file mode 100644 index 0000000..fa80752 --- /dev/null +++ b/cmd/toktape/embed.go @@ -0,0 +1,151 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/midagedev/toktape/internal/card" + "github.com/midagedev/toktape/internal/recorder" + "github.com/midagedev/toktape/internal/tape" +) + +// `toktape embed` (2026-10-15): record an embedding model. Each input of a +// set is sent on its own to /v1/embeddings (or llama-server's /embedding) +// and the vectors go on the tape; `toktape card` shows them and `toktape +// compare` puts two embedding tapes side by side. + +const embedUsage = `toktape embed — record an embedding model, one command + +Usage: + toktape embed [flags] + + Sends each input of a set, one at a time, to an embeddings endpoint and + keeps every returned vector on the tape. The summary shows per input the + dimension, norm and first values, and the cosine similarity between every + pair. ` + "`toktape play a.tape`" + ` replays it on the screen (vector strips and + a coloured cosine matrix), ` + "`--against b.tape`" + ` adds each input's cosine to + the same input in b. ` + "`toktape compare a.tape b.tape`" + ` prints the same + comparison as text: cosine and largest difference per input. + +Flags: + --url URL server to attach to (default: discover) + --api NAME openai (POST /v1/embeddings, default) or llama + (POST /embedding, llama-server's own shape) + --inputs FILE JSONL, one {"id", "input"} per line, optionally with + "role" (doc|query), "topic" and, on a query, "expect" + (the doc id it should find). Default: a built-in + search set, 20 Korean/English docs in 5 topics and + 6 queries + --model NAME "model" to send (openai shape; default: none) + --dimensions N "dimensions" to send (openai shape; default: none) + --encoding-format F float (default) or base64 (openai shape) + --vector-cap N keep at most N values of each vector (default 0 = + all); dim, norm and the first values are always kept + --no-prompts send inputs bare; by default a row with "role" doc or + query goes twice, with EmbeddingGemma 2's task prefix + ("title: none | text: " / "task: search result | + query: ") and bare, so the replay ranks both ways + --gap D, --type-rate D, --hold D + pacing for a watchable replay: idle after each doc + (100ms), per query character (25ms, the replay types + it), after each query (1s); --no-pace for none. + Idle time is never part of a latency + --out DIR where the tape is written (default ~/.toktape/runs) + --tag TEXT, --note TEXT label the run + --host-label TEXT store TEXT as the hostname +` + +// embedRun is the seam the tests use to replace the recorder. +var embedRun = recorder.RecordEmbedding + +func runEmbed(ctx context.Context, c *cli, args []string) int { + fs := newFlagSet("embed") + url := fs.String("url", "", "server base URL (default: discover)") + api := fs.String("api", tape.EmbeddingAPIOpenAI, "openai or llama") + inputsFile := fs.String("inputs", "", "input set file") + model := fs.String("model", "", "model to send") + dims := fs.Int("dimensions", 0, "dimensions to send") + format := fs.String("encoding-format", "float", "float or base64") + vcap := fs.Int("vector-cap", 0, "values kept per vector") + gap := fs.Duration("gap", 100*time.Millisecond, "idle time after each doc") + typeRate := fs.Duration("type-rate", 25*time.Millisecond, "idle time per character before each query") + hold := fs.Duration("hold", time.Second, "idle time after each query") + noPace := fs.Bool("no-pace", false, "no idle time at all") + noPrompts := fs.Bool("no-prompts", false, "send every input bare, without EmbeddingGemma's task prefixes") + outDir := fs.String("out", defaultRunsDir(), "where the tape is written") + tag := fs.String("tag", "", "label the run") + note := fs.String("note", "", "label the run") + hostLabel := fs.String("host-label", "", "store this instead of the machine's hostname") + extra, err := parseArgs(fs, args) + if err != nil { + return c.badFlags("embed", embedUsage, fs, args, err) + } + if len(extra) > 0 { + return c.usagef("toktape embed: unexpected argument %q", extra[0]) + } + switch { + case *api != tape.EmbeddingAPIOpenAI && *api != tape.EmbeddingAPILlama: + return c.usagef("toktape embed: --api %q: use openai or llama", *api) + case *format != "float" && *format != "base64": + return c.usagef("toktape embed: --encoding-format %q: use float or base64", *format) + case *dims < 0: + return c.usagef("toktape embed: --dimensions %d: must be positive", *dims) + case *gap < 0 || *typeRate < 0 || *hold < 0: + return c.usagef("toktape embed: --gap, --type-rate and --hold take 0 or more") + case *vcap < 0: + return c.usagef("toktape embed: --vector-cap %d: use 0 to keep whole vectors", *vcap) + } + opts := recorder.EmbeddingOptions{ + BaseURL: *url, API: *api, Model: *model, Dimensions: *dims, EncodingFormat: *format, + VectorCap: *vcap, Gap: *gap, TypeRate: *typeRate, Hold: *hold, Tail: *hold + time.Second, Tag: *tag, Note: *note, Version: version, + HostLabel: recorder.HostLabel{Set: flagSet(fs, "host-label"), Text: *hostLabel}, + } + if *noPace { + opts.Gap, opts.TypeRate, opts.Hold, opts.Tail = 0, 0, 0, 0 + } + if !*noPrompts { + opts.Prompts = tape.EmbeddingGemmaPrompts + } + if *inputsFile != "" { + raw, err := os.ReadFile(*inputsFile) + if err != nil { + return c.usagef("toktape embed: --inputs: %v", err) + } + opts.Inputs, opts.InputsName = raw, *inputsFile + } + pinned := pinCollectors(recorder.Options{}) + opts.FSRoot, opts.GPU = pinned.FSRoot, pinned.GPU + opts.Progress = func(r tape.EmbeddingRecord) { + if r.Error != "" { + fmt.Fprintf(c.stderr, " %-14s failed: %s\n", r.ID, clipLine(r.Error, 70)) + return + } + fmt.Fprintf(c.stderr, " %-14s %d values %s ms\n", r.ID, r.Dim, fmtMs(float64(r.Latency().Microseconds())/1000)) + } + + tp, err := embedRun(ctx, opts) + if err != nil { + switch { + case errors.Is(err, recorder.ErrUnreachable): + return c.fail(failure{code: exitUnreachable, msg: fmt.Sprintf("toktape: %v", err), + hint: "name the server with --url http://host:port; llama-server needs --embeddings"}) + case errors.Is(err, recorder.ErrAllStreamsFailed): + return c.fail(failure{code: exitStreams, msg: fmt.Sprintf("toktape: %v", err), + hint: "the server was reachable but embedded nothing; llama-server needs --embeddings"}) + default: + return c.usagef("toktape embed: %v", err) + } + } + path := filepath.Join(*outDir, tp.Summary.ID+tape.Ext) + // Only the tape is written: the ledger and the token card refuse an + // embedding tape. + if err := tape.Write(path, tp); err != nil { + return c.usagef("toktape embed: saving the run file: %v", err) + } + fmt.Fprintf(c.stdout, "\n%s\n tape %s\n", card.EmbeddingText(tp), tildePath(path)) + return exitOK +} diff --git a/cmd/toktape/embed_test.go b/cmd/toktape/embed_test.go new file mode 100644 index 0000000..7c33519 --- /dev/null +++ b/cmd/toktape/embed_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/midagedev/toktape/internal/ledger" + "github.com/midagedev/toktape/internal/tape" +) + +func TestEmbedRecordsCardAndCompare(t *testing.T) { + hermetic(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/embeddings" { + http.NotFound(w, r) + return + } + var req struct{ Input string } + _ = json.NewDecoder(r.Body).Decode(&req) + v := []float64{1, 0, 0} + if strings.Contains(req.Input, "b") { + v = []float64{0, 1, 0} + } + json.NewEncoder(w).Encode(map[string]any{"data": []any{map[string]any{"embedding": v}}}) + })) + t.Cleanup(srv.Close) + inputs := filepath.Join(t.TempDir(), "in.jsonl") + if err := os.WriteFile(inputs, []byte(`{"id":"one","input":"a"}`+"\n"+`{"id":"two","input":"b"}`+"\n"), 0o644); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + code, stdout, stderr := exec(t, "embed", "--url", srv.URL, "--inputs", inputs, "--out", dir, "--no-pace") + if code != exitOK { + t.Fatalf("exit %d\n%s", code, stderr) + } + for _, want := range []string{"POST /v1/embeddings", "cosine similarity", " 0 1.00 0.00", "dim 3"} { + if !strings.Contains(stdout, want) { + t.Errorf("summary lacks %q:\n%s", want, stdout) + } + } + tapes, _ := filepath.Glob(filepath.Join(dir, "*"+tape.Ext)) + if len(tapes) != 1 { + t.Fatalf("tapes %v", tapes) + } + tp, err := tape.Read(tapes[0]) + if err != nil || !tp.Summary.IsEmbedding() || len(tp.Embeddings) != 2 { + t.Fatalf("tape: %v", err) + } + if ledger.Holds(tp) { + t.Error("the ledger holds an embedding tape") + } + if code, out, _ := exec(t, "card", tapes[0]); code != exitOK || !strings.Contains(out, "cosine similarity") { + t.Errorf("card exit %d:\n%s", code, out) + } + code, out, _ := exec(t, "compare", tapes[0], tapes[0]) + if code != exitOK || !strings.Contains(out, "1.000000 0.000000") || !strings.Contains(out, "2 inputs matched") { + t.Errorf("compare exit %d:\n%s", code, out) + } + if code, _, _ := exec(t, "compare", tapes[0], filepath.Join("..", "..", "assets", "hero.tape")); code == exitOK { + t.Error("an embedding tape compared with a token tape") + } + if code, _, _ := exec(t, "publish", tapes[0], "--yes"); code == exitOK { + t.Error("an embedding tape was published") + } +} + +func TestEmbedFlagRefusals(t *testing.T) { + hermetic(t) + for name, args := range map[string][]string{ + "bad api": {"embed", "--api", "x"}, + "bad format": {"embed", "--encoding-format", "f16"}, + "neg dims": {"embed", "--dimensions", "-1"}, + "neg cap": {"embed", "--vector-cap", "-1"}, + "stray arg": {"embed", "extra"}, + "missing set": {"embed", "--inputs", filepath.Join(t.TempDir(), "none")}, + } { + if code, _, _ := exec(t, args...); code == exitOK { + t.Errorf("%s: exit 0", name) + } + } +} + +// TestEmbeddingDemoTape: the recorded llama-server tape still reads and +// shows (EmbeddingGemma 2 Q8_0, built-in search set). +func TestEmbeddingDemoTape(t *testing.T) { + code, out, _ := exec(t, "card", filepath.Join("..", "..", "assets", "embedding-demo.tape")) + if code != exitOK || !strings.Contains(out, "embeddinggemma-2") || !strings.Contains(out, "q-starter") { + t.Fatalf("exit %d:\n%s", code, out) + } +} diff --git a/cmd/toktape/help.go b/cmd/toktape/help.go index abb0df5..817ccc2 100644 --- a/cmd/toktape/help.go +++ b/cmd/toktape/help.go @@ -346,6 +346,9 @@ func usageFor(verb string) string { if verb == "decide" { return decideUsage } + if verb == "embed" { + return embedUsage + } return usageText } diff --git a/cmd/toktape/ls.go b/cmd/toktape/ls.go index eb9df89..a4bab9b 100644 --- a/cmd/toktape/ls.go +++ b/cmd/toktape/ls.go @@ -56,6 +56,9 @@ func runLs(c *cli, args []string) int { if s.IsDecision() { n = "decision" } + if s.IsEmbedding() { + n = "embedding" + } rows = append(rows, []string{ s.ID, modelLabel(s), diff --git a/cmd/toktape/play.go b/cmd/toktape/play.go index 8cd5616..cea8f81 100644 --- a/cmd/toktape/play.go +++ b/cmd/toktape/play.go @@ -19,6 +19,8 @@ import ( func runPlay(ctx context.Context, c *cli, args []string) int { fs := newFlagSet("play") speed := fs.Float64("speed", 1, "replay speed multiplier (4 plays four times as fast)") + verify := fs.Bool("verify", false, "embedding tape: the per-input verify screen instead of the search scene") + against := fs.String("against", "", "an embedding tape to compare each input's vector with") grid := fs.String("grid", tui.DefaultGrid.String(), "tile grid per page as COLSxROWS (0 = fit to the terminal)") files, err := parseArgs(fs, args) if err != nil { @@ -38,6 +40,18 @@ func runPlay(ctx context.Context, c *cli, args []string) int { if err != nil { return c.usagef("toktape: %v", err) } + var ref *tape.Tape + if *against != "" { + if !tp.Summary.IsEmbedding() { + return c.usagef("toktape play: --against is for an embedding tape") + } + if ref, err = tape.Read(*against); err != nil { + return c.usagef("toktape: %v", err) + } + if !ref.Summary.IsEmbedding() { + return c.usagef("toktape play: --against %s is not an embedding tape", *against) + } + } if !isTTY(c.stdout) { return c.fail(failure{ code: exitUsage, @@ -54,6 +68,8 @@ func runPlay(ctx context.Context, c *cli, args []string) int { path: tildePath(files[0]), end: tapeDuration(tp), playing: true, + ref: ref, + verify: *verify, } if _, err := tea.NewProgram(p, tea.WithContext(ctx), tea.WithAltScreen(), tea.WithOutput(c.stdout)).Run(); err != nil { if ctx.Err() != nil { @@ -93,6 +109,9 @@ func tapeDuration(tp *tape.Tape) time.Duration { if tp == nil { return end } + if tp.Summary.IsEmbedding() { + return tui.EmbeddingEnd(tp) + } for _, req := range tp.Requests { if n := len(req.Tokens); n > 0 { if t := req.StartedAt + req.Tokens[n-1].T; t > end { @@ -106,15 +125,17 @@ func tapeDuration(tp *tape.Tape) time.Duration { // player is the bubbletea shell for a replay. It owns the wall clock and // nothing else; every frame comes from playModel and tui.View. type player struct { - tp *tape.Tape - speed float64 - grid tui.Grid - theme tui.Theme - path string - end time.Duration - wall time.Duration - start time.Time - w, h int + tp *tape.Tape + ref *tape.Tape // --against, an embedding reference + verify bool // --verify + speed float64 + grid tui.Grid + theme tui.Theme + path string + end time.Duration + wall time.Duration + start time.Time + w, h int // page is which page of tiles the viewer has stepped to. It lives here // rather than in the model because the model is rebuilt from the tape on // every frame; the view takes it as ordinary model state. @@ -125,11 +146,20 @@ type player struct { func (p *player) Init() tea.Cmd { p.start = time.Now() - return playTick() + return p.tick() } -func playTick() tea.Cmd { - return tea.Tick(tui.TickInterval, func(t time.Time) tea.Msg { return playTickMsg(t) }) +// tick is the next redraw. An embedding tape animates at ~30 fps while its +// scene is running and drops to 4 fps once it has settled. +func (p *player) tick() tea.Cmd { + d := tui.TickInterval + if p.tp != nil && p.tp.Summary.IsEmbedding() { + d = tui.EmbedTick + if clipTime(p.wall, p.speed) > p.end+time.Second { + d = 250 * time.Millisecond + } + } + return tea.Tick(d, func(t time.Time) tea.Msg { return playTickMsg(t) }) } type playTickMsg time.Time @@ -159,7 +189,7 @@ func (p *player) Update(msg tea.Msg) (tea.Model, tea.Cmd) { if p.playing { p.wall = time.Since(p.start) } - return p, playTick() + return p, p.tick() } return p, nil } @@ -172,6 +202,8 @@ func (p *player) wallAtPause() time.Duration { return p.wall } // the keys can never step past the last stream. func (p *player) pageAfter(d int) int { m, _ := playModel(p.tp, p.wall, p.speed) + tui.SetEmbedRef(&m, p.ref, refLabel(p.ref)) + m.EmbedVerify = p.verify m.Grid = p.grid m.Page = p.page w, h := p.size() @@ -197,6 +229,8 @@ func (p *player) View() string { } w, h := p.size() m, clip := playModel(p.tp, p.wall, p.speed) + tui.SetEmbedRef(&m, p.ref, refLabel(p.ref)) + m.EmbedVerify = p.verify m.Theme = p.theme // TapePath stays empty, so the footer reads "run complete": p.path is the // file being replayed, and "tape saved " is a claim about where the @@ -206,3 +240,19 @@ func (p *player) View() string { m.Page = p.page return tui.View(m, clip, w, h) } + +// refLabel names a reference tape on the screen: its tag, else its engine +// and endpoint. +func refLabel(t *tape.Tape) string { + if t == nil { + return "" + } + if t.Summary.Tag != "" { + return t.Summary.Tag + } + l := string(t.Summary.Server.Kind) + if e := t.Summary.Embedding; e != nil { + l += " " + e.Endpoint + } + return l +} diff --git a/cmd/toktape/render.go b/cmd/toktape/render.go index f3a4d27..1decb9c 100644 --- a/cmd/toktape/render.go +++ b/cmd/toktape/render.go @@ -224,6 +224,9 @@ func runRender(c *cli, args []string) int { if err != nil { return c.usagef("toktape: %v", err) } + if tp.Summary.IsEmbedding() { + return c.usagef("toktape render: %s is an embedding tape and has no clip; `toktape card` shows it", tapePath) + } // No TapePath, so the footer reads "run complete" (render.Options.TapePath). // // It used to be tildePath(tapePath), the file being replayed, and the diff --git a/cmd/toktape/run.go b/cmd/toktape/run.go index fbd1eda..c61cd4e 100644 --- a/cmd/toktape/run.go +++ b/cmd/toktape/run.go @@ -191,6 +191,8 @@ func Run(ctx context.Context, stdout, stderr io.Writer, args []string) int { return runChat(ctx, c, rest) case "decide": return runDecide(ctx, c, rest) + case "embed": + return runEmbed(ctx, c, rest) case "card": return runCard(c, rest) case "play": @@ -224,7 +226,7 @@ func Run(ctx context.Context, stdout, stderr io.Writer, args []string) int { // verbs are the commands Run dispatches on. The root verb is "record", so // `toktape` and `toktape --url ...` both record. var verbs = map[string]bool{ - "record": true, "chat": true, "decide": true, "card": true, "play": true, "render": true, + "record": true, "chat": true, "decide": true, "embed": true, "card": true, "play": true, "render": true, "ls": true, "log": true, "compare": true, "publish": true, "runs": true, "show": true, "reindex": true, "profile": true, "share": true, "version": true, } diff --git a/internal/card/embedding.go b/internal/card/embedding.go new file mode 100644 index 0000000..0942423 --- /dev/null +++ b/internal/card/embedding.go @@ -0,0 +1,180 @@ +package card + +import ( + "fmt" + "math" + "strings" + "time" + + "github.com/midagedev/toktape/internal/tape" +) + +// EmbeddingText is an embedding tape's view (2026-10-15): who answered, +// one row per input (tokens, latency, dim, norm, first values), and the +// pairwise cosine-similarity matrix over the stored vectors. It reads the +// records, not only the summary, because the vectors are the result. +func EmbeddingText(tp *tape.Tape) string { + s := &tp.Summary + e := s.Embedding + if e == nil { + e = &tape.EmbeddingSummary{} + } + var b strings.Builder + model := orQ(s.Model.FileName) + if model == "?" && e.Model != "" { + model = e.Model + } + fmt.Fprintf(&b, "toktape %s · %s\n", orQ(s.ToktapeVersion), s.ID) + fmt.Fprintf(&b, "MODEL %s\n", model) + engine := orQ(string(s.Server.Kind)) + if s.Server.Commit != "" { + engine += " " + s.Server.Commit + } + fmt.Fprintf(&b, "ENGINE %s · POST %s\n", engine, orQ(e.Endpoint)) + req := []string{fmt.Sprintf("%d inputs", e.Inputs)} + if e.EncodingFormat != "" { + req = append(req, "encoding_format "+e.EncodingFormat) + } + if e.Dimensions > 0 { + req = append(req, fmt.Sprintf("dimensions %d", e.Dimensions)) + } + if e.Model != "" { + req = append(req, "model "+e.Model) + } + if e.VectorCap > 0 { + req = append(req, fmt.Sprintf("vectors capped at %d values", e.VectorCap)) + } + fmt.Fprintf(&b, "REQUEST %s\n", strings.Join(req, " · ")) + fmt.Fprintf(&b, "ANSWER dim %s · latency p50 %s ms max %s ms · %s prompt tokens\n", + intQ(e.Dim), msQ(e.LatencyP50Ms), msQ(e.LatencyMaxMs), intQ(e.PromptTokens)) + + b.WriteString("\n # id tok ms dim norm first values\n") + for i, r := range tp.Embeddings { + if r.Error != "" { + fmt.Fprintf(&b, "%3d %-12s failed: %s\n", i, clip(r.ID, 12), clip(r.Error, 60)) + continue + } + head := make([]string, 0, 4) + for _, x := range r.Head[:min(len(r.Head), 4)] { + head = append(head, fmt.Sprintf("%+.4f", x)) + } + fmt.Fprintf(&b, "%3d %-12s %4s %7s %5d %7.4f %s\n", i, clip(r.ID, 12), intQ(r.PromptTokens), + msQ(float64(r.Latency())/float64(time.Millisecond)), r.Dim, r.Norm, strings.Join(head, " ")) + } + + vecs := make([][]float32, len(tp.Embeddings)) + for i, r := range tp.Embeddings { + vecs[i] = r.Values() + } + if len(vecs) > 1 { + b.WriteString("\ncosine similarity\n ") + for j := range vecs { + fmt.Fprintf(&b, " %5d", j) + } + b.WriteString("\n") + for i := range vecs { + fmt.Fprintf(&b, "%3d ", i) + for j := range vecs { + c := math.NaN() + if vecs[i] != nil && vecs[j] != nil { + c = tape.Cosine(vecs[i], vecs[j]) + } + if math.IsNaN(c) { + b.WriteString(" ?") + } else { + fmt.Fprintf(&b, " %5.2f", c) + } + } + b.WriteString("\n") + } + if e.VectorCap > 0 && e.Dim > e.VectorCap { + fmt.Fprintf(&b, " (over the first %d of %d values only)\n", e.VectorCap, e.Dim) + } + } + for _, w := range s.Warnings { + fmt.Fprintf(&b, "! %s\n", w) + } + return b.String() +} + +// EmbeddingCompare matches two embedding tapes' inputs by id and reports, +// per input, the cosine and the largest absolute difference between the two +// vectors (over the values both stored). a is the run under test, b the +// reference. +func EmbeddingCompare(a, b *tape.Tape) string { + ref := map[string]tape.EmbeddingRecord{} + for _, r := range b.Embeddings { + ref[r.Key()] = r + } + var out strings.Builder + fmt.Fprintf(&out, "A %s · %s %s\n", a.Summary.ID, orQ(string(a.Summary.Server.Kind)), endpointOf(a)) + fmt.Fprintf(&out, "B %s · %s %s\n\n", b.Summary.ID, orQ(string(b.Summary.Server.Kind)), endpointOf(b)) + out.WriteString(" id dim A/B cosine max |diff|\n") + worst, worstID, matched := 1.0, "", 0 + for _, r := range a.Embeddings { + o, ok := ref[r.Key()] + switch { + case !ok: + fmt.Fprintf(&out, " %-12s not in B\n", clip(r.ID, 12)) + continue + case r.Error != "" || o.Error != "": + fmt.Fprintf(&out, " %-12s failed on one side\n", clip(r.ID, 12)) + continue + } + va, vb := r.Values(), o.Values() + c, d := tape.Cosine(va, vb), tape.MaxAbsDiff(va, vb) + note := "" + if r.Input != o.Input { + note = " ! inputs differ" + } + fmt.Fprintf(&out, " %-12s %4d/%-4d %8.6f %10.6f%s\n", clip(r.ID, 12), r.Dim, o.Dim, c, d, note) + matched++ + if !math.IsNaN(c) && c < worst { + worst, worstID = c, r.ID + } + } + if matched > 0 { + fmt.Fprintf(&out, "\n%d inputs matched · lowest cosine %.6f (%s)\n", matched, worst, worstID) + } + return out.String() +} + +func endpointOf(t *tape.Tape) string { + if t.Summary.Embedding == nil { + return "?" + } + return t.Summary.Embedding.Endpoint +} + +func orQ(s string) string { + if s == "" { + return "?" + } + return s +} + +func intQ(n int) string { + if n <= 0 { + return "?" + } + return fmt.Sprint(n) +} + +func msQ(ms float64) string { + switch { + case ms <= 0: + return "?" + case ms < 100: + return fmt.Sprintf("%.1f", ms) + default: + return fmt.Sprintf("%.0f", ms) + } +} + +func clip(s string, n int) string { + s = strings.Join(strings.Fields(s), " ") + if r := []rune(s); len(r) > n { + return string(r[:n-1]) + "…" + } + return s +} diff --git a/internal/card/png/embedding.go b/internal/card/png/embedding.go new file mode 100644 index 0000000..657be7a --- /dev/null +++ b/internal/card/png/embedding.go @@ -0,0 +1,86 @@ +package png + +import ( + "fmt" + "image" + "strconv" + + "github.com/midagedev/toktape/internal/card" + "github.com/midagedev/toktape/internal/tape" +) + +// The embedding card (2026-10-15) is the decision card's frame — header, +// hero split, mid band, identity, strip — with an embedding run's figures: +// the request p50 as the lit hero, recall@1 on the search set beside it, and +// the run's shape in the mid band. Every figure is the summary's. +func (c *canvas) drawEmbedding(s *tape.RunSummary) { + e := tape.EmbeddingSummary{} + if s.Embedding != nil { + e = *s.Embedding + } + ct := &content{} + ct.buildHeader(s) + ct.buildIdent(s) + ct.ident3 = decisionEngineIdent(s.Server) + + c.fill(image.Rect(0, 0, Width, Height), colBase) + panel := image.Rect(panelInset, panelTop, Width-panelInset, panelBottom) + c.roundRect("panel.border", panel, panelRadius, colBorder) + c.roundRect("panel", panel.Inset(1), panelRadius-1, colPanel) + c.drawHeader(ct) + c.hairline("rule.header", contentL, contentR, bandHeroTop, colBorder) + + c.vrule("hero.split", heroSplitX, heroRuleTop, heroRuleBottom, colBorder) + left := heroCol{ + eyebrow: "Embedding", + number: msFig(e.LatencyP50Ms), + unit: "ms", + sub1: "p50 per request · end to end", + sub2: fmt.Sprintf("%d requests · POST %s", e.Inputs, orUnknown(e.Endpoint)), + } + if left.number == unknown { + left.unit = "" + } + right := heroCol{eyebrow: "Recall@1", number: unknown, sub1: "no graded queries"} + if e.Graded > 0 { + right.number = fmt.Sprintf("%d/%d", e.Hits, e.Graded) + right.sub1 = "expected doc ranked first" + if len(e.Prompts) > 0 { + right.sub2 = fmt.Sprintf("with task prompts · %d/%d bare", e.HitsBare, e.Graded) + } + } + c.drawHeroCol("hero.left", left, contentL, heroSplitX-heroGutter, func(x0, x1 int) image.Image { + return hGradient{x0: x0, x1: x1, c0: colAccent, c1: colAccentHigh} + }) + c.drawHeroCol("hero.right", right, heroRightX, contentR, func(int, int) image.Image { + return solid(colText) + }) + c.hairline("rule.hero", contentL, contentR, bandMemTop, colBorder) + + cells := []card.DecisionRow{ + {Key: "dim", Label: "Dim", Parts: []string{strconv.Itoa(e.Dim), "per vector"}}, + {Key: "docs", Label: "Docs", Parts: []string{strconv.Itoa(e.Docs), "indexed"}}, + {Key: "queries", Label: "Queries", Parts: []string{strconv.Itoa(e.Queries), "searched"}}, + {Key: "max", Label: "Slowest", Parts: []string{msFig(e.LatencyMaxMs) + " ms", "one request"}}, + } + w := contentW / len(cells) + for i, r := range cells { + x := contentL + i*w + if i > 0 { + c.vrule("stat.rule"+strconv.Itoa(i), x-14, statRuleTop, statRuleBottom, colBorder) + } + c.drawStat("stat."+r.Key, r, x, w-28) + } + c.drawIdent(ct) + c.drawDecisionStrip(card.DecisionView{Note: s.Note}) +} + +func msFig(ms float64) string { + switch { + case ms <= 0: + return unknown + case ms < 100: + return strconv.FormatFloat(ms, 'f', 1, 64) + } + return strconv.FormatFloat(ms, 'f', 0, 64) +} diff --git a/internal/card/png/embedding_test.go b/internal/card/png/embedding_test.go new file mode 100644 index 0000000..994a61e --- /dev/null +++ b/internal/card/png/embedding_test.go @@ -0,0 +1,23 @@ +package png + +import ( + "testing" + + "github.com/midagedev/toktape/internal/tape" +) + +func TestEmbeddingCard(t *testing.T) { + tp, err := tape.Read("../../../assets/embedding-demo.tape") + if err != nil { + t.Fatal(err) + } + c, err := renderCanvas(&tp.Summary) + if err != nil { + t.Fatal(err) + } + for _, id := range []string{"hero.left.number", "hero.right.number", "stat.dim.label", "stat.max.label", "strip.credit"} { + if _, ok := c.markByID(id); !ok { + t.Errorf("the embedding card has no %s", id) + } + } +} diff --git a/internal/card/png/png.go b/internal/card/png/png.go index 88141f6..8016559 100644 --- a/internal/card/png/png.go +++ b/internal/card/png/png.go @@ -53,6 +53,8 @@ func renderCanvas(s *tape.RunSummary) (*canvas, error) { // (TTP-192, 2026-10-02). if s.IsDecision() { c.drawDecision(s) + } else if s.IsEmbedding() { + c.drawEmbedding(s) } else { c.draw(s) } diff --git a/internal/compare/compare.go b/internal/compare/compare.go index 32c9060..7059400 100644 --- a/internal/compare/compare.go +++ b/internal/compare/compare.go @@ -81,6 +81,9 @@ func Tapes(a, b *tape.Tape) (Report, error) { // A decision-model run has no token figure for the table to put beside // another run's (TTP-192), and two of them compare on latency, which // this table does not have yet. + if a.Summary.IsEmbedding() || b.Summary.IsEmbedding() { + return Report{}, fmt.Errorf("an embedding run compares only with another embedding run, vector by vector (toktape compare does that)") + } if a.Summary.IsDecision() || b.Summary.IsDecision() { return Report{}, fmt.Errorf("a decision-model run is not comparable here: it has no decode or prefill stream, only request latencies") } diff --git a/internal/ledger/file.go b/internal/ledger/file.go index 77b7d42..b1b5008 100644 --- a/internal/ledger/file.go +++ b/internal/ledger/file.go @@ -94,7 +94,7 @@ func Append(dir string, tp *tape.Tape) error { // decision-model run (tape.ModeDecision, TTP-192) is kept out too: every // column of the ledger is a token figure it does not have. func Holds(tp *tape.Tape) bool { - return tp != nil && !tp.Summary.IsChat() && !tp.Summary.IsDecision() + return tp != nil && !tp.Summary.IsChat() && !tp.Summary.IsDecision() && !tp.Summary.IsEmbedding() } // Rebuild regenerates the ledger in dir from every tape it holds and writes diff --git a/internal/publish/embedding_test.go b/internal/publish/embedding_test.go new file mode 100644 index 0000000..0c4a21b --- /dev/null +++ b/internal/publish/embedding_test.go @@ -0,0 +1,25 @@ +package publish + +import ( + "testing" + + "github.com/midagedev/toktape/internal/tape" +) + +func TestEmbeddingIndex(t *testing.T) { + tp, err := tape.Read("../../assets/embedding-demo.tape") + if err != nil { + t.Fatal(err) + } + v := PublicView(tp, WithText) + idx := IndexOf(v) + if idx.Mode != tape.ModeEmbedding || idx.Sessions != 0 || idx.DecisionP50Ms != tp.Summary.Embedding.LatencyP50Ms { + t.Errorf("index row: mode %q sessions %d p50 %v", idx.Mode, idx.Sessions, idx.DecisionP50Ms) + } + if w := PublicView(tp, WithoutText); w.Embeddings[0].Input != "" { + t.Error("without text, an embedding input still travels") + } + if tp.Embeddings[0].Input == "" { + t.Error("PublicView edited the input tape") + } +} diff --git a/internal/publish/index.go b/internal/publish/index.go index 1b23816..4a960c5 100644 --- a/internal/publish/index.go +++ b/internal/publish/index.go @@ -272,6 +272,13 @@ func IndexOf(t *tape.Tape) Index { idx.DecisionColdMs = d.ColdMs idx.DecisionReqPerSec = d.RequestsPerSecond } + if e := s.Embedding; s.IsEmbedding() && e != nil { + // No new column (2026-10-15): an embedding run's one latency figure, + // the request p50, rides decision_p50_ms, and the hub labels it by + // mode. + idx.Sessions = 0 + idx.DecisionP50Ms = e.LatencyP50Ms + } // The rate choice is the one the rest of the repo already makes: above // one stream the figure is the server-wide aggregate, at one stream the diff --git a/internal/publish/preview.go b/internal/publish/preview.go index e14c677..745e34e 100644 --- a/internal/publish/preview.go +++ b/internal/publish/preview.go @@ -156,6 +156,16 @@ func textLine(view *tape.Tape, opts Options) string { if view.Summary.IsDecision() { return decisionTextLine(view, opts) } + if view.Summary.IsEmbedding() { + if opts.Text == WithoutText { + return "nothing · the inputs are removed, the vectors and timings stay" + } + n := 0 + for _, r := range view.Embeddings { + n += len([]rune(r.Input)) + } + return fmt.Sprintf("%d characters across %s, the docs and queries the scene shows", n, plural(len(view.Embeddings), "input")) + } if opts.Text == WithoutText { return "nothing · the prompts and the generated text are removed, the timings stay" } diff --git a/internal/publish/public.go b/internal/publish/public.go index 179b153..4d9e917 100644 --- a/internal/publish/public.go +++ b/internal/publish/public.go @@ -92,6 +92,10 @@ func PublicView(t *tape.Tape, policy TextPolicy) *tape.Tape { // The summary lists the same pictures for the card: the same rule. s.Images = anonymousImages(s.Images) out.Decisions = withoutDecisionText(out.Decisions) + // An embedding run's inputs are text the same way (2026-10-15). + for i := range out.Embeddings { + out.Embeddings[i].Input = "" + } } // A published tape tells the instant, never the time zone (§9.3, // TTP-120): every time.Time becomes UTC, and the run id's leading local diff --git a/internal/recorder/embedding.go b/internal/recorder/embedding.go new file mode 100644 index 0000000..70ee528 --- /dev/null +++ b/internal/recorder/embedding.go @@ -0,0 +1,460 @@ +package recorder + +import ( + "bytes" + "context" + _ "embed" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/http" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/midagedev/toktape/internal/gpu" + "github.com/midagedev/toktape/internal/procmon" + "github.com/midagedev/toktape/internal/server" + "github.com/midagedev/toktape/internal/tape" +) + +// An embedding-model run (2026-10-15): every input of the set is sent on its +// own, one after another, to /v1/embeddings (OpenAI shape) or /embedding +// (llama-server's shape), timed send to the last byte, and the returned +// vector is kept on the tape. One pass, no concurrency: the tape is for +// checking what the vectors are, and the latency beside them is a note. + +//go:embed embedding_inputs.jsonl +var defaultEmbeddingInputs []byte + +// EmbeddingOptions configures one embedding run. +type EmbeddingOptions struct { + BaseURL string + Candidates []string + // API is tape.EmbeddingAPIOpenAI (the default) or tape.EmbeddingAPILlama. + API string + // Inputs is a JSONL set ({"id":..., "input":...} per line), InputsName + // its file name; nil is the built-in set. + Inputs []byte + InputsName string + // Model, Dimensions and EncodingFormat are sent as given on the OpenAI + // shape ("" / 0 = not sent; EncodingFormat "" is "float"). + Model string + Dimensions int + EncodingFormat string + // VectorCap keeps at most this many values per vector; 0 keeps all. + VectorCap int + // Prompts are task prefixes by name ("query", "document"); nil sends + // every input bare. With them, a doc row goes once with the "document" + // prefix and once bare, and a query row likewise with "query", so a + // replay can rank both ways. + Prompts map[string]string + // Pacing, so a replay at 1:1 can be watched; all of it is idle time on + // the timestamps and in no latency. Gap follows each doc (or unlabelled + // input), TypeRate per rune precedes each query (the replay types it), + // Hold follows each query's answers, and Tail ends the run. + Gap, TypeRate, Hold, Tail time.Duration + + Tag, Note string + Version string + Clock Clock + FSRoot string + GPU gpu.Collector + HostLabel HostLabel + HTTP *http.Client + Progress func(tape.EmbeddingRecord) +} + +type embeddingInput struct{ ID, Input, Role, Topic, Expect string } + +func loadEmbeddingInputs(raw []byte) ([]embeddingInput, error) { + var out []embeddingInput + seen := map[string]bool{} + for n, line := range bytes.Split(raw, []byte("\n")) { + line = bytes.TrimSpace(line) + if len(line) == 0 { + continue + } + var row struct { + ID string `json:"id"` + Input string `json:"input"` + Role string `json:"role"` + Topic string `json:"topic"` + Expect string `json:"expect"` + } + if err := json.Unmarshal(line, &row); err != nil { + return nil, fmt.Errorf("inputs line %d: %w", n+1, err) + } + if row.Input == "" { + return nil, fmt.Errorf(`inputs line %d: no "input"`, n+1) + } + if row.ID == "" { + row.ID = fmt.Sprintf("input-%d", len(out)+1) + } + if row.Role != "" && row.Role != tape.EmbeddingRoleDoc && row.Role != tape.EmbeddingRoleQuery { + return nil, fmt.Errorf(`inputs line %d: role %q: use "doc" or "query"`, n+1, row.Role) + } + if seen[row.ID] { + return nil, fmt.Errorf("inputs line %d: id %q is used twice", n+1, row.ID) + } + seen[row.ID] = true + out = append(out, embeddingInput{row.ID, row.Input, row.Role, row.Topic, row.Expect}) + } + if len(out) == 0 { + return nil, errors.New("the input set is empty") + } + return out, nil +} + +// RecordEmbedding runs one embedding pass and returns its tape. +func RecordEmbedding(ctx context.Context, opts EmbeddingOptions) (*tape.Tape, error) { + if opts.API == "" { + opts.API = tape.EmbeddingAPIOpenAI + } + if opts.API != tape.EmbeddingAPIOpenAI && opts.API != tape.EmbeddingAPILlama { + return nil, fmt.Errorf("api %q: use openai or llama", opts.API) + } + if opts.EncodingFormat == "" { + opts.EncodingFormat = "float" + } + if opts.EncodingFormat != "float" && opts.EncodingFormat != "base64" { + return nil, fmt.Errorf("encoding format %q: use float or base64", opts.EncodingFormat) + } + if opts.Clock == nil { + opts.Clock = systemClock{} + } + if opts.HTTP == nil { + opts.HTTP = &http.Client{Timeout: 5 * time.Minute} + } + raw := opts.Inputs + if raw == nil { + raw = defaultEmbeddingInputs + } + inputs, err := loadEmbeddingInputs(raw) + if err != nil { + return nil, err + } + + client := server.New(opts.BaseURL) + base := client.BaseURL() + if base == "" { + if base, err = client.Discover(ctx, opts.Candidates); err != nil { + return nil, fmt.Errorf("%w: %w", ErrUnreachable, err) + } + } + id, err := client.WithBaseURL(base).ProbeEngine(ctx) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrUnreachable, err) + } + // One GPU collector for the inventory and the samples both; the decision + // host opens its own only when handed none, which left the samples bare. + var gpuWarns []string + if opts.GPU == nil { + c, ws := gpu.Open(ctx) + defer c.Close() + opts.GPU, gpuWarns = c, ws + } + host, warnings := collectDecisionHost(ctx, DecisionOptions{FSRoot: opts.FSRoot, GPU: opts.GPU, HostLabel: opts.HostLabel}, base) + warnings = append(gpuWarns, warnings...) + + endpoint := "/v1/embeddings" + if opts.API == tape.EmbeddingAPILlama { + endpoint = "/embedding" + } + startedAt := opts.Clock.Now() + t0 := time.Now() + // The host is read the way a record reads it (the same sampler), so the + // screen's resource pane has a series: the server's process when it is + // on this machine, the GPUs and the load average. + st := newState(1, nil, nil) + stopSampling := make(chan struct{}) + sampled := make(chan struct{}) + fsRoot := opts.FSRoot + if fsRoot == "" { + fsRoot = "/" // procmon joins it with "proc/..." + } + dep := sampleDeps{interval: 500 * time.Millisecond, runStart: t0, fsRoot: fsRoot, gpus: opts.GPU} + if port, err := loopbackPortOf(base); err == nil { + if pid, err := procmon.FindPIDByPort(fsRoot, port); err == nil { + dep.pid = pid + } + } + go func() { st.sampleLoop(ctx, stopSampling, dep); close(sampled) }() + var recs []tape.EmbeddingRecord + send := func(in embeddingInput, prompt string, shown time.Duration) { + rec := sendEmbedding(ctx, opts, base+endpoint, t0, in, prompt) + rec.Index, rec.ShownAt = len(recs), shown + rec.Role, rec.Topic, rec.Expect = in.Role, in.Topic, in.Expect + recs = append(recs, rec) + if opts.Progress != nil { + opts.Progress(rec) + } + } + idle := func(d time.Duration) bool { return d <= 0 || sleepCtx(ctx, d) } + for _, in := range inputs { + if ctx.Err() != nil { + break + } + name := map[string]string{tape.EmbeddingRoleDoc: "document", tape.EmbeddingRoleQuery: "query"}[in.Role] + _, prefixed := opts.Prompts[name] + var shown time.Duration + if in.Role == tape.EmbeddingRoleQuery { + shown = time.Since(t0) + if !idle(opts.TypeRate * time.Duration(len([]rune(in.Input)))) { + break + } + } + if prefixed { + send(in, name, shown) + } + send(in, "", shown) + pause := opts.Gap + if in.Role == tape.EmbeddingRoleQuery { + pause = opts.Hold + } + if !idle(pause) { + break + } + } + outroAt := time.Since(t0) - opts.Hold + if len(inputs) > 0 && inputs[len(inputs)-1].Role != tape.EmbeddingRoleQuery { + outroAt = time.Since(t0) + } + idle(opts.Tail) + sceneEnd := time.Since(t0) + close(stopSampling) + <-sampled + _, samples, _ := st.snapshot() + finishedAt := opts.Clock.Now() + + sum := &tape.EmbeddingSummary{ + API: opts.API, Endpoint: endpoint, Model: opts.Model, Dimensions: opts.Dimensions, + EncodingFormat: opts.EncodingFormat, Inputs: len(recs), VectorCap: opts.VectorCap, + Prompts: opts.Prompts, SceneEnd: sceneEnd, OutroAt: max(0, outroAt), + } + if opts.API == tape.EmbeddingAPILlama { + sum.EncodingFormat = "" // the llama shape has no such field + } + for _, in := range inputs { + switch in.Role { + case tape.EmbeddingRoleDoc: + sum.Docs++ + case tape.EmbeddingRoleQuery: + sum.Queries++ + } + } + for _, r := range recs { + if r.Role != tape.EmbeddingRoleQuery || r.Expect == "" || r.Error != "" { + continue + } + docPrompt := "" + if r.Prompt != "" { + docPrompt = "document" + } + hit := tape.TopHit(recs, r.Values(), docPrompt) == r.Expect + switch { + case r.Prompt == "" && len(opts.Prompts) > 0: + if hit { + sum.HitsBare++ + } + default: + sum.Graded++ + if hit { + sum.Hits++ + } + } + } + if opts.Inputs != nil { + sum.InputSet = filepath.Base(opts.InputsName) + } + var lat []float64 + firstErr := "" + for _, r := range recs { + if r.Error != "" { + sum.Errors++ + if firstErr == "" { + firstErr = r.Error + } + continue + } + if sum.Dim == 0 { + sum.Dim = r.Dim + } else if r.Dim != sum.Dim { + warnings = append(warnings, fmt.Sprintf("%s answered %d values, the first answer had %d", r.ID, r.Dim, sum.Dim)) + } + sum.PromptTokens += r.PromptTokens + lat = append(lat, float64(r.Latency())/float64(time.Millisecond)) + } + if len(lat) == 0 { + if firstErr == "" { + return nil, ctx.Err() + } + return nil, fmt.Errorf("%w: %s", ErrAllStreamsFailed, firstErr) + } + sort.Float64s(lat) + sum.LatencyP50Ms, sum.LatencyMaxMs = lat[len(lat)/2], lat[len(lat)-1] + if sum.Errors > 0 { + warnings = append(warnings, fmt.Sprintf("%d of %d inputs failed; first: %s", sum.Errors, len(recs), firstErr)) + } + + slug := id.ModelFile + if slug == "" { + slug = opts.Model + } + if slug == "" { + slug = "embedding" + } + summary := tape.RunSummary{ + ID: runID(startedAt, slug), + ToktapeVersion: opts.Version, + StartedAt: startedAt, + FinishedAt: finishedAt, + Server: tape.ServerInfo{Kind: id.Kind, URL: base, Host: host.Hostname, Build: id.Build, Commit: id.Commit}, + Model: tape.ModelInfo{Path: id.ModelPath, FileName: id.ModelFile, Quant: id.Quant}, + Host: host, + Concurrency: 1, + Warnings: warnings, + Mode: tape.ModeEmbedding, + Embedding: sum, + Tag: opts.Tag, + Note: opts.Note, + } + if dep.pid > 0 { + summary.Server.PID = dep.pid + } + // The run's last reading is its at-end view, as for a token run, so the + // card and the index have the server's memory and the GPUs. + for _, smp := range samples { + summary.Memory.PeakRSSBytes = max(summary.Memory.PeakRSSBytes, smp.Mem.RSSBytes) + } + if n := len(samples); n > 0 { + summary.Memory.AtEnd = samples[n-1].Mem + summary.GPUsAtEnd = samples[n-1].GPUs + } + return &tape.Tape{Schema: tape.SchemaVersion, Summary: summary, Embeddings: recs, Samples: samples}, nil +} + +func sendEmbedding(ctx context.Context, opts EmbeddingOptions, url string, t0 time.Time, in embeddingInput, prompt string) tape.EmbeddingRecord { + rec := tape.EmbeddingRecord{ID: in.ID, Input: in.Input, Prompt: prompt} + text := opts.Prompts[prompt] + in.Input + var body map[string]any + if opts.API == tape.EmbeddingAPILlama { + body = map[string]any{"content": text} + } else { + body = map[string]any{"input": text, "encoding_format": opts.EncodingFormat} + if opts.Model != "" { + body["model"] = opts.Model + } + if opts.Dimensions > 0 { + body["dimensions"] = opts.Dimensions + } + } + buf, _ := json.Marshal(body) + rec.SentAt = time.Since(t0) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(buf)) + if err != nil { + rec.Error = err.Error() + return rec + } + req.Header.Set("Content-Type", "application/json") + resp, err := opts.HTTP.Do(req) + if err != nil { + rec.Error = err.Error() + return rec + } + raw, err := io.ReadAll(resp.Body) + resp.Body.Close() + answered := time.Since(t0) + if err != nil { + rec.Error = err.Error() + return rec + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + rec.Error = fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(raw[:min(len(raw), 200)]))) + return rec + } + vec, tokens, err := parseEmbedding(raw) + if err != nil { + rec.Error = err.Error() + return rec + } + rec.AnsweredAt = answered + rec.PromptTokens = tokens + rec.Dim = len(vec) + rec.Norm = tape.L2Norm(vec) + rec.Head = append([]float32(nil), vec[:min(len(vec), tape.EmbeddingHead)]...) + keep := vec + if opts.VectorCap > 0 && len(keep) > opts.VectorCap { + keep = keep[:opts.VectorCap] + } + rec.Vector, rec.Stored = tape.EncodeVector(keep), len(keep) + return rec +} + +// parseEmbedding reads the first vector out of either response shape: +// OpenAI {"data":[{"embedding":[...] or "base64"}],"usage":{...}}, or +// llama-server's /embedding [{"index":0,"embedding":[[...]]}] (a pooled +// vector is one row) and its older {"embedding":[...]}. +func parseEmbedding(raw []byte) ([]float32, int, error) { + var oa struct { + Data []struct { + Embedding json.RawMessage `json:"embedding"` + } `json:"data"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + } `json:"usage"` + } + if json.Unmarshal(raw, &oa) == nil && len(oa.Data) > 0 { + v, err := vectorOf(oa.Data[0].Embedding) + return v, oa.Usage.PromptTokens, err + } + var arr []struct { + Embedding json.RawMessage `json:"embedding"` + } + if json.Unmarshal(raw, &arr) == nil && len(arr) > 0 { + v, err := vectorOf(arr[0].Embedding) + return v, 0, err + } + var one struct { + Embedding json.RawMessage `json:"embedding"` + } + if json.Unmarshal(raw, &one) == nil && len(one.Embedding) > 0 { + v, err := vectorOf(one.Embedding) + return v, 0, err + } + return nil, 0, errors.New("the response holds no embedding") +} + +// vectorOf reads a vector written as floats, as base64 float32, or as a +// one-row matrix of floats. +func vectorOf(m json.RawMessage) ([]float32, error) { + var s string + if json.Unmarshal(m, &s) == nil { + return tape.DecodeVector(s) + } + var f []float64 + if json.Unmarshal(m, &f) != nil { + var rows [][]float64 + if err := json.Unmarshal(m, &rows); err != nil || len(rows) == 0 { + return nil, errors.New("embedding is neither a vector nor base64") + } + if len(rows) > 1 { + return nil, fmt.Errorf("embedding has %d rows: the server is not pooling (start it with --pooling)", len(rows)) + } + f = rows[0] + } + if len(f) == 0 { + return nil, errors.New("embedding is empty") + } + v := make([]float32, len(f)) + for i, x := range f { + if math.IsNaN(x) || math.IsInf(x, 0) { + return nil, fmt.Errorf("embedding value %d is %v", i, x) + } + v[i] = float32(x) + } + return v, nil +} diff --git a/internal/recorder/embedding_inputs.jsonl b/internal/recorder/embedding_inputs.jsonl new file mode 100644 index 0000000..e67a7a0 --- /dev/null +++ b/internal/recorder/embedding_inputs.jsonl @@ -0,0 +1,26 @@ +{"id":"cook-1","role":"doc","topic":"cooking","input":"Before a trip, feed the sourdough starter, let it rise for an hour, then refrigerate it; it keeps for about two weeks."} +{"id":"cook-2","role":"doc","topic":"cooking","input":"김치찌개는 잘 익은 김치를 돼지고기와 먼저 볶은 뒤 물을 부어 끓이면 국물 맛이 깊어진다."} +{"id":"cook-3","role":"doc","topic":"cooking","input":"Salt pasta water generously and keep a cup of it to loosen the sauce at the end."} +{"id":"cook-4","role":"doc","topic":"cooking","input":"달걀을 실온에 30분 두었다가 삶으면 껍질이 잘 벗겨지고 노른자가 고르게 익는다."} +{"id":"trav-1","role":"doc","topic":"travel","input":"The JR Pass covers most Shinkansen lines but not the fastest Nozomi trains without a surcharge."} +{"id":"trav-2","role":"doc","topic":"travel","input":"제주도 올레길은 26개 코스로 나뉘며, 7코스는 해안 절벽 풍경으로 가장 인기가 많다."} +{"id":"trav-3","role":"doc","topic":"travel","input":"Carry-on liquids must be in containers of 100 ml or less, all in one clear resealable bag."} +{"id":"trav-4","role":"doc","topic":"travel","input":"해외에서 카드를 쓸 때는 현지 통화로 결제해야 이중 환전 수수료를 피할 수 있다."} +{"id":"code-1","role":"doc","topic":"code","input":"with open(path, encoding=\"utf-8\") as f:\n for line in f:\n print(line.rstrip())"} +{"id":"code-2","role":"doc","topic":"code","input":"git rebase -i HEAD~3 lets you squash the last three commits into one before pushing."} +{"id":"code-3","role":"doc","topic":"code","input":"Rust의 소유권 규칙: 값에는 소유자가 하나뿐이고, 소유자가 스코프를 벗어나면 값은 해제된다."} +{"id":"code-4","role":"doc","topic":"code","input":"SELECT user_id, COUNT(*) FROM orders GROUP BY user_id HAVING COUNT(*) > 5;"} +{"id":"heal-1","role":"doc","topic":"health","input":"Adults should aim for at least 150 minutes of moderate aerobic activity every week."} +{"id":"heal-2","role":"doc","topic":"health","input":"수면 전 한 시간은 스마트폰 화면을 멀리하면 잠드는 데 걸리는 시간이 줄어든다."} +{"id":"heal-3","role":"doc","topic":"health","input":"Drink water steadily through the day; thirst lags behind mild dehydration."} +{"id":"heal-4","role":"doc","topic":"health","input":"허리 통증이 있을 때는 오래 누워 있기보다 가벼운 걷기가 회복에 더 도움이 된다."} +{"id":"fin-1","role":"doc","topic":"finance","input":"An index fund tracks a market index, so its fees are usually far lower than active funds."} +{"id":"fin-2","role":"doc","topic":"finance","input":"연말정산에서 신용카드보다 체크카드 사용액의 소득공제율이 더 높다."} +{"id":"fin-3","role":"doc","topic":"finance","input":"Compound interest means you earn returns on earlier returns, so starting early matters most."} +{"id":"fin-4","role":"doc","topic":"finance","input":"비상금은 보통 생활비 3~6개월치를 입출금이 자유로운 통장에 두는 것이 좋다."} +{"id":"q-starter","role":"query","topic":"cooking","expect":"cook-1","input":"휴가 동안 사워도우 발효종 보관법"} +{"id":"q-file","role":"query","topic":"code","expect":"code-1","input":"python read a file line by line"} +{"id":"q-fx","role":"query","topic":"travel","expect":"trav-4","input":"how to avoid double currency conversion fees abroad"} +{"id":"q-exercise","role":"query","topic":"health","expect":"heal-1","input":"일주일에 운동을 얼마나 해야 하나요?"} +{"id":"q-owner","role":"query","topic":"code","expect":"code-3","input":"what happens to a value when its owner goes out of scope"} +{"id":"q-index","role":"query","topic":"finance","expect":"fin-1","input":"인덱스 펀드 수수료가 낮은 이유"} diff --git a/internal/recorder/embedding_test.go b/internal/recorder/embedding_test.go new file mode 100644 index 0000000..a74ef3d --- /dev/null +++ b/internal/recorder/embedding_test.go @@ -0,0 +1,113 @@ +package recorder + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/midagedev/toktape/internal/tape" +) + +// embedServer answers both shapes with a vector derived from the input's +// length, so two inputs get different vectors and the test can predict them. +func embedServer(t *testing.T, rows int) *httptest.Server { + t.Helper() + vec := func(s string) []float64 { return []float64{float64(len(s)), 1, 2, 3, 4, 5, 6, 7, 8, 9} } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req map[string]any + _ = json.NewDecoder(r.Body).Decode(&req) + switch r.URL.Path { + case "/v1/embeddings": + in, _ := req["input"].(string) + var emb any = vec(in) + if req["encoding_format"] == "base64" { + f := vec(in) + v := make([]float32, len(f)) + for i, x := range f { + v[i] = float32(x) + } + emb = tape.EncodeVector(v) + } + json.NewEncoder(w).Encode(map[string]any{"data": []any{map[string]any{"embedding": emb}}, + "usage": map[string]int{"prompt_tokens": 3}}) + case "/embedding": + in, _ := req["content"].(string) + m := [][]float64{} + for i := 0; i < rows; i++ { + m = append(m, vec(in)) + } + json.NewEncoder(w).Encode([]any{map[string]any{"index": 0, "embedding": m}}) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv +} + +const twoInputs = `{"id":"a","input":"x"} +{"id":"b","input":"hello"} +` + +func TestRecordEmbeddingShapes(t *testing.T) { + srv := embedServer(t, 1) + for _, tc := range []struct{ api, format string }{ + {tape.EmbeddingAPIOpenAI, "float"}, + {tape.EmbeddingAPIOpenAI, "base64"}, + {tape.EmbeddingAPILlama, ""}, + } { + tp, err := RecordEmbedding(context.Background(), EmbeddingOptions{ + BaseURL: srv.URL, API: tc.api, EncodingFormat: tc.format, + Inputs: []byte(twoInputs), InputsName: "/x/set.jsonl", VectorCap: 4, + }) + if err != nil { + t.Fatalf("%s/%s: %v", tc.api, tc.format, err) + } + e := tp.Summary.Embedding + if !tp.Summary.IsEmbedding() || e.Inputs != 2 || e.Dim != 10 || e.InputSet != "set.jsonl" || len(tp.Requests) != 0 { + t.Fatalf("%s: summary %+v", tc.api, e) + } + r := tp.Embeddings[1] + // dim, norm and head come from the whole vector, before the cap. + if r.ID != "b" || r.Dim != 10 || r.Stored != 4 || len(r.Head) != tape.EmbeddingHead || r.Head[0] != 5 { + t.Errorf("%s: record %+v", tc.api, r) + } + if v := r.Values(); len(v) != 4 || v[0] != 5 || v[3] != 3 { + t.Errorf("%s: stored %v", tc.api, v) + } + if r.Norm < 17.60 || r.Norm > 17.61 { // sqrt(25+285) + t.Errorf("%s: norm %v", tc.api, r.Norm) + } + if tc.api == tape.EmbeddingAPIOpenAI && (r.PromptTokens != 3 || e.Endpoint != "/v1/embeddings") { + t.Errorf("openai: tokens %d endpoint %s", r.PromptTokens, e.Endpoint) + } + } +} + +func TestRecordEmbeddingUnpooledFails(t *testing.T) { + srv := embedServer(t, 2) + _, err := RecordEmbedding(context.Background(), EmbeddingOptions{BaseURL: srv.URL, API: tape.EmbeddingAPILlama, Inputs: []byte(twoInputs)}) + if !errors.Is(err, ErrAllStreamsFailed) || !strings.Contains(err.Error(), "--pooling") { + t.Fatalf("err = %v", err) + } +} + +func TestLoadEmbeddingInputs(t *testing.T) { + if in, err := loadEmbeddingInputs(defaultEmbeddingInputs); err != nil || len(in) != 26 { + t.Fatalf("built-in set: %d %v", len(in), err) + } + for name, raw := range map[string]string{ + "empty": "\n", + "no input": `{"id":"a"}`, + "dup id": `{"id":"a","input":"x"}` + "\n" + `{"id":"a","input":"y"}`, + "not json": `x`, + } { + if _, err := loadEmbeddingInputs([]byte(raw)); err == nil { + t.Errorf("%s: no error", name) + } + } +} diff --git a/internal/render/render.go b/internal/render/render.go index f114564..d53108f 100644 --- a/internal/render/render.go +++ b/internal/render/render.go @@ -257,6 +257,10 @@ func RunEnd(tp *tape.Tape) time.Duration { } return end } + if tp.Summary.IsEmbedding() { + // The search scene's own end, holds included (2026-10-15). + return tui.EmbeddingEnd(tp) + } for _, req := range tp.Requests { if n := len(req.Tokens); n > 0 { if e := req.StartedAt + req.Tokens[n-1].T; e > end { @@ -287,6 +291,10 @@ func FirstToken(tp *tape.Tape) time.Duration { } return first } + if tp.Summary.IsEmbedding() { + // Nothing to skip: the scene starts typing at once. + return 0 + } for _, req := range tp.Requests { if len(req.Tokens) == 0 { continue diff --git a/internal/tape/embedding.go b/internal/tape/embedding.go new file mode 100644 index 0000000..54ea377 --- /dev/null +++ b/internal/tape/embedding.go @@ -0,0 +1,226 @@ +package tape + +import ( + "encoding/base64" + "encoding/binary" + "errors" + "math" + "time" +) + +// An embedding tape (2026-10-15) records an embedding model: each input is +// sent on its own to an embeddings endpoint and answered with one vector. +// Nothing is generated, so — like a decision tape — Requests stays empty and +// every token figure is absent; Tape.Embeddings holds the run. Every reader +// that reads tokens branches on IsEmbedding first. + +// ModeEmbedding is RunSummary.Mode for a `toktape embed` run. +const ModeEmbedding = "embedding" + +// IsEmbedding reports whether the run is an embedding-model run. +func (s *RunSummary) IsEmbedding() bool { + return s != nil && s.Mode == ModeEmbedding +} + +// Values of EmbeddingSummary.API: which request shape was sent. +const ( + EmbeddingAPIOpenAI = "openai" // POST /v1/embeddings + EmbeddingAPILlama = "llama" // POST /embedding (llama-server's own) +) + +// EmbeddingHead is how many leading values every record keeps in the clear, +// whatever the vector cap: enough to eyeball two engines against each other. +const EmbeddingHead = 8 + +// EmbeddingRecord is one input: what was sent, what came back, and when. +type EmbeddingRecord struct { + Index int `json:"index"` // send order, from 0 + // ID is the input set row's id ("en-q1"); toktape's own, never sent. + ID string `json:"id"` + Input string `json:"input"` + // Role, Topic and Expect are the input set's own labels, never sent: + // Role is "doc" or "query" ("" = unlabelled), Topic a group name, and + // Expect the id of the doc a query should find. + Role string `json:"role,omitempty"` + Topic string `json:"topic,omitempty"` + Expect string `json:"expect,omitempty"` + // Prompt names the task prefix sent before Input ("query", "document"; + // EmbeddingSummary.Prompts has the strings); "" = Input was sent bare. + Prompt string `json:"prompt,omitempty"` + // ShownAt is when a replay starts showing a query being typed: the + // recorder idled from ShownAt to SentAt. It is pacing, in no latency. + ShownAt time.Duration `json:"shown_at,omitempty"` + // SentAt and AnsweredAt are since run start; AnsweredAt is 0 on a + // failed request. + SentAt time.Duration `json:"sent_at"` + AnsweredAt time.Duration `json:"answered_at,omitempty"` + // PromptTokens is the server's usage figure; 0 = not reported. + PromptTokens int `json:"prompt_tokens,omitempty"` + // Dim and Norm are of the full vector as returned, read before any cap. + Dim int `json:"dim,omitempty"` + Norm float64 `json:"norm,omitempty"` + Head []float32 `json:"head,omitempty"` + // Vector is the first Stored values, little-endian float32, base64 + // (std encoding). Stored == Dim unless the run capped it. + Vector string `json:"vector,omitempty"` + Stored int `json:"stored,omitempty"` + Error string `json:"error,omitempty"` +} + +// Latency is send to the last byte of the answer; 0 for a failed request. +func (r EmbeddingRecord) Latency() time.Duration { + if r.AnsweredAt <= 0 { + return 0 + } + return r.AnsweredAt - r.SentAt +} + +// EmbeddingSummary is an embedding run's summary. +type EmbeddingSummary struct { + API string `json:"api"` // EmbeddingAPI* + Endpoint string `json:"endpoint"` // the path requests went to + // Model is the "model" sent (and echoed back when the server did); "" = + // none sent. + Model string `json:"model,omitempty"` + // Dimensions is the "dimensions" asked for (MRL truncation); 0 = none. + Dimensions int `json:"dimensions,omitempty"` + // EncodingFormat is what was asked for: "float" or "base64". + EncodingFormat string `json:"encoding_format,omitempty"` + InputSet string `json:"input_set,omitempty"` // file name; "" = built-in + Inputs int `json:"inputs"` + // Docs and Queries count the set's labelled rows (each once, however + // many prompts it went with). + Docs int `json:"docs,omitempty"` + Queries int `json:"queries,omitempty"` + // Hits and HitsBare count the queries with an "expect" whose expected + // doc ranked first among the docs — with the task prompts on both + // sides, and bare on both sides (HitsBare is 0 without prompts). Graded + // is how many queries had an "expect". + Hits int `json:"hits,omitempty"` + HitsBare int `json:"hits_bare,omitempty"` + Graded int `json:"graded,omitempty"` + Errors int `json:"errors,omitempty"` + Dim int `json:"dim,omitempty"` // of the first answer + // VectorCap is the per-record cap on stored values; 0 = whole vectors. + VectorCap int `json:"vector_cap,omitempty"` + PromptTokens int `json:"prompt_tokens,omitempty"` // sum of the reported + LatencyP50Ms float64 `json:"latency_p50_ms,omitempty"` + LatencyMaxMs float64 `json:"latency_max_ms,omitempty"` + // Prompts are the task prefixes by name, as sent (a role's input went + // once with its prefix and once bare); nil = none. + Prompts map[string]string `json:"prompts,omitempty"` + // SceneEnd is when the recorder's last idle hold ended: a replay runs + // until then so the last screen can be read. Pacing, in no latency. + SceneEnd time.Duration `json:"scene_end,omitempty"` + // OutroAt is when the final hold began (the last query's answers had + // been read); a replay shows the closing view from there. + OutroAt time.Duration `json:"outro_at,omitempty"` +} + +// Role values of EmbeddingRecord.Role. +const ( + EmbeddingRoleDoc = "doc" + EmbeddingRoleQuery = "query" +) + +// EmbeddingGemmaPrompts are EmbeddingGemma 2's retrieval prefixes, verbatim +// from the "prompts" of google/embeddinggemma-2's +// config_sentence_transformers.json ("query" and "document"), read +// 2026-10-15. +var EmbeddingGemmaPrompts = map[string]string{ + "query": "task: search result | query: ", + "document": "title: none | text: ", +} + +// EncodeVector is the stored form of v: little-endian float32, base64. +func EncodeVector(v []float32) string { + b := make([]byte, 4*len(v)) + for i, x := range v { + binary.LittleEndian.PutUint32(b[4*i:], math.Float32bits(x)) + } + return base64.StdEncoding.EncodeToString(b) +} + +// DecodeVector reads EncodeVector's form (also what an OpenAI server sends +// for encoding_format "base64"). +func DecodeVector(s string) ([]float32, error) { + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return nil, err + } + if len(b)%4 != 0 { + return nil, errors.New("vector bytes are not a whole number of float32") + } + v := make([]float32, len(b)/4) + for i := range v { + v[i] = math.Float32frombits(binary.LittleEndian.Uint32(b[4*i:])) + } + return v, nil +} + +// Values returns the stored vector, nil when none was stored. +func (r EmbeddingRecord) Values() []float32 { + if r.Vector == "" { + return nil + } + v, err := DecodeVector(r.Vector) + if err != nil { + return nil + } + return v +} + +// L2Norm is the Euclidean norm of v. +func L2Norm(v []float32) float64 { + var s float64 + for _, x := range v { + s += float64(x) * float64(x) + } + return math.Sqrt(s) +} + +// Cosine is the cosine similarity of a and b over their common prefix; NaN +// when either is empty or zero. +func Cosine(a, b []float32) float64 { + n := min(len(a), len(b)) + var dot, na, nb float64 + for i := 0; i < n; i++ { + x, y := float64(a[i]), float64(b[i]) + dot += x * y + na += x * x + nb += y * y + } + if na == 0 || nb == 0 { + return math.NaN() + } + return dot / math.Sqrt(na*nb) +} + +// MaxAbsDiff is the largest |a[i]-b[i]| over the common prefix. +func MaxAbsDiff(a, b []float32) float64 { + n := min(len(a), len(b)) + var m float64 + for i := 0; i < n; i++ { + m = math.Max(m, math.Abs(float64(a[i])-float64(b[i]))) + } + return m +} + +// Key is a record's identity across two tapes of the same set: the input's +// id and the prompt it went with. +func (r EmbeddingRecord) Key() string { return r.ID + "|" + r.Prompt } + +// TopHit is the id of the doc in recs (role doc, sent with prompt) nearest +// to q by cosine; "" when there is none. +func TopHit(recs []EmbeddingRecord, q []float32, prompt string) string { + best, id := math.Inf(-1), "" + for _, r := range recs { + if r.Role != EmbeddingRoleDoc || r.Prompt != prompt || r.Error != "" { + continue + } + if c := Cosine(q, r.Values()); c > best { + best, id = c, r.ID + } + } + return id +} diff --git a/internal/tape/embedding_test.go b/internal/tape/embedding_test.go new file mode 100644 index 0000000..ca977d5 --- /dev/null +++ b/internal/tape/embedding_test.go @@ -0,0 +1,37 @@ +package tape + +import ( + "math" + "testing" +) + +func TestVectorRoundTripAndMetrics(t *testing.T) { + v := []float32{0.5, -1.25, 3, 0} + got, err := DecodeVector(EncodeVector(v)) + if err != nil || len(got) != len(v) { + t.Fatalf("decode: %v %v", got, err) + } + for i := range v { + if got[i] != v[i] { + t.Fatalf("value %d = %v, want %v", i, got[i], v[i]) + } + } + if _, err := DecodeVector("AAA="); err == nil { + t.Error("two bytes decoded as a vector") + } + if c := Cosine(v, v); math.Abs(c-1) > 1e-12 { + t.Errorf("self cosine %v", c) + } + if c := Cosine([]float32{1, 0}, []float32{0, 1}); c != 0 { + t.Errorf("orthogonal cosine %v", c) + } + if c := Cosine([]float32{0, 0}, []float32{1, 1}); !math.IsNaN(c) { + t.Errorf("zero vector cosine %v, want NaN", c) + } + if d := MaxAbsDiff([]float32{1, 2, 3}, []float32{1, 2.5, 2}); d != 1 { + t.Errorf("max diff %v", d) + } + if n := L2Norm([]float32{3, 4}); n != 5 { + t.Errorf("norm %v", n) + } +} diff --git a/internal/tape/tape.go b/internal/tape/tape.go index f4bcbea..63f7f62 100644 --- a/internal/tape/tape.go +++ b/internal/tape/tape.go @@ -95,6 +95,9 @@ type Tape struct { // Decisions are the requests of a decision-model run (ModeDecision, // decision.go), in send order. Requests is empty on such a tape. Decisions []DecisionRecord `json:"decisions,omitempty"` + // Embeddings are the requests of an embedding-model run (ModeEmbedding, + // embedding.go), in send order. Requests is empty on such a tape. + Embeddings []EmbeddingRecord `json:"embeddings,omitempty"` } // RequestRecord is one stream: its prompt, every token, and its own timings. @@ -151,6 +154,9 @@ type RunSummary struct { // nil on every other tape. On a decision tape the token figures // (Timings, Aggregate) are empty: nothing was generated. Decision *DecisionSummary `json:"decision,omitempty"` + // Embedding is an embedding-model run's summary (ModeEmbedding), nil on + // every other tape. + Embedding *EmbeddingSummary `json:"embedding,omitempty"` // SpecNMax are the speculative.n_max values of a `record --spec-n-max` // sweep, in the order they ran (TTP-35, 2026-09-13). Each value runs the // whole prompt set, so Rounds = len(SpecNMax) x prompts, and BySpecNMax diff --git a/internal/tape/testdata/embedding-llama.tape b/internal/tape/testdata/embedding-llama.tape new file mode 100644 index 0000000..22e55da Binary files /dev/null and b/internal/tape/testdata/embedding-llama.tape differ diff --git a/internal/tui/embedding.go b/internal/tui/embedding.go new file mode 100644 index 0000000..712d2f1 --- /dev/null +++ b/internal/tui/embedding.go @@ -0,0 +1,495 @@ +package tui + +// The embedding screen (2026-10-15): what View draws for a tape whose +// Summary.IsEmbedding() is true. It follows the decision screen's frame — +// the same border, the same feed of blocks arriving one at a time, the same +// request-latency footer — with what an embedding answers in place of the +// question tiles: each input's vector as a strip of cells (the 768 values +// averaged down to the strip's width, height and colour by sign and size), +// its norm, and below the feed the cosine similarity of every answered pair. +// +// With a reference tape (Model.EmbedRef, `toktape play a --against b`) each +// block also carries a bar of its cosine to the same input in the reference, +// scaled over [EmbedBarFloor, 1] with the band marked: at or above EmbedBand +// the two engines agree to run-to-run noise. +// +// Everything is a pure function of (m, t). + +import ( + "fmt" + "io" + "math" + "strings" + "sync" + "time" + + "github.com/charmbracelet/lipgloss" + "github.com/muesli/termenv" + + "github.com/midagedev/toktape/internal/tape" +) + +const ( + // EmbedBand is the cosine at or above which two vectors of one input + // count as the same answer: two runs of one llama-server endpoint on the + // demo tape agree to >= 0.9995, so 0.999 leaves room for that noise. + EmbedBand = 0.999 + // EmbedBarFloor is where the compare bar starts: below it the bar is + // empty, and nothing that low is "close" anyway. + EmbedBarFloor = 0.998 + embedBlockH = 3 // text row, strip row, separator +) + +// embeddingModel fills m for an embedding tape: the records sent by at. +func embeddingModel(m *Model, tp *tape.Tape, at time.Duration) { + for _, r := range tp.Embeddings { + // A query is on screen from when the replay starts typing it. + if shown := r.ShownAt; r.SentAt <= at || (shown > 0 && shown <= at) { + m.Embeddings = append(m.Embeddings, r) + } + } + if len(tp.Embeddings) > 0 { + m.RunStart, m.runStarted = tp.Embeddings[0].SentAt, true + } + for _, sm := range tp.Samples { + if sm.T > at { + break + } + m.Samples = append(m.Samples, sm) + } + m.seen = seenIn(tp.Samples) + m.RunEnd = EmbeddingEnd(tp) + m.Done = len(tp.Embeddings) > 0 && at >= m.RunEnd +} + +// EmbeddingEnd is when an embedding run's last answer (or failed send) was. +func EmbeddingEnd(tp *tape.Tape) time.Duration { + var end time.Duration + for _, r := range tp.Embeddings { + end = max(end, r.AnsweredAt, r.SentAt) + } + if e := tp.Summary.Embedding; e != nil { + end = max(end, e.SceneEnd) + } + return end +} + +func embedAnswered(r tape.EmbeddingRecord, t time.Duration) bool { + return r.Error == "" && r.AnsweredAt > 0 && r.AnsweredAt <= t +} + +// ---- colour --------------------------------------------------------------- + +// embedLevels is how many steps each side of zero a strip cell has; a +// cosine cell uses the positive side only. +const embedLevels = 4 + +var ( + embedOnce sync.Once + embedNeg [embedLevels]style + embedPos [embedLevels]style +) + +// embedStyle is the diverging scale: clay below zero, teal above, brighter +// with size. level is 1..embedLevels. +func embedStyle(th Theme, neg bool, level int) style { + if !th.colour || level < 1 { + return th.dim + } + embedOnce.Do(func() { + r := lipgloss.NewRenderer(io.Discard) + r.SetColorProfile(termenv.TrueColor) + r.SetHasDarkBackground(true) + negs := []string{"#5a3a34", "#8a4c3f", "#bf5f4a", "#f07a5c"} + poss := []string{"#2c4f4a", "#3c7a70", "#4fae9f", "#7fe3d0"} + for i := 0; i < embedLevels; i++ { + embedNeg[i] = seqOf(r.NewStyle().Foreground(lipgloss.Color(negs[i]))) + embedPos[i] = seqOf(r.NewStyle().Foreground(lipgloss.Color(poss[i]))) + } + }) + if neg { + return embedNeg[level-1] + } + return embedPos[level-1] +} + +// downsample averages v into w buckets. +func downsample(v []float32, w int) []float64 { + if w <= 0 || len(v) == 0 { + return nil + } + w = min(w, len(v)) + out := make([]float64, w) + for i := range out { + lo, hi := i*len(v)/w, (i+1)*len(v)/w + var s float64 + for _, x := range v[lo:hi] { + s += float64(x) + } + out[i] = s / float64(hi-lo) + } + return out +} + +// vectorStrip draws cells (already downsampled) scaled by scale: the glyph's +// height is the value's place between -scale and +scale, the colour its sign +// and size, so the plain rendering still shows the shape. +func vectorStrip(th Theme, cells []float64, scale float64) string { + const glyphs = "▁▂▃▄▅▆▇█" + g := []rune(glyphs) + var b strings.Builder + for _, x := range cells { + f := 0.5 + if scale > 0 { + f = (x/scale + 1) / 2 + } + f = math.Max(0, math.Min(1, f)) + gi := min(len(g)-1, int(f*float64(len(g)))) + mag := 0.0 + if scale > 0 { + mag = math.Abs(x) / scale + } + level := min(embedLevels, int(math.Ceil(mag*embedLevels))) + b.WriteString(th.paint(embedStyle(th, x < 0, level), string(g[gi]))) + } + return b.String() +} + +// ---- body ----------------------------------------------------------------- + +type embedView struct { + m Model + th Theme + t time.Duration + bw int + stripW int + scale float64 + vecs map[int][]float32 // by Index, answered only +} + +func newEmbedView(m Model, th Theme, t time.Duration, bw int) *embedView { + v := &embedView{m: m, th: th, t: t, bw: bw, vecs: map[int][]float32{}} + right := 14 // "norm 1.0000" and its gap + if m.EmbedRef != nil { + right += 32 + } + v.stripW = max(8, bw-4-right) + // One scale for every strip on screen, so strips compare by eye. + for _, r := range m.Embeddings { + if !embedAnswered(r, t) { + continue + } + vals := r.Values() + if vals == nil { + continue + } + v.vecs[r.Index] = vals + for _, x := range downsample(vals, v.stripW) { + v.scale = math.Max(v.scale, math.Abs(x)) + } + } + return v +} + +func (v *embedView) headline() string { + th, e := v.th, v.m.Summary.Embedding + if e == nil { + e = &tape.EmbeddingSummary{} + } + done := 0 + for _, r := range v.m.Embeddings { + if embedAnswered(r, v.t) { + done++ + } + } + l := newLine(th, v.bw) + l.add(th.accentBold, "EMBEDDING") + l.add(th.dim, " POST "+orUnknown(e.Endpoint)) + if e.Dimensions > 0 { + l.add(th.dim, fmt.Sprintf(" · dimensions %d", e.Dimensions)) + } + if e.EncodingFormat != "" { + l.add(th.dim, " · "+e.EncodingFormat) + } + right := fmt.Sprintf("%d/%d inputs · dim %s", done, e.Inputs, fmtTok(e.Dim)) + if v.m.EmbedRef != nil { + right = fmt.Sprintf("vs %s · band ┃ %.3f · %s", v.m.EmbedRefName, EmbedBand, right) + } + l.gapTo(width(right)) + l.add(th.text, right) + return l.String() +} + +// block is one input: id, text and figures; the vector strip, its norm and +// (with a reference) the compare bar; a separator. +func (v *embedView) block(r tape.EmbeddingRecord) []string { + th, t := v.th, v.t + answered := embedAnswered(r, t) + var right string + var rst style + switch { + case answered: + right, rst = fmtLat(msOf(r.Latency()))+" ms", th.accentBold + case r.Error != "": + right, rst = "✗ failed", th.bad + default: + right, rst = "embedding…", pulse(th, t) + } + tok := "input " + fmtTok(r.PromptTokens) + " tok · " + if r.Prompt != "" { + tok = r.Prompt + " prompt · " + tok + } + l := newLine(th, v.bw) + l.add(th.accentBold, fmt.Sprintf("%-11s", r.ID)) + l.space(1) + l.addTrunc(th.text, truncate(oneLine(r.Input), max(0, l.left()-width(tok)-width(right)-2))) + l.gapTo(width(tok) + width(right)) + l.add(th.dim, tok) + l.add(rst, right) + out := []string{l.String()} + + s := newLine(th, v.bw) + s.space(2) + vals, ok := v.vecs[r.Index] + switch { + case ok: + cells := downsample(vals, v.stripW) + // The strip fills left to right as the answer lands; it is painted + // cell by cell, so it goes in pre-measured. + shown := int(math.Round(grow(r.AnsweredAt, t) * float64(len(cells)))) + s.used += shown + s.b.WriteString(vectorStrip(th, cells[:shown], v.scale)) + s.add(th.darkFill, repeat('·', len(cells)-shown)) + s.space(v.stripW - len(cells) + 2) + s.add(th.dim, "norm ") + s.add(th.text, fmt.Sprintf("%.4f", r.Norm)) + if v.m.EmbedRef != nil { + s.space(2) + v.compareCell(s, r, vals) + } + case r.Error != "": + s.addTrunc(th.bad, r.Error) + default: + s.add(th.darkFill, repeat('·', v.stripW)) + } + return append(out, s.String(), blankRow(v.bw)) +} + +// compareCell is "████┃██▌── 0.99988 Δ0.0018" against the reference's +// record of the same id. +func (v *embedView) compareCell(l *lineBuf, r tape.EmbeddingRecord, vals []float32) { + th := v.th + ref, ok := v.m.EmbedRef[r.Key()] + refVals := ref.Values() + if !ok || refVals == nil { + l.add(th.dim, "not in reference") + return + } + c := tape.Cosine(vals, refVals) + const barW = 10 + frac := (c - EmbedBarFloor) / (1 - EmbedBarFloor) + bandAt := int(math.Round((EmbedBand - EmbedBarFloor) / (1 - EmbedBarFloor) * barW)) + bar := []rune(eighthBar(Theme{}, frac, barW, true, '─')) // plain glyphs + st := th.accent + if c < EmbedBand { + st = th.warn + } + // The band is always marked, inside the fill or past it. + bandAt = min(bandAt, barW-1) + l.add(st, string(bar[:bandAt])) + l.add(th.text, "┃") + l.add(st, string(bar[bandAt+1:])) + l.space(1) + l.add(st, fmt.Sprintf("%.5f", c)) + l.add(th.dim, fmt.Sprintf(" Δ%.4f", tape.MaxAbsDiff(vals, refVals))) +} + +// matrix is the cosine section: a header of column numbers and one row per +// input, each cell coloured by its value over the matrix's own range. +func (v *embedView) matrix(room int) []string { + th := v.th + recs := v.m.Embeddings + n := len(recs) + if n < 2 || room < 4 { + return nil + } + n = min(n, room-2) + lo, hi := 1.0, -1.0 + cos := make([][]float64, n) + for i := 0; i < n; i++ { + cos[i] = make([]float64, n) + for j := 0; j < n; j++ { + a, oka := v.vecs[recs[i].Index] + b, okb := v.vecs[recs[j].Index] + cos[i][j] = math.NaN() + if oka && okb { + cos[i][j] = tape.Cosine(a, b) + if i != j { + lo, hi = math.Min(lo, cos[i][j]), math.Max(hi, cos[i][j]) + } + } + } + } + const labelW, cellW = 16, 6 + out := []string{sectionRow(th, v.bw, "COSINE SIMILARITY")} + h := newLine(th, v.bw) + h.space(labelW) + for j := 0; j < n && h.left() >= cellW; j++ { + h.add(th.dim, fmt.Sprintf("%*s", cellW, fmt.Sprintf("%d", j))) + } + out = append(out, h.String()) + for i := 0; i < n; i++ { + l := newLine(th, v.bw) + l.add(th.dim, fmt.Sprintf("%2d ", i)) + l.add(th.text, fmt.Sprintf("%-*s", labelW-3, truncate(recs[i].ID, labelW-4))) + for j := 0; j < n && l.left() >= cellW; j++ { + c := cos[i][j] + switch { + case math.IsNaN(c): + l.add(th.darkFill, fmt.Sprintf("%*s", cellW, "·")) + case i == j: + l.add(th.dim, fmt.Sprintf("%*s", cellW, "1.00")) + default: + level := 1 + if hi > lo { + level = 1 + int(math.Min(embedLevels-1, (c-lo)/(hi-lo)*embedLevels)) + } + st := embedStyle(th, false, level) + if level == embedLevels { + st = th.accentBold + } + l.add(st, fmt.Sprintf("%*.2f", cellW, c)) + } + } + out = append(out, l.String()) + } + return out +} + +func embeddingBody(m Model, th Theme, t time.Duration, bw, bodyH int) []string { + v := newEmbedView(m, th, t, bw) + rows := []string{v.headline(), blankRow(bw), sectionRow(th, bw, "INPUTS")} + mat := v.matrix(min(len(m.Embeddings)+2, bodyH/2)) + room := bodyH - len(rows) - len(mat) - 1 + // A block is three rows with its separator, two without; the separator + // goes first when every input would not fit otherwise. + bh := embedBlockH + if len(m.Embeddings)*bh > room { + bh = embedBlockH - 1 + } + var feed []string + for _, r := range m.Embeddings { + feed = append(feed, v.block(r)[:bh]...) + } + // Newest at the bottom: when the feed still does not fit, the oldest go. + if over := len(feed) - max(0, room); over > 0 { + cut := (over + bh - 1) / bh * bh + feed = feed[min(cut, len(feed)):] + } + rows = append(rows, feed...) + if len(rows)+len(mat) <= bodyH { + rows = append(rows, fitRows(nil, blankRow(bw), max(0, bodyH-len(rows)-len(mat)))...) + rows = append(rows, mat...) + } + return fitRows(rows, blankRow(bw), bodyH) +} + +func embeddingFooter(m Model, th Theme, t time.Duration, w int) string { + var series []float64 + for _, r := range m.Embeddings { + if embedAnswered(r, t) { + series = append(series, msOf(r.Latency())) + } + } + hints := "q quit" + if m.Replay { + hints = "" + } + l := newLine(th, w) + l.add(th.dim, "request latency ") + median := "p50 " + fmtLat(percentile(series, 0.5)) + " ms" + stripW := max(0, l.left()-width(median)-width(hints)-4) + if len(series) > stripW { + series = series[len(series)-stripW:] + } + cells := LatencyStrip(series, stripW) + l.space(stripW - len(cells)) + writeCells(l, th, cells, cellPalette{base: th.dim, warn: th.text, bad: th.warn}) + l.space(2) + if len(cells) > 0 { + l.add(th.dim, median) + } + l.gapTo(width(hints)) + l.add(th.dim, hints) + return l.String() +} + +// embeddingView is View for an embedding tape. +func embeddingView(m Model, t time.Duration, w, h int) string { + th := m.Theme + inner := w - 2 + bodyH := h - chromeH + bar := th.paint(th.dim, "│") + lines := make([]string, 0, h) + lines = append(lines, topBorder(m, th, t, inner)) + body := embeddingBody + if hasScene(m) { + body = sceneBody + } + for _, row := range body(m, th, t, inner-2, bodyH) { + lines = append(lines, bar+" "+row+" "+bar) + } + lines = append(lines, th.paint(th.dim, "├"+repeat('─', inner)+"┤")) + lines = append(lines, bar+" "+embeddingFooter(m, th, t, inner-2)+" "+bar) + lines = append(lines, th.paint(th.dim, "└"+repeat('─', inner)+"┘")) + return strings.Join(lines, "\n") +} + +// SetEmbedRef makes ref the reference m's inputs are compared with, labelled +// name; a nil ref clears it. +func SetEmbedRef(m *Model, ref *tape.Tape, name string) { + if ref == nil { + m.EmbedRef, m.EmbedRefName = nil, "" + return + } + m.EmbedRef = map[string]tape.EmbeddingRecord{} + for _, r := range ref.Embeddings { + m.EmbedRef[r.Key()] = r + } + m.EmbedRefName = name +} + +// embedSpeedRows are the machine pane's SPEED rows on an embedding tape: +// requests answered, their latency p50, and the prompt tokens embedded. +// There is no tok/s to show: an embedding is one prefill, no decode. +func embedSpeedRows(m Model, th Theme, t time.Duration, cw int) []string { + var lat []float64 + n, tok := 0, 0 + for _, r := range m.Embeddings { + if embedAnswered(r, t) { + n++ + tok += r.PromptTokens + lat = append(lat, msOf(r.Latency())) + } + } + total := 0 + if e := m.Summary.Embedding; e != nil { + total = e.Inputs + } + row := func(label, value string, st style) string { + l := newLine(th, cw) + l.add(th.dim, label) + l.gapTo(width(value)) + l.add(st, value) + return l.String() + } + p50 := unknown + if len(lat) > 0 { + p50 = fmtLat(percentile(lat, 0.5)) + " ms" + } + return []string{ + row("requests", fmt.Sprintf("%d/%d", n, total), th.text), + row("latency p50", p50, th.accentBold), + row("prompt tokens", fmtTok(tok), th.text), + } +} diff --git a/internal/tui/embedding_test.go b/internal/tui/embedding_test.go new file mode 100644 index 0000000..3b526d3 --- /dev/null +++ b/internal/tui/embedding_test.go @@ -0,0 +1,160 @@ +package tui + +import ( + "strings" + "testing" + "time" + + "github.com/midagedev/toktape/internal/card" + "github.com/midagedev/toktape/internal/tape" +) + +func embedTapes(t *testing.T) (a, b *tape.Tape) { + t.Helper() + var err error + if a, err = tape.Read("../../assets/embedding-demo.tape"); err != nil { + t.Fatal(err) + } + if b, err = tape.Read("../tape/testdata/embedding-llama.tape"); err != nil { + t.Fatal(err) + } + return a, b +} + +// frame draws m at t plain and in colour, and checks the two agree and the +// plain one is exactly w x h. +func frame(t *testing.T, m Model, at time.Duration, w, h int) string { + t.Helper() + m.Replay = true + p := View(m, at, w, h) + m.Theme = ColourTheme() + if c := View(m, at, w, h); card.StripANSI(c) != p { + t.Fatal("the colour frame is not the plain frame with escapes") + } + lines := strings.Split(p, "\n") + if len(lines) != h { + t.Fatalf("%d lines, want %d", len(lines), h) + } + for i, l := range lines { + if width(l) != w { + t.Fatalf("line %d is %d wide, want %d: %q", i, width(l), w, l) + } + } + return p +} + +func TestEmbeddingVerifyScreen(t *testing.T) { + a, _ := embedTapes(t) + end := EmbeddingEnd(a) + for _, size := range [][2]int{{120, 40}, {MinWidth, MinHeight}} { + m := ModelAt(a, end) + m.EmbedVerify = true + done := frame(t, m, end, size[0], size[1]) + for _, want := range []string{"EMBEDDING", "52/52 inputs", "COSINE SIMILARITY", "q-index", "norm 1.0000", "query prompt"} { + if !strings.Contains(done, want) { + t.Errorf("%v verify frame lacks %q:\n%s", size, want, done) + } + } + } +} + +// TestSearchScene walks the demo through its phases: indexing, a query +// being typed, its ranked answer with the expected hit, and the clusters. +func TestSearchScene(t *testing.T) { + a, _ := embedTapes(t) + var q tape.EmbeddingRecord + for _, r := range a.Embeddings { + if r.Role == tape.EmbeddingRoleQuery && r.Prompt != "" { + q = r + break + } + } + if q.ShownAt == 0 || q.SentAt <= q.ShownAt { + t.Fatalf("the demo's first query has no typing time: %+v", q) + } + e := a.Summary.Embedding + // Mid-typing, on a cursor-on half of the blink. + typing := q.ShownAt + 50*time.Millisecond + for typing%(2*blinkDur) >= blinkDur { + typing += 10 * time.Millisecond + } + if typing >= q.SentAt { + t.Fatalf("no cursor-on instant while %s is typed", q.ID) + } + steps := []struct { + at time.Duration + want []string + }{ + {q.ShownAt / 2, []string{"INDEX", "indexing the corpus"}}, + {typing, []string{"20/20 docs", "▌"}}, + // Settled, before the next query starts (the hold is about 1s). + {q.AnsweredAt + 900*time.Millisecond, []string{"TOP 5", "✓", "PROMPTS with #1", "MRL 768 #1", "top-1 the same", "1›", "RESOURCES", "CPU"}}, + {e.OutroAt + 3*time.Second, []string{"CLUSTERS", "recall@1 6/6", "RESOURCES"}}, + } + for _, size := range [][2]int{{120, 40}, {MinWidth, MinHeight}} { + for _, st := range steps { + f := frame(t, ModelAt(a, st.at), st.at, size[0], size[1]) + for _, w := range st.want { + if !strings.Contains(f, w) { + t.Errorf("%v at %s lacks %q:\n%s", size, st.at, w, f) + } + } + } + } + if end := EmbeddingEnd(a); end < 15*time.Second || end > 25*time.Second { + t.Errorf("the demo replays in %s, want 15-25s", end) + } +} + +func TestEmbeddingAgainst(t *testing.T) { + a, b := embedTapes(t) + m := ModelAt(b, EmbeddingEnd(b)) + SetEmbedRef(&m, a, "v1") + f := frame(t, m, EmbeddingEnd(b), 120, 40) + if !strings.Contains(f, "vs v1 · band") || !strings.Contains(f, "0.999") || !strings.Contains(f, "Δ0.00") { + t.Errorf("compare frame:\n%s", f) + } +} + +func TestDownsampleAndStrip(t *testing.T) { + got := downsample([]float32{1, 3, -2, -4, 0, 0}, 3) + if len(got) != 3 || got[0] != 2 || got[1] != -3 || got[2] != 0 { + t.Fatalf("downsample = %v", got) + } + if s := vectorStrip(Theme{}, []float64{-1, 0, 1}, 1); s != "▁▅█" { + t.Errorf("strip = %q", s) + } +} + +// TestSceneMotion: what moves does so as a function of t — the typing is +// monotonic and complete at send, the bars and strip grow after the answer, +// and once settled the frame stops changing. +func TestSceneMotion(t *testing.T) { + a, _ := embedTapes(t) + var q tape.EmbeddingRecord + for _, r := range a.Embeddings { + if r.Role == tape.EmbeddingRoleQuery && r.Prompt != "" { + q = r + break + } + } + prev := 0 + for at := q.ShownAt; at <= q.SentAt; at += 20 * time.Millisecond { + n := typedCount(q, at) + if n < prev { + t.Fatalf("typing went back at %s: %d after %d", at, n, prev) + } + prev = n + } + if typedCount(q, q.SentAt) != len([]rune(q.Input)) { + t.Error("the query is not fully typed when it is sent") + } + f := func(at time.Duration) string { return frame(t, ModelAt(a, at), at, 120, 40) } + early, settled := f(q.AnsweredAt+60*time.Millisecond), f(q.AnsweredAt+850*time.Millisecond) + if early == settled { + t.Error("nothing moved between the answer landing and settling") + } + if f(q.AnsweredAt+950*time.Millisecond) != settled { + t.Error("the settled frame still moves") + } +} diff --git a/internal/tui/embedscene.go b/internal/tui/embedscene.go new file mode 100644 index 0000000..27c0488 --- /dev/null +++ b/internal/tui/embedscene.go @@ -0,0 +1,659 @@ +package tui + +// The search scene (2026-10-15): the default embedding screen for a tape +// whose inputs carry roles (the built-in search set). It shows what an +// embedding model is for: a corpus is indexed one doc at a time, then each +// query is typed, embedded, and answered with a ranked top-k against the +// index — with the expected doc marked, its rank with and without the task +// prompts, and its rank when the vectors are cut to 256 and 128 dims (MRL: +// truncate, then cosine renormalises). After the last query, the docs' +// cosine matrix ordered by topic shows the clusters. +// +// Every ranking is computed here from the recorded vectors, as a pure +// function of (m, t); the pacing (typing, holds) is the recorder's idle time +// on the tape, never part of a latency. + +import ( + "fmt" + "io" + "math" + "sort" + "strings" + "sync" + "time" + + "github.com/charmbracelet/lipgloss" + "github.com/muesli/termenv" + + "github.com/midagedev/toktape/internal/tape" +) + +const ( + sceneTopK = 5 + sceneStagger = 70 * time.Millisecond // between result rows appearing + // EmbedTick is the replay's redraw period while an embedding scene runs. + EmbedTick = 33 * time.Millisecond + growDur = 350 * time.Millisecond // a bar or strip reaching its value + fadeDur = 200 * time.Millisecond // a row coming up from dim + blinkDur = 530 * time.Millisecond // the typing cursor's half period + clusterLag = 200 * time.Millisecond // between topic blocks fading in +) + +// grow is 0..1 eased progress over growDur from since. +func grow(since, t time.Duration) float64 { + if t <= since { + return 0 + } + return easeOutCubic(float64(t-since) / float64(growDur)) +} + +// fading reports whether a row that appeared at since is still coming up. +func fading(since, t time.Duration) bool { return t < since+fadeDur } + +// pulse is the in-flight item's style: the accent breathing low → high. +func pulse(th Theme, t time.Duration) style { + return []style{th.accentLow, th.accentMid, th.accentHigh}[breathPhase(t)] +} + +// typedCount is how many runes of q are on screen at t. The typing runs +// from ShownAt to SentAt, with a per-rune weight from a hash of the id and +// position so the rate wavers like a hand's, and is the same every replay. +func typedCount(q tape.EmbeddingRecord, t time.Duration) int { + rs := []rune(q.Input) + if q.SentAt <= q.ShownAt || t >= q.SentAt { + return len(rs) + } + if t <= q.ShownAt { + return 0 + } + w := make([]float64, len(rs)) + total := 0.0 + for i := range rs { + h := uint32(2166136261) + for _, c := range q.ID { + h = (h ^ uint32(c)) * 16777619 + } + h = (h ^ uint32(i)) * 16777619 + w[i] = 0.55 + float64(h%1000)/1000 // 0.55..1.55 + if rs[i] == ' ' { + w[i] *= 1.6 // a beat between words + } + total += w[i] + } + f := float64(t-q.ShownAt) / float64(q.SentAt-q.ShownAt) * total + n := 0 + for acc := 0.0; n < len(rs) && acc+w[n] <= f; n++ { + acc += w[n] + } + return n +} + +// sceneMRL are the truncations a query is re-ranked at. +var sceneMRL = []int{768, 256, 128} + +// hasScene reports whether m's tape is a labelled set (rows with roles) +// the scene can draw. +func hasScene(m Model) bool { + if m.EmbedVerify || m.EmbedRef != nil { + return false + } + for _, r := range m.Embeddings { + if r.Role != "" { + return true + } + } + return false +} + +type sceneDoc struct { + rec tape.EmbeddingRecord + vec []float32 // primary (prompted when there are prompts) + bare []float32 +} + +type scene struct { + m Model + th Theme + t time.Duration + docs []sceneDoc // in send order, answered by t + queries []tape.EmbeddingRecord + bareQ map[string][]float32 + byID map[string]int +} + +func primary(r tape.EmbeddingRecord, prompts bool) bool { + return (r.Prompt != "") == prompts +} + +func newScene(m Model, th Theme, t time.Duration) *scene { + s := &scene{m: m, th: th, t: t, bareQ: map[string][]float32{}, byID: map[string]int{}} + prompts := false + for _, r := range m.Embeddings { + if r.Prompt != "" { + prompts = true + } + } + bareDoc := map[string][]float32{} + for _, r := range m.Embeddings { + if r.Prompt == "" && embedAnswered(r, t) { + if r.Role == tape.EmbeddingRoleDoc { + bareDoc[r.ID] = r.Values() + } else { + s.bareQ[r.ID] = r.Values() + } + } + } + for _, r := range m.Embeddings { + if !primary(r, prompts) { + continue + } + switch r.Role { + case tape.EmbeddingRoleDoc: + if embedAnswered(r, t) { + s.byID[r.ID] = len(s.docs) + s.docs = append(s.docs, sceneDoc{rec: r, vec: r.Values(), bare: bareDoc[r.ID]}) + } + case tape.EmbeddingRoleQuery: + s.queries = append(s.queries, r) + } + } + return s +} + +type hit struct { + doc int + score float64 +} + +// rank orders the index by cosine to q, over the first dims values (0 = +// all); bare ranks the bare doc vectors. +func (s *scene) rank(q []float32, dims int, bare bool) []hit { + cut := func(v []float32) []float32 { + if dims > 0 && dims < len(v) { + return v[:dims] + } + return v + } + var hs []hit + for i, d := range s.docs { + v := d.vec + if bare { + v = d.bare + } + if v == nil || q == nil { + continue + } + hs = append(hs, hit{i, tape.Cosine(cut(q), cut(v))}) + } + sort.SliceStable(hs, func(a, b int) bool { return hs[a].score > hs[b].score }) + return hs +} + +// rankOf is the 1-based place of doc id in hs, 0 when absent. +func (s *scene) rankOf(hs []hit, id string) int { + for i, h := range hs { + if s.docs[h.doc].rec.ID == id { + return i + 1 + } + } + return 0 +} + +// active is the query on screen at t: the last one whose typing began. +func (s *scene) active() (tape.EmbeddingRecord, bool) { + var q tape.EmbeddingRecord + ok := false + for _, r := range s.queries { + shown := r.ShownAt + if shown == 0 { + shown = r.SentAt + } + if shown <= s.t { + q, ok = r, true + } + } + return q, ok +} + +func (s *scene) outro() bool { + e := s.m.Summary.Embedding + return e != nil && e.OutroAt > 0 && s.t >= e.OutroAt && len(s.queries) > 0 && + embedAnswered(s.queries[len(s.queries)-1], s.t) +} + +// topicStyle gives each topic its own colour, in the order topics first +// appear; a plain theme leaves them all dim. +func (s *scene) topicStyle(topic string) style { + for i, tp := range s.topics() { + if tp == topic { + return topicColour(s.th, i) + } + } + return s.th.dim +} + +var ( + topicOnce sync.Once + topicStyles []style +) + +func topicColour(th Theme, i int) style { + if !th.colour { + return th.dim + } + topicOnce.Do(func() { + r := lipgloss.NewRenderer(io.Discard) + r.SetColorProfile(termenv.TrueColor) + r.SetHasDarkBackground(true) + for _, c := range []string{"#7fe3d0", "#e0b25c", "#f07a5c", "#9fb4ff", "#c9a0e8", "#d9d9d9"} { + topicStyles = append(topicStyles, seqOf(r.NewStyle().Foreground(lipgloss.Color(c)))) + } + }) + return topicStyles[i%len(topicStyles)] +} + +func (s *scene) topics() []string { + var out []string + seen := map[string]bool{} + for _, r := range s.m.Embeddings { + if r.Role == tape.EmbeddingRoleDoc && !seen[r.Topic] { + seen[r.Topic] = true + out = append(out, r.Topic) + } + } + return out +} + +func (s *scene) totalDocs() int { + if e := s.m.Summary.Embedding; e != nil && e.Docs > 0 { + return e.Docs + } + return len(s.docs) +} + +func (s *scene) prompted() bool { + e := s.m.Summary.Embedding + return e != nil && len(e.Prompts) > 0 +} + +// ---- the index column ------------------------------------------------------- + +func (s *scene) indexColumn(w, h int, hits map[int]int) []string { + th := s.th + title := fmt.Sprintf("INDEX %d/%d docs", len(s.docs), s.totalDocs()) + out := []string{sectionRow(th, w, title)} + for i, d := range s.docs { + l := newLine(th, w) + if r, ok := hits[i]; ok { + l.add(th.accentBold, fmt.Sprintf("%d›", r)) + } else { + l.space(2) + } + l.add(s.topicStyle(d.rec.Topic), "● ") + l.add(th.dim, fmt.Sprintf("%-7s", d.rec.ID)) + ms := fmtLat(msOf(d.rec.Latency())) + " ms" + st := th.text + if d.rec.AnsweredAt > s.t-400*time.Millisecond { + st = th.accentBold // just arrived + } + l.addTrunc(st, truncate(oneLine(d.rec.Input), max(0, l.left()-width(ms)-1))) + l.gapTo(width(ms)) + l.add(th.dim, ms) + out = append(out, l.String()) + } + // The doc in flight. + for _, r := range s.m.Embeddings { + if r.Role == tape.EmbeddingRoleDoc && primary(r, s.prompted()) && !embedAnswered(r, s.t) && r.Error == "" { + l := newLine(th, w) + l.space(2) + l.add(pulse(th, s.t), "● ") + l.add(pulse(th, s.t), fmt.Sprintf("%-7s", r.ID)) + l.addTrunc(pulse(th, s.t), truncate(oneLine(r.Input), l.left())) + out = append(out, l.String()) + break + } + } + return fitRows(out, blankRow(w), h) +} + +// ---- the search column ------------------------------------------------------ + +func (s *scene) searchColumn(w, h int) ([]string, map[int]int) { + th := s.th + out := []string{sectionRow(th, w, "SEARCH")} + hits := map[int]int{} + q, ok := s.active() + if !ok { + l := newLine(th, w) + l.add(th.dim, "› indexing the corpus…") + return fitRows(append(out, l.String()), blankRow(w), h), hits + } + // Typing: the text appears over ShownAt..SentAt at a wavering rate. + typed := []rune(q.Input)[:typedCount(q, s.t)] + l := newLine(th, w) + l.add(th.accentBold, "› ") + l.addTrunc(th.text, truncate(string(typed), max(0, l.left()-12))) + switch { + case s.t < q.SentAt: + cur := "▌" + if int(s.t/blinkDur)%2 == 1 { + cur = " " + } + l.add(th.accent, cur) + case !embedAnswered(q, s.t): + l.add(pulse(th, s.t), " embedding…") + default: + ms := fmtLat(msOf(q.Latency())) + " ms" + l.gapTo(width(ms)) + l.add(th.dim, ms) + } + out = append(out, l.String()) + if q.Prompt != "" { + p := newLine(th, w) + p.add(th.dim, " prefix ") + p.addTrunc(th.text, fmt.Sprintf("%q", s.m.Summary.Embedding.Prompts[q.Prompt])) + out = append(out, p.String()) + } + out = append(out, blankRow(w)) + if !embedAnswered(q, s.t) { + return fitRows(append(out, s.history(w, q.ID)...), blankRow(w), h), hits + } + + qv := q.Values() + // The query's vector fills in left to right as the answer lands. + sv := newLine(th, w) + sv.add(th.dim, " vec ") + cells := downsample(qv, max(0, sv.left())) + scale := 0.0 + for _, x := range cells { + scale = math.Max(scale, math.Abs(x)) + } + shown := int(math.Round(grow(q.AnsweredAt, s.t) * float64(len(cells)))) + sv.used += shown + sv.b.WriteString(vectorStrip(th, cells[:shown], scale)) + sv.add(th.darkFill, repeat('·', len(cells)-shown)) + out = append(out[:len(out)-1], sv.String(), blankRow(w)) + + ranked := s.rank(qv, 0, false) + out = append(out, sectionRow(th, w, fmt.Sprintf("TOP %d", sceneTopK))) + barW := min(12, max(4, w/5)) + for i, hh := range ranked[:min(sceneTopK, len(ranked))] { + at := q.AnsweredAt + time.Duration(i)*sceneStagger + if s.t < at { + out = append(out, blankRow(w)) + continue + } + d := s.docs[hh.doc] + hits[hh.doc] = i + 1 + good := d.rec.ID == q.Expect + r := newLine(th, w) + st := th.text + mark := " " + if good { + st, mark = th.accentBold, "✓ " + } + if fading(at, s.t) { + st = th.dim + } + // Slide in from the right: the row starts two cells over. + r.space(int(math.Round(2 * (1 - grow(at, s.t))))) + r.add(th.dim, fmt.Sprintf("%d ", i+1)) + // The bar grows to its score; painted cell by cell, so pre-measured. + bw := min(barW, r.left()) + r.b.WriteString(eighthBar(th, (hh.score-0.3)/0.7*grow(at, s.t), bw, good, ' ')) + r.used += bw + r.add(st, fmt.Sprintf(" %.3f ", hh.score*grow(at, s.t))) + r.add(st, mark) + r.add(s.topicStyle(d.rec.Topic), "● ") + r.addTrunc(st, truncate(oneLine(d.rec.Input), r.left())) + out = append(out, r.String()) + } + vAt := q.AnsweredAt + sceneTopK*sceneStagger + if s.t < vAt { + return fitRows(out, blankRow(w), h), hits + } + out = append(out, blankRow(w)) + out = append(out, s.verdict(w, q, qv, ranked, vAt)...) + out = append(out, blankRow(w)) + out = append(out, s.history(w, q.ID)...) + return fitRows(out, blankRow(w), h), hits +} + +// verdict is the expected doc's rank three ways: as ranked, without the +// task prompts, and at the MRL truncations. +func (s *scene) verdict(w int, q tape.EmbeddingRecord, qv []float32, ranked []hit, from time.Duration) []string { + th := s.th + if q.Expect == "" { + return nil + } + // Each figure comes up in turn, from dim, a beat after the one before. + k := 0 + place := func(l *lineBuf, r int) { + at := from + time.Duration(k)*sceneStagger + k++ + if s.t < at { + l.add(th.darkFill, "··") + return + } + st := th.accentBold + if r != 1 { + st = th.warn + } + if fading(at, s.t) { + st = th.dim + } + if r == 0 { + l.add(th.dim, "–") + return + } + l.add(st, fmt.Sprintf("#%d", r)) + } + var out []string + p := newLine(th, w) + p.add(th.dim, "PROMPTS ") + if s.prompted() { + bare := s.rank(s.bareQ[q.ID], 0, true) + p.add(th.dim, "with ") + place(p, s.rankOf(ranked, q.Expect)) + p.add(th.dim, " "+margin(s, ranked, q.Expect, grow(from, s.t))) + p.add(th.dim, " bare ") + place(p, s.rankOf(bare, q.Expect)) + p.add(th.dim, " "+margin(s, bare, q.Expect, grow(from+sceneStagger, s.t))) + } else { + p.add(th.dim, "none sent (--no-prompts)") + } + out = append(out, p.String()) + m := newLine(th, w) + m.add(th.dim, "MRL ") + top1 := "" + stable := true + for i, d := range sceneMRL { + hs := s.rank(qv, d, false) + if i > 0 { + m.add(th.dim, " · ") + } + m.add(th.dim, fmt.Sprintf("%d ", d)) + place(m, s.rankOf(hs, q.Expect)) + if len(hs) > 0 { + id := s.docs[hs[0].doc].rec.ID + if top1 == "" { + top1 = id + } else if id != top1 { + stable = false + } + } + } + out = append(out, m.String()) + st := newLine(th, w) + st.space(8) + if s.t >= from+time.Duration(k)*sceneStagger { + if stable { + st.add(th.dim, "top-1 the same at every size") + } else { + st.add(th.warn, "top-1 changes with the size") + } + } + return append(out, st.String()) +} + +// history is one line per query answered before current. +func (s *scene) history(w int, current string) []string { + th := s.th + var out []string + for _, q := range s.queries { + if q.ID == current || !embedAnswered(q, s.t) { + continue + } + if len(out) == 0 { + out = append(out, sectionRow(th, w, "EARLIER")) + } + ranked := s.rank(q.Values(), 0, false) + r := s.rankOf(ranked, q.Expect) + l := newLine(th, w) + if r == 1 { + l.add(th.accentBold, "✓ ") + } else { + l.add(th.warn, "✗ ") + } + mrl := make([]string, 0, len(sceneMRL)) + for _, d := range sceneMRL { + mrl = append(mrl, fmt.Sprintf("#%d", s.rankOf(s.rank(q.Values(), d, false), q.Expect))) + } + tail := fmt.Sprintf(" #%d · bare #%d · %s", r, s.rankOf(s.rank(s.bareQ[q.ID], 0, true), q.Expect), strings.Join(mrl, "/")) + l.addTrunc(th.text, truncate(q.Input, max(0, l.left()-width(tail)))) + l.gapTo(width(tail)) + l.add(th.dim, tail) + out = append(out, l.String()) + } + return out +} + +// ---- the outro: clusters ---------------------------------------------------- + +func (s *scene) clusters(w, h int) []string { + th := s.th + docs := append([]sceneDoc(nil), s.docs...) + order := s.topics() + idx := map[string]int{} + for i, tp := range order { + idx[tp] = i + } + sort.SliceStable(docs, func(a, b int) bool { return idx[docs[a].rec.Topic] < idx[docs[b].rec.Topic] }) + n := len(docs) + lo, hi := 1.0, -1.0 + for i := 0; i < n; i++ { + for j := 0; j < n; j++ { + if i != j { + c := tape.Cosine(docs[i].vec, docs[j].vec) + lo, hi = math.Min(lo, c), math.Max(hi, c) + } + } + } + const labelW, cellW = 18, 3 + // Topic blocks fade in one after another: a cell is up once both its + // row's and its column's topics are. + outro := s.m.Summary.Embedding.OutroAt + upAt := func(d sceneDoc) time.Duration { return outro + time.Duration(idx[d.rec.Topic])*clusterLag } + out := []string{sectionRow(th, w, "CLUSTERS doc × doc cosine ×100, ordered by topic")} + for i := 0; i < n && len(out) < h-2; i++ { + l := newLine(th, w) + if s.t < upAt(docs[i]) { + out = append(out, l.String()) + continue + } + l.add(s.topicStyle(docs[i].rec.Topic), "● ") + l.add(th.text, fmt.Sprintf("%-*s", labelW-2, truncate(docs[i].rec.ID+" "+docs[i].rec.Topic, labelW-3))) + for j := 0; j < n && l.left() >= cellW; j++ { + c := tape.Cosine(docs[i].vec, docs[j].vec) + cell := fmt.Sprintf("%*d", cellW, int(math.Round(c*100))) + at := max(upAt(docs[i]), upAt(docs[j])) + if s.t < at { + l.add(th.darkFill, fmt.Sprintf("%*s", cellW, "·")) + continue + } + if fading(at, s.t) { + l.add(th.dim, cell) + continue + } + if i == j { + l.add(th.darkFill, fmt.Sprintf("%*s", cellW, "▪▪")) + continue + } + level := 1 + int(math.Min(embedLevels-1, (c-lo)/math.Max(1e-9, hi-lo)*embedLevels)) + st := embedStyle(th, false, level) + if docs[i].rec.Topic == docs[j].rec.Topic { + st = th.accentBold + } + l.add(st, cell) + } + out = append(out, l.String()) + } + // Recall line. + with, bare, n1 := 0, 0, 0 + for _, q := range s.queries { + if q.Expect == "" || !embedAnswered(q, s.t) { + continue + } + n1++ + if s.rankOf(s.rank(q.Values(), 0, false), q.Expect) == 1 { + with++ + } + if s.rankOf(s.rank(s.bareQ[q.ID], 0, true), q.Expect) == 1 { + bare++ + } + } + out = append(out, blankRow(w)) + l := newLine(th, w) + if s.t < outro+time.Duration(len(order))*clusterLag { + return fitRows(out, blankRow(w), h) + } + l.add(th.dim, "same-topic pairs bold · ") + l.add(th.accentBold, fmt.Sprintf("recall@1 %d/%d", with, n1)) + if s.prompted() { + l.add(th.dim, fmt.Sprintf(" with task prompts · %d/%d bare", bare, n1)) + } + out = append(out, l.String()) + return fitRows(out, blankRow(w), h) +} + +// sceneBody is the scene's body rows. +func sceneBody(m Model, th Theme, t time.Duration, bw, bodyH int) []string { + s := newScene(m, th, t) + // The machine pane on the right is the record screen's own (rightPane): + // speed, placement, memory, and the resource graphs from the samples. + paneW := rightWidth(bw + 4) + mainW := bw - paneW - 3 + pane := rightPane(m, th, t, paneW, bodyH, false) + sep := th.paint(th.dim, " │ ") + var main []string + if s.outro() { + main = s.clusters(mainW, bodyH) + } else { + lw := mainW * 45 / 100 + rw := mainW - lw - 3 + right, hits := s.searchColumn(rw, bodyH) + left := s.indexColumn(lw, bodyH, hits) + for i := 0; i < bodyH; i++ { + main = append(main, left[i]+sep+right[i]) + } + } + rows := make([]string, bodyH) + for i := range rows { + rows[i] = main[i] + sep + pane[i] + } + return rows +} + +// margin is the expected doc's lead over the best other doc ("+0.08"), or +// how far behind the top it is ("-0.03"). +// The figure counts up from zero as f goes 0..1. +func margin(s *scene, hs []hit, id string, f float64) string { + r := s.rankOf(hs, id) + if r == 0 || len(hs) < 2 { + return "" + } + if r == 1 { + return "(+" + strings.TrimPrefix(fmt.Sprintf("%.3f", (hs[0].score-hs[1].score)*f), "0") + ")" + } + return "(-" + strings.TrimPrefix(fmt.Sprintf("%.3f", (hs[0].score-hs[r-1].score)*f), "0") + ")" +} diff --git a/internal/tui/model.go b/internal/tui/model.go index 7835337..34b1cff 100644 --- a/internal/tui/model.go +++ b/internal/tui/model.go @@ -176,6 +176,16 @@ type Model struct { // Decisions are a decision run's requests sent by At (TTP-192), empty on // every other tape. The view asks of each whether its answer is in by t. Decisions []tape.DecisionRecord + + // Embeddings are an embedding run's requests sent by At, empty on every + // other tape. EmbedRef, when set, is a reference tape's records by id + // (`toktape play --against`), and EmbedRefName its label. + Embeddings []tape.EmbeddingRecord + EmbedRef map[string]tape.EmbeddingRecord + EmbedRefName string + // EmbedVerify draws the per-input verify screen instead of the search + // scene (`toktape play --verify`). + EmbedVerify bool } // ModelAt builds the state of tp as of clip time at: every token, sample and @@ -195,6 +205,10 @@ func ModelAt(tp *tape.Tape, at time.Duration) Model { decisionModel(&m, tp, at) return m } + if tp.Summary.IsEmbedding() { + embeddingModel(&m, tp, at) + return m + } // A rounds run is drawn one round at a time: the round active at `at`, // and only its requests become tiles (TTP-38). A single-round tape takes diff --git a/internal/tui/right.go b/internal/tui/right.go index 59b544b..54fa50b 100644 --- a/internal/tui/right.go +++ b/internal/tui/right.go @@ -83,7 +83,11 @@ func buildRightPane(m Model, th Theme, t time.Duration, cw, h int, boardDrawn bo } section("SPEED", "", th.dim) - out = append(out, speedRows(m, th, t, cw, boardDrawn)...) + if m.Summary.IsEmbedding() { + out = append(out, embedSpeedRows(m, th, t, cw)...) + } else { + out = append(out, speedRows(m, th, t, cw, boardDrawn)...) + } section("PLACEMENT", "", th.dim) out = append(out, placementRows(m, th, t, cw)...) section("MEMORY", "", th.dim) diff --git a/internal/tui/view.go b/internal/tui/view.go index bb2102c..3b894a3 100644 --- a/internal/tui/view.go +++ b/internal/tui/view.go @@ -51,6 +51,9 @@ func View(m Model, t time.Duration, w, h int) string { if m.Summary.IsDecision() && m.Err == "" { return decisionView(m, t, w, h) } + if m.Summary.IsEmbedding() && m.Err == "" { + return embeddingView(m, t, w, h) + } th := m.Theme inner := w - 2 bodyH := h - chromeH diff --git a/web/player/smoke.mjs b/web/player/smoke.mjs index 4246db2..34e0d97 100644 --- a/web/player/smoke.mjs +++ b/web/player/smoke.mjs @@ -89,6 +89,17 @@ check("the summary is an object carrying caveats", typeof d.summary === "object" // A tape it cannot read must say so rather than throw: a page that fetched a // truncated file has to be able to tell its reader what happened. +// An embedding tape (2026-10-15) plays its search scene through the same +// load() and frame(), with the resource panel on the right. +{ + const eb = readFileSync(join(here, "..", "..", "assets", "embedding-demo.tape")); + const el = api.load(new Uint8Array(eb)); + check("embedding-demo.tape loads", el.ok === true, el.error || `${eb.length} bytes in`); + check("the embedding scene has a duration", el.durationMs > 10000, `${el.durationMs} ms`); + const f = api.frame(Math.floor(el.durationMs * 0.6), COLS, ROWS).replace(/\x1b\[[0-9;]*m/g, ""); + check("the frame is the search scene", /SEARCH/.test(f) && /INDEX/.test(f), f.split("\n").find((l) => /INDEX/.test(l))?.trim().slice(0, 70)); +} + const bad = api.load(new Uint8Array([1, 2, 3, 4])); check("a file that is not a tape is refused by name", bad.ok === false && bad.error.length > 0, bad.error); diff --git a/web/src/run.js b/web/src/run.js index 527035a..6f8708e 100644 --- a/web/src/run.js +++ b/web/src/run.js @@ -146,7 +146,9 @@ function runPage(row, idx, base) { ["host", [idx.os, idx.host_class].filter(Boolean).join(" · ") || "?"], ["gpus", gpusRow(idx)], ["memory", memoryRow(idx)], - isDecision(idx) + isEmbedding(idx) + ? ["mode", "embedding model · POST /v1/embeddings or /embedding"] + : isDecision(idx) ? ["mode", `decision model · POST /v1/systemone${ idx.decision_req_per_sec ? ` · ${fmt(idx.decision_req_per_sec)} req/s back to back` : "" }`] @@ -166,7 +168,9 @@ function runPage(row, idx, base) { ]; // A decision run (TTP-192) has no token figure; its figures are the // card's, every latency send to last byte on the client's clock. - const figures = isDecision(idx) + const figures = isEmbedding(idx) + ? [["request p50", idx.decision_p50_ms, "ms"]] + : isDecision(idx) ? [ ["decision p50", idx.decision_p50_ms, "ms"], ["engine p50", idx.decision_engine_p50_ms, "ms"], @@ -457,7 +461,9 @@ function caveatBlock(idx) { // head() escapes it. No name, no credit — never the token handle. function shareTitle(idx, fallback, noteTitle, authorName) { const parts = []; - if (isDecision(idx)) { + if (isEmbedding(idx)) { + if (idx.decision_p50_ms) parts.push(`${fmt(idx.decision_p50_ms)} ms p50 embedding`); + } else if (isDecision(idx)) { if (idx.decision_p50_ms) parts.push(`${fmt(idx.decision_p50_ms)} ms p50 decision`); } else if (idx.decode_per_sec) parts.push(`${fmt(idx.decode_per_sec)} tok/s`); // The file name stands in when the model never normalised, without its @@ -547,9 +553,17 @@ function isDecision(idx) { return idx.mode === "decision"; } +// An embedding run (2026-10-15) carries its request p50 in decision_p50_ms +// and no token figure. +function isEmbedding(idx) { + return idx.mode === "embedding"; +} + function summaryLine(idx) { const parts = []; - if (isDecision(idx)) { + if (isEmbedding(idx)) { + if (idx.decision_p50_ms) parts.push(`${fmt(idx.decision_p50_ms)} ms p50 embedding`); + } else if (isDecision(idx)) { if (idx.decision_p50_ms) parts.push(`${fmt(idx.decision_p50_ms)} ms p50 decision`); } else if (idx.decode_per_sec) parts.push(`${fmt(idx.decode_per_sec)} tok/s decode`); if (idx.sessions) parts.push(`${idx.sessions} stream${idx.sessions === 1 ? "" : "s"}`); @@ -605,7 +619,7 @@ function prefillFigure(idx) { // A short prompt leads with the reason. Neither a short prompt nor a probe // means no row. A decision row is not given one. function prefillFacts(idx) { - if (isDecision(idx)) return ""; + if (isDecision(idx) || isEmbedding(idx)) return ""; const probe = probeClause(idx); if (idx.short_prompt) { const n = idx.prompt_n > 0 ? String(idx.prompt_n) : "?";