From 519b8f6fd7dee0c99ba009d1e57ab65b3bfcf32f Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Tue, 6 Oct 2026 15:52:58 -0400 Subject: [PATCH 01/12] fix(merge): the same gloss on a re-add is not a new form Slice 07's merge rule says an incoming gloss becomes a form only when it differs from the word's own gloss. Both runtimes added it whenever it wasn't already a form, so a form the learner had removed came back on the next re-add, and restoring a backup without ids a second time reported its words as merged (and changed them) instead of identical. A property test (restoring the same backup twice changes nothing) found it. The shared merge fixture gains the case, so the server and the extension are both held to it, and the backup tests gain a regression test for the restore. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ayla Croft --- extension/lib/word-merge.js | 6 ++++-- server/lib/kotiko/word_merge.ex | 17 +++++++++++++++-- spec/fixtures/merge.json | 6 ++++++ test/unit/backup.test.mjs | 15 +++++++++++++++ 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/extension/lib/word-merge.js b/extension/lib/word-merge.js index 6405597..3f3ba24 100644 --- a/extension/lib/word-merge.js +++ b/extension/lib/word-merge.js @@ -47,10 +47,12 @@ out.pronunciation_source = incoming.pronunciation_source || "model"; } // Union by case-insensitive text; existing forms keep their flags and are never dropped - // by the cap; a different incoming gloss becomes a form. + // by the cap; a different incoming gloss becomes a form (the same one doesn't, so a + // form the learner removed stays removed). const old = (existing.forms ?? []).map(form); const seen = new Set(old.filter((f) => f.text !== null).map((f) => fold(f.text))); - const candidates = [...(incoming.forms ?? []).map(form), ...(incoming.gloss ? [form({ text: incoming.gloss })] : [])]; + const newGloss = incoming.gloss && !(typeof existing.gloss === "string" && fold(existing.gloss) === fold(incoming.gloss)); + const candidates = [...(incoming.forms ?? []).map(form), ...(newGloss ? [form({ text: incoming.gloss })] : [])]; const added = []; for (const f of candidates) { if (f.text === null || seen.has(fold(f.text))) continue; diff --git a/server/lib/kotiko/word_merge.ex b/server/lib/kotiko/word_merge.ex index ce75845..c931e22 100644 --- a/server/lib/kotiko/word_merge.ex +++ b/server/lib/kotiko/word_merge.ex @@ -54,10 +54,16 @@ defmodule Kotiko.WordMerge do # Union by case-insensitive text. Existing forms keep their flags (a disabled form stays # disabled); new ones are appended with theirs. A different incoming gloss becomes a - # form. Existing forms win when the cap is hit, and are never dropped by it. + # form; the same one doesn't, so a form the learner removed stays removed. Existing + # forms win when the cap is hit, and are never dropped by it. defp forms(acc, existing, incoming) do old = Enum.map(existing[:forms] || [], &Forms.form/1) - gloss = if incoming[:gloss], do: [Forms.form(%{text: incoming[:gloss]})], else: [] + + gloss = + if new_gloss?(existing[:gloss], incoming[:gloss]), + do: [Forms.form(%{text: incoming[:gloss]})], + else: [] + seen = MapSet.new(old, &Text.fold(&1.text)) new = @@ -69,6 +75,13 @@ defmodule Kotiko.WordMerge do if new == [], do: acc, else: Map.put(acc, :forms, old ++ new) end + defp new_gloss?(_existing, nil), do: false + + defp new_gloss?(existing, incoming) when is_binary(existing), + do: Text.fold(existing) != Text.fold(incoming) + + defp new_gloss?(_existing, _incoming), do: true + defp status(acc, existing, incoming, opts) do to = case {existing[:status], incoming[:status]} do diff --git a/spec/fixtures/merge.json b/spec/fixtures/merge.json index 041b9a8..4ab228e 100644 --- a/spec/fixtures/merge.json +++ b/spec/fixtures/merge.json @@ -49,6 +49,12 @@ "incoming": { "status": "active", "gloss": "hound", "forms": [{ "text": "hound" }, { "text": "hounds" }] }, "changes": { "forms": [{ "text": "dog", "enabled": true, "case": "any", "ambiguous": false }, { "text": "hound", "enabled": true, "case": "any", "ambiguous": false }, { "text": "hounds", "enabled": true, "case": "any", "ambiguous": false }] } }, + { + "name": "the same gloss is not a new form: a form the learner removed stays removed", + "existing": { "status": "active", "gloss": "cat", "forms": [{ "text": "kitty" }] }, + "incoming": { "status": "active", "gloss": "Cat", "forms": [{ "text": "kitty" }] }, + "changes": {} + }, { "name": "the cap keeps existing forms and drops only new ones", "existing": { "status": "active", "gloss": "a", "forms": [{ "text": "f1" }, { "text": "f2" }, { "text": "f3" }, { "text": "f4" }, { "text": "f5" }, { "text": "f6" }, { "text": "f7" }, { "text": "f8" }, { "text": "f9" }] }, diff --git a/test/unit/backup.test.mjs b/test/unit/backup.test.mjs index b59670a..0c2376a 100644 --- a/test/unit/backup.test.mjs +++ b/test/unit/backup.test.mjs @@ -217,6 +217,21 @@ describe("what a restore does (§5 merge rules)", () => { assert.deepEqual(p.items.map((i) => i.kind), ["create"]); }); + // Found by the restore-twice property (test/unit/properties/backup.test.mjs): a file + // without ids, whose word's meaning isn't one of its forms, came back "merged" on every + // restore after the first, its meaning added as a form each time it had been removed. + test("restoring a file without ids a second time changes nothing (07: the same gloss is no new form)", () => { + const raw = { lang: "el", native: "γάτα", base_lang: "en", gloss: "cat", forms: ["kitty"] }; + const words = B.read({ format: B.FORMAT, schemaVersion: 2, words: [raw] }, { now: T0 }).words; + assert.equal(words[0].id, null); + const once = B.plan(words, [], { now: T0 }); + assert.deepEqual(once.items.map((i) => i.kind), ["create"]); + const here = once.writes.map((w) => w.record); + const twice = B.plan(words, here, { now: T0 + 1000 }); + assert.deepEqual(twice.items, [{ kind: "identical", id: here[0].id }]); + assert.deepEqual(twice.writes, []); + }); + test("a word deleted here after the backup was made is skipped unless asked", () => { const [w] = file(); const here = [{ ...w, deleted_at: iso(T0 - DAY), updated_at: iso(T0 - DAY) }, { ...file()[1] }]; From 1a28dd6d7529263d0345c966800d187031edba6b Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Tue, 6 Oct 2026 15:58:02 -0400 Subject: [PATCH 02/12] test(helpers): load extension libraries without wrapper text requireExt wrapped each library in "(function (module, exports) {" before evaluating it, which shifted every code offset by the wrapper's length. When the same file was also run unwrapped (importScripts in the background's vm context, or a page's jsdom window) in another test process, Node's coverage merge could not line the two up and reported lines as uncovered that tests do run: lib/local-mode.js showed 40 % of its lines covered while its own tests cover 89 %. vm.compileFunction takes the file as the function body, so the offsets match the file whichever way it is loaded. The libraries behave exactly as before; only the coverage numbers become right (extension branch coverage 79.1 % -> 84.0 %, lines 92.2 % -> 93.7 %, with no test added). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ayla Croft --- test/helpers/load-script.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/helpers/load-script.mjs b/test/helpers/load-script.mjs index 04cd6a0..7daa6a8 100644 --- a/test/helpers/load-script.mjs +++ b/test/helpers/load-script.mjs @@ -21,11 +21,11 @@ export const manifest = () => JSON.parse(readExt("manifest.json")); // Loads a lib file in this realm, like `require`, and returns its module.exports. // (The root package.json says "type": "module", so Node's own require would treat a .js // file as ESM; this wraps it the way CommonJS does instead.) +// vm.compileFunction takes the file as the function's body, with no wrapper text before +// it, so code coverage offsets line up with the file whichever way a test loaded it. export function requireExt(rel) { const module = { exports: {} }; - const wrapper = vm.runInThisContext(`(function (module, exports) {${readExt(rel)}\n})`, { - filename: extPath(rel), - }); + const wrapper = vm.compileFunction(readExt(rel), ["module", "exports"], { filename: extPath(rel) }); wrapper(module, module.exports); return module.exports; } From 96401626efb289f028c879f07b1f5ec59f3d1ed9 Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Tue, 6 Oct 2026 15:58:46 -0400 Subject: [PATCH 03/12] test(dom): cover the privacy page, the theme script and the connect page privacy.js, ui/theme.js and content/connect.js were the three extension scripts no test loaded. The new tests check that the policy renders as elements (HTML in the file stays text), follows Kotiko's language and falls back to English, and says so when nothing can be read; that the theme is applied from the cache before storage answers, refreshed from storage, follows changes, and survives blocked site data and a failing storage read; and that the connect page hands the code to the background, reports done, expired or failed, takes the code out of the address, and does nothing on any other page. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ayla Croft --- test/dom/pages.test.mjs | 258 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 test/dom/pages.test.mjs diff --git a/test/dom/pages.test.mjs b/test/dom/pages.test.mjs new file mode 100644 index 0000000..d981035 --- /dev/null +++ b/test/dom/pages.test.mjs @@ -0,0 +1,258 @@ +// SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +// SPDX-License-Identifier: Apache-2.0 + +// The three small page scripts: the privacy policy page (privacy.js with lib/policy.js, +// slice 28 §2), the theme applied before first paint (ui/theme.js, slice 06 §2) and the +// "Connect OpenRouter" return page's content script (content/connect.js, slice 11 §4). +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { JSDOM } from "jsdom"; +import { createFakeChrome } from "../helpers/fake-chrome.mjs"; +import { createI18n } from "../helpers/fake-i18n.mjs"; +import { readExt, runInWindow, sleep } from "../helpers/load-script.mjs"; + +// Polls `fn` until it returns something truthy (no fixed waits: a slow machine only waits longer). +async function until(fn, ms = 5000) { + const end = Date.now() + ms; + for (;;) { + const v = await fn(); + if (v) return v; + if (Date.now() > end) throw new Error("timed out waiting"); + await sleep(5); + } +} + +describe("privacy page (slice 28 §2)", () => { + // The page as the browser opens it. `files` answers the extension's own files by path + // (missing ones are a 404); `failing` paths make fetch reject, as a blocked request does. + function openPrivacy({ locale = "en", sync = {}, files = null, failing = [] } = {}) { + const fake = createFakeChrome({ sync }); + // The browser's language; a locale Kotiko has no translation for shows English text. + const i18n = createI18n(locale === "es" ? "es" : "en"); + fake.chrome.i18n = { ...i18n, getMessage: (key, subs) => (key === "@@ui_locale" || key === "ui_locale" ? locale.replace("-", "_") : i18n.getMessage(key, subs)) }; + const dom = new JSDOM(readExt("privacy.html"), { url: "chrome-extension://fake-extension-id/privacy.html", runScripts: "outside-only" }); + dom.window.chrome = fake.chrome; + const asked = []; + dom.window.fetch = async (url) => { + const rel = String(url).replace(/^chrome-extension:\/\/[^/]+\//, ""); + asked.push(rel); + if (failing.includes(rel)) throw new TypeError("Failed to fetch"); + if (files && rel in files) return files[rel] === null ? new Response("", { status: 404 }) : new Response(files[rel]); + try { + return new Response(readExt(rel)); + } catch { + return new Response("", { status: 404 }); + } + }; + for (const rel of ["lib/i18n.js", "lib/policy.js", "privacy.js"]) runInWindow(dom, rel); + const root = dom.window.document.getElementById("policy"); + const done = () => until(() => !root.hasAttribute("aria-busy")); + return { dom, root, asked, done, doc: dom.window.document }; + } + + test("renders the shipped policy as elements, never as HTML from the file", async () => { + const { root, done, doc } = openPrivacy(); + await done(); + assert.equal(root.lang, "en"); + assert.equal(doc.title, createI18n("en").getMessage("privacy_title")); + assert.equal(doc.documentElement.lang, "en"); + assert.match(root.querySelector("h1").textContent, /Kotiko privacy policy/); + assert.ok(root.querySelectorAll("h2").length >= 3, "the sections are headings"); + assert.ok(root.querySelectorAll("ul li").length >= 5, "the short version is a list"); + for (const a of root.querySelectorAll("a")) { + assert.match(a.href, /^https:\/\//); + assert.equal(a.rel, "noopener noreferrer"); + assert.equal(a.target, "_blank"); + } + }); + + test("a policy line with HTML in it stays text", async () => { + const md = "# Title\n\nSome **bold** and `code` at .\n\n- one\n continued\n- two"; + const { root, done } = openPrivacy({ files: { "privacy/en.md": md } }); + await done(); + assert.equal(root.querySelector("img"), null); + const p = root.querySelector("p"); + assert.match(p.textContent, //); + assert.equal(p.querySelector("strong").textContent, "bold"); + assert.equal(p.querySelector("code").textContent, "code"); + assert.equal(p.querySelector("a").href, "https://kotiko.org/"); + assert.deepEqual([...root.querySelectorAll("li")].map((li) => li.textContent), ["one continued", "two"]); + }); + + test("follows Kotiko's language setting, and falls back to English when the policy isn't translated", async () => { + const { root, asked, done, doc } = openPrivacy({ sync: { ui: { uiLang: "es" } }, files: { "privacy/es.md": null } }); + await done(); + assert.equal(doc.documentElement.lang, "es"); + assert.equal(doc.title, createI18n("es").getMessage("privacy_title")); + assert.deepEqual(asked.filter((p) => p.startsWith("privacy/")), ["privacy/es.md", "privacy/en.md"]); + assert.equal(root.lang, "en", "the English text says it's English"); + }); + + test("a regional interface language tries its own policy, then the language's, then English", async () => { + const { root, asked, done } = openPrivacy({ locale: "pt-BR", files: { "privacy/pt.md": "# Política" }, failing: ["privacy/pt-BR.md"] }); + await done(); + assert.deepEqual(asked.filter((p) => p.startsWith("privacy/")), ["privacy/pt-BR.md", "privacy/pt.md"]); + assert.equal(root.lang, "pt"); + assert.equal(root.querySelector("h1").textContent, "Política"); + }); + + test("says so when no policy can be read", async () => { + const { root, done } = openPrivacy({ files: { "privacy/en.md": null } }); + await done(); + assert.equal(root.children.length, 1); + assert.equal(root.firstElementChild.localName, "p"); + assert.equal(root.textContent, createI18n("en").getMessage("privacy_unavailable")); + }); +}); + +describe("theme before first paint (slice 06 §2)", () => { + // A page with ui/theme.js run first. `cached` is what localStorage held from last time; + // `blocked` makes localStorage throw, as it does with site data blocked. + function openPage({ cached = {}, prefs = undefined, blocked = false, chrome = true } = {}) { + const fake = createFakeChrome({ local: prefs === undefined ? {} : { prefs } }); + const dom = new JSDOM("", { url: "https://kotiko.test/dashboard.html", runScripts: "outside-only" }); + if (chrome) dom.window.chrome = fake.chrome; + for (const [k, v] of Object.entries(cached)) dom.window.localStorage.setItem(k, v); + if (blocked) { + Object.defineProperty(dom.window, "localStorage", { + get() { + throw new dom.window.DOMException("The operation is insecure.", "SecurityError"); + }, + }); + } + runInWindow(dom, "ui/theme.js"); + const root = dom.window.document.documentElement; + return { dom, fake, root, stored: (k) => (blocked ? null : dom.window.localStorage.getItem(k)) }; + } + + test("applies the cached theme and motion synchronously, before storage answers", () => { + const { root } = openPage({ cached: { "kotiko.theme": "dark", "kotiko.motion": "reduce" }, prefs: { theme: "dark", motion: "reduce" } }); + assert.equal(root.dataset.theme, "dark"); + assert.equal(root.dataset.motion, "reduce"); + }); + + test("an unknown cached value follows the system", () => { + const { root } = openPage({ cached: { "kotiko.theme": "sepia", "kotiko.motion": "fast" } }); + assert.equal(root.dataset.theme, undefined); + assert.equal(root.dataset.motion, undefined); + }); + + test("the stored preference replaces a stale cache and is cached for next time", async () => { + const { root, stored } = openPage({ cached: { "kotiko.theme": "dark" }, prefs: { theme: "light", motion: "reduce" } }); + await until(() => root.dataset.theme === "light"); + assert.equal(root.dataset.motion, "reduce"); + assert.equal(stored("kotiko.theme"), "light"); + assert.equal(stored("kotiko.motion"), "reduce"); + }); + + test("no preference saved yet means the system's theme", async () => { + const { root, stored } = openPage({ cached: { "kotiko.theme": "dark", "kotiko.motion": "reduce" } }); + await until(() => stored("kotiko.theme") === "system"); + assert.equal(root.dataset.theme, undefined); + assert.equal(root.dataset.motion, undefined); + assert.equal(stored("kotiko.motion"), "system"); + }); + + test("follows a change made in another page, and ignores other areas and keys", async () => { + const { root, fake, stored } = openPage({ prefs: { theme: "light" } }); + await until(() => root.dataset.theme === "light"); + await fake.chrome.storage.sync.set({ prefs: { theme: "dark" } }); + await fake.chrome.storage.local.set({ words: [] }); + await fake.idle(); + assert.equal(root.dataset.theme, "light"); + await fake.chrome.storage.local.set({ prefs: { theme: "dark", motion: "reduce" } }); + await fake.idle(); + assert.equal(root.dataset.theme, "dark"); + assert.equal(root.dataset.motion, "reduce"); + assert.equal(stored("kotiko.theme"), "dark"); + }); + + test("blocked site data: no cache, but the stored preference still applies", async () => { + const { root } = openPage({ blocked: true, prefs: { theme: "dark" } }); + await until(() => root.dataset.theme === "dark"); + }); + + test("a storage read that fails keeps the cached theme, and a page without storage events still loads", async () => { + const fake = createFakeChrome(); + fake.chrome.storage.local.get = async () => { + throw new Error("storage unavailable"); + }; + fake.chrome.storage.onChanged = { + addListener() { + throw new Error("no storage events in this context"); + }, + }; + const dom = new JSDOM("", { url: "https://kotiko.test/popup.html", runScripts: "outside-only" }); + dom.window.chrome = fake.chrome; + dom.window.localStorage.setItem("kotiko.theme", "light"); + runInWindow(dom, "ui/theme.js"); + const root = dom.window.document.documentElement; + assert.equal(root.dataset.theme, "light"); + await fake.idle(); + await sleep(0); + assert.equal(root.dataset.theme, "light"); + assert.equal(dom.window.localStorage.getItem("kotiko.theme"), "light"); + }); +}); + +describe("the Connect OpenRouter return page (slice 11 §4)", () => { + const STATUS = '

'; + function openReturnPage({ url = "https://kotiko.org/connect/?code=abc123", answer = () => ({ ok: true }), body = STATUS } = {}) { + const fake = createFakeChrome({ onSendMessage: (msg) => answer(msg) }); + fake.chrome.i18n = createI18n("en"); + const dom = new JSDOM(`${body}`, { url, runScripts: "outside-only" }); + dom.window.chrome = fake.chrome; + for (const rel of ["lib/i18n.js", "content/connect.js"]) runInWindow(dom, rel); + const status = () => dom.window.document.getElementById("kotiko-connect-status")?.textContent; + return { dom, fake, status, sent: fake.calls.sendMessage }; + } + const msg = (key) => createI18n("en").getMessage(key); + + test("hands the code to the background, says it worked and takes the code out of the address", async () => { + const { dom, status, sent } = openReturnPage(); + assert.equal(status(), msg("connect_working")); + await until(() => status() === msg("connect_done")); + assert.deepEqual(sent, [{ type: "oauth.code", code: "abc123" }]); + await until(() => dom.window.location.href === "https://kotiko.org/connect/"); + }); + + test("an expired code says so; any other refusal is a plain failure", async () => { + const expired = openReturnPage({ answer: () => ({ ok: false, code: "key_rejected", details: { reason: "expired" } }) }); + await until(() => expired.status() === msg("connect_expired")); + const rejected = openReturnPage({ answer: () => ({ ok: false, code: "key_rejected", details: { reason: "invalid" } }) }); + await until(() => rejected.status() === msg("connect_failed")); + const nothing = openReturnPage({ answer: () => undefined }); + await until(() => nothing.status() === msg("connect_failed")); + }); + + test("a background that can't be reached is a failure, and the code still leaves the address", async () => { + const { dom, status } = openReturnPage({ + answer: () => { + throw new Error("Could not establish connection. Receiving end does not exist."); + }, + }); + await until(() => status() === msg("connect_failed")); + await until(() => dom.window.location.search === ""); + }); + + test("works without a status line on the page", async () => { + const { dom, sent } = openReturnPage({ body: "" }); + await until(() => dom.window.location.search === ""); + assert.equal(sent.length, 1); + }); + + for (const [why, url] of [ + ["another site", "https://example.com/connect/?code=abc123"], + ["another page of the docs site", "https://kotiko.org/other/?code=abc123"], + ["no code", "https://kotiko.org/connect/"], + ["an empty code", "https://kotiko.org/connect/?code="], + ]) { + test(`does nothing on ${why}`, async () => { + const { dom, status, sent, fake } = openReturnPage({ url }); + await fake.idle(); + assert.equal(sent.length, 0); + assert.equal(status(), ""); + assert.equal(dom.window.location.href, url); + }); + } +}); From bf1708685b860c13b752da3e0cf675f1c592a403 Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Tue, 6 Oct 2026 15:58:58 -0400 Subject: [PATCH 04/12] test: property-based tests with fast-check, and an assertion mode fast-check (a pinned dev dependency; the extension still ships no npm code) runs properties over generated input, 100 cases each with a new seed on every run, so CI keeps exploring: the matcher on any page and vocabulary (bounds, order, no overlaps, exactly the vocabulary's words, case-blind, leftmost-longest phrases, nothing across a block edge), server address normalising (never throws, idempotent, no user info, http only where it is local), the bulk list reader (any bytes and text, RFC 4180 round trip, UTF-8 and UTF-16 decoding, line lists), the backup (any file is safe to read, export then read is a fixed point, restoring twice changes nothing, no secret reaches a backup), casing, the word-list response check, and the privacy policy's Markdown. A failure prints the seed and path; FC_SEED and FC_PATH replay it. Assertion mode: npm test loads test/helpers/assert-mode.mjs, which sets globalThis.__KOTIKO_ASSERT__; the test loaders pass it on to every jsdom window and vm context. With it set, the matcher checks every scan's matches, the backup checks every restore plan, and the word-list check checks every filtered list, throwing when an invariant breaks. The shipped extension never sets the flag, and npm run perf doesn't load the file, so the budgets measure the code as it ships. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ayla Croft --- extension/lib/backup.js | 26 ++- extension/lib/matcher.js | 22 +++ extension/lib/validate-words.js | 14 ++ package-lock.json | 41 +++++ package.json | 3 +- test/helpers/assert-mode.mjs | 11 ++ test/helpers/load-script.mjs | 6 + test/helpers/properties.mjs | 20 +++ test/unit/properties/backup.test.mjs | 178 +++++++++++++++++++ test/unit/properties/bulk-parse.test.mjs | 160 +++++++++++++++++ test/unit/properties/casing.test.mjs | 83 +++++++++ test/unit/properties/matcher.test.mjs | 168 +++++++++++++++++ test/unit/properties/policy.test.mjs | 51 ++++++ test/unit/properties/url.test.mjs | 127 +++++++++++++ test/unit/properties/validate-words.test.mjs | 87 +++++++++ 15 files changed, 995 insertions(+), 2 deletions(-) create mode 100644 test/helpers/assert-mode.mjs create mode 100644 test/helpers/properties.mjs create mode 100644 test/unit/properties/backup.test.mjs create mode 100644 test/unit/properties/bulk-parse.test.mjs create mode 100644 test/unit/properties/casing.test.mjs create mode 100644 test/unit/properties/matcher.test.mjs create mode 100644 test/unit/properties/policy.test.mjs create mode 100644 test/unit/properties/url.test.mjs create mode 100644 test/unit/properties/validate-words.test.mjs diff --git a/extension/lib/backup.js b/extension/lib/backup.js index 7804e95..0ec359f 100644 --- a/extension/lib/backup.js +++ b/extension/lib/backup.js @@ -304,7 +304,31 @@ items.push({ kind: local ? "restore" : "create", id: rec.id }); counts.new++; } - return { items, counts, writes: [...writes.values()], restoreDeleted: restoreAll }; + const out = { items, counts, writes: [...writes.values()], restoreDeleted: restoreAll }; + if (globalThis.__KOTIKO_ASSERT__ === true) checkPlan(fileWords, out); + return out; + } + + // Assertion mode (test/helpers/assert-mode.mjs): what a restore plan promises the store's + // one transaction and its Undo, checked under the tests only. + function checkPlan(fileWords, { items, counts, writes }) { + const fail = (what) => { + throw new Error(`KotikoBackup.plan invariant: ${what}`); + }; + if (items.length !== fileWords.length) fail(`${items.length} items for ${fileWords.length} words`); + const kinds = {}; + for (const i of items) kinds[i.kind] = (kinds[i.kind] ?? 0) + 1; + if ((kinds.create ?? 0) + (kinds.restore ?? 0) !== counts.new) fail("new words miscounted"); + for (const k of ["merge", "identical", "kept"]) if ((kinds[k] ?? 0) !== counts[k]) fail(`${k} miscounted`); + if (counts.restoredDeleted > counts.deletedLater) fail("restored more deleted words than there were"); + const ids = new Set(); + for (const { record, previous } of writes) { + if (typeof record.id !== "string" || !record.id) fail("a write has no id"); + if (ids.has(record.id)) fail(`${record.id} is written twice`); + ids.add(record.id); + if (previous && previous.id !== record.id) fail("a write replaces another word"); + if (record.deleted_at) fail("a restore writes a deleted word"); + } } function pick(w, fields) { diff --git a/extension/lib/matcher.js b/extension/lib/matcher.js index 9f16d96..fd9414f 100644 --- a/extension/lib/matcher.js +++ b/extension/lib/matcher.js @@ -273,9 +273,31 @@ } if (index.symbolicRe) addSymbolic(text, index, matches); matches.sort((a, b) => a.start - b.start); + if (globalThis.__KOTIKO_ASSERT__ === true) checkScan(text, index, matches, tokenCount); return { matches, tokenCount }; } + // Assertion mode (test/helpers/assert-mode.mjs): what every scan promises the content + // script, checked after each scan under the tests. The shipped extension never sets the + // flag, so this never runs there. + function checkScan(text, index, matches, tokenCount) { + const fail = (what, m) => { + throw new Error(`KotikoMatcher invariant: ${what} (${JSON.stringify({ text, start: m?.start, end: m?.end, surface: m?.surface })})`); + }; + let last = 0; + let tokens = 0; + for (const m of matches) { + if (!(Number.isInteger(m.start) && Number.isInteger(m.end) && m.start >= 0 && m.start < m.end && m.end <= text.length)) fail("a match lies outside its text", m); + if (text.slice(m.start, m.end) !== m.surface) fail("a match's surface isn't the text it covers", m); + if (m.start < last) fail("matches overlap or are out of order", m); + last = m.end; + const entry = (m.tokenIndex === -1 ? index.symbolic : index.entries).get(m.key); + if (!entry || m.entry !== entry || !entry.candidates.length) fail("a match has no entry in the index", m); + if (m.tokenIndex !== -1) tokens += m.tokens; + } + if (tokens > tokenCount) fail(`matches cover ${tokens} words of ${tokenCount}`); + } + // English "the house's roof": the word before 's, with 's left as written. After "it", // "let" and the rest, 's is "is" or "us" and the token matches only whole. function possessive(full, t, i, index, base) { diff --git a/extension/lib/validate-words.js b/extension/lib/validate-words.js index 61bd446..61eaccb 100644 --- a/extension/lib/validate-words.js +++ b/extension/lib/validate-words.js @@ -121,9 +121,23 @@ } } const dropped = Object.values(reasons).reduce((a, b) => a + b, 0); + if (globalThis.__KOTIKO_ASSERT__ === true) checkFiltered(list, words, dropped); return { words, dropped, reasons, droppedForms }; } + // Assertion mode (test/helpers/assert-mode.mjs): every word is kept or counted, and every + // kept word passes the checks again. Under the tests only. + function checkFiltered(list, words, dropped) { + const fail = (what) => { + throw new Error(`WordValidator.filterWords invariant: ${what}`); + }; + if (words.length + dropped !== list.length) fail(`${words.length} kept and ${dropped} dropped of ${list.length}`); + for (const w of words) { + if (checkWord(w) !== null || extraFixes(w) !== null) fail(`a kept word fails the checks (${JSON.stringify(w).slice(0, 200)})`); + if ((w.forms ?? []).length !== cleanForms(w.forms ?? []).length) fail("a kept word has a bad form"); + } + } + const error = (code, message, details = {}) => ({ ok: false, code, message, details }); // raw: { status, contentType, body } where body is the response text. diff --git a/package-lock.json b/package-lock.json index 2661dbe..deb52f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "eslint": "10.11.0", "eslint-plugin-no-unsanitized": "4.1.5", "fake-indexeddb": "6.2.5", + "fast-check": "4.10.2", "fflate": "0.8.3", "globals": "17.13.0", "jsdom": "30.1.1", @@ -2510,6 +2511,29 @@ "node": ">=18" } }, + "node_modules/fast-check": { + "version": "4.10.2", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.10.2.tgz", + "integrity": "sha512-iK2f+YrcmoeGqk6fA0ea2bptcu/itMIm4NfEozq6N25+aG6h7s5HZbB/k1aV7b5w5sFLMCbbtRUsTVR+BgC3xw==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^8.0.0" + }, + "engines": { + "node": ">=12.17.0" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -4112,6 +4136,23 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pure-rand": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.2.tgz", + "integrity": "sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, "node_modules/qified": { "version": "0.10.1", "resolved": "https://registry.npmjs.org/qified/-/qified-0.10.1.tgz", diff --git a/package.json b/package.json index c2f3452..9b65c36 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "node": ">=22.22.2" }, "scripts": { - "test": "node --test 'test/{unit,dom,bg}/**/*.test.mjs'", + "test": "node --test --import ./test/helpers/assert-mode.mjs 'test/{unit,dom,bg}/**/*.test.mjs'", "e2e": "playwright test", "lint": "eslint . && web-ext lint --source-dir extension --self-hosted", "perf": "node test/perf/run.mjs" @@ -23,6 +23,7 @@ "eslint": "10.11.0", "eslint-plugin-no-unsanitized": "4.1.5", "fake-indexeddb": "6.2.5", + "fast-check": "4.10.2", "fflate": "0.8.3", "globals": "17.13.0", "jsdom": "30.1.1", diff --git a/test/helpers/assert-mode.mjs b/test/helpers/assert-mode.mjs new file mode 100644 index 0000000..1cbd03b --- /dev/null +++ b/test/helpers/assert-mode.mjs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +// SPDX-License-Identifier: Apache-2.0 + +// Assertion mode (OpenSSF dynamic_analysis_enable_assertions). `npm test` and +// `npm run coverage` load this file first (node --import), so every test process runs the +// extension's invariant checks: lib/matcher.js checks every scan's matches, lib/backup.js +// every restore plan, lib/validate-words.js every filtered word list, and they throw when +// one doesn't hold. load-script.mjs passes the flag on to the jsdom windows and vm contexts +// the extension's scripts run in. The shipped extension never sets it, and `npm run perf` +// doesn't load this file, so the budgets measure the code as it ships. +globalThis.__KOTIKO_ASSERT__ = true; diff --git a/test/helpers/load-script.mjs b/test/helpers/load-script.mjs index 7daa6a8..09810f7 100644 --- a/test/helpers/load-script.mjs +++ b/test/helpers/load-script.mjs @@ -30,8 +30,13 @@ export function requireExt(rel) { return module.exports; } +// Assertion mode (helpers/assert-mode.mjs): the flag is passed on to every window and vm +// context an extension script runs in, so its invariant checks run there too. +const assertMode = () => globalThis.__KOTIKO_ASSERT__ === true; + // Runs an extension script inside a jsdom window (created with runScripts: "outside-only"). export function runInWindow(dom, rel) { + if (assertMode()) dom.window.__KOTIKO_ASSERT__ = true; return vm.runInContext(readExt(rel), dom.getInternalVMContext(), { filename: extPath(rel) }); } @@ -114,6 +119,7 @@ export function runInVm(rel, globals = {}) { indexedDB: new IDBFactory(), IDBKeyRange, fetch: () => Promise.reject(new TypeError("fetch is not stubbed in this test")), + ...(assertMode() ? { __KOTIKO_ASSERT__: true } : {}), ...globals, }); ctx.fetch = withOwnFiles(ctx.fetch, globals.wiktionary); diff --git a/test/helpers/properties.mjs b/test/helpers/properties.mjs new file mode 100644 index 0000000..450816c --- /dev/null +++ b/test/helpers/properties.mjs @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +// SPDX-License-Identifier: Apache-2.0 + +// Property-based tests (OpenSSF dynamic_analysis) with fast-check: each property runs on +// 100 generated inputs per test run, a new random seed each time, so CI keeps exploring. +// A failure prints the seed and the shrunk counterexample; to replay it exactly: +// +// FC_SEED= FC_PATH= node --test --import ./test/helpers/assert-mode.mjs test/unit/properties/ +// +// FC_NUM_RUNS= runs more (or fewer) inputs, for a longer local hunt. +import fc from "fast-check"; + +const env = (name) => (process.env[name] ? process.env[name] : undefined); +fc.configureGlobal({ + numRuns: Number(env("FC_NUM_RUNS") ?? 100), + ...(env("FC_SEED") ? { seed: Number(env("FC_SEED")) } : {}), + ...(env("FC_PATH") ? { path: env("FC_PATH") } : {}), +}); + +export { fc }; diff --git a/test/unit/properties/backup.test.mjs b/test/unit/properties/backup.test.mjs new file mode 100644 index 0000000..b26d48f --- /dev/null +++ b/test/unit/properties/backup.test.mjs @@ -0,0 +1,178 @@ +// SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +// SPDX-License-Identifier: Apache-2.0 + +// Properties of the JSON backup (lib/backup.js, slice 12) on generated files and word +// lists: reading any file is safe, a backup reads back as it was written, restoring the +// same backup twice changes nothing the second time, and no secret reaches a backup. +// Assertion mode checks every restore plan's own invariants as well. +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { fc } from "../../helpers/properties.mjs"; +import { loadLocalLibs } from "../../helpers/local-libs.mjs"; +import { requireExt } from "../../helpers/load-script.mjs"; + +const L = loadLocalLibs(); +globalThis.KotikoStore = L.Store; +const B = requireExt("lib/backup.js"); +const T0 = Date.parse("2026-10-01T12:00:00.000Z"); +const CODES = new Set(["unreadable", "not_backup", "backup_newer", "too_big", "too_many"]); +const byId = (list) => [...list].sort((a, b) => (String(a.id) < String(b.id) ? -1 : 1)); + +// Words as a backup holds them: a language and a native word in its script, an English +// meaning, forms, and the optional fields, some of them odd. +const NATIVE = { es: /^[a-zñáéíóú]{1,12}$/, fr: /^[a-zàâçéèêëîïôûù]{1,12}$/, ru: /^[а-яё]{1,12}$/, el: /^[α-ω]{1,10}$/, ja: /^[ぁ-ゖ]{1,6}$/ }; +const iso = fc.integer({ min: T0 - 400 * 86_400_000, max: T0 }).map((ms) => new Date(ms).toISOString()); +const text = fc.stringMatching(/^[a-z][a-z ]{0,14}$/); +const rawWord = fc + .constantFrom(...Object.keys(NATIVE)) + .chain((lang) => + fc.record( + { + id: fc.oneof(fc.uuid(), fc.constant(null), fc.constant("not-a-uuid")), + lang: fc.constant(lang), + native: fc.stringMatching(NATIVE[lang]), + base_lang: fc.constant("en"), + sense: fc.oneof(fc.constant(""), text), + gloss: text.filter((s) => s.trim()), + forms: fc.array(fc.oneof(text, fc.record({ text, enabled: fc.boolean(), case: fc.constantFrom("any", "lower", "exact", "proper", "weird") })), { maxLength: 4 }), + note: fc.oneof(fc.constant(null), text, fc.string({ minLength: 201, maxLength: 210 })), + status: fc.constantFrom("active", "well_known", "paused"), + origin: fc.constantFrom("add", "bulk", "import", "somewhere"), + created_at: fc.oneof(iso, fc.constant(null), fc.constant("yesterday")), + updated_at: fc.oneof(iso, fc.constant(null)), + future_field: fc.oneof(fc.constant(undefined), fc.string({ maxLength: 5 })), + }, + { requiredKeys: ["lang", "native", "base_lang", "gloss"] }, + ), + ); +const file = (words, version = 2) => JSON.stringify({ format: B.FORMAT, schemaVersion: version, exportedAt: new Date(T0).toISOString(), words }); + +// What a restore's writes leave in the store. +function apply(existing, writes) { + const out = new Map(existing.map((r) => [r.id, r])); + for (const { record } of writes) out.set(record.id, record); + return [...out.values()]; +} + +describe("reading any file", () => { + test("never throws: a known refusal, or words that all passed the checks", () => { + const doc = fc.oneof( + fc.string({ maxLength: 40 }), + fc.jsonValue().map((v) => JSON.stringify(v)), + fc.record({ format: fc.oneof(fc.constant(B.FORMAT), fc.string()), schemaVersion: fc.oneof(fc.integer({ min: -1, max: 4 }), fc.double(), fc.string()), words: fc.oneof(fc.array(fc.oneof(rawWord, fc.anything()), { maxLength: 6 }), fc.anything()) }, { requiredKeys: [] }), + fc.anything(), + ); + fc.assert( + fc.property(doc, (input) => { + const r = B.read(input, { now: T0 }); + if (!r.ok) return assert.ok(CODES.has(r.code), r.code); + assert.equal(r.total, r.words.length + r.invalid.length); + for (const w of r.words) { + assert.equal(typeof w.native, "string"); + assert.ok(w.forms.length >= 1 && w.forms.length <= 10); + assert.equal(w.deleted_at, null); + assert.ok(!("english" in w)); + } + }), + ); + }); +}); + +describe("a backup reads back as it was written", () => { + test("export then read is a fixed point of read", () => { + fc.assert( + fc.property(fc.array(rawWord, { maxLength: 8 }), (raw) => { + const first = B.read(file(raw), { now: T0 }); + assert.equal(first.ok, true); + const again = B.read(B.stringify(B.exportDoc({ words: first.words, now: T0 })), { now: T0 }); + assert.equal(again.ok, true); + assert.deepEqual(again.invalid, []); + assert.deepEqual(again.dropped, []); + assert.deepEqual(byId(again.words), byId(first.words)); + }), + ); + }); + + test("a word the checks accept keeps every one of its fields", () => { + fc.assert( + fc.property(rawWord, (raw) => { + const r = B.read(file([raw]), { now: T0 }); + if (!r.words.length) return; + const [w] = r.words; + assert.equal(w.native, raw.native.normalize("NFC")); + assert.equal(w.gloss, raw.gloss.trim().replace(/\s+/g, " ")); + assert.equal(w.lang, raw.lang); + if (raw.future_field !== undefined) assert.equal(w.future_field, raw.future_field); + if (typeof raw.note === "string" && raw.note.length > 200) assert.equal(w.note, null); + }), + ); + }); +}); + +describe("restoring", () => { + test("restoring the same backup twice changes nothing the second time", () => { + fc.assert( + fc.property(fc.array(rawWord, { maxLength: 8 }), fc.array(rawWord, { maxLength: 4 }), (raw, here) => { + const fileWords = B.read(file(raw), { now: T0 }).words; + const existing = B.plan(B.read(file(here), { now: T0 }).words, [], { now: T0 }).writes.map((w) => w.record); + const once = B.plan(fileWords, existing, { now: T0 + 1000 }); + const after = apply(existing, once.writes); + const twice = B.plan(fileWords, after, { now: T0 + 2000 }); + assert.deepEqual(twice.writes, []); + assert.ok(twice.items.every((i) => i.kind === "identical" || i.kind === "kept"), JSON.stringify(twice.items)); + }), + ); + }); + + test("into an empty list every word is new, unless two share a natural key", () => { + fc.assert( + fc.property(fc.array(rawWord, { maxLength: 8 }), (raw) => { + const words = B.read(file(raw), { now: T0 }).words; + const p = B.plan(words, [], { now: T0 }); + const keys = new Set(words.map((w) => JSON.stringify(L.Merge.naturalKey(w)))); + assert.equal(p.writes.length, keys.size); + assert.equal(p.counts.new, keys.size); + assert.equal(p.counts.new + p.counts.merge + p.counts.identical, words.length); + }), + ); + }); +}); + +describe("settings in a backup (slice 39's allowlist)", () => { + const secret = fc.stringMatching(/^[0-9a-f]{24}$/).map((h) => `SECRET${h}`); + const anyValue = fc.oneof(fc.string({ maxLength: 6 }), fc.boolean(), fc.integer(), fc.constant(null)); + test("no secret ever reaches a backup, and addresses lose their user name, query and fragment", () => { + fc.assert( + fc.property(secret, fc.dictionary(fc.string({ maxLength: 8 }), anyValue, { maxKeys: 5 }), fc.webUrl({ withQueryParameters: true, withFragments: true }), (s, extra, url) => { + const local = { + ...extra, + token: s, + apiKey: s, + server: { url: url.replace("://", `://user:${s}@`), token: s }, + lookup: { provider: "openrouter", baseUrl: `${url}?key=${s}`, apiKey: s, model: "m" }, + oauth: { verifier: s }, + enabled: true, + seedSalt: "0123456789abcdef0123456789abcdef", + }; + const out = B.settingsOf({ local, sync: { ui: { uiLang: "es", token: s } } }); + assert.ok(!JSON.stringify(out).includes(s)); + for (const k of Object.keys(out)) assert.ok(["enabled", "pausedHosts", "hiddenLangs", "prefs", "mixing", "speech", "seedSalt", "lookup", "server", "ui"].includes(k), k); + if (out.server) assert.equal(new URL(out.server.url).username, ""); + }), + ); + }); + + test("restoring settings never changes where requests go", () => { + fc.assert( + fc.property(fc.webUrl(), fc.webUrl(), fc.constantFrom("openrouter", "ollama", "custom"), fc.constantFrom("openrouter", "ollama", "custom"), (fileUrl, hereUrl, fileProvider, hereProvider) => { + const current = { lookup: { provider: hereProvider, baseUrl: hereUrl, model: "here" }, server: { url: hereUrl } }; + const { local, sync } = B.settingsPatch({ lookup: { provider: fileProvider, baseUrl: fileUrl, model: "file" }, server: { url: fileUrl }, ui: { baseLangs: ["es", "en", "xx-invalid-tag-that-is-long"] } }, { current }); + assert.equal(local.lookup.provider, hereProvider); + assert.equal(local.lookup.baseUrl, hereUrl); + assert.equal(local.lookup.model, fileProvider === hereProvider ? "file" : "here"); + assert.ok(!("server" in local)); + assert.ok(sync.baseLangs.length <= 4); + }), + ); + }); +}); diff --git a/test/unit/properties/bulk-parse.test.mjs b/test/unit/properties/bulk-parse.test.mjs new file mode 100644 index 0000000..81b9ed0 --- /dev/null +++ b/test/unit/properties/bulk-parse.test.mjs @@ -0,0 +1,160 @@ +// SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +// SPDX-License-Identifier: Apache-2.0 + +// Properties of reading a pasted list or a dropped file (bulk/parse.js, slice 13 §3) on +// generated input: any bytes, any text, any table. +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fc } from "../../helpers/properties.mjs"; +import { EXT_DIR, requireExt } from "../../helpers/load-script.mjs"; + +const P = requireExt("bulk/parse.js"); +const langs = JSON.parse(fs.readFileSync(path.join(EXT_DIR, "spec/languages.json"), "utf8")).languages; +const FORMATS = new Set(["kotiko-backup", "json", "anki", "csv", "tsv", "lines"]); +const ERRORS = new Set(["spreadsheet", "apkg", "unreadable"]); + +// RFC 4180 as a spreadsheet writes it: a cell with the separator, a quote or a line break +// is quoted, its quotes doubled. +const quote = (cell, sep) => (cell.includes(sep) || cell.includes('"') || cell.includes("\n") ? `"${cell.replaceAll('"', '""')}"` : cell); +const write = (rows, sep) => rows.map((r) => r.map((c) => quote(c, sep)).join(sep)).join("\n"); + +const cell = fc.oneof(fc.string({ maxLength: 10 }), fc.string({ unit: "grapheme", maxLength: 6 }), fc.constantFrom('"', '""', ",", "\t", "；", "\n", "a,b", 'say "hi"', " padded ")); +const page = fc.oneof( + fc.string({ maxLength: 200 }), + fc.string({ unit: "grapheme", maxLength: 120 }), + fc.array(fc.constantFrom("gato = cat", "perro - dog", "1. casa: house", "#separator:tab", "#html:true", "a\tb", '"x","y"', "[", "{", '{"schemaVersion":2}', "[{}]", "", " ", "собака (sobaka) = dog # note", "猫：cat"), { maxLength: 12 }).map((ls) => ls.join("\n")), +); +const filename = fc.constantFrom("", "list.txt", "list.csv", "list.tsv", "words.json", "deck.apkg", "sheet.xlsx", "notes", "x.CSV"); + +describe("decode", () => { + test("any bytes: text without a NUL, or an error, never a throw", () => { + fc.assert( + fc.property(fc.uint8Array({ maxLength: 300 }), (bytes) => { + const r = P.decode(bytes); + if (r.error) return assert.ok(["too_big", "unreadable"].includes(r.error)); + assert.equal(typeof r.text, "string"); + assert.ok(!r.text.includes("\0") && !r.text.includes("\r")); + }), + ); + }); + + test("UTF-8 and UTF-16 text (with its byte-order mark) comes back as written, line breaks as \\n", () => { + const text = fc.string({ unit: "binary", maxLength: 80 }).filter((s) => !s.includes("\0")); + fc.assert( + fc.property(text, fc.constantFrom("utf8", "utf8bom", "utf16le", "utf16be"), (s, enc) => { + let bytes; + if (enc.startsWith("utf8")) bytes = Buffer.concat([Buffer.from(enc === "utf8bom" ? [0xef, 0xbb, 0xbf] : []), Buffer.from(s, "utf8")]); + else { + const le = Buffer.from(s, "utf16le"); + if (enc === "utf16be") le.swap16(); + bytes = Buffer.concat([Buffer.from(enc === "utf16le" ? [0xff, 0xfe] : [0xfe, 0xff]), le]); + } + const r = P.decode(new Uint8Array(bytes)); + assert.equal(r.warning, null); + assert.equal(r.text, s.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n").normalize("NFC")); + }), + ); + }); +}); + +describe("csv", () => { + test("reads back any table written the RFC 4180 way, for every separator", () => { + fc.assert( + fc.property(fc.array(fc.array(cell, { minLength: 1, maxLength: 5 }), { maxLength: 8 }), fc.constantFrom(",", "\t", "；", ";"), (rows, sep) => { + const want = rows.filter((r) => r.some((c) => c.trim())); + assert.deepEqual(P.csv(write(rows, sep), sep), want); + }), + ); + }); +}); + +describe("read and build", () => { + test("any text and file name gives a known format or a known error, never a throw", async () => { + await fc.assert( + fc.asyncProperty(page, filename, fc.constantFrom("en", "es", "ja"), async (text, name, base) => { + const r = P.read(text, { filename: name }); + if (r.error) return assert.ok(ERRORS.has(r.error), r.error); + assert.ok(FORMATS.has(r.format), r.format); + const built = await P.build(r, { base, langs }); + assert.ok(Array.isArray(built.rows)); + assert.ok(built.rows.length <= P.MAX_ROWS); + for (const row of built.rows) { + assert.equal(typeof row.native, "string"); + assert.equal(typeof row.gloss, "string"); + assert.ok(Array.isArray(row.forms)); + assert.ok(row.forms.every((f) => typeof f === "string" && f.length > 0)); + } + }), + ); + }); + + test("a list of word = meaning lines gives one row per line, the word and its meanings in order", async () => { + const latin = fc.stringMatching(/^[a-zà-ÿ]{2,10}$/); + const cyrillic = fc.stringMatching(/^[а-я]{2,10}$/); + await fc.assert( + fc.asyncProperty(fc.array(fc.tuple(cyrillic, fc.uniqueArray(latin, { minLength: 1, maxLength: 3 })), { minLength: 1, maxLength: 10 }), fc.constantFrom(" = ", " - ", " → ", "：", ": "), fc.boolean(), async (pairs, sep, wordFirst) => { + const text = pairs.map(([w, ms]) => (wordFirst ? `${w}${sep}${ms.join(", ")}` : `${ms.join(", ")}${sep}${w}`)).join("\n"); + const built = await P.build(P.read(text), { base: "en", target: "ru", langs }); + if (!wordFirst && sep === ": ") return; // "meaning: word" is ambiguous with a language prefix; covered by the examples + assert.deepEqual( + built.rows.map((r) => [r.native, r.forms]), + pairs.map(([w, ms]) => [w, ms]), + ); + assert.equal(built.decidedBy, "script"); + }), + ); + }); + + test("a JSON array of objects keeps every row and every key as a column", () => { + const obj = fc.dictionary(fc.constantFrom("native", "gloss", "note", "lang", "extra"), fc.oneof(fc.string({ maxLength: 8 }), fc.integer(), fc.constant(null)), { minKeys: 1 }); + fc.assert( + fc.property(fc.array(obj, { minLength: 1, maxLength: 10 }), (list) => { + const r = P.read(JSON.stringify(list)); + assert.equal(r.format, "json"); + assert.equal(r.rows.length, list.length); + assert.deepEqual(new Set(r.header), new Set(list.flatMap((o) => Object.keys(o)))); + r.rows.forEach((row, i) => r.header.forEach((k, j) => assert.equal(row[j], list[i][k] == null ? "" : String(list[i][k])))); + }), + ); + }); +}); + +describe("one line and its parts", () => { + test("line never throws; it gives one or two sides and keeps the source", () => { + fc.assert( + fc.property(fc.oneof(fc.string({ maxLength: 60 }), page), (raw) => { + const l = P.line(raw, langs); + if (l === null) return; + assert.ok(l.sides.length === 1 || l.sides.length === 2); + assert.equal(l.source, String(raw).trim()); + for (const s of l.sides) assert.equal(typeof s.text, "string"); + assert.ok(l.lang === null || typeof l.lang === "string"); + }), + ); + }); + + test("ankiText strips sound tags, cloze markup and HTML, and never leaves a tag", () => { + fc.assert( + fc.property(fc.array(fc.oneof(fc.string({ maxLength: 8 }), fc.constantFrom("", "", "
", "
", "[sound:x.mp3]", "{{c1::gato}}", "{{c2::perro::dog}}", "&")), { maxLength: 10 }), (parts) => { + const out = P.ankiText(parts.join(""), true, null); + assert.ok(!/<[^>]*>/.test(out)); + assert.ok(!out.includes("[sound:")); + assert.equal(out, out.trim()); + assert.ok(!/\s{2}/.test(out)); + }), + ); + }); + + test("scriptOf names the script of most letters; languageNamed only answers tags it knows", () => { + fc.assert( + fc.property(fc.string({ unit: "grapheme", maxLength: 20 }), (s) => { + const iso = P.scriptOf(s); + assert.ok(iso === null || /^[A-Z][a-z]{3}$/.test(iso)); + const named = P.languageNamed(s, langs); + assert.ok(named === null || langs[named.split("-")[0].toLowerCase()] !== undefined, named); + }), + ); + }); +}); diff --git a/test/unit/properties/casing.test.mjs b/test/unit/properties/casing.test.mjs new file mode 100644 index 0000000..d0b8edf --- /dev/null +++ b/test/unit/properties/casing.test.mjs @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +// SPDX-License-Identifier: Apache-2.0 + +// Properties of how a swapped word is written (lib/casing.js, slice 17) on generated words +// in many scripts. +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { fc } from "../../helpers/properties.mjs"; +import { requireExt } from "../../helpers/load-script.mjs"; + +const C = requireExt("lib/casing.js"); + +const word = fc.oneof( + fc.string({ unit: "grapheme", maxLength: 12 }), + fc.stringMatching(/^[a-zçñäöüß]{1,10}$/), + fc.stringMatching(/^[а-яё]{1,10}$/), + fc.stringMatching(/^[α-ωάέήίόύώϊϋΐΰ]{1,10}$/), + fc.stringMatching(/^[ა-ჰ]{1,8}$/), + fc.stringMatching(/^[ぁ-ゖ一-龯]{1,6}$/), + fc.constantFrom("ijs", "džungla", "Ljubav", "istanbul", "ıslak", "ἄνθρωπος", "Monday", "COVID", "iPhone"), +); +const lang = fc.constantFrom("en", "es", "de", "tr", "az", "lt", "el", "ru", "ka", "ja", "nl", "sr-Latn", "hr", "xx"); +const shape = fc.constantFrom("lower", "title", "upper", "mixed", undefined); + +describe("display", () => { + test("a capital is only passed on at a sentence start or in a shouted line; otherwise the word shows as stored", () => { + fc.assert( + fc.property(word, lang, shape, fc.boolean(), fc.boolean(), (native, l, s, sentenceStart, shouting) => { + const out = C.display({ shape: s, sentenceStart, shouting, native, lang: l }); + const passesOn = (s === "upper" && shouting) || (s === "title" && sentenceStart); + if (!passesOn || !C.hasCase(native)) assert.equal(out, native); + else assert.equal(out, s === "upper" ? C.upper(native, l) : C.title(native, l)); + }), + ); + }); + + test("title case changes only the first letter (and its marks), never the rest of the word", () => { + fc.assert( + fc.property(word, lang, (native, l) => { + fc.pre(native.length > 0 && !(l === "nl" && native.startsWith("ij"))); + const out = C.title(native, l); + const first = native.match(/^\P{M}\p{M}*/u)?.[0] ?? native.charAt(0); + assert.ok(out.endsWith(native.slice(first.length)), JSON.stringify({ native, out })); + }), + ); + }); + + test("upper case is idempotent, and in Greek drops accents and breathings but keeps the word's letters", () => { + fc.assert( + fc.property(word, lang, (native, l) => { + const once = C.upper(native, l); + assert.equal(C.upper(once, l), once); + }), + ); + fc.assert( + fc.property(fc.stringMatching(/^[α-ωάέήίόύώϊϋΐΰἀ-ἇ\u1F70-\u1F7D]{1,10}$/), (native) => { + const out = C.greekUpper(native).normalize("NFD"); + assert.ok(!/[̀́̓̔͂]/u.test(out), out); + const letters = (s) => s.normalize("NFD").replace(/\p{M}/gu, ""); + assert.equal(letters(out), letters(native).toUpperCase()); + }), + ); + }); + + test("scripts without case in everyday writing, Georgian included, are never changed", () => { + fc.assert( + fc.property(fc.stringMatching(/^[ა-ჰぁ-ゖ一-龯ก-๛]{1,8}$/), lang, shape, (native, l, s) => { + assert.equal(C.hasCase(native), false); + assert.equal(C.display({ shape: s, sentenceStart: true, shouting: true, native, lang: l }), native); + }), + ); + }); + + test("Turkish and Azerbaijani capitalise i as İ; every other language as I", () => { + fc.assert( + fc.property(fc.stringMatching(/^i[a-z]{0,8}$/), lang, (native, l) => { + const dotted = ["tr", "az"].includes(l); + assert.equal(C.title(native, l)[0], dotted ? "İ" : "I"); + assert.equal(C.upper(native, l)[0], dotted ? "İ" : "I"); + }), + ); + }); +}); diff --git a/test/unit/properties/matcher.test.mjs b/test/unit/properties/matcher.test.mjs new file mode 100644 index 0000000..021644e --- /dev/null +++ b/test/unit/properties/matcher.test.mjs @@ -0,0 +1,168 @@ +// SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +// SPDX-License-Identifier: Apache-2.0 + +// Properties of the matcher (lib/matcher.js, slice 14) on generated vocabularies and +// pages. Assertion mode is on under the tests, so every scan here also runs the matcher's +// own invariant checks. +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fc } from "../../helpers/properties.mjs"; +import { ROOT, requireExt } from "../../helpers/load-script.mjs"; + +const T = requireExt("lib/text.js"); +const M = requireExt("lib/matcher.js"); +const TABLE = JSON.parse(fs.readFileSync(path.join(ROOT, "spec/lang/_generic/boundaries.json"), "utf8")); +const rules = (base) => T.rulesFor(TABLE, base); +const BASES = ["en", "es", "fr", "it", "de", "tr", "ja", "zh", "th"]; + +let nextId = 1; +const word = (native, lang, base, forms) => ({ id: nextId++, lang, native, base_lang: base, gloss: forms[0] ?? null, forms, status: "active" }); + +// Vocabulary words use the letters a–m only and filler words n–z only, so a filler word is +// never a form and the expected matches are known. +const vocabWord = fc.stringMatching(/^[a-m]{2,8}$/); +const fillerWord = fc.stringMatching(/^[n-z]{2,8}$/); +// Anything a page or a word list might hold: any text, the glue and punctuation the +// tokenizer treats specially, apostrophes, hyphens and invisible characters. +const tricky = fc.oneof( + fc.string({ unit: "grapheme", maxLength: 12 }), + fc.constantFrom("c++", "c#", ".net", "u.s.", "I", "I-95", "l'eau", "dell'acqua", "it's", "house's", "ice-cream", "ice cream", "¿qué?", "Ｃafé", "猫", "ภาษา", "\u00AD", "\u200B", "\u2029", "?", "!", " ", "\n", "'s"), +); + +describe("scan on any page and any vocabulary", () => { + test("never throws, and every match covers its own text, in order, without overlaps", () => { + fc.assert( + fc.property( + fc.constantFrom(...BASES), + fc.array(fc.array(tricky, { minLength: 1, maxLength: 3 }), { maxLength: 12 }), + fc.array(tricky, { maxLength: 30 }), + fc.string({ maxLength: 8 }), + fc.string({ maxLength: 8 }), + (base, formLists, pieces, before, after) => { + const words = formLists.map((forms, i) => word(`nat${i}`, "xx", base, forms)); + const index = M.buildIndex(words, { base, rules: rules(base) }); + // Pages built partly from the forms themselves, so matches actually happen. + const text = pieces.map((p, i) => (i % 3 === 0 && formLists.length ? formLists[i % formLists.length][0] : p)).join(" "); + const { matches, tokenCount } = M.scan(text, { base, before, after, count: true }, index); + assert.ok(Number.isInteger(tokenCount) && tokenCount >= 0); + let last = 0; + for (const m of matches) { + assert.ok(m.start >= last && m.start < m.end && m.end <= text.length); + assert.equal(text.slice(m.start, m.end), m.surface); + assert.ok(m.entry.candidates.every((c) => words.includes(c.word))); + last = m.end; + } + }, + ), + ); + }); + + test("finds exactly the vocabulary's words in a page of words separated by spaces", () => { + fc.assert( + fc.property( + fc.uniqueArray(vocabWord, { minLength: 1, maxLength: 8 }), + fc.array(fc.tuple(fc.boolean(), fc.nat(), fillerWord), { minLength: 1, maxLength: 25 }), + fc.constantFrom("en", "es", "fr", "de"), + (vocab, slots, base) => { + const words = vocab.map((v, i) => word(`n${i}`, "xx", base, [v])); + const tokens = slots.map(([isWord, k, filler]) => (isWord ? vocab[k % vocab.length] : filler)); + const text = tokens.join(" "); + const { matches, tokenCount } = M.scan(text, { base }, M.buildIndex(words, { base, rules: rules(base) })); + assert.equal(tokenCount, tokens.length); + const want = []; + let at = 0; + for (const t of tokens) { + if (vocab.includes(t)) want.push({ start: at, surface: t, native: `n${vocab.indexOf(t)}` }); + at += t.length + 1; + } + assert.deepEqual( + matches.map((m) => ({ start: m.start, surface: m.surface, native: m.entry.candidates[0].word.native })), + want, + ); + }, + ), + ); + }); + + test("case and the page's own capitals don't change what is found", () => { + fc.assert( + fc.property(fc.uniqueArray(vocabWord, { minLength: 1, maxLength: 6 }), fc.array(fc.oneof(vocabWord, fillerWord), { minLength: 1, maxLength: 20 }), (vocab, tokens) => { + const index = M.buildIndex(vocab.map((v, i) => word(`n${i}`, "xx", "en", [v])), { base: "en", rules: rules("en") }); + const at = (text) => M.scan(text, { base: "en" }, index).matches.map((m) => [m.start, m.end, m.key]); + const text = tokens.join(" "); + assert.deepEqual(at(text.toUpperCase()), at(text)); + assert.deepEqual(at(tokens.map((t) => t[0].toUpperCase() + t.slice(1)).join(" ")), at(text)); + }), + ); + }); + + test("a phrase wins over its first word (leftmost-longest), and both are found apart", () => { + fc.assert( + fc.property(vocabWord, vocabWord, fillerWord, (a, b, filler) => { + fc.pre(a !== b); + const words = [word("one", "xx", "en", [a]), word("two", "xx", "en", [`${a} ${b}`])]; + const index = M.buildIndex(words, { base: "en", rules: rules("en") }); + const found = (text) => M.scan(text, { base: "en" }, index).matches.map((m) => [m.surface, m.entry.candidates[0].word.native]); + assert.deepEqual(found(`${filler} ${a} ${b} ${filler}`), [[`${a} ${b}`, "two"]]); + assert.deepEqual(found(`${a} ${filler} ${b}`), [[a, "one"]]); + // Across a block boundary (U+2029 in the context) a phrase is never matched. + assert.deepEqual(M.scan(a, { base: "en", after: `\u2029${b}` }, index).matches.map((m) => m.surface), [a]); + }), + ); + }); + + test("a word that runs on past the edge of the text is never matched", () => { + fc.assert( + fc.property(vocabWord, fc.stringMatching(/^[a-z]{1,4}$/), fc.boolean(), (v, extra, atEnd) => { + const index = M.buildIndex([word("n", "xx", "en", [v])], { base: "en", rules: rules("en") }); + const ctx = atEnd ? { base: "en", after: extra } : { base: "en", before: extra }; + assert.deepEqual(M.scan(v, ctx, index).matches, []); + }), + ); + }); + + test("words learned in a base, or whose meaning is in another base, never enter its index", () => { + fc.assert( + fc.property(vocabWord, fc.constantFrom("en", "en-GB", "es", "pt-BR", "zh-Hant", "zh-TW"), fc.constantFrom("en", "es", "pt", "zh", "zh-Hans"), (v, wordBase, base) => { + const words = [word("n", "xx", wordBase, [v]), word("own", base, base, [v])]; + const index = M.buildIndex(words, { base, rules: rules(base) }); + const natives = [...index.entries.values()].flatMap((e) => e.candidates.map((c) => c.word.native)); + assert.deepEqual(natives, M.sameBase(wordBase, base) ? ["n"] : []); + }), + ); + }); +}); + +describe("language tags", () => { + const tag = fc.oneof( + fc.constantFrom("en", "EN", "en-US", "en_GB", "pt-BR", "pt", "zh", "zh-Hans", "zh-Hant", "zh-TW", "zh-HK", "zh-CN", "iw", "he", "in", "id", "sr-Latn", "sr-Cyrl", "sr", "cmn"), + fc.stringMatching(/^[a-z]{2,3}(-[A-Za-z0-9]{2,8}){0,2}$/), + ); + + test("sameBase is reflexive and symmetric; baseOf only returns a base of the same language", () => { + fc.assert( + fc.property(tag, tag, fc.array(tag, { maxLength: 4 }), (a, b, bases) => { + assert.equal(M.sameBase(a, a), true); + assert.equal(M.sameBase(a, b), M.sameBase(b, a)); + const found = M.baseOf(a, bases); + if (found === null) assert.ok(!bases.some((x) => M.sameBase(a, x))); + else assert.ok(bases.includes(found) && M.sameBase(a, found)); + }), + ); + }); + + test("an empty tag is no base", () => { + fc.assert(fc.property(fc.constantFrom("", null, undefined), tag, (none, t) => assert.equal(M.sameBase(none, t), false))); + }); +}); + +describe("assertion mode", () => { + test("is on under the tests, and catches an index that breaks the matcher's promises", () => { + assert.equal(globalThis.__KOTIKO_ASSERT__, true, "npm test loads test/helpers/assert-mode.mjs"); + const index = M.buildIndex([word("n", "xx", "en", ["gato"])], { base: "en", rules: rules("en") }); + index.entries.get("gato").candidates.length = 0; + assert.throws(() => M.scan("el gato", { base: "en" }, index), /KotikoMatcher invariant: a match has no entry/); + }); +}); diff --git a/test/unit/properties/policy.test.mjs b/test/unit/properties/policy.test.mjs new file mode 100644 index 0000000..a26659e --- /dev/null +++ b/test/unit/properties/policy.test.mjs @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +// SPDX-License-Identifier: Apache-2.0 + +// Properties of the privacy policy's Markdown reader (lib/policy.js, slice 28 §2): it +// loses no text, and nothing in the file ever becomes HTML. +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { JSDOM } from "jsdom"; +import { fc } from "../../helpers/properties.mjs"; +import { requireExt } from "../../helpers/load-script.mjs"; + +const P = requireExt("lib/policy.js"); +const piece = fc.oneof( + fc.string({ maxLength: 12 }), + fc.stringMatching(/^\*\*[a-z ]{1,8}\*\*$/), + fc.stringMatching(/^`[a-z<>&"]{1,8}`$/), + fc.stringMatching(/^$/), + fc.constantFrom("", "", "", "", "**", "`", "&", "\n", "\n\n", "# ", "## ", "- ", ""), +); + +// The spans written back as Markdown. +const back = (spans) => spans.map((s) => (s.text !== undefined ? s.text : s.strong !== undefined ? `**${s.strong}**` : s.code !== undefined ? `\`${s.code}\`` : `<${s.link}>`)).join(""); + +describe("policy Markdown", () => { + test("inline loses no character: its spans written back are the line", () => { + fc.assert(fc.property(fc.array(piece, { maxLength: 10 }), (parts) => { + const line = parts.join(""); + assert.equal(back(P.inline(line)), line); + })); + }); + + test("only https links become links", () => { + fc.assert(fc.property(fc.array(piece, { maxLength: 10 }), (parts) => { + for (const s of P.inline(parts.join(""))) if (s.link !== undefined) assert.match(s.link, /^https:\/\/[^\s<>]+$/); + })); + }); + + test("rendering any file makes only the policy's own elements, and no script, handler or foreign link", () => { + const { window } = new JSDOM("
"); + const doc = window.document; + fc.assert(fc.property(fc.array(piece, { maxLength: 30 }), (parts) => { + const root = doc.createElement("main"); + P.render(doc, root, P.parse(parts.join(""))); + for (const el of root.querySelectorAll("*")) { + assert.ok(["h1", "h2", "p", "ul", "li", "strong", "code", "a"].includes(el.localName), el.localName); + for (const a of el.attributes) assert.ok(["href", "target", "rel"].includes(a.name), a.name); + if (el.localName === "a") assert.match(el.getAttribute("href"), /^https:\/\//); + } + })); + }); +}); diff --git a/test/unit/properties/url.test.mjs b/test/unit/properties/url.test.mjs new file mode 100644 index 0000000..b0bfa27 --- /dev/null +++ b/test/unit/properties/url.test.mjs @@ -0,0 +1,127 @@ +// SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +// SPDX-License-Identifier: Apache-2.0 + +// Properties of the server address check (lib/url.js, slice 26) on generated input: what +// a learner might type or paste, and what an attacker might plant in storage. +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { fc } from "../../helpers/properties.mjs"; +import { requireExt } from "../../helpers/load-script.mjs"; + +const U = requireExt("lib/url.js"); + +const octet = fc.integer({ min: 0, max: 255 }); +const ipv4 = fc.tuple(octet, octet, octet, octet).map((p) => p.join(".")); +const port = fc.integer({ min: 1, max: 65535 }); +const label = fc.stringMatching(/^[a-z0-9]([a-z0-9-]{0,10}[a-z0-9])?$/); +const domain = fc.array(label, { minLength: 1, maxLength: 3 }).chain((ls) => fc.constantFrom("com", "org", "net", "dev", "example").map((tld) => [...ls, tld].join("."))); +const host = fc.oneof(ipv4, domain, fc.constantFrom("localhost", "kotiko.local", "box.localhost", "[::1]", "[fd7a:115c:a1e0::1]", "[2001:db8::1]", "pi.tail1234.ts.net", "100.100.1.2")); +const pathPart = fc.array(fc.stringMatching(/^[a-z0-9_-]{1,8}$/), { maxLength: 3 }).map((ps) => ps.map((p) => `/${p}`).join("")); +// Addresses as typed: with or without a scheme, port, path and trailing slashes, padded. +const typed = fc + .tuple(fc.constantFrom("", "http://", "https://", "HTTP://", "HtTpS://"), host, fc.option(port), pathPart, fc.constantFrom("", "/", "//"), fc.constantFrom("", " ", "\t")) + .map(([scheme, h, p, pa, slash, pad]) => `${pad}${scheme}${h}${p === null ? "" : `:${p}`}${pa}${slash}${pad}`); +const anything = fc.oneof(typed, fc.string(), fc.string({ unit: "grapheme" }), fc.webUrl({ withQueryParameters: true, withFragments: true }), fc.anything()); + +describe("normalizeServerUrl", () => { + test("never throws, and answers either a clean http(s) address or the one error code", () => { + fc.assert( + fc.property(anything, (input) => { + const r = U.normalizeServerUrl(input); + if (!r.ok) { + assert.equal(r.code, "server_address_invalid"); + assert.equal(typeof r.hint, "string"); + assert.ok(r.hint.length > 0); + return; + } + const u = new URL(r.url); + assert.match(u.protocol, /^https?:$/); + assert.equal(u.username + u.password + u.search + u.hash, ""); + assert.ok(!r.url.endsWith("/"), "no trailing slash"); + assert.ok(u.hostname.length > 0); + }), + ); + }); + + test("is idempotent: a normalised address normalises to itself", () => { + fc.assert( + fc.property(anything, (input) => { + const r = U.normalizeServerUrl(input); + if (r.ok) assert.deepEqual(U.normalizeServerUrl(r.url), r); + }), + ); + }); + + test("keeps the host, port and path the learner typed", () => { + fc.assert( + fc.property(host, fc.option(port), pathPart, (h, p, pa) => { + const r = U.normalizeServerUrl(`${h}${p === null ? "" : `:${p}`}${pa}/`); + assert.equal(r.ok, true); + const u = new URL(r.url); + assert.equal(u.hostname, new URL(`http://${h}`).hostname); + assert.equal(u.pathname === "/" ? "" : u.pathname, pa); + if (p !== null && !(p === 80 && u.protocol === "http:") && !(p === 443 && u.protocol === "https:")) assert.equal(u.port, String(p)); + }), + ); + }); + + test("an address without a scheme gets http only for this machine, IP literals, .local and Tailscale", () => { + fc.assert( + fc.property(host, (h) => { + const r = U.normalizeServerUrl(h); + assert.equal(r.ok, true); + assert.equal(r.url.startsWith("http://"), U.isLocalHost(h)); + }), + ); + fc.assert( + fc.property(domain, (d) => { + assert.equal(U.normalizeServerUrl(d).url, `https://${d}`); + }), + ); + }); + + test("never accepts a user name or password, however it is written", () => { + fc.assert( + fc.property(fc.constantFrom("", "http://", "https://"), fc.stringMatching(/^[a-z0-9]{1,8}$/), fc.option(fc.stringMatching(/^[a-z0-9]{1,8}$/)), host, (scheme, user, pass, h) => { + const r = U.normalizeServerUrl(`${scheme}${user}${pass === null ? "" : `:${pass}`}@${h}`); + assert.equal(r.ok, false); + }), + ); + }); + + test("refuses a query or fragment, and any scheme but http and https", () => { + fc.assert( + fc.property(typed, fc.constantFrom("?", "#", "?a=1", "#x"), (t, tail) => assert.equal(U.normalizeServerUrl(t.trim() + tail).ok, false)), + ); + fc.assert( + fc.property(fc.constantFrom("ftp", "file", "javascript", "data", "ws", "chrome-extension"), host, (scheme, h) => assert.equal(U.normalizeServerUrl(`${scheme}://${h}`).ok, false)), + ); + }); +}); + +describe("sendsInClear (slice 28 §5)", () => { + test("only plain http to another machine, outside Tailscale, crosses the network in clear", () => { + fc.assert( + fc.property(anything, (input) => { + const clear = U.sendsInClear(input); + const r = U.normalizeServerUrl(input); + if (!r.ok || r.url.startsWith("https://")) return assert.equal(clear, false); + const h = new URL(r.url).hostname; + const local = h === "localhost" || h.endsWith(".localhost") || h.endsWith(".ts.net") || /^127\./.test(h) || h === "[::1]" || h.startsWith("[fd7a:115c:a1e0:"); + const tailscale = /^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./.test(h); + assert.equal(clear, !(local || tailscale)); + }), + ); + }); + + test("loopback and Tailscale IPv4 addresses never warn; the rest of private space does", () => { + fc.assert( + fc.property(octet, octet, octet, port, (b, c, d, p) => { + assert.equal(U.sendsInClear(`http://127.${b}.${c}.${d}:${p}`), false); + assert.equal(U.sendsInClear(`http://100.${64 + (b % 64)}.${c}.${d}:${p}`), false); + assert.equal(U.sendsInClear(`http://10.${b}.${c}.${d}:${p}`), true); + assert.equal(U.sendsInClear(`https://10.${b}.${c}.${d}:${p}`), false); + }), + ); + }); +}); diff --git a/test/unit/properties/validate-words.test.mjs b/test/unit/properties/validate-words.test.mjs new file mode 100644 index 0000000..cad88ab --- /dev/null +++ b/test/unit/properties/validate-words.test.mjs @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +// SPDX-License-Identifier: Apache-2.0 + +// Properties of the check every server word list goes through before it is cached or put +// on a page (lib/validate-words.js, slice 26), on generated responses: what a broken or +// hostile server might send. +import { describe, test } from "node:test"; +import assert from "node:assert/strict"; +import { fc } from "../../helpers/properties.mjs"; +import { requireExt } from "../../helpers/load-script.mjs"; + +const V = requireExt("lib/validate-words.js"); +const CODES = new Set(["server_key_rejected", "server_address_invalid", "server_unreachable", "internal", "not_kotiko_server"]); + +const text = (max) => fc.oneof(fc.string({ maxLength: max }), fc.string({ unit: "grapheme", maxLength: max }), fc.constantFrom("", " ", "a\nb", "\u2028", "x".repeat(max + 1))); +const form = fc.oneof(text(45), fc.record({ text: text(45), enabled: fc.boolean() }, { requiredKeys: [] }), fc.anything()); +// Words close to valid, so most generated ones are kept or fixed rather than all dropped. +const serverWord = fc.record( + { + id: fc.oneof(fc.integer(), fc.uuid(), fc.constant(""), fc.constant(null)), + lang: fc.oneof(fc.constantFrom("es", "ja", "ru", "pt-BR"), text(40)), + base_lang: fc.oneof(fc.constantFrom("en", "es"), fc.constant(undefined), fc.integer()), + native: text(70), + forms: fc.oneof(fc.array(form, { maxLength: 14 }), fc.constant(null), fc.string()), + english: fc.oneof(text(45), fc.constant(undefined)), // base-neutral-ok: a 0.2 server's words + pronunciation: fc.oneof(text(100), fc.constant(null), fc.integer()), + pronunciation_careful: fc.oneof(text(100), fc.constant(null)), + reading: fc.oneof(text(70), fc.constant(null)), + native_vocalized: fc.oneof(text(20), fc.constant(null)), + pronunciation_source: fc.oneof(fc.constantFrom("model", "user", "wiktionary", "elsewhere"), fc.constant(null)), + verification: fc.oneof(fc.constant(null), fc.record({ pronunciation: fc.record({ status: fc.constantFrom("verified", "corrected", "differs", "no_data", "maybe"), source: text(45), stressed: text(20) }, { requiredKeys: [] }) }, { requiredKeys: [] }), fc.anything()), + }, + { requiredKeys: ["id", "lang", "native"] }, +); +const list = fc.array(fc.oneof({ weight: 6, arbitrary: serverWord }, { weight: 1, arbitrary: fc.anything() }), { maxLength: 20 }); + +describe("filterWords", () => { + test("every word is kept or counted, and every kept word passes the checks again unchanged", () => { + fc.assert( + fc.property(list, (words) => { + const r = V.filterWords(words); + assert.equal(r.words.length + r.dropped, words.length); + for (const w of r.words) { + assert.equal(V.checkWord(w), null); + assert.equal(V.extraFixes(w), null); + assert.ok((w.forms ?? []).length <= V.LIMITS.maxForms); + } + // Idempotent: a filtered list goes through again untouched. + const again = V.filterWords(r.words); + assert.deepEqual(again.words, r.words); + assert.equal(again.dropped, 0); + assert.equal(again.droppedForms, 0); + }), + ); + }); + + test("a clean word is passed on as the very same object", () => { + fc.assert( + fc.property(fc.uuid(), fc.constantFrom("es", "ja"), fc.stringMatching(/^[a-z]{1,20}$/), fc.array(fc.stringMatching(/^[a-z]{1,20}$/), { minLength: 1, maxLength: 10 }), (id, lang, native, forms) => { + const w = { id, lang, native, forms }; + assert.equal(V.filterWords([w]).words[0], w); + }), + ); + }); +}); + +describe("validateWordsResponse", () => { + test("any response, however broken, gives a word list or one of the known errors", () => { + const response = fc.record( + { + status: fc.oneof(fc.constantFrom(200, 401, 404, 421, 500, 502, 503, 504), fc.integer({ min: 100, max: 599 }), fc.anything()), + contentType: fc.oneof(fc.constantFrom("application/json", "application/json; charset=utf-8", "text/html", ""), fc.anything()), + body: fc.oneof(list.map((words) => JSON.stringify({ words })), fc.jsonValue().map((v) => JSON.stringify(v)), fc.string(), fc.anything()), + }, + { requiredKeys: [] }, + ); + fc.assert( + fc.property(fc.oneof(response, fc.anything()), (raw) => { + const r = V.validateWordsResponse(raw); + if (r.ok) return assert.ok(Array.isArray(r.words)); + assert.ok(CODES.has(r.code), r.code); + assert.equal(typeof r.message, "string"); + if (r.details.error !== undefined) assert.ok(r.details.error.length <= 200); + }), + ); + }); +}); From 0f4bb9c432f47e77f497ceb472f80260a7bb032e Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Tue, 6 Oct 2026 16:01:53 -0400 Subject: [PATCH 05/12] =?UTF-8?q?fix(server):=20words=20with=20=C3=85=20or?= =?UTF-8?q?=20=C5=85=20no=20longer=20break=20a=20lookup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit prepare_input/1 replaced line breaks with ~r/[\t\n\r\v\f]/ in byte mode, where \v is any vertical space, the byte 0x85 (NEL) included. That byte is inside the UTF-8 of Å (C3 85), Ņ (C5 85) and other letters, so the text became invalid and the next step raised: POST /api/v1/words with "Åland" answered 500, and the bot replied with an internal error. The pattern is now Unicode-aware and names the vertical tab itself (\x0B), as \v means in the extension's JavaScript. A StreamData property on prepare_input/1 found it; the shared input fixture gains the case, so both runtimes check it, and the server has its own regression test. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ayla Croft --- server/lib/kotiko/word_spec.ex | 5 ++++- server/test/kotiko/word_spec_test.exs | 10 ++++++++++ spec/fixtures/input.json | 8 ++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/server/lib/kotiko/word_spec.ex b/server/lib/kotiko/word_spec.ex index de1516b..eff7259 100644 --- a/server/lib/kotiko/word_spec.ex +++ b/server/lib/kotiko/word_spec.ex @@ -32,7 +32,10 @@ defmodule Kotiko.WordSpec do t = text |> :unicode.characters_to_nfc_binary() - |> String.replace(~r/[\t\n\r\v\f]/, " ") + # Unicode mode: in byte mode \v also matched the byte 0x85, which is inside the + # UTF-8 of letters such as Å and Ņ, and broke them. \x0B is the vertical tab + # alone, as \v is in the extension's JavaScript. + |> String.replace(~r/[\t\n\r\x0B\f]/u, " ") |> String.replace(~r/\p{Cc}/u, "") |> String.trim() diff --git a/server/test/kotiko/word_spec_test.exs b/server/test/kotiko/word_spec_test.exs index c022920..72b1575 100644 --- a/server/test/kotiko/word_spec_test.exs +++ b/server/test/kotiko/word_spec_test.exs @@ -151,6 +151,16 @@ defmodule Kotiko.WordSpecTest do assert {:ok, _} = WordSpec.prepare_input(String.duplicate("a", 200)) end + # Found by the prepare_input property: a byte-mode \v matched 0x85, a byte inside the + # UTF-8 of Å (C3 85) and Ņ (C5 85), so the text became invalid and the next step raised. + test "letters whose UTF-8 holds the byte 0x85 or 0xA0 pass through unchanged" do + for text <- ["Åland", "Ņ", "Рим", "à la carte", "\u{4D1C5}"] do + assert WordSpec.prepare_input(text) == {:ok, text} + end + + assert WordSpec.prepare_input("a\vb\fc\u0085d") == {:ok, "a b cd"} + end + test "no tag in lang-tags.json crashes the pipeline (F17)" do tags = @fixtures diff --git a/spec/fixtures/input.json b/spec/fixtures/input.json index dd8dfcc..6c7f338 100644 --- a/spec/fixtures/input.json +++ b/spec/fixtures/input.json @@ -17,6 +17,14 @@ "input": "a\u0000b\u0007c", "text": "abc" }, + { + "input": "Åland Ņ", + "text": "Åland Ņ" + }, + { + "input": "a\u000bb\u0085c", + "text": "a bc" + }, { "input": "", "error": "empty_input" From eba134380bbe07cddadf42f4147d5958a6a2e114 Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Tue, 6 Oct 2026 16:02:03 -0400 Subject: [PATCH 06/12] fix(server): a repeated query parameter is a 400, not a 500 GET /api/v1/words?lang[]=ru (and base, status, limit) and GET /api/v1/export?include[]=pending reach the router as a list or a map, which String.split/3 and Integer.parse/1 can't take, so the server answered 500. A non-string value is now refused with invalid_request, naming the field. Found by the new StreamData properties on the v1 router: any word body for POST /words, /words/batch and PATCH /words/:id, any profile and job control body, and any query string for /words, /export and the legacy list are answered below 500, with slice 25's error shape, and a batch answers every word by its index. test/support/gen.ex holds the shared generators. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ayla Croft --- server/lib/kotiko/router_v1.ex | 64 ++++---- server/test/kotiko/router_property_test.exs | 159 ++++++++++++++++++++ server/test/support/gen.ex | 110 ++++++++++++++ 3 files changed, 307 insertions(+), 26 deletions(-) create mode 100644 server/test/kotiko/router_property_test.exs create mode 100644 server/test/support/gen.ex diff --git a/server/lib/kotiko/router_v1.ex b/server/lib/kotiko/router_v1.ex index 079074e..859c346 100644 --- a/server/lib/kotiko/router_v1.ex +++ b/server/lib/kotiko/router_v1.ex @@ -64,33 +64,11 @@ defmodule Kotiko.RouterV1 do get "/export" do conn = fetch_query_params(conn) q = conn.query_params - pending? = "pending" in String.split(q["include"] || "", ",", trim: true) - conn = - if q["download"] == "1" do - name = "kotiko-backup-#{Date.to_iso8601(Date.utc_today())}.json" - put_resp_header(conn, "content-disposition", ~s(attachment; filename="#{name}")) - else - conn - end - - conn = - conn - |> put_resp_content_type("application/json") - |> put_resp_header("cache-control", "no-store") - |> send_chunked(200) - - {_count, conn} = - Backup.reduce( - conn, - fn data, conn -> - {:ok, conn} = chunk(conn, data) - conn - end, - include_pending: pending? - ) - - conn + case q["include"] || "" do + include when is_binary(include) -> export(conn, q, "pending" in String.split(include, ",")) + _ -> invalid(conn, %{field: "include"}) + end end # Deletes every word, pending lookup and tombstone; only with the confirmation in the @@ -411,6 +389,35 @@ defmodule Kotiko.RouterV1 do error(conn, 404, "not_found") end + # The backup, streamed in chunks as Kotiko.Backup reads it. + defp export(conn, q, pending?) do + conn = + if q["download"] == "1" do + name = "kotiko-backup-#{Date.to_iso8601(Date.utc_today())}.json" + put_resp_header(conn, "content-disposition", ~s(attachment; filename="#{name}")) + else + conn + end + + conn = + conn + |> put_resp_content_type("application/json") + |> put_resp_header("cache-control", "no-store") + |> send_chunked(200) + + {_count, conn} = + Backup.reduce( + conn, + fn data, conn -> + {:ok, conn} = chunk(conn, data) + conn + end, + include_pending: pending? + ) + + conn + end + # ── parameters ─────────────────────────────────────────────────────── # Pending words (Telegram lookups not yet added) are server-local: never shown here. @@ -427,12 +434,16 @@ defmodule Kotiko.RouterV1 do defp tags(nil, _normalize, _field), do: {:ok, nil} defp tags("", _normalize, _field), do: {:ok, nil} + # A repeated or nested parameter (lang[]=ru) arrives as a list or a map: refused. + defp tags(value, _normalize, field) when not is_binary(value), do: {:error, field} + defp tags(value, normalize, field) do tags = value |> String.split(",", trim: true) |> Enum.map(normalize) if Enum.all?(tags, &is_binary/1), do: {:ok, tags}, else: {:error, field} end defp statuses(nil), do: {:ok, nil} + defp statuses(value) when not is_binary(value), do: {:error, "status"} defp statuses(value) do statuses = String.split(value, ",", trim: true) @@ -443,6 +454,7 @@ defmodule Kotiko.RouterV1 do end defp limit(nil), do: {:ok, @max_limit} + defp limit(value) when not is_binary(value), do: {:error, "limit"} defp limit(value) do case Integer.parse(value) do diff --git a/server/test/kotiko/router_property_test.exs b/server/test/kotiko/router_property_test.exs new file mode 100644 index 0000000..2e0c430 --- /dev/null +++ b/server/test/kotiko/router_property_test.exs @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +# SPDX-License-Identifier: Apache-2.0 + +defmodule Kotiko.RouterPropertyTest do + # Dynamic analysis (OpenSSF dynamic_analysis): requests with generated bodies and query + # strings, as a buggy or hostile client could send them, through the whole router + # (token check, body parsing, /api/v1). The server answers each one; a mistake in the + # request is a 4xx with slice 25's error shape, never a 500. + use Kotiko.ConnCase, async: false + use ExUnitProperties + import ExUnit.CaptureLog + alias Kotiko.{Gen, LLMStub} + + # Each run touches the database, so fewer runs than a pure property. + @runs 60 + + setup do + LLMStub.stub(fn _, conn -> LLMStub.answer(conn, LLMStub.words([LLMStub.word()], "add")) end) + :ok + end + + defp send_request(method, path, body) do + opts = if body == nil, do: [], else: [body: body] + {conn, _log} = with_log(fn -> request(method, path, auth(), opts) end) + conn + end + + defp assert_answered(conn) do + assert conn.status in 200..499, "#{conn.status}: #{conn.resp_body}" + + if conn.status >= 400 do + assert %{"error" => %{"code" => code, "message" => message}} = json_body(conn) + assert is_binary(code) and is_binary(message) + end + end + + defp request_id do + frequency([ + {3, constant(nil)}, + {3, map(constant(nil), fn _ -> Ecto.UUID.generate() end)}, + {1, Gen.scalar()} + ]) + end + + property "POST /api/v1/words: any word object or body is answered without a 500" do + check all( + body <- + one_of([ + fixed_map(%{"word" => Gen.word(), "client_request_id" => request_id()}), + Gen.object( + ~w(word text base_langs hint_lang preview client_request_id), + Gen.json() + ) + ]), + max_runs: @runs + ) do + assert_answered(send_request("POST", "/api/v1/words", body)) + end + end + + property "POST /api/v1/words/batch: any list of words is answered per word" do + check all( + words <- list_of(frequency([{4, Gen.word()}, {1, Gen.json()}]), max_length: 4), + max_runs: @runs + ) do + conn = send_request("POST", "/api/v1/words/batch", %{"words" => words}) + assert_answered(conn) + + if conn.status == 200 do + %{"results" => results, "rejected" => rejected} = json_body(conn) + indexes = Enum.map(results ++ rejected, & &1["index"]) + # Every word is either saved (one result per base) or rejected, by its index. + assert Enum.uniq(indexes) |> Enum.sort() == Enum.to_list(0..(length(words) - 1)//1) + end + end + end + + property "PATCH /api/v1/words/:id: any patch body is applied or refused with a 4xx" do + word = word_fixture() + + check all( + body <- + Gen.object( + ~w(lang native sense romanization native_vocalized gloss forms + pronunciation pronunciation_careful pronunciation_source note + status if_updated_at), + Gen.json() + ), + max_runs: @runs + ) do + assert_answered(send_request("PATCH", "/api/v1/words/#{word.uuid}", body)) + end + end + + property "PUT /api/v1/profile and the job control: any body gets a 200 or a 400" do + check all( + body <- Gen.object(~w(base_langs locale action), Gen.json()), + path <- member_of(["/api/v1/profile", "/api/v1/jobs/pronunciation-refresh"]), + max_runs: @runs + ) do + method = if path == "/api/v1/profile", do: "PUT", else: "POST" + conn = send_request(method, path, body) + assert_answered(conn) + assert conn.status in [200, 400] + end + end + + property "GET /api/v1/words: any query string is answered without a 500" do + key = member_of(~w(lang base status limit include download)) + shape = member_of(["~s=~s", "~s[]=~s", "~s[x]=~s", "~s"]) + + check all( + params <- list_of(tuple({key, shape, Gen.text(max_length: 10)}), max_length: 4), + max_runs: @runs + ) do + query = + Enum.map_join(params, "&", fn {k, s, v} -> + :io_lib.format( + s, + [k, URI.encode_www_form(v)] |> Enum.take(if s == "~s", do: 1, else: 2) + ) + |> IO.iodata_to_binary() + end) + + for path <- ["/api/v1/words?" <> query, "/api/v1/export?" <> query, "/api/words?" <> query] do + assert_answered(send_request("GET", path, nil)) + end + end + end + + describe "regressions the properties found" do + # lang[]=ru reaches the router as a list; String.split/3 raised and the client got a 500. + test "a repeated or nested query parameter is a 400 naming it, not a 500" do + for {query, field} <- [ + {"lang[]=ru", "lang"}, + {"base[x]=en", "base"}, + {"status[]=active", "status"}, + {"limit[]=5", "limit"} + ] do + conn = send_request("GET", "/api/v1/words?" <> query, nil) + assert conn.status == 400, query + + assert %{"error" => %{"code" => "invalid_request", "details" => %{"field" => ^field}}} = + json_body(conn) + end + + conn = send_request("GET", "/api/v1/export?include[]=pending", nil) + assert conn.status == 400 + assert json_body(conn)["error"]["details"] == %{"field" => "include"} + end + + # Å is C3 85 in UTF-8; a byte-mode \v matched the 85 (see word_spec_test.exs). + test "a lookup of a word with Å reaches the model instead of failing with a 500" do + conn = send_request("POST", "/api/v1/words", %{"text" => "Åland", "base_langs" => ["en"]}) + assert conn.status == 200 + assert [_ | _] = LLMStub.requests() + end + end +end diff --git a/server/test/support/gen.ex b/server/test/support/gen.ex new file mode 100644 index 0000000..cac2ce2 --- /dev/null +++ b/server/test/support/gen.ex @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +# SPDX-License-Identifier: Apache-2.0 + +defmodule Kotiko.Gen do + @moduledoc """ + StreamData generators for the property tests: JSON values as a client could send them + (after Plug's JSON parser), and words that are almost right, so a property reaches the + checks past the first one instead of failing on a missing field every time. + """ + import StreamData + import ExUnitProperties, only: [gen: 2] + + @doc "Text with the awkward cases mixed in: control and invisible characters, other scripts." + def text(opts \\ []) do + max = Keyword.get(opts, :max_length, 30) + + one_of([ + string(:printable, max_length: max), + string(:utf8, max_length: max), + member_of(["", " ", "\t\n", "​", "да", "犬", "perro", "ı", "ς", "á", "\u0000x"]) + ]) + end + + @doc "A scalar JSON value." + def scalar do + one_of([ + constant(nil), + boolean(), + integer(), + float(), + text() + ]) + end + + @doc "Any JSON value: scalars, lists and objects with string keys, a few levels deep." + def json do + tree(scalar(), fn inner -> + one_of([ + list_of(inner, max_length: 4), + map_of(text(max_length: 12), inner, max_length: 4) + ]) + end) + end + + @doc "An object with any of `keys`, each with a value from `value`." + def object(keys, value), do: optional_map(Map.new(keys, &{&1, value})) + + @doc "A language tag: a real one, a malformed one or something that isn't a string." + def lang do + one_of([ + member_of(~w(ru ja es en de ar zh-Hant pt-BR sr-Latn tr el he hi ase EN es_419 x qqq-zz)), + text(max_length: 12), + scalar() + ]) + end + + # Words that pass the checks with a matching language, so most generated words get past + # the first check: [lang, native, gloss]. + @plausible [ + ["ru", "да", "yes"], + ["ja", "犬", "dog"], + ["es", "casa", "house"], + ["de", "Haus", "house"], + ["ar", "شكرا", "thanks"], + ["sv", "Åland", "Åland"], + ["zh-Hant", "謝謝", "thank you"], + ["el", "ναι", "yes"] + ] + + @doc """ + A word object from a client: mostly a plausible word with a few fields replaced by junk + or missing, sometimes junk throughout. + """ + def word do + gen all( + [lang, native, gloss] <- member_of(@plausible), + base <- frequency([{4, member_of(~w(en es fr pt-BR))}, {1, lang()}]), + lang <- frequency([{4, constant(lang)}, {1, lang()}]), + native <- frequency([{4, constant(native)}, {1, text()}, {1, scalar()}]), + gloss <- frequency([{4, constant(gloss)}, {1, text()}, {1, scalar()}]), + forms <- + frequency([ + {3, list_of(member_of(["perro", "dogs", "Yes", gloss, ""]), max_length: 5)}, + {1, list_of(one_of([text(), json()]), max_length: 5)}, + {1, json()} + ]), + extra <- + object( + ~w(sense romanization native_vocalized pronunciation pronunciation_careful + pronunciation_source note status origin source_text id), + frequency([{2, text()}, {1, json()}]) + ), + drop <- + frequency([ + {4, constant([])}, + {1, list_of(member_of(~w(lang base_lang native gloss forms)), max_length: 2)} + ]) + ) do + %{ + "lang" => lang, + "base_lang" => base, + "native" => native, + "gloss" => gloss, + "forms" => forms + } + |> Map.merge(extra) + |> Map.drop(drop) + end + end +end From bd016da4200b0d28093d8360304e46d844ef8f23 Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Tue, 6 Oct 2026 16:02:04 -0400 Subject: [PATCH 07/12] test(server): 90 % line coverage, enforced, and more properties mix test --cover now fails below 90 % of lines (test_coverage in mix.exs); the helpers in test/support are test code and don't count. Coverage goes from 87.8 % to 91.4 % with tests of what had none: the Telegram voice path end to end (download, transcription, lookup), the bot's commands, the janitor, pronunciation refresh and Wiktionary pass processes running a step when their timer fires and surviving one that raises, the lookup cache, word edge cases and the boot summary. Transcriber reads optional Req options (empty in production) so a test can stub it like Telegram and the model. StreamData properties, run by plain mix test: word validation and patches, input preparation and answer extraction (WordSpec), any .env parses to typed settings or lists its problems without echoing a secret (Config), language tags, text cleaning, UUIDv7, log redaction, and backup import and export round trips. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ayla Croft --- server/.gitignore | 2 + server/lib/kotiko/transcriber.ex | 7 +- server/mix.exs | 15 +- server/test/kotiko/background_jobs_test.exs | 153 ++++++++++++++ server/test/kotiko/backup_property_test.exs | 51 +++++ server/test/kotiko/bot_commands_test.exs | 186 ++++++++++++++++++ server/test/kotiko/config_property_test.exs | 111 +++++++++++ server/test/kotiko/llm/cache_test.exs | 137 +++++++++++++ server/test/kotiko/parsers_property_test.exs | 115 +++++++++++ server/test/kotiko/startup_summary_test.exs | 80 ++++++++ server/test/kotiko/voice_test.exs | 170 ++++++++++++++++ .../test/kotiko/word_spec_property_test.exs | 125 ++++++++++++ server/test/kotiko/words_edges_test.exs | 76 +++++++ 13 files changed, 1226 insertions(+), 2 deletions(-) create mode 100644 server/test/kotiko/background_jobs_test.exs create mode 100644 server/test/kotiko/backup_property_test.exs create mode 100644 server/test/kotiko/bot_commands_test.exs create mode 100644 server/test/kotiko/config_property_test.exs create mode 100644 server/test/kotiko/llm/cache_test.exs create mode 100644 server/test/kotiko/parsers_property_test.exs create mode 100644 server/test/kotiko/startup_summary_test.exs create mode 100644 server/test/kotiko/voice_test.exs create mode 100644 server/test/kotiko/word_spec_property_test.exs create mode 100644 server/test/kotiko/words_edges_test.exs diff --git a/server/.gitignore b/server/.gitignore index 832d002..b2ee380 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -4,3 +4,5 @@ *.db *.db-* /tmp/ +# mix test --cover +/cover/ diff --git a/server/lib/kotiko/transcriber.ex b/server/lib/kotiko/transcriber.ex index 154ef3c..9c52c48 100644 --- a/server/lib/kotiko/transcriber.ex +++ b/server/lib/kotiko/transcriber.ex @@ -27,7 +27,12 @@ defmodule Kotiko.Transcriber do response_format: "json" ] - case Req.post(url, form_multipart: form, headers: headers, receive_timeout: 120_000) do + # The options are empty in production; tests route the request to a Req.Test stub. + options = + [form_multipart: form, headers: headers, receive_timeout: 120_000] ++ + Application.get_env(:kotiko, :transcribe_req_options, []) + + case Req.post(url, options) do {:ok, %Req.Response{status: 200, body: %{"text" => text}}} -> {:ok, String.trim(text)} diff --git a/server/mix.exs b/server/mix.exs index 28b7677..1827ffb 100644 --- a/server/mix.exs +++ b/server/mix.exs @@ -15,7 +15,20 @@ defmodule Kotiko.MixProject do # The VM exits when the app stops, so systemd (or Docker) restarts it. start_permanent: true, deps: deps(), - aliases: aliases() + aliases: aliases(), + # `mix test --cover` fails below 90 % of lines (OpenSSF test_statement_coverage90). + # The helpers in test/support are test code, not the server, so they don't count. + test_coverage: [ + summary: [threshold: 90], + ignore_modules: [ + Kotiko.ConnCase, + Kotiko.DataCase, + Kotiko.Gen, + Kotiko.LegacyDb, + Kotiko.LLMStub, + Kotiko.TelegramStub + ] + ] ] end diff --git a/server/test/kotiko/background_jobs_test.exs b/server/test/kotiko/background_jobs_test.exs new file mode 100644 index 0000000..2f0ae3f --- /dev/null +++ b/server/test/kotiko/background_jobs_test.exs @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +# SPDX-License-Identifier: Apache-2.0 + +defmodule Kotiko.BackgroundJobsTest do + # The background processes around the job steps the other tests call directly: each + # one runs a step when its timer fires, survives a step that raises, and schedules the + # next step from the result. config/test.exs doesn't start them; each test starts its + # own, with the first timer far away, and fires the timer itself. + use Kotiko.DataCase, async: false + import ExUnit.CaptureLog + alias Kotiko.{Janitor, PronunciationRefresh, WiktionaryPass} + + # Fires `msg` in the process and waits until it has been handled. + defp fire(pid, msg) do + send(pid, msg) + :sys.get_state(pid) + end + + defp next_in(%{timer: nil}), do: nil + defp next_in(%{timer: ref}), do: Process.read_timer(ref) + + defp job_row(name, fields) do + sets = Enum.map_join(fields, ", ", fn {k, _} -> "#{k} = ?" end) + + Repo.query!( + "UPDATE maintenance_jobs SET #{sets} WHERE name = ?", + Keyword.values(fields) ++ [name] + ) + end + + defp ensure_job(name) do + Repo.query!( + "INSERT OR IGNORE INTO maintenance_jobs (name, state, done, total, attempts) " <> + "VALUES (?, 'running', 0, 0, '{}')", + [name] + ) + end + + describe "Janitor" do + test "a run clears old tombstones and the process keeps going" do + pid = start_supervised!(Janitor) + word = word_fixture() + {:ok, _} = Words.delete(word) + long_ago = Word.timestamp(DateTime.add(Words.now(), -200, :day)) + Repo.query!("UPDATE words SET deleted_at = ? WHERE id = ?", [long_ago, word.id]) + + # The summary is an info line; the test config logs warnings and up. + level = Logger.level() + Logger.configure(level: :info) + on_exit(fn -> Logger.configure(level: level) end) + log = capture_log(fn -> fire(pid, :run) end) + + assert Repo.get(Word, word.id) == nil + assert log =~ "removed 1 deleted over 180 days ago" + assert Process.alive?(pid) + end + + test "run/0 cleans up as of now" do + assert %{scrubbed: 0, purged: 0, add_requests: _, lookup_cache: _} = Janitor.run() + end + end + + describe "WiktionaryPass" do + setup do + ensure_job("wiktionary_pronunciations") + :ok + end + + test "with no word left it is done and looks again in six hours" do + pid = start_supervised!({WiktionaryPass, first_run: :timer.hours(1)}) + state = fire(pid, :tick) + assert_in_delta next_in(state), :timer.hours(6), 1_000 + end + + test "while Wikimedia asked it to wait, it waits until then" do + at = DateTime.add(Words.now(), 120, :second) + job_row("wiktionary_pronunciations", state: "waiting", retry_at: Word.timestamp(at)) + word_fixture() + + pid = start_supervised!({WiktionaryPass, first_run: :timer.hours(1)}) + state = fire(pid, :tick) + assert_in_delta next_in(state), 120_000, 5_000 + end + + test "a step that raises is logged and tried again in ten minutes" do + job_row("wiktionary_pronunciations", attempts: "not json") + pid = start_supervised!({WiktionaryPass, first_run: :timer.hours(1)}) + + log = capture_log(fn -> fire(pid, :tick) |> next_in() |> send_to_self() end) + + assert_received {:next, ms} + assert_in_delta ms, :timer.minutes(10), 1_000 + assert log =~ "Wiktionary pronunciations failed" + assert Process.alive?(pid) + end + end + + defp send_to_self(ms), do: send(self(), {:next, ms}) + + describe "PronunciationRefresh" do + test "a finished job schedules nothing more" do + job_row("pronunciation_refresh", state: "done") + pid = start_supervised!({PronunciationRefresh, first_run: :timer.hours(1)}) + assert next_in(fire(pid, :tick)) == nil + end + + test "without a model key for OpenRouter it waits an hour" do + put_app_env(:llm_api_key, nil) + put_app_env(:llm_url, "https://openrouter.ai/api/v1") + job_row("pronunciation_refresh", state: "running", retry_at: nil) + + pid = start_supervised!({PronunciationRefresh, first_run: :timer.hours(1)}) + assert_in_delta next_in(fire(pid, :tick)), :timer.hours(1), 1_000 + assert PronunciationRefresh.status().state == "waiting" + end + + test "a waiting job waits until its time" do + at = DateTime.add(Words.now(), 300, :second) + job_row("pronunciation_refresh", state: "waiting", retry_at: Word.timestamp(at)) + + pid = start_supervised!({PronunciationRefresh, first_run: :timer.hours(1)}) + assert_in_delta next_in(fire(pid, :tick)), 300_000, 5_000 + assert %{state: "waiting", retry_at: _} = PronunciationRefresh.status() + end + + test "a step that raises is logged and tried again in an hour" do + job_row("pronunciation_refresh", state: "running", attempts: "not json") + pid = start_supervised!({PronunciationRefresh, first_run: :timer.hours(1)}) + + log = capture_log(fn -> send_to_self(next_in(fire(pid, :tick))) end) + + assert_received {:next, ms} + assert_in_delta ms, :timer.hours(1), 1_000 + assert log =~ "Pronunciation refresh failed" + end + + test "resuming a paused job wakes the process" do + job_row("pronunciation_refresh", state: "paused") + pid = start_supervised!({PronunciationRefresh, first_run: :timer.hours(1)}) + :erlang.trace(pid, true, [:receive]) + + assert %{state: "running"} = PronunciationRefresh.control(:resume) + assert_receive {:trace, ^pid, :receive, :tick} + # Let the step finish before the test's sandbox goes away. + :sys.get_state(pid) + end + + test "with no job row the status is done, with nothing to do" do + Repo.query!("DELETE FROM maintenance_jobs WHERE name = 'pronunciation_refresh'") + assert PronunciationRefresh.status() == %{state: "done", done: 0, total: 0} + end + end +end diff --git a/server/test/kotiko/backup_property_test.exs b/server/test/kotiko/backup_property_test.exs new file mode 100644 index 0000000..41d744b --- /dev/null +++ b/server/test/kotiko/backup_property_test.exs @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +# SPDX-License-Identifier: Apache-2.0 + +defmodule Kotiko.BackupPropertyTest do + # Dynamic analysis (OpenSSF dynamic_analysis): restoring generated backup files. A + # backup is a file the learner picks, so it can hold anything: the import counts every + # word it was given as created, updated, unchanged or rejected, and never crashes. + use Kotiko.DataCase, async: false + use ExUnitProperties + import ExUnit.CaptureLog + alias Kotiko.{Backup, Gen} + + property "any backup document is imported word by word or refused as a whole" do + check all( + doc <- + one_of([ + fixed_map(%{ + "schemaVersion" => one_of([integer(0..3), Gen.scalar()]), + "words" => list_of(frequency([{4, Gen.word()}, {1, Gen.json()}]), max_length: 5) + }), + Gen.json() + ]), + max_runs: 60 + ) do + {result, _log} = with_log(fn -> Backup.import(doc) end) + + case result do + {:ok, counts} -> + live = Enum.reject(doc["words"], &(is_map(&1) and &1["deleted_at"] not in [nil, ""])) + + assert counts.created + counts.updated + counts.unchanged + counts.rejected == + length(live) + + {:error, reason} -> + assert reason == :not_backup or match?({:backup_newer, _}, reason) + end + end + end + + property "an exported backup imports back with every word unchanged" do + check all(words <- list_of(Gen.word(), max_length: 4), max_runs: 30) do + {:ok, _} = Backup.import(%{"schemaVersion" => Backup.schema_version(), "words" => words}) + before = Repo.aggregate(Word, :count) + + doc = Jason.decode!(Backup.export_binary()) + assert {:ok, counts} = Backup.import(doc) + assert counts.created == 0 and counts.rejected == 0 + assert Repo.aggregate(Word, :count) == before + end + end +end diff --git a/server/test/kotiko/bot_commands_test.exs b/server/test/kotiko/bot_commands_test.exs new file mode 100644 index 0000000..c5f8c4f --- /dev/null +++ b/server/test/kotiko/bot_commands_test.exs @@ -0,0 +1,186 @@ +# SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +# SPDX-License-Identifier: Apache-2.0 + +defmodule Kotiko.BotCommandsTest do + # The bot's commands, buttons and refusals that bot_test.exs doesn't cover, and the + # poller. Updates go straight to Bot.handle_update/1; Telegram and the model are stubs. + use Kotiko.DataCase, async: false + import ExUnit.CaptureLog + alias Kotiko.{Bot, I18n, LLMStub, TelegramStub} + + @user 4242 + + setup do + put_app_env(:allowed_ids, [@user]) + TelegramStub.stub() + :ok + end + + defp t(key, vars \\ %{}), do: I18n.t("en", key, vars) + + defp text_update(text) do + %{ + "update_id" => 1, + "message" => %{ + "message_id" => 10, + "from" => %{"id" => @user, "language_code" => "en"}, + "chat" => %{"id" => @user}, + "text" => text + } + } + end + + defp callback_update(fields) do + %{ + "update_id" => 2, + "callback_query" => + Map.merge( + %{ + "id" => "cb1", + "from" => %{"id" => @user, "language_code" => "en"}, + "message" => %{"message_id" => 11, "chat" => %{"id" => @user}} + }, + fields + ) + } + end + + defp handle(update) do + capture_log(fn -> Bot.handle_update(update) end) + TelegramStub.calls() + end + + defp replies(text), do: text |> text_update() |> handle() |> TelegramStub.texts() + + describe "commands" do + test "/languages lists each language with its count, or says there are none" do + assert replies("/languages") == [t("bot_languages_none")] + + word_fixture() + word_fixture(%{native: "нет", gloss: "no", forms: ["no"]}) + word_fixture(%{lang: "ja", native: "犬", romanization: "inu", gloss: "dog", forms: ["dog"]}) + + assert [text] = replies("/languages") + lines = String.split(text, "\n") + assert t("bot_languages_line", %{language: "Russian", count: 2}) in lines + assert t("bot_languages_line", %{language: "Japanese", count: 1}) in lines + end + + test "/list says when there are no words, or none in the language asked for" do + assert replies("/list") == [t("bot_list_none")] + assert replies("/list klingonish") == [t("bot_list_none_in", %{term: "klingonish"})] + end + + test "/remove asks which word, and says when nothing matches" do + assert replies("/remove") == [t("bot_remove_which")] + assert replies("/remove nichts") == [t("bot_remove_none", %{term: "nichts"})] + end + + test "/add saves the word at once; /add alone and unknown commands show the help" do + LLMStub.stub(fn _, conn -> LLMStub.answer(conn, LLMStub.words([LLMStub.word()])) end) + + assert [added] = replies("/add da") + assert added =~ t("bot_card_added") + assert [%Word{status: "active"}] = Repo.all(Word) + + assert replies("/add") == [t("bot_help")] + assert replies("/frobnicate") == [t("bot_help")] + assert replies("/help@KotikoBot") == [t("bot_help")] + end + end + + describe "refusals" do + # The model's word fails a check: the learner reads why, per word. + for {reason, attrs} <- [ + {"script_mismatch", %{"lang" => "ru", "native" => "dog"}}, + {"invalid_lang", %{"lang" => "1"}}, + {"sign_language_unsupported", %{"lang" => "ase", "native" => "HELLO"}} + ] do + @attrs attrs + @reason reason + test "a word refused for #{reason} is explained" do + LLMStub.stub(fn _, conn -> + LLMStub.answer(conn, LLMStub.words([LLMStub.word(@attrs)])) + end) + + assert [text] = replies("what's da") + + reason = t("bot_rejected_#{@reason}") + word = @attrs["native"] || "да" + + assert text == + t("bot_rejected_intro") <> + "\n" <> t("bot_rejected_line", %{word: word, reason: reason}) + + assert Repo.all(Word) == [] + end + end + end + + describe "buttons" do + test "Remove deletes every record of the word and edits the card" do + word = word_fixture() + + calls = handle(callback_update(%{"data" => "del:#{word.uuid}"})) + + assert Repo.get(Word, word.id).deleted_at + assert {"editMessageText", %{"text" => text}} = List.keyfind(calls, "editMessageText", 0) + assert text =~ t("bot_card_removed") + + removed = t("bot_toast_removed") + assert Enum.any?(calls, &match?({"answerCallbackQuery", %{"text" => ^removed}}, &1)) + end + + test "an unknown action or a button without data is only acknowledged" do + word = word_fixture() + + assert [{"answerCallbackQuery", %{"callback_query_id" => "cb1"}}] = + handle(callback_update(%{"data" => "zap:#{word.uuid}"})) + + assert [{"answerCallbackQuery", %{"callback_query_id" => "cb1"}}] = + handle(callback_update(%{})) + + assert Repo.get(Word, word.id).deleted_at == nil + end + + test "a button for a word that is gone says so" do + word = word_fixture() + Words.delete(word) + + calls = handle(callback_update(%{"data" => "add:#{word.uuid}"})) + gone = t("bot_toast_gone") + assert Enum.any?(calls, &match?({"answerCallbackQuery", %{"text" => ^gone}}, &1)) + assert [{_, _}] = handle(callback_update(%{"data" => "add:not-an-id"})) + end + end + + describe "the poller" do + test "asks for updates after the last one it saw, and polls again" do + TelegramStub.stub(%{"getUpdates" => []}) + + assert {:noreply, 7} = Bot.handle_info(:poll, 7) + assert [{"getUpdates", %{"offset" => 7, "timeout" => 30}}] = TelegramStub.calls() + assert_received :poll + end + + test "moves the offset past the updates it hands to tasks" do + # Updates from someone not allowed are ignored, so the tasks touch nothing. + update = %{"update_id" => 41, "message" => %{"from" => %{"id" => 1}, "text" => "hi"}} + TelegramStub.stub(%{"getUpdates" => [update, %{update | "update_id" => 42}]}) + + assert {:noreply, 43} = Bot.handle_info(:poll, 7) + assert_received :poll + end + + test "sets the command menu when it starts" do + assert {:noreply, 3} = Bot.handle_info(:commands, 3) + assert [{"setMyCommands", _} | _] = TelegramStub.calls() + end + + test "a failed setMyCommands is logged, not raised" do + Req.Test.stub(Kotiko.Telegram, &Req.Test.json(Plug.Conn.put_status(&1, 500), %{ok: false})) + log = capture_log(fn -> assert Bot.set_commands(["en"]) == :ok end) + assert log =~ "setMyCommands failed" + end + end +end diff --git a/server/test/kotiko/config_property_test.exs b/server/test/kotiko/config_property_test.exs new file mode 100644 index 0000000..121ba98 --- /dev/null +++ b/server/test/kotiko/config_property_test.exs @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +# SPDX-License-Identifier: Apache-2.0 + +defmodule Kotiko.ConfigPropertyTest do + # Dynamic analysis (OpenSSF dynamic_analysis): Kotiko.Config.parse/2 with generated + # .env files. Whatever the owner writes, the server either starts with typed settings + # or prints every problem; it never crashes on a setting, and never prints a secret. + use ExUnit.Case, async: true + use ExUnitProperties + alias Kotiko.{Config, Gen} + + defp resolver("localhost"), do: {:ok, {127, 0, 0, 1}} + defp resolver(_), do: {:error, :nxdomain} + + defp parse(vars) do + Config.parse(vars, + resolver: &resolver/1, + ensure_dir: fn _ -> :ok end, + default_data_dir: "/srv/kotiko-default" + ) + end + + # Plausible values for each setting, mixed with junk. + defp value("PORT"), do: one_of([map(integer(-5..70_000), &to_string/1), Gen.text()]) + + defp value("BIND"), + do: member_of(["127.0.0.1", "0.0.0.0", "::1", "localhost", "nope.invalid", "1.2.3"]) + + defp value("ALLOWED_TELEGRAM_IDS"), + do: one_of([map(list_of(integer()), &Enum.join(&1, ",")), Gen.text()]) + + defp value(name) when name in ~w(LLM_URL PUBLIC_URL TRANSCRIBE_URL), + do: + member_of([ + "https://openrouter.ai/api/v1", + "http://127.0.0.1:11434/v1", + "ftp://x", + "http://", + "openrouter.ai" + ]) + + defp value("LOG_LEVEL"), do: member_of(~w(debug info warning error loud INFO)) + + defp value(name) when name in ~w(LOG_LOOKUPS KOTIKO_LOG_SQL KOTIKO_WIKTIONARY), + do: member_of(~w(true false 1 0 yes no maybe)) + + defp value(_name), do: Gen.text() + + # A secret that can't appear by chance anywhere else. + defp secret do + gen(all(s <- string(:alphanumeric, min_length: 4, max_length: 40)), do: "SeCrEt" <> s) + end + + defp env do + plain = Config.documented_vars() -- Config.secret_vars() + + gen all( + values <- fixed_map(Map.new(plain, &{&1, value(&1)})), + keep <- list_of(member_of(plain), max_length: length(plain)), + secrets <- optional_map(Map.new(Config.secret_vars(), &{&1, secret()})) + ) do + values |> Map.take(keep) |> Map.merge(secrets) + end + end + + property "any .env starts with typed settings or lists its problems, never echoing a secret" do + check all(vars <- env()) do + secrets = Map.take(vars, Config.secret_vars()) |> Map.values() + + case parse(vars) do + {:ok, config, warnings} -> + assert config[:port] in 1..65_535 + assert is_tuple(config[:bind_ip]) + assert Enum.all?(config[:allowed_ids], &is_integer/1) + assert config[:log_level] in [:debug, :info, :warning, :error] + assert is_boolean(config[:log_lookups]) and is_boolean(config[:log_sql]) + assert [_ | _] = config[:llm_models] + assert config[:llm_model_source] in [:env, :default] + for w <- warnings, s <- secrets, do: refute(w =~ s) + + {:error, [_ | _] = problems, warnings} -> + message = Config.format_problems(problems, nil, false) + for s <- secrets, text <- [message | warnings], do: refute(text =~ s) + end + end + end + + property "PORT is a whole number from 1 to 65535, or a problem that names PORT" do + check all(raw <- one_of([map(integer(), &to_string/1), Gen.text()])) do + case parse(%{"PORT" => raw}) do + # An empty line is the default. + {:ok, config, _} -> + expected = + if String.trim(raw) == "", do: {4747, ""}, else: Integer.parse(String.trim(raw)) + + assert {config[:port], ""} == expected + + {:error, [{label, _lines}], _} -> + assert label =~ ~r/^PORT=/ + end + end + end + + property "ALLOWED_TELEGRAM_IDS keeps every listed ID, in order" do + check all(ids <- list_of(integer(), min_length: 1), spaces <- member_of(["", " ", ", "])) do + raw = Enum.join(ids, "," <> spaces) + assert {:ok, config, _} = parse(%{"ALLOWED_TELEGRAM_IDS" => raw}) + assert config[:allowed_ids] == ids + end + end +end diff --git a/server/test/kotiko/llm/cache_test.exs b/server/test/kotiko/llm/cache_test.exs new file mode 100644 index 0000000..9ccc5dc --- /dev/null +++ b/server/test/kotiko/llm/cache_test.exs @@ -0,0 +1,137 @@ +# SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +# SPDX-License-Identifier: Apache-2.0 + +defmodule Kotiko.LLM.CacheTest do + # Slice 10's lookup cache: what it keeps, and the single flight that makes concurrent + # lookups of the same text share one model call. llm_test.exs covers the cache through + # whole lookups; these tests drive Cache.run/3 directly, with each caller in its own + # process and every wait a condition, not a sleep. + use Kotiko.DataCase, async: false + alias Kotiko.LLM.Cache + + defp key, do: "test-#{System.unique_integer([:positive])}" + + # A leader that runs until told to go on, then returns or raises. + defp start_leader(key) do + test = self() + + pid = + spawn(fn -> + result = + try do + Cache.run(key, 5_000, fn -> + send(test, {:leading, self()}) + + receive do + :return -> :leader_result + :raise -> raise "model call failed" + end + end) + rescue + e -> {:raised, Exception.message(e)} + end + + send(test, {:leader_done, result}) + end) + + assert_receive {:leading, ^pid} + pid + end + + defp waiter(key, wait_ms, fun \\ fn -> :waiter_ran_it end), + do: Task.async(fn -> Cache.run(key, wait_ms, fun) end) + + # Waits until `n` callers wait on `key`. + defp await_waiters(key, n) do + case :sys.get_state(Cache).flights do + %{^key => %{waiters: w}} when length(w) == n -> :ok + _ -> await_waiters(key, n) + end + end + + test "callers that join a running lookup share the leader's result" do + key = key() + leader = start_leader(key) + a = waiter(key, 5_000) + b = waiter(key, 5_000) + await_waiters(key, 2) + + send(leader, :return) + + assert Task.await(a) == {:shared, :leader_result} + assert Task.await(b) == {:shared, :leader_result} + assert_receive {:leader_done, :leader_result} + refute Map.has_key?(:sys.get_state(Cache).flights, key) + end + + test "when the leader's call raises, the leader sees the error and a waiter runs it itself" do + key = key() + leader = start_leader(key) + task = waiter(key, 5_000) + await_waiters(key, 1) + + send(leader, :raise) + + assert_receive {:leader_done, {:raised, "model call failed"}} + assert Task.await(task) == :waiter_ran_it + end + + test "when the leader dies, a waiter runs the lookup itself" do + key = key() + leader = start_leader(key) + task = waiter(key, 5_000) + await_waiters(key, 1) + + Process.exit(leader, :kill) + + assert Task.await(task) == :waiter_ran_it + refute Map.has_key?(:sys.get_state(Cache).flights, key) + end + + test "a waiter that waits too long gives up and leaves the flight" do + key = key() + leader = start_leader(key) + + assert Task.await(waiter(key, 20)) == {:error, :wait_timeout} + await_waiters(key, 0) + + send(leader, :return) + assert_receive {:leader_done, :leader_result} + end + + test "an unknown process going down, or leaving a finished flight, changes nothing" do + state = :sys.get_state(Cache) + send(Cache, {:DOWN, make_ref(), :process, self(), :normal}) + GenServer.cast(Cache, {:leave, key(), make_ref()}) + assert :sys.get_state(Cache) == state + end + + describe "the kept lookups" do + test "a result without words isn't kept" do + key = key() + assert Cache.put(key, %{words: []}, "m1") == :ok + assert Cache.get(key) == :miss + end + + test "a row that can't be read is a miss, not an error" do + key = key() + at = Word.timestamp(Words.now()) + + Repo.insert_all("lookup_cache", [ + %{key: key, result: "not base64!", model: "m1", inserted_at: at, last_hit_at: at, hits: 0} + ]) + + assert Cache.get(key) == :miss + end + + test "clear/0 removes every entry, prune/0 keeps the recent ones" do + assert Cache.put(key(), %{words: [%{native: "да"}]}, "m1") == :ok + assert Cache.put(key(), %{words: [%{native: "нет"}]}, "m1") == :ok + assert Cache.prune() == 0 + assert Cache.count() == 2 + + assert Cache.clear() == 2 + assert Cache.count() == 0 + end + end +end diff --git a/server/test/kotiko/parsers_property_test.exs b/server/test/kotiko/parsers_property_test.exs new file mode 100644 index 0000000..af8305e --- /dev/null +++ b/server/test/kotiko/parsers_property_test.exs @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +# SPDX-License-Identifier: Apache-2.0 + +defmodule Kotiko.ParsersPropertyTest do + # Dynamic analysis (OpenSSF dynamic_analysis): the small parsers every request goes + # through, with generated input. Language tags, text cleaning, word ids and the log + # filter that keeps keys out of the logs. + use ExUnit.Case, async: true + use ExUnitProperties + alias Kotiko.{Gen, Lang, Text, UUID7} + alias Kotiko.Log.Redact + + property "Lang: any input canonicalises to a stable tag or an error code" do + check all(input <- one_of([Gen.lang(), Gen.text(max_length: 20), Gen.json()])) do + case Lang.canonical(input) do + {:ok, tag} -> + assert Lang.canonical(tag) == {:ok, tag} + assert is_binary(Lang.name(tag, "en")) and is_binary(Lang.endonym(tag)) + + {:error, code} -> + assert code in [:invalid_lang, :sign_language_unsupported] + end + + case Lang.base_tag(input) do + nil -> :ok + base -> assert Lang.base_tag(base) == base and Lang.same_base?(base, base) + end + end + end + + property "Text.clean: idempotent, trimmed, single-spaced, nil for blank" do + check all(input <- one_of([Gen.text(max_length: 40), Gen.scalar()])) do + case Text.clean(input) do + # Only non-text, invalid UTF-8, or nothing but whitespace, control and invisible + # characters. + nil -> + if is_binary(input) and String.valid?(input) do + assert String.replace(input, ~r/[\p{Cc}\s\x{200B}\x{FEFF}\x{00AD}]/u, "") == "" + end + + s -> + assert Text.clean(s) == s + assert s == String.trim(s) and s != "" + refute s =~ ~r/\s{2}|[\x{200B}\x{FEFF}\x{00AD}]/u + assert Text.length(s) == length(String.to_charlist(s)) + end + end + end + + property "UUID7: valid, ordered by time, and the time reads back" do + check all( + a <- integer(0..(2 ** 48 - 2)), + gap <- integer(1..1_000_000) + ) do + b = min(a + gap, 2 ** 48 - 1) + ida = UUID7.generate(a) + idb = UUID7.generate(b) + + assert UUID7.valid?(ida) and UUID7.valid?(idb) + assert UUID7.timestamp_ms(ida) == a and UUID7.timestamp_ms(idb) == b + assert String.at(ida, 14) == "7" and String.at(ida, 19) in ~w(8 9 a b) + assert ida < idb + end + end + + property "UUID7.valid?/1 accepts only lowercase hyphenated UUIDs" do + check all(input <- one_of([Gen.text(max_length: 40), Gen.json()])) do + if UUID7.valid?(input), do: assert(input =~ ~r/\A[0-9a-f-]{36}\z/) + refute UUID7.valid?(if is_binary(input), do: String.upcase(input) <> "G", else: input) + end + end + + # A token that can't occur in the text around it by chance. + defp token(chars), do: map(string(chars, min_length: 20, max_length: 60), &("Zq" <> &1)) + + property "Redact: keys and tokens never survive in a log line" do + alnum = Enum.concat([?a..?z, ?A..?Z, ?0..?9]) + + check all( + bearer <- token(alnum ++ ~c"._~+/=-"), + key <- token(alnum), + bot <- token(alnum ++ ~c"_-"), + id <- positive_integer(), + around <- Gen.text(max_length: 20) + ) do + line = + "#{around} Authorization: Bearer #{bearer} key sk-or-v1-#{key} " <> + "GET /bot#{id}:#{bot}/getUpdates sk-#{key} #{around}" + + redacted = Redact.redact(line) + + for secret <- [bearer, key, bot], do: refute(redacted =~ secret) + assert Redact.redact(redacted) == redacted + + event = %{level: :warning, msg: {:string, line}, meta: %{}} + assert %{msg: {:string, filtered}} = Redact.filter(event, []) + refute IO.chardata_to_string(filtered) =~ bearer + end + end + + property "Redact.filter: any log event passes through, never raising" do + check all( + msg <- + one_of([ + map(Gen.text(), &{:string, &1}), + map(list_of(Gen.text(max_length: 5), max_length: 4), &{:string, &1}), + map(map_of(atom(:alphanumeric), Gen.scalar(), max_length: 3), &{:report, &1}), + map(list_of(Gen.scalar(), max_length: 3), &{"~p ~p ~p", &1}) + ]) + ) do + event = %{level: :info, msg: msg, meta: %{}} + assert %{msg: _} = Redact.filter(event, []) + end + end +end diff --git a/server/test/kotiko/startup_summary_test.exs b/server/test/kotiko/startup_summary_test.exs new file mode 100644 index 0000000..8139ef6 --- /dev/null +++ b/server/test/kotiko/startup_summary_test.exs @@ -0,0 +1,80 @@ +# SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +# SPDX-License-Identifier: Apache-2.0 + +defmodule Kotiko.StartupSummaryTest do + # The info block the server logs at boot (Kotiko.Application.summary_lines/3): what an + # owner reads to see the server is set up as they meant. The boot itself runs in + # boot_test.exs, in a separate VM. + use ExUnit.Case, async: false + import Kotiko.DataCase, only: [put_app_env: 2] + alias Kotiko.Application, as: App + + defp line(lines, prefix), do: Enum.find(lines, &String.starts_with?(&1, prefix)) + + test "names the version, the data folder, the database and the active words" do + lines = App.summary_lines(:env, 1, "HTTP API: off") + data_dir = Elixir.Application.fetch_env!(:kotiko, :data_dir) + + assert hd(lines) == "Starting the Kotiko server, version #{Kotiko.Health.version()}" + assert line(lines, "Data:") == "Data: #{data_dir} (kotiko.db, 1 active word)" + assert line(lines, "Listening:") == "Listening: HTTP API: off" + assert line(lines, "Data:") =~ "1 active word)" + assert App.summary_lines(:env, 12, "x") |> line("Data:") =~ "12 active words)" + end + + test "says where the API token comes from, never the token itself" do + token = Elixir.Application.fetch_env!(:kotiko, :api_token) + + assert line(App.summary_lines(:env, 0, ""), "API token:") == "API token: from .env" + + assert line(App.summary_lines({:saved, "/data/api-token"}, 0, ""), "API token:") == + "API token: saved in /data/api-token" + + assert line(App.summary_lines({:generated, "/data/api-token"}, 0, ""), "API token:") == + "API token: saved in /data/api-token" + + refute Enum.any?(App.summary_lines(:env, 0, ""), &(&1 =~ token)) + end + + test "describes the model: OpenRouter's free list, or the models from LLM_MODEL" do + put_app_env(:llm_url, "https://openrouter.ai/api/v1") + put_app_env(:llm_model_source, :default) + + assert line(App.summary_lines(:env, 0, ""), "Model:") == + "Model: OpenRouter, its current free models (checked 10 s after start)" + + put_app_env(:llm_url, "http://127.0.0.1:11434/v1") + put_app_env(:llm_model_source, :env) + put_app_env(:llm_models, ["llama3"]) + + assert line(App.summary_lines(:env, 0, ""), "Model:") == + "Model: http://127.0.0.1:11434/v1, 1 model from LLM_MODEL" + + put_app_env(:llm_models, ["a", "b"]) + assert line(App.summary_lines(:env, 0, ""), "Model:") =~ ", 2 models from LLM_MODEL" + end + + test "describes the bot: off, waiting for IDs, or on with its accounts" do + put_app_env(:telegram_token, nil) + + assert line(App.summary_lines(:env, 0, ""), "Telegram:") =~ + "off (TELEGRAM_BOT_TOKEN is empty)" + + put_app_env(:telegram_token, "123:abc") + put_app_env(:allowed_ids, []) + assert line(App.summary_lines(:env, 0, ""), "Telegram:") =~ "waiting for ALLOWED_TELEGRAM_IDS" + + put_app_env(:allowed_ids, [1, 2]) + + assert line(App.summary_lines(:env, 0, ""), "Telegram:") == + "Telegram: on, 2 allowed accounts" + end + + test "says whether voice notes are transcribed" do + put_app_env(:transcribe_url, nil) + assert List.last(App.summary_lines(:env, 0, "")) =~ "off (TRANSCRIBE_URL is empty)" + + put_app_env(:transcribe_url, "http://127.0.0.1:8080/inference") + assert List.last(App.summary_lines(:env, 0, "")) == "Voice notes: on" + end +end diff --git a/server/test/kotiko/voice_test.exs b/server/test/kotiko/voice_test.exs new file mode 100644 index 0000000..72cdabb --- /dev/null +++ b/server/test/kotiko/voice_test.exs @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +# SPDX-License-Identifier: Apache-2.0 + +defmodule Kotiko.VoiceTest do + # Telegram voice notes: the file download (Kotiko.Telegram), speech to text + # (Kotiko.Transcriber) and the bot turning what was heard into a lookup. Telegram, the + # transcriber and the model are Req.Test stubs. + use Kotiko.DataCase, async: false + import ExUnit.CaptureLog + alias Kotiko.{Bot, LLMStub, Telegram, Transcriber} + + @user 4242 + + setup do + put_app_env(:allowed_ids, [@user]) + put_app_env(:transcribe_url, "http://whisper.test/inference") + put_app_env(:transcribe_req_options, plug: {Req.Test, Kotiko.Transcriber}) + :ok + end + + # Telegram: getFile answers with a path, the file URL with the audio; other methods ok. + defp stub_telegram(file_status \\ 200) do + test = self() + + Req.Test.stub(Kotiko.Telegram, fn + %{method: "GET", request_path: path} = conn -> + send(test, {:download, path}) + Plug.Conn.send_resp(conn, file_status, "OggS-audio") + + conn -> + method = List.last(conn.path_info) + {:ok, raw, conn} = Plug.Conn.read_body(conn) + send(test, {:telegram, method, Jason.decode!(raw)}) + result = if method == "getFile", do: %{file_path: "voice/file_1.oga"}, else: true + Req.Test.json(conn, %{ok: true, result: result}) + end) + end + + defp stub_transcriber(fun) do + test = self() + + Req.Test.stub(Kotiko.Transcriber, fn conn -> + {:ok, body, conn} = Plug.Conn.read_body(conn) + send(test, {:transcribe, Plug.Conn.get_req_header(conn, "authorization"), body}) + fun.(conn) + end) + end + + defp texts do + receive do + {:telegram, m, %{"text" => text}} when m in ["sendMessage", "editMessageText"] -> + [text | texts()] + + {:telegram, _, _} -> + texts() + after + 0 -> [] + end + end + + defp voice_update(kind \\ "voice") do + %{ + "update_id" => 5, + "message" => %{ + "message_id" => 10, + "from" => %{"id" => @user, "language_code" => "en"}, + "chat" => %{"id" => @user}, + kind => %{"file_id" => "f1"} + } + } + end + + describe "Transcriber.transcribe/1" do + test "uploads the audio with the model and the key, and trims the text" do + put_app_env(:transcribe_api_key, "whisper-key-123") + stub_transcriber(&Req.Test.json(&1, %{text: " shukran \n"})) + + assert Transcriber.transcribe("OggS") == {:ok, "shukran"} + assert_received {:transcribe, ["Bearer whisper-key-123"], body} + assert body =~ ~s(name="file"; filename="voice.ogg") + assert body =~ "whisper-1" + end + + test "sends no authorization without a key" do + put_app_env(:transcribe_api_key, nil) + stub_transcriber(&Req.Test.json(&1, %{text: "da"})) + + assert Transcriber.transcribe("OggS") == {:ok, "da"} + assert_received {:transcribe, [], _} + end + + test "reads JSON sent as text, and plain text" do + stub_transcriber(&Plug.Conn.send_resp(&1, 200, ~s({"text": " gracias "}))) + assert Transcriber.transcribe("OggS") == {:ok, "gracias"} + + stub_transcriber(&Plug.Conn.send_resp(&1, 200, " merci \n")) + assert Transcriber.transcribe("OggS") == {:ok, "merci"} + end + + test "an error status or no connection is an error with the reason" do + stub_transcriber(&Plug.Conn.send_resp(&1, 413, "too big")) + assert {:error, "transcriber returned 413: \"too big\""} = Transcriber.transcribe("OggS") + + stub_transcriber(&Req.Test.transport_error(&1, :econnrefused)) + assert {:error, message} = Transcriber.transcribe("OggS") + assert message == "connection refused" + end + end + + describe "Telegram" do + test "download_file/1 asks for the path, then fetches the file" do + stub_telegram() + + assert Telegram.download_file("f1") == {:ok, "OggS-audio"} + assert_received {:telegram, "getFile", %{"file_id" => "f1"}} + assert_received {:download, path} + assert String.ends_with?(path, "/voice/file_1.oga") + end + + test "download_file/1 reports a missing file" do + stub_telegram(404) + assert Telegram.download_file("f1") == {:error, "file download returned 404"} + end + + test "an answer without ok: true, or no connection, is an error with the reason" do + Req.Test.stub(Kotiko.Telegram, &Req.Test.json(Plug.Conn.put_status(&1, 401), %{ok: false})) + assert {:error, "telegram 401: " <> _} = Telegram.call("getMe", %{}) + assert {:error, "telegram 401: " <> _} = Telegram.download_file("f1") + + Req.Test.stub(Kotiko.Telegram, &Req.Test.transport_error(&1, :timeout)) + assert {:error, message} = Telegram.call("getMe", %{}) + assert message =~ "timeout" + end + end + + describe "voice notes in the bot" do + test "a voice note is transcribed, echoed, and looked up like typed text" do + stub_telegram() + stub_transcriber(&Req.Test.json(&1, %{text: "da"})) + LLMStub.stub(fn _, conn -> LLMStub.answer(conn, LLMStub.words([LLMStub.word()])) end) + + capture_log(fn -> Bot.handle_update(voice_update()) end) + + assert [heard, card] = texts() + assert heard =~ "da" + assert card =~ "да" + assert [%Word{status: "pending", native: "да"}] = Repo.all(Word) + end + + test "an audio file is handled like a voice note" do + stub_telegram() + stub_transcriber(&Req.Test.json(&1, %{text: "da"})) + LLMStub.stub(fn _, conn -> LLMStub.answer(conn, LLMStub.words([LLMStub.word()])) end) + + capture_log(fn -> Bot.handle_update(voice_update("audio")) end) + assert_received {:transcribe, _, _} + end + + test "without TRANSCRIBE_URL the bot says voice notes are off" do + put_app_env(:transcribe_url, nil) + stub_telegram() + + capture_log(fn -> Bot.handle_update(voice_update()) end) + + assert [off] = texts() + assert off == Kotiko.I18n.t("en", "bot_voice_off") + refute_received {:download, _} + end + end +end diff --git a/server/test/kotiko/word_spec_property_test.exs b/server/test/kotiko/word_spec_property_test.exs new file mode 100644 index 0000000..9138324 --- /dev/null +++ b/server/test/kotiko/word_spec_property_test.exs @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +# SPDX-License-Identifier: Apache-2.0 + +defmodule Kotiko.WordSpecPropertyTest do + # Dynamic analysis (OpenSSF dynamic_analysis): Kotiko.WordSpec checks everything a + # client or the model sends before it reaches the database. Whatever arrives, it answers + # with a result or a reason, never an exception, and what it accepts keeps the rules. + use ExUnit.Case, async: true + use ExUnitProperties + alias Kotiko.{Gen, Lang, Spec, Text, Word, WordSpec} + + @fields ~w(lang native sense romanization native_vocalized gloss forms pronunciation + pronunciation_careful pronunciation_source note status) + + defp fold(s), do: WordSpec.fold(s) + + property "validate_word/2 answers every object with a valid word or a reason" do + check all(word <- Gen.word(), base_langs <- member_of([nil, [], ["en"], ["es", "en"]])) do + case WordSpec.validate_word(word, base_langs: base_langs) do + {:ok, attrs, dropped} -> + assert Lang.canonical(attrs.lang) == {:ok, attrs.lang} + assert attrs.base_lang == Lang.base_tag(attrs.base_lang) + refute Lang.same_base?(attrs.lang, attrs.base_lang) + if base_langs not in [nil, []], do: assert(attrs.base_lang in base_langs) + + assert attrs.native == Text.clean(attrs.native) + assert attrs.gloss == Text.clean(attrs.gloss) + assert Text.length(attrs.native) <= Spec.rule(:max_native_chars) + assert Text.length(attrs.gloss) <= Spec.rule(:max_gloss_chars) + assert attrs.note == nil or Text.length(attrs.note) <= Spec.rule(:max_note_chars) + assert Text.length(attrs.sense) <= Spec.rule(:max_sense_chars) + assert attrs.sense == String.downcase(attrs.sense) + + assert attrs.forms != [] and length(attrs.forms) <= Spec.rule(:max_forms) + assert Enum.any?(attrs.forms, & &1.enabled) + assert Enum.uniq_by(attrs.forms, &fold(&1.text)) == attrs.forms + + for f <- attrs.forms do + assert is_binary(f.text) and Text.length(f.text) <= Spec.rule(:max_form_chars) + end + + assert attrs.status in ~w(active paused) + assert attrs.origin in Word.origins() + # No pronunciation, no source. + if attrs.pronunciation, + do: assert(attrs.pronunciation_source in ~w(model user wiktionary)), + else: assert(attrs.pronunciation_source == nil) + + assert attrs.id == nil or Kotiko.UUID7.valid?(attrs.id) + assert is_list(dropped) + + {:error, reason, summary} -> + assert is_atom(reason) + assert Map.keys(summary) |> Enum.sort() == [:base_lang, :gloss, :native] + end + end + end + + property "a word validate_word/2 accepted is accepted again unchanged" do + check all(word <- Gen.word()) do + with {:ok, attrs, _} <- WordSpec.validate_word(word) do + assert {:ok, again, _} = WordSpec.validate_word(attrs) + assert again == attrs + end + end + end + + property "patch/1 returns only the fields the body has, or names the one it refused" do + # Values each field could take, so a patch often gets past its first field. + plausible = + member_of([nil, "active", "paused", "user", "wiktionary", "ru", "Haus", "da", ["dog"]]) + + check all( + body <- + Gen.object(@fields ++ ["id", "other"], frequency([{2, plausible}, {1, Gen.json()}])) + ) do + case WordSpec.patch(body) do + {:ok, patch} -> + assert patch |> Map.keys() |> Enum.map(&to_string/1) |> Enum.all?(&(&1 in @fields)) + for {k, _} <- patch, do: assert(Map.has_key?(body, to_string(k))) + if Map.has_key?(patch, :status), do: assert(patch.status in ~w(active paused)) + if is_binary(patch[:native]), do: assert(patch.native == Text.clean(patch.native)) + + {:error, %{field: field, reason: reason}} -> + assert field in @fields and Map.has_key?(body, field) + assert is_binary(reason) + end + end + end + + property "prepare_input/1 gives trimmed NFC text without control characters, or a reason" do + check all(input <- one_of([Gen.text(max_length: 250), Gen.json()])) do + case WordSpec.prepare_input(input) do + {:ok, text} -> + assert text == String.trim(text) and text != "" + assert text == :unicode.characters_to_nfc_binary(text) + refute text =~ ~r/\p{Cc}/u + assert Text.length(text) <= Spec.rule(:max_input_chars) + assert WordSpec.prepare_input(text) == {:ok, text} + + {:error, reason} -> + assert reason in [:empty_input, :input_too_long] + end + end + end + + property "extract/2 finds the JSON object in any model answer, or nothing" do + check all( + obj <- map_of(Gen.text(max_length: 8), Gen.json(), max_length: 3), + before <- Gen.text(), + after_text <- Gen.text(), + fence? <- boolean() + ) do + object = Map.put(obj, "words", []) + json = Jason.encode!(object) + json = if fence?, do: "```json\n" <> json <> "\n```", else: json + content = "" <> before <> "" <> json + + assert WordSpec.extract(content) == object + # Text around it never makes it raise. + result = WordSpec.extract(before <> json <> after_text) + assert result == nil or is_map(result) + end + end +end diff --git a/server/test/kotiko/words_edges_test.exs b/server/test/kotiko/words_edges_test.exs new file mode 100644 index 0000000..7c4d9f0 --- /dev/null +++ b/server/test/kotiko/words_edges_test.exs @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +# SPDX-License-Identifier: Apache-2.0 + +defmodule Kotiko.WordsEdgesTest do + # Kotiko.Words at its edges: ids that aren't ids, words that are pending or deleted, + # and the cleanup steps with their default clock. words_test.exs covers the main paths. + use Kotiko.DataCase, async: false + alias Kotiko.UUID7 + + test "get/1 and get_row/1 answer nil for anything that isn't an id" do + for id <- [nil, 42, "42", "not-a-uuid", String.upcase(UUID7.generate())], + do: assert(Words.get(id) == nil, inspect(id)) + + for id <- [nil, "7", 1.0], do: assert(Words.get_row(id) == nil, inspect(id)) + end + + test "get_by_key/3 finds a live word with no sense, in any case of the native" do + w = word_fixture(%{native: "Да"}) + assert Words.get_by_key("ru", "да", "en").id == w.id + assert Words.get_by_key("ru", "ДА", nil, "en").id == w.id + assert Words.get_by_key("ru", "да", "es") == nil + end + + test "get_many/1 returns the live words among the ids" do + a = word_fixture() + b = word_fixture(%{native: "нет", gloss: "no", forms: ["no"]}) + {:ok, _} = Words.delete(b) + + assert [%Word{id: id}] = Words.get_many([a.uuid, b.uuid, UUID7.generate()]) + assert id == a.id + end + + test "recent/0 lists the newest active words first" do + old = word_fixture() + new = word_fixture(%{native: "нет", gloss: "no", forms: ["no"]}) + word_fixture(%{native: "спасибо", gloss: "thanks", forms: ["thanks"]}, "pending") + + assert Enum.map(Words.recent(), & &1.id) == [new.id, old.id] + end + + test "update/3 refuses a pending word as not found and a tombstone as deleted" do + pending = word_fixture(%{}, "pending") + assert Words.update(pending.uuid, %{gloss: "yeah"}) == {:error, :not_found} + + gone = word_fixture(%{native: "нет", gloss: "no", forms: ["no"]}) + {:ok, _} = Words.delete(gone) + assert Words.update(gone.uuid, %{gloss: "nope"}) == {:error, :deleted} + assert Words.update(UUID7.generate(), %{gloss: "x"}) == {:error, :not_found} + end + + test "update/3 moves the tag to the script the edited native is written in" do + w = word_fixture(%{lang: "sr", native: "хвала", romanization: "hvala", gloss: "thanks"}) + assert w.lang == "sr" + + assert {:ok, updated} = Words.update(w.uuid, %{native: "hvala"}) + assert {updated.lang, updated.native} == {"sr-Latn", "hvala"} + end + + test "delete/1 of an unknown id is not found; deleting twice keeps the first time" do + assert Words.delete(UUID7.generate()) == {:error, :not_found} + + w = word_fixture() + {:ok, first} = Words.delete(w) + assert {:ok, again} = Words.delete(w.uuid) + assert again.deleted_at == first.deleted_at + end + + test "the tombstone cleanups default to now and leave fresh tombstones alone" do + w = word_fixture() + {:ok, _} = Words.delete(w) + + assert Words.scrub_tombstones() == 0 + assert Words.purge_tombstones() == 0 + assert Repo.get(Word, w.id).native == "да" + end +end From aa8bca694eaf6195532870612a36bd488aa446bb Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Tue, 6 Oct 2026 16:02:51 -0400 Subject: [PATCH 08/12] ci: gate coverage, and run every test before a release CI's extension job runs npm run coverage instead of npm test: the same tests, failing below 90 % of lines or 80 % of branches in extension/ (Node 22's --test-coverage-lines and --test-coverage-branches). The server job runs mix test --cover on the newest Elixir, failing below 90 % of lines (mix.exs), and plain mix test on 1.15.8, where line counts differ a little. The release build ran the extension's tests only. It now runs the same coverage gates for both halves, so the property tests (the dynamic analysis) run on every tree that is released. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ayla Croft --- .github/workflows/ci.yml | 9 ++++++++- .github/workflows/release.yml | 13 +++++++++++-- package.json | 1 + 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6631010..00a0469 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,6 +73,11 @@ jobs: - run: mix deps.audit - run: mix deps.unlock --check-unused - run: mix test + if: ${{ !matrix.current }} + # The coverage gate (90 % of lines, mix.exs) on one compiler only: line counts differ + # a little between Elixir versions, and cover makes the run slower. + - run: mix test --cover + if: matrix.current extension: runs-on: ubuntu-24.04 @@ -84,7 +89,9 @@ jobs: cache: npm - run: npm ci - run: npm run lint - - run: npm test + # Every unit, DOM and background test (property tests and assertion mode included), + # failing below 90 % line or 80 % branch coverage of extension/. + - run: npm run coverage - run: npm run perf # Slices 08 and 09: the shared spec. The extension's copy is current, the CLDR-derived diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b9cd66c..ffbad47 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -105,9 +105,18 @@ jobs: echo "Building $v with Node $(node --version)" - run: node scripts/check-versions.mjs - run: npm ci - # The same checks as CI, on the tree being released. + # The same checks as CI, on the tree being released: the extension's and the server's + # tests with their coverage gates. The property tests run in both, so the dynamic + # analysis runs before every release. - run: npm run lint - - run: npm test + - run: npm run coverage + - name: Server tests + working-directory: server + env: + MIX_ENV: test + run: | + mix deps.get + mix test --cover - name: Build the zips twice and compare env: VERSION: ${{ steps.version.outputs.version }} diff --git a/package.json b/package.json index 9b65c36..54bc784 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ }, "scripts": { "test": "node --test --import ./test/helpers/assert-mode.mjs 'test/{unit,dom,bg}/**/*.test.mjs'", + "coverage": "node --test --import ./test/helpers/assert-mode.mjs --experimental-test-coverage --test-coverage-include='extension/**' --test-coverage-lines=90 --test-coverage-branches=80 'test/{unit,dom,bg}/**/*.test.mjs'", "e2e": "playwright test", "lint": "eslint . && web-ext lint --source-dir extension --self-hosted", "perf": "node test/perf/run.mjs" From 77a414aa2770578fd4838f0fd132ad83bba8b4c7 Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Tue, 6 Oct 2026 16:02:51 -0400 Subject: [PATCH 09/12] docs: coverage gates, property tests and assertion mode CONTRIBUTING.md's Tests section says which gates CI enforces and how to run them, that the server has no branch number (Elixir's cover counts lines), where the property tests live and how to replay a failure from its seed, and what the extension's assertion mode checks. The docs site's developer setup lists the coverage commands. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ayla Croft --- CONTRIBUTING.md | 50 +++++++++++++++++++++-- site/src/content/docs/contribute/setup.md | 5 ++- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ac1c44e..4e44272 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,11 +20,11 @@ You need Elixir 1.15 or newer and Node 22. # server cd server mix deps.get -mix test +mix test # or mix test --cover, with the coverage gate # extension and end-to-end tests (from the repository root) npm ci -npm test +npm test # or npm run coverage, with the coverage gate npx playwright install chromium npm run e2e ``` @@ -60,7 +60,51 @@ site in `site/` (its own `package.json`, build and tests), in practical, say why in the pull request; a maintainer has to agree. - Tests never use the network: the server stubs HTTP with `Req.Test`, and the extension tests serve their own pages. -- How to run them is under [Setup](#setup); CI runs them on every pull request. +- How to run them is under [Setup](#setup); CI runs them on every pull request, and the + release workflow runs them again on the tree it releases. + +### Coverage gates + +CI fails when coverage drops below these numbers, and prints them on every run: + +| Part | Command | Gate | +|---|---|---| +| Extension | `npm run coverage` | 90 % of lines and 80 % of branches in `extension/` (Node's built-in coverage) | +| Server | `cd server && mix test --cover` | 90 % of lines (`test_coverage` in `server/mix.exs`; the helpers in `test/support` don't count) | + +Elixir's cover tool counts lines, not branches, so the server has no branch number. The +server gate runs on the newest Elixir in the CI matrix only, because line counts differ a +little between compiler versions. A gate is never lowered; raise it when coverage grows. +Coverage comes from tests of behaviour: a test that only runs code without checking what +it does is not accepted. + +### Property tests + +Besides examples, both halves have property-based tests, which check a rule on many +generated inputs: + +- Extension: [fast-check](https://fast-check.dev/) (a dev dependency) in + `test/unit/properties/`, for the matcher, server address checks, the bulk list reader, + the backup, casing, the word-list check and the policy page's Markdown. +- Server: StreamData in `server/test/**/*_property_test.exs`, for word validation and + input preparation, the v1 routes, `.env` parsing, language tags, text cleaning, UUIDv7, + log redaction and backups; shared generators are in `server/test/support/gen.ex`. + +They run as part of `npm test` and `mix test`, so in CI and before every release. Each run +uses a new random seed. When one fails, it prints the seed and the smallest input it found; +replay it with `FC_SEED= FC_PATH= npm test` (extension; `FC_NUM_RUNS=` +tries more inputs) or `mix test --seed ` (server). Fix the bug and add a plain +regression test with that input, so the case is checked on every run. + +### Assertion mode + +`npm test` and `npm run coverage` load `test/helpers/assert-mode.mjs`, which turns on the +extension's invariant checks in every test process, jsdom window and vm context: the +matcher checks each scan's matches, the backup each restore plan, and the word-list check +each filtered list, and they throw when an invariant breaks. The shipped extension never +turns them on, and `npm run perf` measures the code as it ships. Add a check to this mode +when a function makes a promise its callers rely on. The end-to-end tests run the real +extension, which has no way to turn the mode on. ## Commits and pull requests diff --git a/site/src/content/docs/contribute/setup.md b/site/src/content/docs/contribute/setup.md index 3032393..f24e269 100644 --- a/site/src/content/docs/contribute/setup.md +++ b/site/src/content/docs/contribute/setup.md @@ -29,7 +29,8 @@ From the repository root: ```bash npm ci npm run lint # ESLint and web-ext lint -npm test # unit, DOM and background tests +npm test # unit, DOM, background and property tests +npm run coverage # the same, failing below 90 % of lines or 80 % of branches npx playwright install chromium npm run e2e # end-to-end tests in Chromium with the unpacked extension ``` @@ -42,7 +43,7 @@ CI runs more checks (the shared spec, versions, licenses, the old name); the ful ```bash cd server mix deps.get -mix test +mix test # or mix test --cover: fails below 90 % of lines cp .env.example .env && chmod 600 .env # then fill in LLM_API_KEY ./run.sh ``` From d01c8db978243b5ea442a18f89a9c8dfb340468c Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Tue, 6 Oct 2026 17:27:01 -0400 Subject: [PATCH 10/12] docs(best-practices): the coverage gates and property tests earn their answers With this branch merged, these are Met, each with its evidence: - gold test_statement_coverage90 (server 91.6 % lines, extension 93.9 %, both gated) and test_branch_coverage80 (extension 84.5 %, gated; no FLOSS branch tool exists for Elixir); - dynamic_analysis and dynamic_analysis_enable_assertions (passing and gold, one shared answer): fast-check and StreamData properties in CI and the release workflow, and the extension's test-only assertion mode; - passing dynamic_analysis_fixed: every defect the properties found was fixed with a regression test. test_most and silver test_statement_coverage80 carry the new numbers, and the passing summary is recounted (62 Met, 1 Unmet, 4 N/A). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ayla Croft --- docs/best-practices.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/best-practices.md b/docs/best-practices.md index 224a881..28671cc 100644 --- a/docs/best-practices.md +++ b/docs/best-practices.md @@ -46,14 +46,14 @@ request, then on the form. The plan behind it is slice | | Met | Unmet | N/A | Total | |---|---|---|---|---| -| MUST | 39 | 0 | 4 | 43 | +| MUST | 40 | 0 | 3 | 43 | | SHOULD | 10 | 0 | 0 | 10 | -| SUGGESTED | 10 | 3 | 1 | 14 | -| **All** | **59** | **3** | **5** | **67** | +| SUGGESTED | 12 | 1 | 1 | 14 | +| **All** | **62** | **1** | **4** | **67** | Passing needs every MUST met or N/A, every SHOULD met or explained, and every SUGGESTED -answered. Every MUST and SHOULD is met or N/A today. The three unmet SUGGESTED criteria -(`version_tags`, `dynamic_analysis`, `dynamic_analysis_enable_assertions`) don't block passing. +answered. Every MUST and SHOULD is met or N/A today. The one unmet SUGGESTED criterion +(`version_tags`: no release tagged yet) doesn't block passing. What has to happen for the badge, and to keep it, and who does it: @@ -119,7 +119,7 @@ What has to happen for the badge, and to keep it, and who does it: | `build_floss_tools` | SHOULD | Met | Elixir, Erlang/OTP, Mix, Node.js and npm are all FLOSS. | – | – | | `test` | MUST | Met | ExUnit tests for the server, Node's built-in test runner for the extension (unit, DOM and background suites) and Playwright end-to-end tests, all in the repository under Apache-2.0. How to run them: CONTRIBUTING.md, Setup; CI runs them on every pull request. | `server/test/`; `test/`; `CONTRIBUTING.md:15-30`; `.github/workflows/ci.yml:75,87,177` | – | | `test_invocation` | SHOULD | Met | npm test (Node's built-in test runner) for the extension and mix test for the server, the standard commands of each ecosystem; npm run e2e runs the Playwright end-to-end tests. https://github.com/ScriptKittyOS/kotiko/blob/main/CONTRIBUTING.md#setup | `package.json` `scripts.test`; `CONTRIBUTING.md:19-30` | – | -| `test_most` | SUGGESTED | Met | Measured 2026-10-06: the server tests cover 87.8 % of lines (Elixir's cover tool) and the extension's unit, DOM and background tests 91.4 % of lines and 79.1 % of branches of the extension code they load. | `cd server && mix test --cover`: 1000 tests, 2 properties, 0 failures, 87.78 %; `node --test --experimental-test-coverage --test-coverage-include='extension/**' 'test/{unit,dom,bg}/**/*.test.mjs'`: 1857 tests pass, all files 91.42 % lines, 79.13 % branches | A coverage gate in CI is planned (slice 02 addition) | +| `test_most` | SUGGESTED | Met | Measured 2026-10-06 and enforced in CI: the server's tests cover 91.6 % of lines (mix test --cover, which fails below 90 %), and the extension's tests 93.9 % of lines and 84.5 % of branches (npm run coverage, which fails below 90 % of lines or 80 % of branches). | `server/mix.exs` `test_coverage`; `package.json` `coverage`; `.github/workflows/ci.yml` | – | | `test_continuous_integration` | SUGGESTED | Met | Every change reaches main through a short-lived pull request, and GitHub Actions runs every test suite, the linters and the audits on each pull request and each push to main; main requires 8 checks to pass. https://github.com/ScriptKittyOS/kotiko/actions/workflows/ci.yml | `.github/workflows/ci.yml:5-8`; ruleset 24531669 (`required_status_checks`) | – | | `test_policy` | MUST | Met | Every pull request that adds or changes behaviour adds or updates automated tests in the same pull request; every bug fix adds a regression test (CONTRIBUTING.md, Tests). | `CONTRIBUTING.md:55-63`; `.github/pull_request_template.md:9-10` | – | | `tests_are_added` | MUST | Met | The most recent major features came with tests, for example backup, restore, export and delete (commit c5144de: server, background, DOM and end-to-end tests), the accessibility baseline (59af37e: axe and screen-reader end-to-end tests) and the docs site (97e2943: build and page tests). | `git show --stat c5144de 59af37e 97e2943` | – | @@ -157,10 +157,10 @@ What has to happen for the badge, and to keep it, and who does it: | `static_analysis_common_vulnerabilities` | SUGGESTED | Met | Two security analysers look for common vulnerabilities on every pull request and push to main, and a finding fails the build: Sobelow for the Elixir server (injection, traversal, unsafe deserialization, atom exhaustion; every finding fails, false positives are skipped per function with a written reason) and CodeQL with the security-extended queries for JavaScript and the GitHub Actions workflows (a required check). For the extension, eslint-plugin-no-unsanitized and a ban on HTML string sinks cover DOM cross-site scripting. The policy says what blocks a merge or a release: https://github.com/ScriptKittyOS/kotiko/blob/main/docs/security/dependency-and-static-analysis-policy.md | `server/.sobelow-conf`; `.github/workflows/codeql.yml`; `.github/workflows/ci.yml` (server job, `mix sobelow`); `eslint.config.js`; ruleset 24531669 requires `analyze (actions)`, `analyze (javascript-typescript)` | – | | `static_analysis_fixed` | MUST | Met | No static analysis finding is open: CI fails on any, and pull requests are merged only when CI passes. None so far was an exploitable vulnerability. | `.github/workflows/ci.yml`; ruleset 24531669 | – | | `static_analysis_often` | SUGGESTED | Met | On every push to main and every pull request. | `.github/workflows/ci.yml:5-8` | – | -| `dynamic_analysis` | SUGGESTED | Unmet | Not yet for every release. CI runs property-based tests (StreamData) that feed generated input to two server parsers, the end-to-end tests drive the real extension against the real server, and axe scans every interface, but no fuzzer or scanner is applied to each proposed release yet. Planned: property-based tests for the request parsers, the URL and response validators and the matcher, run in the release workflow, or an OWASP ZAP baseline scan of the server. | `server/test/kotiko/word_test.exs`, `server/test/kotiko/text_test.exs` (`property`) | Slice 53 section 8 (gold groundwork) | +| `dynamic_analysis` | SUGGESTED | Met | Property-based testing is the dynamic analysis, and it runs on every pull request and again in the release workflow before anything is built. The extension's pure code (the page matcher, URL checks, bulk-add parsing, backup read and restore, casing, the word-list check) is tested with fast-check, and the server's request validation, word spec, settings parsing, language tags, text cleaning, redaction and backup round trips with StreamData; each run uses a new seed, and a failing seed can be replayed. It has found real defects: a word with Å or Ņ made the server answer 500, a repeated query parameter did the same, and a re-add brought back a form the learner had removed; each was fixed with a regression test. https://github.com/ScriptKittyOS/kotiko/blob/main/CONTRIBUTING.md#tests | `test/unit/properties/`; `server/test/` (StreamData properties); `.github/workflows/ci.yml`; `.github/workflows/release.yml` (`npm run coverage`, `mix test --cover`) | – | | `dynamic_analysis_unsafe` | SUGGESTED | N/A | Kotiko's code is Elixir and JavaScript, both memory-safe. The only memory-unsafe code is SQLite inside a dependency (exqlite), which the project does not write or build itself. | – | – | -| `dynamic_analysis_enable_assertions` | SUGGESTED | Unmet | Not yet. The tests assert heavily, but the shipped code has no run-time assertions that are switched on during dynamic analysis. Planned with the property-based tests above. | – | Slice 53 section 8 | -| `dynamic_analysis_fixed` | MUST | N/A | No dynamic analysis tool is run yet, so none has reported a vulnerability. | – | – | +| `dynamic_analysis_enable_assertions` | SUGGESTED | Met | Under test, the extension runs with an assertion mode (globalThis.__KOTIKO_ASSERT__, set only by the test harness and never by the shipped extension): the matcher checks every scan's matches (bounds, text, order, no overlap), the backup restore plan checks its counts and that it never writes a deleted word, and the word-list check verifies every word is kept or counted. The property-based tests drive these paths with generated input. The server relies on Elixir's pattern matches and guards, which already stop on a broken invariant in production. https://github.com/ScriptKittyOS/kotiko/blob/main/CONTRIBUTING.md#tests | `test/helpers/assert-mode.mjs`; `extension/lib/matcher.js`; `extension/lib/backup.js`; `test/unit/properties/` | – | +| `dynamic_analysis_fixed` | MUST | Met | Every defect the property-based tests have found was fixed in the same change, with a regression test: two inputs that made the server answer 500 (a word with Å or Ņ, and a repeated query parameter) and a merge rule that brought back a removed form. None was an exploitable vulnerability. Any medium or higher vulnerability they find is fixed within the deadlines in SECURITY.md. https://github.com/ScriptKittyOS/kotiko/blob/main/SECURITY.md | `git log --grep 'fix(server)'`; `spec/fixtures/input.json`; `spec/fixtures/merge.json` | – | ## Silver and gold @@ -223,7 +223,7 @@ earned, and gold once silver is; the answers can be entered before that. | `interfaces_current` | SHOULD [J] [N/A J] | Met | The extension uses Manifest V3 and current WebExtension APIs, and web-ext lint reports no deprecated API. The server runs on current Elixir and Erlang/OTP (CI tests the oldest supported, Elixir 1.15 on OTP 26, and the newest, 1.19 on OTP 28) and compiles with warnings as errors, so a deprecation warning fails the build. | `extension/manifest.json:2`; `.github/workflows/ci.yml:44-47,69`; `npx web-ext lint --source-dir extension --self-hosted`: 0 errors, 0 notices, 1 warning (Firefox ignores `background.service_worker`, intended) | – | | `automated_integration_testing` | MUST [J] | Met | GitHub Actions runs on every push to main and every pull request: the server tests on the oldest and newest supported Elixir, the extension's unit, DOM and background suites, the shared-spec checks, and Playwright end-to-end tests against the real server. Each job reports pass or fail on the pull request, and main accepts only pull requests whose 8 required checks pass. https://github.com/ScriptKittyOS/kotiko/actions/workflows/ci.yml | `.github/workflows/ci.yml:5-8,39-183`; ruleset 24531669 (8 required status checks); `gh run list --workflow ci.yml`: push to main 2026-10-06T18:38:55Z success | – | | `regression_tests_added50` | MUST [J] [N/A J] | Met | Measured with scripts/regression-audit.mjs on 2026-10-06: 11 of the 15 fix commits on main in the last six months changed a test in the same commit (73 %). CONTRIBUTING.md requires a regression test that fails without the fix with every bug fix. | `node scripts/regression-audit.mjs`: 15 fix commits, 11 with a test change (73 %); `CONTRIBUTING.md:59-60` | – | -| `test_statement_coverage80` | MUST [J] [N/A J] | Met | Measured 2026-10-06 with FLOSS tools: the server's ExUnit suite (1000 tests and 2 properties) covers 87.8 % of lines (mix test --cover, Erlang's cover tool), and the extension's Node test suites cover 92.6 % of the lines of the extension code they load (node --test --experimental-test-coverage); three small extension files (120 lines) are not loaded by them. The end-to-end tests come on top. A coverage gate in CI is planned. | Scratch copy of HEAD: `MIX_ENV=test mix test --cover` gives 1000 tests, 2 properties, 0 failures, Total 87.78 %; `node --test --experimental-test-coverage --test-coverage-include='extension/**' 'test/{unit,dom,bg}/**/*.test.mjs'` gives all files 92.59 % lines, 79.13 % branches (1856 of 1857 pass; the one failure needs a git commit, absent in the scratch copy); not loaded: `extension/content/connect.js`, `extension/privacy.js`, `extension/ui/theme.js` | Agent (optional): CI coverage gate so the number can't slip | +| `test_statement_coverage80` | MUST [J] [N/A J] | Met | Statement coverage is above 80 % and enforced by FLOSS tools on every pull request and before every release: the server's ExUnit suite covers 91.6 % of lines (mix test --cover with a 90 % threshold in mix.exs, Erlang's cover tool), and the extension's Node test suites cover 93.9 % of lines (node --test coverage with a 90 % line threshold). https://github.com/ScriptKittyOS/kotiko/blob/main/CONTRIBUTING.md#tests | `server/mix.exs` `test_coverage: [summary: [threshold: 90]]`; `package.json` `coverage`; `.github/workflows/ci.yml`; `.github/workflows/release.yml` | – | | `test_policy_mandated` | MUST [J] [N/A J] | Met | CONTRIBUTING.md, Tests: every pull request that adds or changes behaviour adds or updates automated tests in the same pull request, and major new functionality is not merged without tests. https://github.com/ScriptKittyOS/kotiko/blob/main/CONTRIBUTING.md#tests | `CONTRIBUTING.md:55-63` | – | | `tests_documented_added` | MUST [J] [N/A J] | Met | The instructions for change proposals say it: CONTRIBUTING.md, Tests (https://github.com/ScriptKittyOS/kotiko/blob/main/CONTRIBUTING.md#tests), requires tests with every behaviour change and says major new functionality is not merged without them, and the pull request template's checklist asks for them. | `CONTRIBUTING.md:55-63`; `.github/pull_request_template.md:9-10` | – | | `warnings_strict` | MUST [J] [N/A J] | Met | Warnings are as strict as practical, and any warning fails CI on every pull request: the server compiles and runs its tests with --warnings-as-errors; Credo runs in strict mode (server/.credo.exs sets strict: true, so a plain mix credo reports the same); ESLint adds about 60 rules beyond its recommended set (among them eqeqeq, no-shadow, no-unused-vars for every argument, no-param-reassign, no-implicit-coercion, no-loop-func, prefer-const, no-var, no-eval) and treats unused disable directives as errors; web-ext lint, ShellCheck and Sobelow fail on findings too. The few rules not enabled, and why, are written down. https://github.com/ScriptKittyOS/kotiko/blob/main/docs/CODING_STANDARDS.md | `server/.credo.exs`; `.github/workflows/ci.yml` (`mix credo --strict`, `mix test --warnings-as-errors`); `eslint.config.js` (STRICTER block); `docs/CODING_STANDARDS.md` | – | @@ -281,8 +281,8 @@ earned, and gold once silver is; the answers can be entered before that. | `build_reproducible` | MUST [URL] [N/A J] | Met | The extension zips, the only built release files, are reproducible by anyone: the build uses entries sorted by path, the commit's timestamp, fixed permissions and a pinned pure-JavaScript deflate, so the bytes don't depend on the machine or time zone. The release workflow builds twice and compares, and docs/reproducible-builds.md shows how to rebuild a release from its tag and check it against the published SHA256SUMS. The server is released as source. https://github.com/ScriptKittyOS/kotiko/blob/main/docs/reproducible-builds.md | `docs/reproducible-builds.md:1-57`; `.github/workflows/release.yml:111-118`; two local builds on 2026-10-06 gave identical SHA-256 | Once v1.0.0 exists, have someone outside the project rebuild it and compare (slice 53 section 7.5) | | `test_invocation` | MUST [URL] | Met | npm test (Node's built-in test runner) for the extension and mix test for the server, the standard commands of each ecosystem; npm run e2e runs the Playwright end-to-end tests. https://github.com/ScriptKittyOS/kotiko/blob/main/CONTRIBUTING.md#setup | `package.json:10-11`; `CONTRIBUTING.md:19-30` | – | | `test_continuous_integration` | MUST [URL] | Met | Every change reaches main through a short-lived pull request, and GitHub Actions runs every test suite, the linters and the audits on each pull request and each push to main; main requires 8 checks to pass. https://github.com/ScriptKittyOS/kotiko/actions/workflows/ci.yml | `.github/workflows/ci.yml:5-8`; ruleset 24531669; `gh run list --workflow ci.yml` | – | -| `test_statement_coverage90` | MUST [J] [N/A J] | Unmet | Not yet. Measured 2026-10-06: the server's tests cover 87.8 % of lines (mix test --cover) and the extension's 92.6 % (node --test --experimental-test-coverage). Raising the server above 90 % (the Telegram client, the transcriber, the janitor and application start-up are the gaps) and adding a coverage gate to CI are planned. | Server cover table: Kotiko.Janitor 30.77 %, Kotiko.Application 36.99 %, Kotiko.TaskEnv 42.31 %, Kotiko.Transcriber 43.75 %, Kotiko.Telegram 69.57 %, Total 87.78 % | Agent: server tests to 90 %+ and a CI gate | -| `test_branch_coverage80` | MUST [J] [N/A J] | Unmet | Not yet. Measured 2026-10-06: the extension's tests cover 79.1 % of branches (node --test --experimental-test-coverage). For the server, Erlang's cover tool measures lines, not branches, and we know of no FLOSS branch-coverage tool for Elixir. | Extension coverage: all files 79.13 % branches; lowest `extension/ui/popover-style.js` 42.86 %, `extension/data-tools.js` 64.57 %, `extension/lib/backup.js` 69.10 % | Agent: extension branch coverage above 80 % with a CI gate. Check again for an Elixir branch-coverage tool at gold time | +| `test_statement_coverage90` | MUST [J] [N/A J] | Met | Statement coverage is above 90 % and enforced by FLOSS tools in CI and in the release workflow: the server's ExUnit suite covers 91.6 % of lines and fails below 90 % (mix test --cover, threshold in mix.exs), and the extension's Node test suites cover 93.9 % of lines and fail below 90 % (npm run coverage). https://github.com/ScriptKittyOS/kotiko/blob/main/CONTRIBUTING.md#tests | `server/mix.exs`; `package.json` `coverage`; `.github/workflows/ci.yml`; `.github/workflows/release.yml` | – | +| `test_branch_coverage80` | MUST [J] [N/A J] | Met | The extension's Node test suites cover 84.5 % of branches and fail below 80 % (npm run coverage, node --test with --test-coverage-branches=80), in CI and before every release. For the Elixir server no FLOSS tool measures branch coverage (Erlang's cover tool counts lines only), so its 90 % line gate applies instead. https://github.com/ScriptKittyOS/kotiko/blob/main/CONTRIBUTING.md#tests | `package.json` `coverage`; `.github/workflows/ci.yml`; `.github/workflows/release.yml` | – | ## Gold: Security @@ -298,8 +298,8 @@ earned, and gold once silver is; the answers can be entered before that. | Criterion | Level | Status | Answer | Evidence | Needs | |---|---|---|---|---|---| -| `dynamic_analysis` | MUST [J] [N/A J] | Unmet | Not yet for every release. CI runs property-based tests (StreamData) that feed generated input to two server parsers, the end-to-end tests drive the real extension against the real server, and axe scans every interface, but no fuzzer or scanner is applied to each proposed release yet. Planned: property-based tests for the request parsers, the URL and response validators and the matcher, run in the release workflow, or an OWASP ZAP baseline scan of the server. | `server/test/kotiko/text_test.exs:21`, `server/test/kotiko/word_test.exs:12` (the only `property` tests); `.github/workflows/release.yml:107-110` (build job runs lint and npm test only) | Agent: fast-check (dev dependency) and more StreamData properties, run in CI and the release build job | -| `dynamic_analysis_enable_assertions` | SHOULD [J] [N/A J] | Unmet | Not yet. The tests assert heavily, but the shipped code has no run-time assertions that are switched on during dynamic analysis. Planned with the property-based tests above. | – | Agent: invariant checks behind a test-only flag, exercised by the property and end-to-end tests | +| `dynamic_analysis` | MUST [J] [N/A J] | Met | Property-based testing is the dynamic analysis, and it runs on every pull request and again in the release workflow before anything is built. The extension's pure code (the page matcher, URL checks, bulk-add parsing, backup read and restore, casing, the word-list check) is tested with fast-check, and the server's request validation, word spec, settings parsing, language tags, text cleaning, redaction and backup round trips with StreamData; each run uses a new seed, and a failing seed can be replayed. It has found real defects: a word with Å or Ņ made the server answer 500, a repeated query parameter did the same, and a re-add brought back a form the learner had removed; each was fixed with a regression test. https://github.com/ScriptKittyOS/kotiko/blob/main/CONTRIBUTING.md#tests | `test/unit/properties/`; `server/test/` (StreamData properties); `.github/workflows/ci.yml`; `.github/workflows/release.yml` (`npm run coverage`, `mix test --cover`) | – | +| `dynamic_analysis_enable_assertions` | SHOULD [J] [N/A J] | Met | Under test, the extension runs with an assertion mode (globalThis.__KOTIKO_ASSERT__, set only by the test harness and never by the shipped extension): the matcher checks every scan's matches (bounds, text, order, no overlap), the backup restore plan checks its counts and that it never writes a deleted word, and the word-list check verifies every word is kept or counted. The property-based tests drive these paths with generated input. The server relies on Elixir's pattern matches and guards, which already stop on a broken invariant in production. https://github.com/ScriptKittyOS/kotiko/blob/main/CONTRIBUTING.md#tests | `test/helpers/assert-mode.mjs`; `extension/lib/matcher.js`; `extension/lib/backup.js`; `test/unit/properties/` | – | ## Baseline From 86be7c18eb1ccc74ddaa481f78bf8f696d17cfc7 Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Tue, 6 Oct 2026 17:39:03 -0400 Subject: [PATCH 11/12] test(server): a fake key that looks fake, and the old one recorded for gitleaks CI's gitleaks scan (full history) flagged "whisper-key-123" in voice_test.exs as a generic API key. It's a test value, but it broke the CONTRIBUTING rule that fake keys contain 0123456789, which is what tells the scanner they're fake. The test now uses whisper-key-0123456789, and .gitleaksignore records the one historical finding by its fingerprint with the reason, instead of rewriting signed commits. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ayla Croft --- .gitleaksignore | 9 +++++++++ server/test/kotiko/voice_test.exs | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 .gitleaksignore diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 0000000..311f505 --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,9 @@ +# SPDX-FileCopyrightText: 2026 ScriptKittyOS and the Kotiko contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Findings gitleaks may skip. Each line is a fingerprint (commit:file:rule:line) of a value +# that is not a secret, with the reason above it. New test keys follow CONTRIBUTING.md +# ("0123456789" in every fake key), so this list should stay short. + +# A fake Whisper key in a test, written before it followed that rule; renamed since. +bd016da4200b0d28093d8360304e46d844ef8f23:server/test/kotiko/voice_test.exs:generic-api-key:75 diff --git a/server/test/kotiko/voice_test.exs b/server/test/kotiko/voice_test.exs index 72cdabb..8f4e38e 100644 --- a/server/test/kotiko/voice_test.exs +++ b/server/test/kotiko/voice_test.exs @@ -72,11 +72,11 @@ defmodule Kotiko.VoiceTest do describe "Transcriber.transcribe/1" do test "uploads the audio with the model and the key, and trims the text" do - put_app_env(:transcribe_api_key, "whisper-key-123") + put_app_env(:transcribe_api_key, "whisper-key-0123456789") stub_transcriber(&Req.Test.json(&1, %{text: " shukran \n"})) assert Transcriber.transcribe("OggS") == {:ok, "shukran"} - assert_received {:transcribe, ["Bearer whisper-key-123"], body} + assert_received {:transcribe, ["Bearer whisper-key-0123456789"], body} assert body =~ ~s(name="file"; filename="voice.ogg") assert body =~ "whisper-1" end From f02766b606aca935c817d8a25cd001df87e0d4bc Mon Sep 17 00:00:00 2001 From: Ayla Croft Date: Tue, 6 Oct 2026 17:39:47 -0400 Subject: [PATCH 12/12] ci: gitleaks scans the history being merged, not every branch The secrets job checks out with fetch-depth 0, which fetches every branch, and ran gitleaks with --log-opts=--all. So a finding on one pull request's branch failed every other pull request (#22 failed on a test key in #21's branch). It now scans HEAD: the whole history of main plus the pull request's own commits, which is everything that would land. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Ayla Croft --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 697a057..e95da66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,9 @@ jobs: echo "${GITLEAKS_SHA256} $RUNNER_TEMP/gitleaks.tar.gz" | sha256sum -c - tar -xzf "$RUNNER_TEMP/gitleaks.tar.gz" -C "$RUNNER_TEMP" gitleaks - name: Scan the full history - run: '"$RUNNER_TEMP/gitleaks" git --log-opts="--all" --redact --config .gitleaks.toml --exit-code 1 .' + # HEAD's full history (all of main plus this pull request's commits). Not --all: the + # checkout fetches every branch, and another branch's finding would fail this one. + run: '"$RUNNER_TEMP/gitleaks" git --log-opts="HEAD" --redact --config .gitleaks.toml --exit-code 1 .' - name: No secret, database or oversized files env: BASE: ${{ github.event.pull_request.base.sha }}