From e9bb14a3e2c2184d695c5d799eb1e387afc85415 Mon Sep 17 00:00:00 2001 From: pirumpi Date: Fri, 16 Oct 2026 12:23:27 -0600 Subject: [PATCH 1/3] Add unpublished MiniCPM-V 4.7 still-image candidate --- .github/workflows/build.yml | 4 + THIRD_PARTY_NOTICES.md | 12 + catalog/candidates/minicpm-v-4.7.json | 62 +++++ desktop/package.json | 7 + desktop/scripts/prepare-runtime.ts | 12 +- desktop/src/app/adapters.test.ts | 18 ++ desktop/src/app/adapters.ts | 9 +- .../src/features/chat/ChatWorkspace.test.ts | 44 ++++ desktop/src/features/chat/ChatWorkspace.tsx | 81 +++++- desktop/src/features/chat/chat.css | 33 ++- desktop/src/features/chat/state.ts | 2 +- desktop/src/features/chat/types.ts | 3 +- desktop/src/shared/ipc.ts | 16 +- desktop/src/shared/vision-ipc.test.ts | 13 + docs/guides/minicpm-v-4.7.md | 165 +++++++++++++ .../guides/model-integration-qualification.md | 7 + internal/app/app.go | 49 +++- internal/app/artifact_integrity.go | 37 +-- internal/app/vision_test.go | 139 +++++++++++ internal/catalog/catalog.go | 55 ++++- internal/catalog/integrity.go | 38 +++ internal/catalog/vision_test.go | 73 ++++++ internal/config/config.go | 18 +- internal/control/features.go | 11 +- internal/control/features_test.go | 37 +++ internal/server/anthropic.go | 15 +- internal/server/responses.go | 15 +- internal/server/server.go | 91 ++++++- internal/server/testdata/vision/ocr.png | Bin 0 -> 9953 bytes internal/server/testdata/vision/shapes.png | Bin 0 -> 1623 bytes internal/server/vision_input.go | 134 ++++++++++ internal/server/vision_runtime.go | 98 ++++++++ internal/server/vision_test.go | 232 ++++++++++++++++++ scripts/model_watch.py | 4 +- scripts/prepare-vision-runtime.sh | 18 ++ scripts/qualify-minicpm-vision.py | 196 +++++++++++++++ scripts/test-model-watch.py | 8 + 37 files changed, 1680 insertions(+), 76 deletions(-) create mode 100644 catalog/candidates/minicpm-v-4.7.json create mode 100644 desktop/src/shared/vision-ipc.test.ts create mode 100644 docs/guides/minicpm-v-4.7.md create mode 100644 internal/app/vision_test.go create mode 100644 internal/catalog/integrity.go create mode 100644 internal/catalog/vision_test.go create mode 100644 internal/server/testdata/vision/ocr.png create mode 100644 internal/server/testdata/vision/shapes.png create mode 100644 internal/server/vision_input.go create mode 100644 internal/server/vision_runtime.go create mode 100644 internal/server/vision_test.go create mode 100755 scripts/prepare-vision-runtime.sh create mode 100644 scripts/qualify-minicpm-vision.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 365e7d6..8b1ff73 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -102,12 +102,16 @@ jobs: tar -xzf llama.cpp.tar.gz -C dist/runtime/llama.cpp --strip-components=1 test -x dist/runtime/llama.cpp/llama-server + - name: Prepare isolated Apple Silicon vision runtime + run: sh scripts/prepare-vision-runtime.sh dist/runtime + - name: Build Tapioca Desktop env: CSC_IDENTITY_AUTO_DISCOVERY: "false" run: | mkdir -p desktop/runtime cp -R dist/runtime/llama.cpp desktop/runtime/llama.cpp + cp -R dist/runtime/llama-vision-b11696 desktop/runtime/llama-vision-b11696 npm ci --prefix desktop npm run package --prefix desktop diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index bb83275..6b20700 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -77,3 +77,15 @@ modifying, or redistributing model files or generated outputs. This notice is informational and is not legal advice. The license files shipped with a particular dependency or model are authoritative. + +### Optional MiniCPM-V 4.7 1B candidate + +OpenBMB's source and official GGUF model cards declare exactly +`license: apache-2.0`. Source: ; +GGUF card pinned at +. +The candidate pull retains that exact card as `MODEL_CARD.md`, with its own +revision, byte count and SHA-256. No separate model LICENSE file existed in the +pinned repository. The isolated llama.cpp b11696 archive retains its upstream +LICENSE and notices. This candidate is still-image understanding/OCR only; +its presence here does not claim platform or hardware qualification. diff --git a/catalog/candidates/minicpm-v-4.7.json b/catalog/candidates/minicpm-v-4.7.json new file mode 100644 index 0000000..2d3dac1 --- /dev/null +++ b/catalog/candidates/minicpm-v-4.7.json @@ -0,0 +1,62 @@ +{ + "schema": 1, + "generated_at": "2026-10-16T16:00:00Z", + "models": { + "minicpm-v-4.7": { + "name": "minicpm-v-4.7", + "repo": "openbmb/MiniCPM-V-4.7-1B-gguf", + "files": { + "1b-q4-k-m": "MiniCPM-V-4_7-1B-Q4_K_M.gguf" + }, + "default": "1b-q4-k-m", + "backends": { + "1b-q4-k-m": "llama-vision" + }, + "context": 4096, + "sizes": { + "1b-q4-k-m": "1.66 GiB (model + projector)" + }, + "memory": { + "1b-q4-k-m": "Qualification target: 8 GiB; 16 GiB preferred; measured minimum pending" + }, + "gpus": { + "1b-q4-k-m": "Apple Silicon Metal or CPU; other platforms unqualified" + }, + "features": { + "1b-q4-k-m": "Experimental still-image understanding/OCR; PNG/JPEG data URLs; no video; reasoning off by default" + }, + "license": "Apache-2.0", + "license_url": "https://huggingface.co/openbmb/MiniCPM-V-4.7-1B-gguf/blob/6c2f4cb8bde51a00247e11003bbf66ca7c4f9249/README.md", + "downloads": { + "1b-q4-k-m": { + "revision": "6c2f4cb8bde51a00247e11003bbf66ca7c4f9249", + "size_bytes": 672003680, + "sha256": "588c6c94c82f756b86dba35c42ed9ba9ec187d65824eb52f039c347df563e63d" + } + }, + "projectors": { + "1b-q4-k-m": { + "repo": "openbmb/MiniCPM-V-4.7-1B-gguf", + "filename": "mmproj-model-f16.gguf", + "target": "mmproj-model-f16.gguf", + "revision": "6c2f4cb8bde51a00247e11003bbf66ca7c4f9249", + "size_bytes": 1110117984, + "sha256": "fb9cc047a3cdad5317fd8433a4aeda2d22f00f02ca4bd93b34ad84165b25a900" + } + }, + "artifacts": { + "1b-q4-k-m": [ + { + "repo": "openbmb/MiniCPM-V-4.7-1B-gguf", + "filename": "README.md", + "target": "MODEL_CARD.md", + "revision": "6c2f4cb8bde51a00247e11003bbf66ca7c4f9249", + "sha256": "a8730753941a10d236343ef977d744648ed8a88c0f50debd6a74089122041bb8", + "size_bytes": 17895 + } + ] + }, + "kind": "text" + } + } +} diff --git a/desktop/package.json b/desktop/package.json index 0e6175b..7f90440 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -81,6 +81,13 @@ { "from": "../THIRD_PARTY_NOTICES.md", "to": "THIRD_PARTY_NOTICES.md" + }, + { + "from": "runtime", + "to": "sidecar/runtime", + "filter": [ + "llama-vision-b11696/**/*" + ] } ], "asar": true, diff --git a/desktop/scripts/prepare-runtime.ts b/desktop/scripts/prepare-runtime.ts index 4c08c5e..21ca079 100644 --- a/desktop/scripts/prepare-runtime.ts +++ b/desktop/scripts/prepare-runtime.ts @@ -103,5 +103,15 @@ export async function prepareRuntime( const invoked = process.argv[1] ? path.resolve(process.argv[1]) : ""; if (invoked === fileURLToPath(import.meta.url)) { const desktopRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); - console.log(`Prepared ${await prepareRuntime(desktopRoot)}`); + const prepared = await prepareRuntime(desktopRoot); + if (process.platform === "darwin" && process.arch === "arm64") { + const vision = spawnSync( + "sh", + [path.resolve(desktopRoot, "../scripts/prepare-vision-runtime.sh"), path.resolve(desktopRoot, "runtime")], + { shell: false, stdio: "inherit" }, + ); + if (vision.error) throw vision.error; + if (vision.status !== 0) throw new Error("Preparing pinned vision runtime failed"); + } + console.log(`Prepared ${prepared}`); } diff --git a/desktop/src/app/adapters.test.ts b/desktop/src/app/adapters.test.ts index 870a331..1ee9d96 100644 --- a/desktop/src/app/adapters.test.ts +++ b/desktop/src/app/adapters.test.ts @@ -569,3 +569,21 @@ describe("renderer adapters", () => { expect(onEvent).toHaveBeenCalledWith({ type: "exit", code: 0 }); }); }); + +it("forwards ordered image parts and retains them when reopening a session", async () => { + const name = "minicpm-v-4.7:1b-q4-k-m"; + const status = [{id: "desktop-chat", model: name, endpoint: "http://127.0.0.1:11435", state: "running" as const}]; + const api = apiFixture({ + serverStatus: vi.fn().mockResolvedValue(status), + chatStatus: vi.fn().mockResolvedValue({installed: [{name, repo: "openbmb/model", backend: "llama-vision"}], servers: status}), + chatRequest: vi.fn().mockResolvedValue({id: "answer", object: "chat.completion", created: 1, model: name, choices: [{index: 0, message: {role: "assistant", content: "Red circle"}, finish_reason: "stop"}]}), + }); + const adapter = createRendererAdapters(api, vi.fn()).chat; + const images = ["data:image/png;base64,AA==", "data:image/jpeg;base64,AQ=="]; + const result = await adapter.send({modelId: name, messages: [{role: "user", content: "Describe", images}]}, vi.fn()); + expect(api.chatRequest).toHaveBeenCalledWith(expect.objectContaining({messages: [{role: "user", content: [{type: "text", text: "Describe"}, ...images.map((url) => ({type: "image_url", image_url: {url}}))]}]})); + const session = await adapter.loadSession(result.sessionId); + expect(session.messages[0].images).toEqual(images); + await adapter.send({modelId: name, sessionId: result.sessionId, messages: [...session.messages, {role: "user", content: "Which color?"}]}, vi.fn()); + expect(api.chatRequest).toHaveBeenLastCalledWith(expect.objectContaining({messages: expect.arrayContaining([expect.objectContaining({role: "user", content: expect.arrayContaining([{type: "image_url", image_url: {url: images[0]}}])})])})); +}); diff --git a/desktop/src/app/adapters.ts b/desktop/src/app/adapters.ts index abf9b33..7181848 100644 --- a/desktop/src/app/adapters.ts +++ b/desktop/src/app/adapters.ts @@ -203,7 +203,13 @@ export function createRendererAdapters( const response = await api.chatRequest({ jobId, model: request.modelId, - messages: request.messages, + messages: request.messages.map(({ role, content, images }) => ({ + role, + content: images?.length ? [ + { type: "text" as const, text: content }, + ...images.map((url) => ({ type: "image_url" as const, image_url: { url } })), + ] : content, + })), }); const message = response.choices[0]?.message; const content = textContent(message?.content); @@ -230,6 +236,7 @@ export function createRendererAdapters( id: `${sessionId}-${index}`, role: item.role, content: item.content, + ...(item.images?.length ? {images: item.images} : {}), createdAt: new Date().toISOString(), })), { diff --git a/desktop/src/features/chat/ChatWorkspace.test.ts b/desktop/src/features/chat/ChatWorkspace.test.ts index eefa47d..24b2794 100644 --- a/desktop/src/features/chat/ChatWorkspace.test.ts +++ b/desktop/src/features/chat/ChatWorkspace.test.ts @@ -81,3 +81,47 @@ describe("ChatWorkspace", () => { expect(container.textContent).toContain("Conversation ended with /bye."); }); }); + +it("sends a selected attachment with the prompt and keeps its transcript preview", async () => { + Element.prototype.scrollIntoView = vi.fn(); + class DecodedImage { + width = 512; + height = 256; + onload?: () => void; + set src(_value: string) { queueMicrotask(() => this.onload?.()); } + } + vi.stubGlobal("Image", DecodedImage); + const send = vi.fn().mockResolvedValue({sessionId: "session", assistantMessageId: "answer"}); + const adapter: ChatAdapter = { + connection: vi.fn().mockResolvedValue("ready"), + reconnect: vi.fn().mockResolvedValue("ready"), + models: vi.fn().mockResolvedValue([{id: "minicpm", name: "MiniCPM", ready: true}]), + sessions: vi.fn().mockResolvedValue([]), + loadSession: vi.fn(), send, stop: vi.fn(), + }; + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + try { + await act(async () => root?.render(createElement(ChatWorkspace, {adapter}))); + const picker = container.querySelector('input[type="file"]'); + const file = new File(["fixture"], "scene.png", {type: "image/png"}); + Object.defineProperty(picker, "files", {value: [file]}); + await act(async () => { + picker?.dispatchEvent(new Event("change", {bubbles: true})); + await new Promise((resolve) => setTimeout(resolve, 20)); + }); + expect(container.querySelector('img[alt="Pending image 1"]')).not.toBeNull(); + const input = container.querySelector("#chat-prompt"); + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value")?.set?.call(input,"Describe"); + input?.dispatchEvent(new Event("input", {bubbles: true})); + }); + await act(async () => container?.querySelector('button[type="submit"]')?.click()); + expect(send).toHaveBeenCalledWith(expect.objectContaining({messages: [expect.objectContaining({content: "Describe", images: ["data:image/png;base64,Zml4dHVyZQ=="]})]}), expect.any(Function)); + expect(container.querySelector('img[alt="Attached image 1"]')).not.toBeNull(); + expect(container.querySelector('img[alt="Pending image 1"]')).toBeNull(); + } finally { + vi.unstubAllGlobals(); + } +}); diff --git a/desktop/src/features/chat/ChatWorkspace.tsx b/desktop/src/features/chat/ChatWorkspace.tsx index 8270e59..9a18c23 100644 --- a/desktop/src/features/chat/ChatWorkspace.tsx +++ b/desktop/src/features/chat/ChatWorkspace.tsx @@ -1,4 +1,5 @@ import { + type ChangeEvent, type FormEvent, type KeyboardEvent, useCallback, @@ -40,6 +41,8 @@ export function ChatWorkspace({ adapter }: ChatWorkspaceProps) { const [messages, setMessages] = useState([]); const [draft, setDraft] = useState(); const [input, setInput] = useState(""); + const [images, setImages] = useState([]); + const [readingImages, setReadingImages] = useState(false); const [busy, setBusy] = useState(false); const [closed, setClosed] = useState(false); const [error, setError] = useState(); @@ -78,12 +81,13 @@ export function ChatWorkspace({ adapter }: ChatWorkspaceProps) { ); const openSession = async (id: string) => { - if (busy) return; + if (busy || readingImages) return; setError(undefined); try { const session = await adapter.loadSession(id); setSessionId(session.id); setMessages(session.messages); + setImages([]); if (session.modelId) setModelId(session.modelId); setClosed(false); } catch (cause) { @@ -91,9 +95,30 @@ export function ChatWorkspace({ adapter }: ChatWorkspaceProps) { } }; + const attachImages = async (event: ChangeEvent) => { + const files = Array.from(event.target.files ?? []); + event.target.value = ""; + const historyImages = messages.reduce((total, message) => total + (message.images?.length ?? 0), 0); + if (historyImages + images.length + files.length > 4 || files.some((file) => + file.size > 128 * 1024 || !["image/png", "image/jpeg"].includes(file.type))) { + setError("Choose at most four images per conversation, each a PNG/JPEG no larger than 128 KiB and 1024 pixels per side."); + return; + } + setReadingImages(true); + try { + const urls = await Promise.all(files.map(readChatImage)); + setImages((current) => [...current, ...urls]); + setError(undefined); + } catch (cause) { + setError(cause instanceof Error ? cause.message : String(cause)); + } finally { + setReadingImages(false); + } + }; + const sendPrompt = async (prompt: string) => { const text = prompt.trim(); - if (!text || busy) return; + if (!text || busy || readingImages) return; if (isByeCommand(text)) { try { await adapter.stop(); @@ -116,10 +141,12 @@ export function ChatWorkspace({ adapter }: ChatWorkspaceProps) { id: crypto.randomUUID(), role: "user", content: text, + ...(images.length ? {images} : {}), createdAt: now(), }; const requestMessages = [...messages, userMessage]; setMessages(requestMessages); + setImages([]); setInput(""); setError(undefined); setClosed(false); @@ -214,10 +241,12 @@ export function ChatWorkspace({ adapter }: ChatWorkspaceProps) { onClick={() => { setSessionId(undefined); setMessages([]); + setImages([]); setDraft(undefined); setClosed(false); setError(undefined); }} + disabled={busy || readingImages} aria-label="Start a new conversation" > + @@ -322,6 +351,7 @@ export function ChatWorkspace({ adapter }: ChatWorkspaceProps) { )} + {message.images?.map((url, index) => {`Attached)}

{message.content}

))} @@ -366,6 +396,30 @@ export function ChatWorkspace({ adapter }: ChatWorkspaceProps) { ) : (
+ + {images.length > 0 && ( +
+ {images.map((url, index) => ( + + ))} +
+ )}