Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ jobs:
go-version-file: go.mod
- run: go vet ./...
- run: go test ./...
- name: Test MiniCPM acceptance cleanup
run: python3 scripts/test-minicpm-qualification.py
- name: Test Sopro candidate validation
run: python3 -m unittest discover -s internal/speechruntime -p 'test_sopro_qualification.py'
- name: Verify published catalog manifest
Expand Down Expand Up @@ -102,12 +104,16 @@ jobs:
tar -xzf llama.cpp.tar.gz -C dist/runtime/llama.cpp --strip-components=1
test -x dist/runtime/llama.cpp/llama-server

- name: Prepare isolated Apple Silicon vision runtime
run: sh scripts/prepare-vision-runtime.sh dist/runtime

- name: Build Tapioca Desktop
env:
CSC_IDENTITY_AUTO_DISCOVERY: "false"
run: |
mkdir -p desktop/runtime
cp -R dist/runtime/llama.cpp desktop/runtime/llama.cpp
cp -R dist/runtime/llama-vision-b11696 desktop/runtime/llama-vision-b11696
npm ci --prefix desktop
npm run package --prefix desktop

Expand Down
12 changes: 12 additions & 0 deletions THIRD_PARTY_NOTICES.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,15 @@ modifying, or redistributing model files or generated outputs.

This notice is informational and is not legal advice. The license files shipped
with a particular dependency or model are authoritative.

### Optional MiniCPM-V 4.7 1B candidate

OpenBMB's source and official GGUF model cards declare exactly
`license: apache-2.0`. Source: <https://huggingface.co/openbmb/MiniCPM-V-4.7-1B>;
GGUF card pinned at
<https://huggingface.co/openbmb/MiniCPM-V-4.7-1B-gguf/blob/6c2f4cb8bde51a00247e11003bbf66ca7c4f9249/README.md>.
The candidate pull retains that exact card as `MODEL_CARD.md`, with its own
revision, byte count and SHA-256. No separate model LICENSE file existed in the
pinned repository. The isolated llama.cpp b11696 archive retains its upstream
LICENSE and notices. This candidate is still-image understanding/OCR only;
its presence here does not claim platform or hardware qualification.
62 changes: 62 additions & 0 deletions catalog/candidates/minicpm-v-4.7.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
{
"schema": 1,
"generated_at": "2026-10-16T16:00:00Z",
"models": {
"minicpm-v-4.7": {
"name": "minicpm-v-4.7",
"repo": "openbmb/MiniCPM-V-4.7-1B-gguf",
"files": {
"1b-q4-k-m": "MiniCPM-V-4_7-1B-Q4_K_M.gguf"
},
"default": "1b-q4-k-m",
"backends": {
"1b-q4-k-m": "llama-vision"
},
"context": 4096,
"sizes": {
"1b-q4-k-m": "1.66 GiB (model + projector)"
},
"memory": {
"1b-q4-k-m": "Qualification target: 8 GiB; 16 GiB preferred; measured minimum pending"
},
"gpus": {
"1b-q4-k-m": "Apple Silicon Metal or CPU; other platforms unqualified"
},
"features": {
"1b-q4-k-m": "Experimental still-image understanding/OCR; PNG/JPEG data URLs; no video; reasoning off by default"
},
"license": "Apache-2.0",
"license_url": "https://huggingface.co/openbmb/MiniCPM-V-4.7-1B-gguf/blob/6c2f4cb8bde51a00247e11003bbf66ca7c4f9249/README.md",
"downloads": {
"1b-q4-k-m": {
"revision": "6c2f4cb8bde51a00247e11003bbf66ca7c4f9249",
"size_bytes": 672003680,
"sha256": "588c6c94c82f756b86dba35c42ed9ba9ec187d65824eb52f039c347df563e63d"
}
},
"projectors": {
"1b-q4-k-m": {
"repo": "openbmb/MiniCPM-V-4.7-1B-gguf",
"filename": "mmproj-model-f16.gguf",
"target": "mmproj-model-f16.gguf",
"revision": "6c2f4cb8bde51a00247e11003bbf66ca7c4f9249",
"size_bytes": 1110117984,
"sha256": "fb9cc047a3cdad5317fd8433a4aeda2d22f00f02ca4bd93b34ad84165b25a900"
}
},
"artifacts": {
"1b-q4-k-m": [
{
"repo": "openbmb/MiniCPM-V-4.7-1B-gguf",
"filename": "README.md",
"target": "MODEL_CARD.md",
"revision": "6c2f4cb8bde51a00247e11003bbf66ca7c4f9249",
"sha256": "a8730753941a10d236343ef977d744648ed8a88c0f50debd6a74089122041bb8",
"size_bytes": 17895
}
]
},
"kind": "text"
}
}
}
4 changes: 2 additions & 2 deletions desktop/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 8 additions & 1 deletion desktop/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@tapioca/desktop",
"version": "0.13.0",
"version": "0.14.0",
"private": true,
"description": "Tapioca desktop application",
"author": "Kyndlo",
Expand Down Expand Up @@ -81,6 +81,13 @@
{
"from": "../THIRD_PARTY_NOTICES.md",
"to": "THIRD_PARTY_NOTICES.md"
},
{
"from": "runtime",
"to": "sidecar/runtime",
"filter": [
"llama-vision-b11696/**/*"
]
}
],
"asar": true,
Expand Down
12 changes: 11 additions & 1 deletion desktop/scripts/prepare-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,5 +103,15 @@ export async function prepareRuntime(
const invoked = process.argv[1] ? path.resolve(process.argv[1]) : "";
if (invoked === fileURLToPath(import.meta.url)) {
const desktopRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
console.log(`Prepared ${await prepareRuntime(desktopRoot)}`);
const prepared = await prepareRuntime(desktopRoot);
if (process.platform === "darwin" && process.arch === "arm64") {
const vision = spawnSync(
"sh",
[path.resolve(desktopRoot, "../scripts/prepare-vision-runtime.sh"), path.resolve(desktopRoot, "runtime")],
{ shell: false, stdio: "inherit" },
);
if (vision.error) throw vision.error;
if (vision.status !== 0) throw new Error("Preparing pinned vision runtime failed");
}
console.log(`Prepared ${prepared}`);
}
18 changes: 18 additions & 0 deletions desktop/src/app/adapters.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -569,3 +569,21 @@ describe("renderer adapters", () => {
expect(onEvent).toHaveBeenCalledWith({ type: "exit", code: 0 });
});
});

it("forwards ordered image parts and retains them when reopening a session", async () => {
const name = "minicpm-v-4.7:1b-q4-k-m";
const status = [{id: "desktop-chat", model: name, endpoint: "http://127.0.0.1:11435", state: "running" as const}];
const api = apiFixture({
serverStatus: vi.fn().mockResolvedValue(status),
chatStatus: vi.fn().mockResolvedValue({installed: [{name, repo: "openbmb/model", backend: "llama-vision"}], servers: status}),
chatRequest: vi.fn().mockResolvedValue({id: "answer", object: "chat.completion", created: 1, model: name, choices: [{index: 0, message: {role: "assistant", content: "Red circle"}, finish_reason: "stop"}]}),
});
const adapter = createRendererAdapters(api, vi.fn()).chat;
const images = ["data:image/png;base64,AA==", "data:image/jpeg;base64,AQ=="];
const result = await adapter.send({modelId: name, messages: [{role: "user", content: "Describe", images}]}, vi.fn());
expect(api.chatRequest).toHaveBeenCalledWith(expect.objectContaining({messages: [{role: "user", content: [{type: "text", text: "Describe"}, ...images.map((url) => ({type: "image_url", image_url: {url}}))]}]}));
const session = await adapter.loadSession(result.sessionId);
expect(session.messages[0].images).toEqual(images);
await adapter.send({modelId: name, sessionId: result.sessionId, messages: [...session.messages, {role: "user", content: "Which color?"}]}, vi.fn());
expect(api.chatRequest).toHaveBeenLastCalledWith(expect.objectContaining({messages: expect.arrayContaining([expect.objectContaining({role: "user", content: expect.arrayContaining([{type: "image_url", image_url: {url: images[0]}}])})])}));
});
9 changes: 8 additions & 1 deletion desktop/src/app/adapters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,13 @@ export function createRendererAdapters(
const response = await api.chatRequest({
jobId,
model: request.modelId,
messages: request.messages,
messages: request.messages.map(({ role, content, images }) => ({
role,
content: images?.length ? [
{ type: "text" as const, text: content },
...images.map((url) => ({ type: "image_url" as const, image_url: { url } })),
] : content,
})),
});
const message = response.choices[0]?.message;
const content = textContent(message?.content);
Expand All @@ -230,6 +236,7 @@ export function createRendererAdapters(
id: `${sessionId}-${index}`,
role: item.role,
content: item.content,
...(item.images?.length ? {images: item.images} : {}),
createdAt: new Date().toISOString(),
})),
{
Expand Down
62 changes: 62 additions & 0 deletions desktop/src/features/chat/ChatWorkspace.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,3 +81,65 @@ describe("ChatWorkspace", () => {
expect(container.textContent).toContain("Conversation ended with /bye.");
});
});

it("sends and retries an attachment without duplicating the user message", async () => {
Element.prototype.scrollIntoView = vi.fn();
class DecodedImage {
width = 512;
height = 256;
onload?: () => void;
set src(_value: string) { queueMicrotask(() => this.onload?.()); }
}
vi.stubGlobal("Image", DecodedImage);
const send = vi.fn()
.mockRejectedValueOnce(new Error("Temporary failure"))
.mockResolvedValueOnce({sessionId: "session", assistantMessageId: "answer"});
const adapter: ChatAdapter = {
connection: vi.fn().mockResolvedValue("ready"),
reconnect: vi.fn().mockResolvedValue("ready"),
models: vi.fn().mockResolvedValue([{id: "minicpm", name: "MiniCPM", ready: true}]),
sessions: vi.fn().mockResolvedValue([]),
loadSession: vi.fn(), send, stop: vi.fn(),
};
container = document.createElement("div");
document.body.append(container);
root = createRoot(container);
let completeRead!: () => void;
const readComplete = new Promise<void>((resolve) => { completeRead = resolve; });
const nativeRead = FileReader.prototype.readAsDataURL;
const readSpy = vi.spyOn(FileReader.prototype, "readAsDataURL").mockImplementation(function (this: FileReader, file: Blob) {
this.addEventListener("loadend", () => queueMicrotask(completeRead), {once: true});
nativeRead.call(this, file);
});
try {
await act(async () => root?.render(createElement(ChatWorkspace, {adapter})));
const picker = container.querySelector<HTMLInputElement>('input[type="file"]');
const file = new File(["fixture"], "scene.png", {type: "image/png"});
Object.defineProperty(picker, "files", {value: [file]});
await act(async () => {
picker?.dispatchEvent(new Event("change", {bubbles: true}));
await readComplete;
});
expect(container.querySelector('img[alt="Pending image 1"]')).not.toBeNull();
const input = container.querySelector<HTMLTextAreaElement>("#chat-prompt");
await act(async () => {
Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, "value")?.set?.call(input,"Describe");
input?.dispatchEvent(new Event("input", {bubbles: true}));
});
await act(async () => container?.querySelector<HTMLButtonElement>('button[type="submit"]')?.click());
expect(send).toHaveBeenCalledWith(expect.objectContaining({messages: [expect.objectContaining({content: "Describe", images: ["data:image/png;base64,Zml4dHVyZQ=="]})]}), expect.any(Function));
expect(container.querySelector('img[alt="Attached image 1"]')).not.toBeNull();
expect(container.querySelector('img[alt="Pending image 1"]')).toBeNull();
expect(container.textContent).toContain("Temporary failure");
const original = send.mock.calls[0][0].messages;
const retry = Array.from(container.querySelectorAll("button")).find((button) => button.textContent === "Try again");
await act(async () => retry?.click());
expect(send).toHaveBeenCalledTimes(2);
expect(send.mock.calls[1][0].messages).toEqual(original);
expect(container.querySelectorAll('.chat-message--user')).toHaveLength(1);
expect(container.querySelectorAll('img[alt="Attached image 1"]')).toHaveLength(1);
} finally {
readSpy.mockRestore();
vi.unstubAllGlobals();
}
});
Loading
Loading