Self-contained desktop installers with first-run model setup

.github/workflows/ci.yml

@@ -19,6 +19,44 @@ jobs:
         run: node scripts/check-release-notes.mjs
       - name: One version across core, app and desktop
         run: node scripts/version.mjs
+  desktop:
+    # Builds the self-contained desktop app for each platform and checks it runs with nothing else installed.
+    # Pull requests only: it takes several minutes per platform.
+    name: desktop app (${{ matrix.os }})
+    if: github.event_name == 'pull_request'
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest, ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: |
+            app/package-lock.json
+            desktop/package-lock.json
+      - uses: astral-sh/setup-uv@v6
+      - name: Build web app
+        working-directory: app
+        run: npm ci && npm run build
+      - name: Bundle and package
+        working-directory: desktop
+        shell: bash
+        env:
+          CSC_IDENTITY_AUTO_DISCOVERY: false
+        run: |
+          npm ci
+          npm test
+          npm run bundle
+          npx electron-builder --dir ${{ runner.os == 'macOS' && '--arm64' || '' }}
+      - name: Check the packaged app works on its own
+        working-directory: desktop
+        shell: bash
+        run: ${{ runner.os == 'Linux' && 'xvfb-run -a --server-args="-screen 0 1600x1000x24"' || '' }} node scripts/check-package.mjs
   core:
     runs-on: ubuntu-latest
     steps:

.github/workflows/release.yml

@@ -64,9 +64,14 @@ jobs:
             echo "Tag $GITHUB_REF_NAME does not match version $version"; exit 1
           fi
           node scripts/changelog-section.mjs "$version" --require > /dev/null
+      - uses: astral-sh/setup-uv@v6
       - name: Build web app
         working-directory: app
         run: npm ci && npm run build
+      # Installers carry their own Python, core, web app and FFmpeg so they work with nothing else installed.
+      - name: Bundle Python, core, web app and FFmpeg
+        working-directory: desktop
+        run: npm ci && npm test && npm run bundle
       - name: Build installers (unsigned until certificates are configured, see OPEN_QUESTIONS Q3)
         working-directory: desktop
         shell: bash
@@ -75,10 +80,11 @@ jobs:
           # APPLE_TEAM_ID (notarisation) as repository secrets, then remove "identity": null from package.json.
           # Until then macOS builds are ad-hoc signed by desktop/scripts/adhoc-sign-mac.mjs.
           CSC_IDENTITY_AUTO_DISCOVERY: false
-        run: |
-          npm ci
-          npm test
-          npx electron-builder --publish never ${{ runner.os == 'macOS' && '--arm64' || '' }}
+        run: npx electron-builder --publish never ${{ runner.os == 'macOS' && '--arm64' || '' }}
+      - name: Check the packaged app works on its own
+        working-directory: desktop
+        shell: bash
+        run: ${{ runner.os == 'Linux' && 'xvfb-run -a --server-args="-screen 0 1600x1000x24"' || '' }} node scripts/check-package.mjs
       - uses: actions/upload-artifact@v4
         with:
           name: installers-${{ runner.os }}

.gitignore

@@ -29,3 +29,7 @@ app/playwright-report/
 .coverage
 coverage.xml
 ci-gates.json
+
+# Desktop bundle (desktop/scripts/bundle.mjs) and its download cache
+desktop/bundle/
+desktop/.cache/

CHANGELOG.md

@@ -8,6 +8,14 @@ still pre-1.0, minor bumps carry new features and patch bumps carry fixes.
 
 ## [Unreleased]
 
+### Added
+
+- **The desktop app installs everything it needs.** The Windows installer, the macOS disk image and the Linux AppImage now include their own Python, the Metachlorian engine, FFmpeg and the web app, so nothing else has to be installed first.
+- The first time it opens, the desktop app downloads the analysis models (about 1.7 GB, once) and shows its progress. If the download fails it says so and offers **Try again**, keeping anything already downloaded.
+
 ### Fixed
 
+- Metachlorian now runs on Windows. It previously stopped at start-up there, because it used a file lock that only exists on macOS and Linux.
+- Semantic search and zero-shot labels now work on Apple Silicon Macs. Embedding shots failed when analysis used Apple's Core ML accelerator; it now runs on the CPU unless `METACHLORIAN_COREML=1` is set.
+
 - The macOS desktop download no longer opens with "Metachlorian is damaged and can't be opened". The app is now ad-hoc signed, so macOS shows its usual unidentified-developer prompt and **System Settings → Privacy & Security → Open Anyway** works. It is still not signed with an Apple Developer ID or notarised.

README.md

@@ -131,9 +131,12 @@ cd core && uv venv -p 3.11 && uv pip install -e ".[analysis]"
 cd ../app && npm ci && npm run build           # the web app the core serves
 ```
 
-**Desktop app:** `cd desktop && npm ci && npm start` runs the web app in Electron. In solo mode it starts a local core; in team
-mode it connects to a shared one. It adds native drag-out of real clip files and one-click hand-off to Cutawan. Installers are
-built with `npm run package` (unsigned until signing certificates are provided, see [OPEN_QUESTIONS.md](OPEN_QUESTIONS.md)).
+**Desktop app (Windows, macOS, Linux):** download the installer from the [releases page](https://github.com/JeremySNR/Metachlorian/releases)
+and open it. It includes everything (its own Python, the core, FFmpeg and the web app), so nothing else needs installing. The
+first launch downloads the analysis models (about 1.7 GB, once). In solo mode it runs the library on your computer; in team
+mode it connects to a shared server. It adds native drag-out of real clip files and one-click hand-off to Cutawan. Installers
+are not yet signed (see [OPEN_QUESTIONS.md](OPEN_QUESTIONS.md)): on macOS use **System Settings → Privacy & Security → Open
+Anyway** the first time; on Windows choose **More info → Run anyway**. From source: `cd desktop && npm ci && npm start`.
 
 ## Community sharing
 

core/metachlorian/__main__.py

@@ -0,0 +1,4 @@
+"""`python -m metachlorian` runs the CLI (the desktop app starts its bundled core this way)."""
+from .cli import main
+
+main()

core/metachlorian/cli.py

@@ -181,6 +181,28 @@ def cmd_models(args) -> None:
             print(f"{'✓' if m['installed'] else '·'} {m['name']:28s} {m['licence']:12s} {m['size_mb']:>6} MB  {m['purpose']}")
     elif args.action == "fetch":
         names = args.names or [n for n, sp in models.SPECS.items() if sp.default]
+        if args.json:
+            # One JSON object per line, for the desktop app's first-run setup screen.
+            def emit(**event):
+                print(json.dumps(event), flush=True)
+            missing = [n for n in names if not models.installed(root, n)]
+            emit(event="plan", models=[{"name": n, "size_mb": models.SPECS[n].size_mb} for n in missing])
+            for n in missing:
+                last = [0.0]
+
+                def on_bytes(done, total, n=n, last=last):
+                    if done - last[0] >= 4 << 20 or done == total:
+                        last[0] = done
+                        emit(event="progress", model=n, done=done, total=total)
+                emit(event="start", model=n)
+                try:
+                    models.fetch(root, n, progress=lambda _m: None, on_bytes=on_bytes)
+                except Exception as e:  # noqa: BLE001 - reported to the caller, which offers a retry
+                    emit(event="error", model=n, message=str(e)[:300])
+                    sys.exit(1)
+                emit(event="done", model=n)
+            emit(event="complete")
+            return
         for n in names:
             print(f"fetching {n} …", flush=True)
             models.fetch(root, n)
@@ -247,6 +269,7 @@ def main(argv: list[str] | None = None) -> None:
     sp = sub.add_parser("models")
     sp.add_argument("action", choices=["list", "fetch", "install-from"])
     sp.add_argument("names", nargs="*")
+    sp.add_argument("--json", action="store_true", help="fetch: report progress as JSON lines")
     sp.set_defaults(fn=cmd_models)
     args = ap.parse_args(argv)
     logging.basicConfig(level=os.environ.get("METACHLORIAN_LOG", "WARNING"))

core/metachlorian/llm.py

@@ -15,7 +15,6 @@
 import base64
 import copy
 import datetime as dt
-import fcntl
 import hashlib
 import json
 import os
@@ -29,6 +28,26 @@
 from pathlib import Path
 from typing import Any
 
+try:
+    import fcntl
+except ImportError:  # Windows
+    fcntl = None
+    import msvcrt
+
+
+def _lock_exclusive(f) -> None:
+    """Block until this process holds an exclusive lock on the open file (released when it is closed)."""
+    if fcntl:
+        fcntl.flock(f, fcntl.LOCK_EX)
+    else:
+        f.seek(0)
+        while True:
+            try:
+                msvcrt.locking(f.fileno(), msvcrt.LK_LOCK, 1)  # retries for ~10 s, then raises
+                return
+            except OSError:
+                continue
+
 import httpx
 import jsonschema
 
@@ -276,7 +295,7 @@ def __init__(self, settings: Settings):
     def _locked(self, fn):
         self.path.parent.mkdir(parents=True, exist_ok=True)
         with open(self.path.with_suffix(".lock"), "w") as lk:
-            fcntl.flock(lk, fcntl.LOCK_EX)
+            _lock_exclusive(lk)
             try:
                 data = json.loads(self.path.read_text()) if self.path.exists() else {}
             except json.JSONDecodeError:

core/metachlorian/models.py

@@ -101,7 +101,8 @@ def require(models_root: Path, name: str) -> Path:
     return model_dir(models_root, name)
 
 
-def fetch(models_root: Path, name: str, progress=print) -> Path:
+def fetch(models_root: Path, name: str, progress=print, on_bytes=None) -> Path:
+    """Download a model. ``on_bytes(done, total)`` is called as each archive downloads (total is 0 if unknown)."""
     spec = SPECS[name]
     d = model_dir(models_root, name)
     if installed(models_root, name):
@@ -111,7 +112,12 @@ def fetch(models_root: Path, name: str, progress=print) -> Path:
         tmp = d / (".download-" + hashlib.sha1(url.encode()).hexdigest()[:8])
         progress(f"downloading {url}")
         with urllib.request.urlopen(url) as r, open(tmp, "wb") as f:
-            shutil.copyfileobj(r, f, 1 << 20)
+            total, done = int(r.headers.get("Content-Length") or 0), 0
+            while chunk := r.read(1 << 20):
+                f.write(chunk)
+                done += len(chunk)
+                if on_bytes:
+                    on_bytes(done, total)
         if url.endswith((".tar.bz2", ".tar.gz", ".tgz")):
             with tarfile.open(tmp) as tf:
                 for m in tf.getmembers():
@@ -156,7 +162,12 @@ def ort_session(path: Path, threads: int | None = None):
     so.intra_op_num_threads = threads or int(os.environ.get("METACHLORIAN_THREADS", "0") or 0) or max(1, (os.cpu_count() or 2) // 2)
     so.inter_op_num_threads = 1
     so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
-    providers = [p for p in ("CUDAExecutionProvider", "CoreMLExecutionProvider", "DmlExecutionProvider") if p in ort.get_available_providers()]
+    # CoreML is opt-in (METACHLORIAN_COREML=1): it fails at inference on SigLIP's vision head on Apple Silicon.
+    wanted = ("CUDAExecutionProvider", "DmlExecutionProvider") + (("CoreMLExecutionProvider",) if os.environ.get("METACHLORIAN_COREML") == "1" else ())
+    providers = [p for p in wanted if p in ort.get_available_providers()]
+    # METACHLORIAN_ACCELERATORS=false forces CPU everywhere.
+    if os.environ.get("METACHLORIAN_ACCELERATORS", "").lower() in ("0", "false", "no", "off"):
+        providers = []
     return ort.InferenceSession(str(path), sess_options=so, providers=providers + ["CPUExecutionProvider"])
 
 

desktop/package.json

@@ -9,7 +9,9 @@
     "start": "electron .",
     "test": "node --test src/*.test.js",
     "package": "electron-builder",
-    "package:dir": "electron-builder --dir"
+    "package:dir": "electron-builder --dir",
+    "bundle": "node scripts/bundle.mjs",
+    "check-package": "node scripts/check-package.mjs"
   },
   "devDependencies": {
     "electron": "^39.0.0",
@@ -28,13 +30,16 @@
     ],
     "extraResources": [
       {
-        "from": "../core",
-        "to": "core",
-        "filter": [
-          "metachlorian/**",
-          "pyproject.toml",
-          "README.md"
-        ]
+        "from": "bundle/python",
+        "to": "python"
+      },
+      {
+        "from": "bundle/app",
+        "to": "app"
+      },
+      {
+        "from": "bundle/ffmpeg",
+        "to": "ffmpeg"
       }
     ],
     "mac": {

desktop/scripts/bundle.mjs

@@ -0,0 +1,83 @@
+/**
+ * Builds everything the desktop app ships alongside Electron, for the platform
+ * it runs on, into desktop/bundle/ (packaged as Resources/{python,app,ffmpeg}):
+ *
+ *   python/   a relocatable CPython 3.11 (python-build-standalone, via uv) with
+ *             the core and its analysis dependencies installed into it
+ *   app/      the built web app (app/dist), which the core serves
+ *   ffmpeg/   static ffmpeg + ffprobe builds (ffmpeg-static's release assets)
+ *
+ * Needs uv on PATH and app/dist built (`cd app && npm run build`).
+ * Usage: node scripts/bundle.mjs        (from desktop/; `npm run bundle`)
+ */
+import { execFileSync, spawnSync } from 'node:child_process'
+import { chmodSync, cpSync, createWriteStream, existsSync, mkdirSync, readdirSync, realpathSync, rmSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+import { pipeline } from 'node:stream/promises'
+import { Readable } from 'node:stream'
+import { createGunzip } from 'node:zlib'
+
+const desktop = resolve(import.meta.dirname, '..')
+const repo = resolve(desktop, '..')
+const out = join(desktop, 'bundle')
+const win = process.platform === 'win32'
+const PYTHON = '3.11'
+// ffmpeg-static's GitHub release with prebuilt binaries per platform (GPL builds; licences are copied alongside).
+const FFMPEG_RELEASE = 'https://github.com/eugeneware/ffmpeg-static/releases/download/b6.1.1'
+
+const run = (cmd, args, opts = {}) => execFileSync(cmd, args, { stdio: 'inherit', ...opts })
+const step = (msg) => console.log(`\n▸ ${msg}`)
+
+rmSync(out, { recursive: true, force: true })
+mkdirSync(out, { recursive: true })
+
+// ---------------------------------------------------------------- python + core
+step(`CPython ${PYTHON} (standalone)`)
+const cache = join(desktop, '.cache', 'python')
+run('uv', ['python', 'install', PYTHON, '--install-dir', cache])
+// uv adds a minor-version alias next to the real install; copy the real directory.
+const installs = readdirSync(cache).filter((d) => d.startsWith(`cpython-${PYTHON}.`))
+if (!installs.length) throw new Error(`uv did not install CPython ${PYTHON} into ${cache}`)
+const pythonHome = join(out, 'python')
+cpSync(realpathSync(join(cache, installs.sort().at(-1))), pythonHome, { recursive: true, verbatimSymlinks: true })
+const python = win ? join(pythonHome, 'python.exe') : join(pythonHome, 'bin', 'python3')
+// uv marks its interpreters externally managed; this copy is ours to install into.
+rmSync(join(pythonHome, win ? 'Lib' : `lib/python${PYTHON}`, 'EXTERNALLY-MANAGED'), { force: true })
+
+step('Metachlorian core and analysis dependencies')
+run('uv', ['pip', 'install', '--python', python, '--no-cache', `${join(repo, 'core')}[analysis]`])
+// Precompile so the read-only (and, on macOS, signed) bundle is never written to at runtime. A few files in the
+// standard library and in dependencies' test data aren't valid Python 3.11 and fail to compile; that is expected,
+// so list them rather than failing the build.
+const compiled = spawnSync(python, ['-m', 'compileall', '-q', '-j', '0', pythonHome], { encoding: 'utf8' })
+if (compiled.error) throw compiled.error
+const failed = (compiled.stdout || '').split(/\r?\n/).filter((l) => /^\*\*\* Error compiling/.test(l))
+if (failed.length) console.log(`  (${failed.length} file(s) not precompiled: ${failed.slice(0, 5).map((l) => l.replace(/^\*\*\* Error compiling /, '')).join(', ')}${failed.length > 5 ? ', …' : ''})`)
+run(python, ['-c', 'import metachlorian, onnxruntime, cv2, sherpa_onnx, usearch; print("core", metachlorian.__version__, "imports ok")'])
+
+// ---------------------------------------------------------------- web app
+step('Web app')
+const dist = join(repo, 'app', 'dist')
+if (!existsSync(join(dist, 'index.html'))) throw new Error('app/dist is missing: run `npm run build` in app/ first')
+cpSync(dist, join(out, 'app'), { recursive: true })
+
+// ---------------------------------------------------------------- ffmpeg
+step('FFmpeg')
+const target = { darwin: 'darwin', win32: 'win32', linux: 'linux' }[process.platform] + '-' + process.arch
+mkdirSync(join(out, 'ffmpeg'), { recursive: true })
+async function download(name, dest, gz) {
+  const r = await fetch(`${FFMPEG_RELEASE}/${name}`)
+  if (!r.ok) throw new Error(`Download failed (${r.status}): ${FFMPEG_RELEASE}/${name}`)
+  const body = Readable.fromWeb(r.body)
+  await (gz ? pipeline(body, createGunzip(), createWriteStream(dest)) : pipeline(body, createWriteStream(dest)))
+}
+for (const tool of ['ffmpeg', 'ffprobe']) {
+  const dest = join(out, 'ffmpeg', tool + (win ? '.exe' : ''))
+  await download(`${tool}-${target}.gz`, dest, true)
+  chmodSync(dest, 0o755)
+  run(dest, ['-hide_banner', '-version'], { stdio: ['ignore', 'ignore', 'inherit'] })
+}
+await download(`${target}.LICENSE`, join(out, 'ffmpeg', 'LICENSE'), false)
+await download(`${target}.README`, join(out, 'ffmpeg', 'README'), false)
+
+console.log(`\nBundle ready in ${out}`)