diff --git a/.gitignore b/.gitignore
index 2d7905928..cff64d92b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -98,3 +98,7 @@ docs/wire-captures/
# Scratch renders derived from scripts/e2e/fixtures (regenerable, not versioned)
/tmp/
+
+# Generated native video runtime
+.video-build/
+native/video/*.xcframework/
diff --git a/App.tsx b/App.tsx
index 862b92582..841c57f63 100644
--- a/App.tsx
+++ b/App.tsx
@@ -1,3 +1,5 @@
+import { videoGenerationService } from './src/services/videoGenerationService';
+import { videoProvider } from './src/services/modelDownloadService/providers/videoProvider';
/**
* Off Grid - On-Device AI Chat Application
* Private AI assistant that runs entirely on your device
@@ -136,6 +138,7 @@ function App() {
const applyPendingProRedirect = useProExpiryRedirect();
const [isInitializing, setIsInitializing] = useState(true);
const startupGeneration = useRef(0);
+ const showSyncStatusBar = useAppStore(s => s.showSyncStatusBar);
const setDeviceInfo = useAppStore((s) => s.setDeviceInfo);
const setModelRecommendation = useAppStore((s) => s.setModelRecommendation);
const setDownloadedModels = useAppStore((s) => s.setDownloadedModels);
@@ -234,6 +237,8 @@ function App() {
// becomes the SINGLE owner only once the Download Manager consumes the service
// and the old recovery paths are folded into the providers.
registerCoreDownloadProviders();
+ await videoProvider.reconcile?.().catch(error => logger.error('[Video] Download recovery failed:', error));
+ await videoGenerationService.restore().catch(error => logger.error('[Video] Job recovery failed:', error));
// Re-surface QUEUED downloads that never started before an app kill. A queued item (waiting for
// one of the 3 concurrency slots) has no native row, so hydrateDownloadStore can't recover it —
@@ -406,7 +411,7 @@ function App() {
- {__DEV__ ? : null}
+ {__DEV__ && showSyncStatusBar ? : null}
);
@@ -419,7 +424,7 @@ function App() {
- {__DEV__ ? : null}
+ {__DEV__ && showSyncStatusBar ? : null}
);
@@ -465,7 +470,7 @@ function App() {
>
- {__DEV__ ? : null}
+ {__DEV__ && showSyncStatusBar ? : null}
);
diff --git a/__tests__/integration/settings/modelSettingsSurfaceParity.test.tsx b/__tests__/integration/settings/modelSettingsSurfaceParity.test.tsx
index d8caed028..dd53ecfeb 100644
--- a/__tests__/integration/settings/modelSettingsSurfaceParity.test.tsx
+++ b/__tests__/integration/settings/modelSettingsSurfaceParity.test.tsx
@@ -40,6 +40,40 @@ describe('model settings surface parity', () => {
_clearSlotsForTesting();
});
+ it('keeps shared controls in General on both surfaces and preserves saved values', () => {
+ useAppStore.getState().updateSettings({
+ modelLoadingMode: 'aggressive',
+ showGenerationDetails: true,
+ });
+ const chatSettings = render( {}} />);
+ fireEvent.press(chatSettings.getByText('TEXT GENERATION'));
+ expect(chatSettings.queryByText('Model Loading')).toBeNull();
+ expect(chatSettings.queryByText('Show Generation Details')).toBeNull();
+ fireEvent.press(chatSettings.getByTestId('modal-general-accordion'));
+ expect(chatSettings.getByText('Inference Backend')).toBeTruthy();
+ expect(chatSettings.getByTestId('model-loading-mode-aggressive-button')).toBeTruthy();
+ expect(chatSettings.getByTestId('show-gen-details-on-button')).toBeTruthy();
+ chatSettings.unmount();
+
+ const modelSettings = renderModelSettings();
+ fireEvent.press(modelSettings.getByTestId('text-generation-accordion'));
+ expect(modelSettings.queryByText('Model Loading')).toBeNull();
+ fireEvent.press(modelSettings.getByTestId('general-generation-accordion'));
+ expect(modelSettings.getByText('Inference Backend')).toBeTruthy();
+ expect(modelSettings.getByTestId('model-loading-mode-aggressive-button')).toBeTruthy();
+ expect(modelSettings.getByTestId('show-gen-details-on-button')).toBeTruthy();
+ });
+
+ it('shows and edits the saved default system prompt in chat Text settings', () => {
+ useAppStore.getState().updateSettings({ systemPrompt: 'Saved instructions' });
+ const chatSettings = render( {}} />);
+ fireEvent.press(chatSettings.getByText('TEXT GENERATION'));
+ const input = chatSettings.getByTestId('default-system-prompt-input');
+ expect(input.props.value).toBe('Saved instructions');
+ fireEvent.changeText(input, 'New instructions');
+ expect(useAppStore.getState().settings.systemPrompt).toBe('New instructions');
+ });
+
it('caps output by context on both surfaces and writes one shared setting state', () => {
useAppStore.getState().setModelMaxContext(262144);
// A context wide enough for the output this test chooses. Max tokens is capped BY the context,
diff --git a/__tests__/rntl/screens/ModelSettingsScreen.test.tsx b/__tests__/rntl/screens/ModelSettingsScreen.test.tsx
index 25f008bda..ceabf7bcd 100644
--- a/__tests__/rntl/screens/ModelSettingsScreen.test.tsx
+++ b/__tests__/rntl/screens/ModelSettingsScreen.test.tsx
@@ -59,6 +59,9 @@ const renderWithSections = (...sections: ('prompt' | 'image' | 'text')[]) => {
text: 'text-advanced-toggle',
};
for (const section of sections) {
+ if (section === 'text') {
+ fireEvent.press(result.getByTestId('general-generation-accordion'));
+ }
fireEvent.press(result.getByTestId(testIDMap[section]));
if (advancedMap[section]) {
fireEvent.press(result.getByTestId(advancedMap[section]));
diff --git a/__tests__/unit/sync/sharedFileMaterializer.test.ts b/__tests__/unit/sync/sharedFileMaterializer.test.ts
index f98508349..d9192f26c 100644
--- a/__tests__/unit/sync/sharedFileMaterializer.test.ts
+++ b/__tests__/unit/sync/sharedFileMaterializer.test.ts
@@ -65,10 +65,36 @@ describe('making a transferred file appear in the app', () => {
?.messages.find(({ uuid }) => uuid === messageUuid)?.attachments ?? [];
beforeEach(() => {
- useAppStore.setState({ generatedImages: [] });
+ useAppStore.setState({ generatedImages: [], generatedVideos: [] });
useChatStore.setState({ conversations: [] });
});
+ describe('a video generated on another device', () => {
+ const video = () => record({
+ mimeType: 'video/mp4', name: 'lake.mp4', localPath: '/docs/shared_files/lake.mp4',
+ conversationId: 'chat-7', messageId: 'message-1', width: 320, height: 192,
+ durationSeconds: 2.125,
+ metadataJson: JSON.stringify({ prompt: 'A moving lake', steps: 20, guidance: 6, seed: 42, modelId: 'wan', fps: 8, frames: 17 }),
+ });
+ it('appears once in the video gallery and on its message after repeated delivery', () => {
+ useChatStore.setState({ conversations: [conversation()] });
+ materializeSharedFile(video());
+ materializeSharedFile(video());
+ expect(gallery()).toEqual([]);
+ expect(useAppStore.getState().generatedVideos).toHaveLength(1);
+ expect(useAppStore.getState().generatedVideos[0]).toMatchObject({ prompt: 'A moving lake', durationSeconds: 2.125, fps: 8, seed: 42 });
+ expect(attachmentsOn('chat-7', 'message-1')).toHaveLength(1);
+ expect(attachmentsOn('chat-7', 'message-1')[0]).toMatchObject({ type: 'video', mimeType: 'video/mp4' });
+ });
+ it('removes the gallery entry and chat attachment when the shared video is deleted', () => {
+ useChatStore.setState({ conversations: [conversation()] });
+ materializeSharedFile(video());
+ removeMaterializedSharedFile(video());
+ expect(useAppStore.getState().generatedVideos).toEqual([]);
+ expect(attachmentsOn('chat-7', 'message-1')).toEqual([]);
+ });
+ });
+
describe('a picture generated on another device', () => {
it('appears in the gallery, with the prompt that made it', () => {
materializeSharedFile(
diff --git a/__tests__/unit/utils/proPricing.test.ts b/__tests__/unit/utils/proPricing.test.ts
index b922408f1..b982fd31b 100644
--- a/__tests__/unit/utils/proPricing.test.ts
+++ b/__tests__/unit/utils/proPricing.test.ts
@@ -3,14 +3,13 @@ import { getPricingCopy } from '../../../src/utils/proPricing';
describe('getPricingCopy', () => {
const copy = getPricingCopy();
- it('offers the two current plans - $49/yr and $69 lifetime - and no monthly', () => {
- expect(copy.title).toBe('$49/yr or $69 lifetime');
- expect(copy.sheetSubheadline).toMatch(/\$49 a year/);
+ it('offers the two current plans - $4.99/month and $69 lifetime - and no yearly', () => {
+ expect(copy.title).toBe('$4.99/month or $69 lifetime');
+ expect(copy.sheetSubheadline).toMatch(/\$4\.99 a month/);
expect(copy.sheetSubheadline).toMatch(/\$69 once/);
- // The retired monthly plan must not resurface anywhere in the copy.
+ // The retired yearly plan must not resurface anywhere in the copy.
const all = Object.values(copy).join(' ');
- expect(all).not.toMatch(/month/i);
- expect(all).not.toMatch(/\$39/);
+ expect(all).not.toMatch(/year|annual|\$49/i);
});
it('keeps the Get Pro CTA (the web pay-page trigger the Pro surfaces assert)', () => {
diff --git a/android/app/build.gradle b/android/app/build.gradle
index 88875fc44..acfa0e975 100644
--- a/android/app/build.gradle
+++ b/android/app/build.gradle
@@ -89,6 +89,8 @@ android {
coreLibraryDesugaringEnabled true
}
+ externalNativeBuild { cmake { path "src/main/cpp/CMakeLists.txt"; version "3.22.1" } }
+ sourceSets.main.assets.srcDir("../../.video-build/hexagon-assets")
defaultConfig {
applicationId "ai.offgridmobile"
minSdkVersion rootProject.ext.minSdkVersion
@@ -195,3 +197,18 @@ dependencies {
testImplementation("org.mockito.kotlin:mockito-kotlin:5.4.0")
testImplementation("androidx.test:core:1.6.1")
}
+
+// Native video setup is shared by command-line, IDE, and CI builds.
+// Configure tasks need the source tree before CMake evaluates it.
+def videoRoot = rootProject.projectDir.parentFile
+def prepareVideoRuntime = tasks.register("prepareVideoRuntime", Exec) {
+ workingDir videoRoot
+ commandLine "bash", new File(videoRoot, "scripts/prepare-video-runtime.sh").absolutePath, "--android"
+ inputs.files(fileTree(new File(videoRoot, "scripts/video")), new File(videoRoot, "scripts/prepare-video-runtime.sh"))
+ outputs.files(new File(videoRoot, ".video-build/runtime-ready"), new File(videoRoot, ".video-build/shaderc/bin/glslc"),
+ new File(videoRoot, ".video-build/hexagon-runtime/htp_iface_stub.c"), new File(videoRoot, ".video-build/hexagon-runtime/htp_iface.h"))
+ outputs.dir(new File(videoRoot, ".video-build/hexagon-assets/video-hexagon"))
+}
+tasks.configureEach { task ->
+ if (task.name.startsWith("configureCMake") || (task.name.startsWith("merge") && task.name.endsWith("Assets"))) task.dependsOn(prepareVideoRuntime)
+}
diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml
index 04fef1ba4..41b474c28 100644
--- a/android/app/src/main/AndroidManifest.xml
+++ b/android/app/src/main/AndroidManifest.xml
@@ -47,6 +47,8 @@
+
+
@@ -78,6 +80,7 @@
+
+
+
diff --git a/android/app/src/main/cpp/CMakeLists.txt b/android/app/src/main/cpp/CMakeLists.txt
new file mode 100644
index 000000000..848be61ae
--- /dev/null
+++ b/android/app/src/main/cpp/CMakeLists.txt
@@ -0,0 +1,54 @@
+cmake_minimum_required(VERSION 3.22)
+project(appmodules LANGUAGES C CXX)
+include(${REACT_ANDROID_DIR}/cmake-utils/ReactNative-application.cmake)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_C_VISIBILITY_PRESET hidden)
+set(CMAKE_CXX_VISIBILITY_PRESET hidden)
+set(SD_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
+set(SD_BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
+set(SD_BUILD_SHARED_GGML_LIB OFF CACHE BOOL "" FORCE)
+set(GGML_NATIVE OFF CACHE BOOL "" FORCE)
+set(SD_WEBP OFF CACHE BOOL "" FORCE)
+set(SD_WEBM OFF CACHE BOOL "" FORCE)
+set(SD_METAL OFF CACHE BOOL "" FORCE)
+set(SD_VULKAN ON CACHE BOOL "" FORCE)
+set(SD_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../.video-build/source")
+set(GGML_HEXAGON ON CACHE BOOL "" FORCE)
+set(OFFGRID_VIDEO_HEXAGON_CMAKE "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../scripts/video/hexagon-prebuilt.cmake")
+if(NOT EXISTS "${SD_SOURCE}/CMakeLists.txt")
+ message(FATAL_ERROR "Run scripts/prepare-video-runtime.sh before building Android.")
+endif()
+set(SPIRV-Headers_DIR "${SD_SOURCE}/../vulkan-header-prefix/share/cmake/SPIRV-Headers")
+set(Vulkan_INCLUDE_DIR "${SD_SOURCE}/../vulkan-header-prefix/include" CACHE PATH "Pinned Vulkan headers" FORCE)
+string(TOLOWER "${CMAKE_HOST_SYSTEM_NAME}" VIDEO_HOST_OS)
+# The NDK shader compiler can be too old for the pinned runtime. Use the host
+# shaderc installation, or an explicit -DVulkan_GLSLC_EXECUTABLE path.
+find_program(Vulkan_GLSLC_EXECUTABLE NAMES glslc glslc.exe
+ HINTS "${SD_SOURCE}/../shaderc/bin" NO_CMAKE_FIND_ROOT_PATH REQUIRED)
+if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
+ # The NDK prepends its compiler directory even for host-program discovery.
+ # Its clang cannot compile against the current macOS SDK. Use Xcode's host
+ # toolchain for the shader generator; model libraries still use the NDK.
+ execute_process(COMMAND xcrun --sdk macosx --find clang
+ OUTPUT_VARIABLE VIDEO_HOST_CC OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY)
+ execute_process(COMMAND xcrun --sdk macosx --find clang++
+ OUTPUT_VARIABLE VIDEO_HOST_CXX OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY)
+else()
+ find_program(VIDEO_HOST_CC NAMES clang gcc cl NO_CMAKE_FIND_ROOT_PATH REQUIRED)
+ find_program(VIDEO_HOST_CXX NAMES clang++ g++ cl NO_CMAKE_FIND_ROOT_PATH REQUIRED)
+endif()
+set(GGML_VULKAN_SHADERS_GEN_TOOLCHAIN "${CMAKE_BINARY_DIR}/video-host-toolchain.cmake")
+file(WRITE "${GGML_VULKAN_SHADERS_GEN_TOOLCHAIN}"
+ "set(CMAKE_C_COMPILER \"${VIDEO_HOST_CC}\")\nset(CMAKE_CXX_COMPILER \"${VIDEO_HOST_CXX}\")\nset(CMAKE_MAKE_PROGRAM \"${CMAKE_MAKE_PROGRAM}\" CACHE FILEPATH \"Host build tool\" FORCE)\n")
+add_subdirectory("${SD_SOURCE}" runtime)
+# Keep model execution representative in debug apps. Unoptimized tensor kernels
+# make CPU conditioning take minutes even though the app remains responsive.
+foreach(VIDEO_RUNTIME_TARGET stable-diffusion ggml-base ggml-cpu ggml-vulkan ggml-hexagon)
+ if(TARGET ${VIDEO_RUNTIME_TARGET})
+ target_compile_options(${VIDEO_RUNTIME_TARGET} PRIVATE $<$:-O3>)
+ endif()
+endforeach()
+add_library(offgrid_video SHARED video/video-jni.cpp)
+target_include_directories(offgrid_video PRIVATE "${SD_SOURCE}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../native/video/common")
+target_link_libraries(offgrid_video PRIVATE stable-diffusion android log)
+target_link_options(offgrid_video PRIVATE "-Wl,--exclude-libs,ALL" "-Wl,-z,max-page-size=16384")
diff --git a/android/app/src/main/cpp/video/video-jni.cpp b/android/app/src/main/cpp/video/video-jni.cpp
new file mode 100644
index 000000000..cd12491c5
--- /dev/null
+++ b/android/app/src/main/cpp/video/video-jni.cpp
@@ -0,0 +1,216 @@
+#include
+#include
+#include
+#include "VideoRuntime.hpp"
+
+static offgrid::VideoRuntime runtime;
+
+class AttachedEnv {
+ JavaVM *vm;
+ bool attached = false;
+public:
+ JNIEnv *env = nullptr;
+ explicit AttachedEnv(JavaVM *value) : vm(value) {
+ if (vm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6) == JNI_EDETACHED) {
+ attached = vm->AttachCurrentThread(&env, nullptr) == JNI_OK;
+ if (!attached) env = nullptr;
+ }
+ }
+ ~AttachedEnv() { if (attached) vm->DetachCurrentThread(); }
+};
+
+static std::string string(JNIEnv *env, jstring value) {
+ if (!value) throw std::runtime_error("Missing video argument.");
+ const char *bytes = env->GetStringUTFChars(value, nullptr);
+ if (!bytes) throw std::runtime_error("Missing video argument.");
+ std::string result(bytes);
+ env->ReleaseStringUTFChars(value, bytes);
+ return result;
+}
+
+extern "C" JNIEXPORT void JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativePrepare(JNIEnv *, jobject) {
+#ifndef NDEBUG
+ sd_set_log_callback([](sd_log_level_t level, const char *text, void *) {
+ const int priority = level == SD_LOG_ERROR ? ANDROID_LOG_ERROR :
+ level == SD_LOG_WARN ? ANDROID_LOG_WARN : ANDROID_LOG_DEBUG;
+ __android_log_write(priority, "OffgridVideo", text);
+ }, nullptr);
+#endif
+ runtime.cancelled.store(false);
+}
+extern "C" JNIEXPORT void JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeCancel(JNIEnv *, jobject) { runtime.cancel(); }
+extern "C" JNIEXPORT void JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeSetRuntimeDirectory(JNIEnv *env, jobject, jstring path) {
+ try {
+ auto directory = string(env, path);
+ const char *existing = getenv("ADSP_LIBRARY_PATH");
+ std::string search = directory + ";" + (existing ? existing : "/vendor/lib/rfsa/adsp;/vendor/dsp;/system/lib/rfsa/adsp;/dsp");
+ if (setenv("ADSP_LIBRARY_PATH", search.c_str(), 1) != 0 ||
+ setenv("OFFGRID_VIDEO_HTP_DIR", directory.c_str(), 1) != 0)
+ throw std::runtime_error("Could not configure the NPU runtime.");
+ } catch (const std::exception &error) {
+ if (!env->ExceptionCheck()) {
+ auto klass = env->FindClass("java/lang/IllegalStateException");
+ if (klass) { env->ThrowNew(klass, error.what()); env->DeleteLocalRef(klass); }
+ }
+ }
+}
+extern "C" JNIEXPORT void JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeGenerate(
+ JNIEnv *env, jobject self, jstring weight, jstring vae, jstring encoder, jstring prompt, jstring negative,
+ jint width, jint height, jint frames, jint fps, jint steps, jdouble guidance, jdouble seed, jstring llm, jstring embeddings, jstring audioVae, jdouble flowShift, jstring diagnosticBackend) {
+ jobject owner = nullptr;
+ try {
+ offgrid::VideoRequest request{string(env, weight), string(env, vae), string(env, encoder), string(env, prompt), string(env, negative), width, height, frames, fps, steps, (float)guidance, (int64_t)seed};
+ request.llm = string(env, llm);
+ request.embeddings = string(env, embeddings);
+ request.audioVae = string(env, audioVae);
+ request.flowShift = (float)flowShift;
+ const auto backend = string(env, diagnosticBackend);
+ request.cpuOnly = backend == "cpu";
+ request.skipNpu = backend == "gpu";
+ JavaVM *vm = nullptr;
+ if (env->GetJavaVM(&vm) != JNI_OK) throw std::runtime_error("Could not access the video host.");
+ owner = env->NewGlobalRef(self);
+ if (!owner) throw std::runtime_error("Not enough memory to start video generation.");
+ auto klass = env->GetObjectClass(self);
+ auto progress = env->GetMethodID(klass, "progress", "(II)V");
+ auto conditioning = env->GetMethodID(klass, "conditioning", "(Ljava/lang/String;)V");
+ auto frame = env->GetMethodID(klass, "frame", "([BIII)V");
+ auto decodeProgress = env->GetMethodID(klass, "decoding", "(II)V");
+ auto preview = env->GetMethodID(klass, "preview", "([BII)V");
+ env->DeleteLocalRef(klass);
+ if (!progress || !frame || !conditioning || !decodeProgress || !preview) throw std::runtime_error("Missing video host callbacks.");
+ std::atomic_bool callbackFailed{false};
+ runtime.run(request, [&](int step, int total) {
+ // The engine may report progress from a worker thread. JNI environments
+ // are thread-local; never retain the caller's environment in this callback.
+ AttachedEnv thread(vm);
+ if (!thread.env) { callbackFailed.store(true); runtime.cancel(); return; }
+ thread.env->CallVoidMethod(owner, progress, step, total);
+ if (thread.env->ExceptionCheck()) {
+ thread.env->ExceptionClear();
+ callbackFailed.store(true);
+ runtime.cancel();
+ }
+ }, [&](sd_image_t *images, int count, int) {
+ if (callbackFailed.load()) throw std::runtime_error("Could not report video progress.");
+ for (int i = 0; i < count; ++i) {
+ if (runtime.cancelled.load()) throw std::runtime_error("Video generation stopped.");
+ auto &image = images[i];
+ const uint64_t size = uint64_t(image.width) * image.height * image.channel;
+ if (!image.data || image.width != uint32_t(width) || image.height != uint32_t(height) ||
+ image.channel != 3 || size > uint64_t(std::numeric_limits::max()))
+ throw std::runtime_error("The video engine returned an invalid frame.");
+ auto bytes = env->NewByteArray(static_cast(size));
+ if (!bytes) throw std::runtime_error("Not enough memory to encode video.");
+ env->SetByteArrayRegion(bytes, 0, static_cast(size), reinterpret_cast(image.data));
+ if (!env->ExceptionCheck()) env->CallVoidMethod(owner, frame, bytes, image.width, image.height, image.channel);
+ env->DeleteLocalRef(bytes);
+ if (env->ExceptionCheck()) throw std::runtime_error("Video encoder failed.");
+ }
+ }, [&](const char *backend) {
+ auto hardware = env->NewStringUTF(backend);
+ env->CallVoidMethod(owner, conditioning, hardware);
+ env->DeleteLocalRef(hardware);
+ if (env->ExceptionCheck()) throw std::runtime_error("Could not report prompt processing.");
+ }, [&](int completed, int total) {
+ AttachedEnv thread(vm);
+ if (!thread.env) return;
+ thread.env->CallVoidMethod(owner, decodeProgress, completed, total);
+ if (thread.env->ExceptionCheck()) thread.env->ExceptionClear();
+ }, [&](const sd_image_t &image) {
+ // A preview is optional. Its failure must not discard a completed decode.
+ const uint64_t size = uint64_t(image.width) * image.height * image.channel;
+ if (!image.data || image.channel != 3 || size > uint64_t(std::numeric_limits::max())) return;
+ AttachedEnv thread(vm);
+ if (!thread.env || runtime.cancelled.load()) return;
+ auto bytes = thread.env->NewByteArray(static_cast(size));
+ if (bytes) {
+ thread.env->SetByteArrayRegion(bytes, 0, static_cast(size), reinterpret_cast(image.data));
+ if (!thread.env->ExceptionCheck()) thread.env->CallVoidMethod(owner, preview, bytes, image.width, image.height);
+ thread.env->DeleteLocalRef(bytes);
+ }
+ if (thread.env->ExceptionCheck()) thread.env->ExceptionClear();
+ });
+ } catch (const std::exception &error) {
+ if (!env->ExceptionCheck()) {
+ auto klass = env->FindClass("java/lang/IllegalStateException");
+ if (klass) { env->ThrowNew(klass, error.what()); env->DeleteLocalRef(klass); }
+ }
+ }
+ if (owner) env->DeleteGlobalRef(owner);
+}
+
+extern "C" JNIEXPORT void JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeLoadImage(
+ JNIEnv *env, jobject, jstring path, jstring weight, jstring vae, jstring llm, jint threads, jboolean cpuOnly, jstring family, jstring sampler, jstring scheduler) {
+ try {
+ offgrid::VideoRequest request{};
+ request.weight = string(env, weight); request.vae = string(env, vae); request.llm = string(env, llm);
+ request.threads = threads; request.cpuOnly = cpuOnly;
+ request.imageFamily = string(env, family); request.imageSampler = string(env, sampler); request.imageScheduler = string(env, scheduler);
+ runtime.loadImage(request, string(env, path));
+ } catch (const std::exception &error) {
+ auto klass = env->FindClass("java/lang/IllegalStateException");
+ if (klass) { env->ThrowNew(klass, error.what()); env->DeleteLocalRef(klass); }
+ }
+}
+extern "C" JNIEXPORT jstring JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeImagePath(JNIEnv *env, jobject) {
+ return env->NewStringUTF(runtime.loadedImagePath().c_str());
+}
+extern "C" JNIEXPORT void JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeUnloadImage(JNIEnv *env, jobject) {
+ try { runtime.unloadImage(); }
+ catch (const std::exception &error) {
+ auto klass = env->FindClass("java/lang/IllegalStateException");
+ if (klass) { env->ThrowNew(klass, error.what()); env->DeleteLocalRef(klass); }
+ }
+}
+extern "C" JNIEXPORT jbyteArray JNICALL Java_ai_offgridmobile_video_VideoGenerationModule_nativeGenerateImage(
+ JNIEnv *env, jobject self, jstring prompt, jstring negative, jint width, jint height, jint steps, jdouble guidance, jdouble seed, jint previewInterval) {
+ jbyteArray output = nullptr;
+ jobject owner = nullptr;
+ try {
+ offgrid::VideoRequest request{};
+ request.prompt = string(env, prompt); request.negative = string(env, negative);
+ request.width = width; request.height = height; request.steps = steps; request.guidance = guidance; request.seed = seed;
+ JavaVM *vm = nullptr;
+ if (env->GetJavaVM(&vm) != JNI_OK) throw std::runtime_error("Could not access the image host.");
+ owner = env->NewGlobalRef(self);
+ if (!owner) throw std::runtime_error("Could not retain the image host.");
+ auto klass = env->GetObjectClass(self);
+ auto progress = env->GetMethodID(klass, "imageProgress", "(II)V");
+ auto preview = env->GetMethodID(klass, "imagePreview", "([BIIII)V");
+ env->DeleteLocalRef(klass);
+ if (!progress || !preview) throw std::runtime_error("Missing image progress callback.");
+ runtime.image(request, [&](int step, int total) {
+ AttachedEnv thread(vm);
+ if (!thread.env) { runtime.cancel(); return; }
+ thread.env->CallVoidMethod(owner, progress, step, total);
+ if (thread.env->ExceptionCheck()) { thread.env->ExceptionClear(); runtime.cancel(); }
+ }, [&](const sd_image_t &image) {
+ const uint64_t size = uint64_t(image.width) * image.height * image.channel;
+ if (!image.data || image.width != uint32_t(width) || image.height != uint32_t(height) || (image.channel != 3 && image.channel != 4) || size > INT32_MAX)
+ throw std::runtime_error("The image engine returned invalid pixels.");
+ output = env->NewByteArray(static_cast(size));
+ if (!output) throw std::runtime_error("Not enough memory to save the image.");
+ env->SetByteArrayRegion(output, 0, size, reinterpret_cast(image.data));
+ }, previewInterval, [&](int step, const sd_image_t &image) {
+ const uint64_t size = uint64_t(image.width) * image.height * image.channel;
+ if (!image.data || (image.channel != 3 && image.channel != 4) || size > INT32_MAX) return;
+ AttachedEnv thread(vm);
+ if (!thread.env) return;
+ jbyteArray bytes = thread.env->NewByteArray(static_cast(size));
+ if (bytes) {
+ thread.env->SetByteArrayRegion(bytes, 0, size, reinterpret_cast(image.data));
+ thread.env->CallVoidMethod(owner, preview, bytes, image.width, image.height, image.channel, step);
+ thread.env->DeleteLocalRef(bytes);
+ }
+ if (thread.env->ExceptionCheck()) thread.env->ExceptionClear();
+ });
+ } catch (const std::exception &error) {
+ if (!env->ExceptionCheck()) {
+ auto klass = env->FindClass("java/lang/IllegalStateException");
+ if (klass) { env->ThrowNew(klass, error.what()); env->DeleteLocalRef(klass); }
+ }
+ }
+ if (owner) env->DeleteGlobalRef(owner);
+ return output;
+}
diff --git a/android/app/src/main/java/ai/offgridmobile/MainApplication.kt b/android/app/src/main/java/ai/offgridmobile/MainApplication.kt
index 258707112..3040aef5e 100644
--- a/android/app/src/main/java/ai/offgridmobile/MainApplication.kt
+++ b/android/app/src/main/java/ai/offgridmobile/MainApplication.kt
@@ -28,6 +28,7 @@ class MainApplication : Application(), ReactApplication {
// Packages that cannot be autolinked yet can be added manually here, for example:
add(DownloadManagerPackage())
add(LocalDreamPackage())
+ add(ai.offgridmobile.video.VideoGenerationPackage())
add(PDFExtractorPackage())
add(LiteRTPackage())
add(DeviceMemoryPackage())
diff --git a/android/app/src/main/java/ai/offgridmobile/downloads/SyncDownloadsModule.kt b/android/app/src/main/java/ai/offgridmobile/downloads/SyncDownloadsModule.kt
index abdcff1e1..55a035086 100644
--- a/android/app/src/main/java/ai/offgridmobile/downloads/SyncDownloadsModule.kt
+++ b/android/app/src/main/java/ai/offgridmobile/downloads/SyncDownloadsModule.kt
@@ -1,6 +1,7 @@
package ai.offgridmobile.downloads
import android.Manifest
+import android.content.ContentValues
import android.content.Intent
import android.content.pm.PackageManager
import android.net.Uri
@@ -43,6 +44,38 @@ class SyncDownloadsModule(
override fun getName(): String = "SyncDownloadsModule"
+ /** Export an app-owned file through scoped storage without broad media access. */
+ @ReactMethod
+ fun saveFileToDownloads(sourcePath: String, displayName: String, mimeType: String, promise: Promise) {
+ Thread({
+ var created: Uri? = null
+ try {
+ require(Build.VERSION.SDK_INT >= 29) { "Scoped Downloads requires Android 10." }
+ val source = File(sourcePath).canonicalFile
+ val roots = listOf(context.filesDir.canonicalFile, context.cacheDir.canonicalFile)
+ require(roots.any { source.path.startsWith(it.path + File.separator) } && source.isFile) { "The file is outside app storage." }
+ require(displayName == File(displayName).name && displayName.isNotBlank() && displayName.length <= 200) { "Invalid export filename." }
+ require(mimeType in listOf("video/mp4", "image/png", "image/jpeg", "image/webp")) { "Unsupported media type." }
+ val values = ContentValues().apply {
+ put(MediaStore.MediaColumns.DISPLAY_NAME, displayName)
+ put(MediaStore.MediaColumns.MIME_TYPE, mimeType)
+ put(MediaStore.MediaColumns.RELATIVE_PATH, Environment.DIRECTORY_DOWNLOADS)
+ put(MediaStore.MediaColumns.IS_PENDING, 1)
+ }
+ val uri = checkNotNull(context.contentResolver.insert(MediaStore.Downloads.EXTERNAL_CONTENT_URI, values))
+ created = uri
+ checkNotNull(context.contentResolver.openOutputStream(uri)).use { output -> source.inputStream().use { it.copyTo(output) } }
+ values.clear()
+ values.put(MediaStore.MediaColumns.IS_PENDING, 0)
+ context.contentResolver.update(uri, values, null, null)
+ promise.resolve(uri.toString())
+ } catch (error: Throwable) {
+ created?.let { runCatching { context.contentResolver.delete(it, null, null) } }
+ promise.reject("MEDIA_EXPORT_FAILED", error.message, error)
+ }
+ }, "offgrid-media-export").start()
+ }
+
@ReactMethod
fun hasPermission(promise: Promise) {
promise.resolve(granted() || allFilesAccess())
diff --git a/android/app/src/main/java/ai/offgridmobile/video/VideoEncoder.kt b/android/app/src/main/java/ai/offgridmobile/video/VideoEncoder.kt
new file mode 100644
index 000000000..db8f3b2c4
--- /dev/null
+++ b/android/app/src/main/java/ai/offgridmobile/video/VideoEncoder.kt
@@ -0,0 +1,90 @@
+package ai.offgridmobile.video
+
+import android.media.MediaCodec
+import android.media.MediaCodecInfo
+import android.media.MediaFormat
+import android.media.MediaMuxer
+import java.io.Closeable
+
+/** Platform MP4 adapter. Diffusion and job state remain in the shared runtime. */
+class VideoEncoder(path: String, private val width: Int, private val height: Int, private val fps: Int) : Closeable {
+ private val codec = MediaCodec.createEncoderByType("video/avc")
+ private val muxer = MediaMuxer(path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
+ private var track = -1
+ private var started = false
+ private var frame = 0
+ private val info = MediaCodec.BufferInfo()
+ init {
+ try {
+ val format = MediaFormat.createVideoFormat("video/avc", width, height).apply {
+ setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible)
+ setInteger(MediaFormat.KEY_BIT_RATE, width * height * fps * 2)
+ setInteger(MediaFormat.KEY_FRAME_RATE, fps)
+ setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
+ }
+ codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
+ codec.start()
+ } catch (error: Throwable) { codec.release(); muxer.release(); throw error }
+ }
+ fun append(rgb: ByteArray, channels: Int, cancelled: () -> Boolean) {
+ require(channels >= 3 && rgb.size == width * height * channels)
+ var index: Int
+ val deadline = System.nanoTime() + 30_000_000_000L
+ do {
+ check(!cancelled()) { "Video generation stopped." }
+ index = codec.dequeueInputBuffer(10_000)
+ drain(false, cancelled)
+ check(System.nanoTime() < deadline) { "Video encoder stalled." }
+ } while (index < 0)
+ val image = checkNotNull(codec.getInputImage(index)) { "Video encoder has no YUV input." }
+ for (y in 0 until height) for (x in 0 until width) {
+ val p = (y * width + x) * channels
+ val r = rgb[p].toInt() and 255; val g = rgb[p + 1].toInt() and 255; val b = rgb[p + 2].toInt() and 255
+ fun put(plane: Int, px: Int, py: Int, value: Int) {
+ val target = image.planes[plane]
+ target.buffer.put(py * target.rowStride + px * target.pixelStride, value.coerceIn(0, 255).toByte())
+ }
+ put(0, x, y, ((66 * r + 129 * g + 25 * b + 128) shr 8) + 16)
+ if (x % 2 == 0 && y % 2 == 0) {
+ put(1, x / 2, y / 2, ((-38 * r - 74 * g + 112 * b + 128) shr 8) + 128)
+ put(2, x / 2, y / 2, ((112 * r - 94 * g - 18 * b + 128) shr 8) + 128)
+ }
+ }
+ image.close()
+ codec.queueInputBuffer(index, 0, width * height * 3 / 2, frame++ * 1_000_000L / fps, 0)
+ drain(false, cancelled)
+ }
+ fun finish(cancelled: () -> Boolean) {
+ val deadline = System.nanoTime() + 30_000_000_000L
+ var index: Int
+ do {
+ check(!cancelled() && System.nanoTime() < deadline) { "Video encoding stopped." }
+ index = codec.dequeueInputBuffer(10_000); drain(false, cancelled)
+ } while (index < 0)
+ codec.queueInputBuffer(index, 0, 0, frame * 1_000_000L / fps, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
+ drain(true, cancelled)
+ }
+ private fun drain(finish: Boolean, cancelled: () -> Boolean) {
+ val deadline = System.nanoTime() + 30_000_000_000L
+ while (true) {
+ check(!cancelled() && System.nanoTime() < deadline) { "Video encoding stopped." }
+ val index = codec.dequeueOutputBuffer(info, if (finish) 10_000 else 0)
+ if (index == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
+ check(!started); track = muxer.addTrack(codec.outputFormat); muxer.start(); started = true
+ } else if (index >= 0) {
+ val buffer = checkNotNull(codec.getOutputBuffer(index))
+ if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) info.size = 0
+ if (info.size > 0) { check(started); buffer.position(info.offset); buffer.limit(info.offset + info.size); muxer.writeSampleData(track, buffer, info) }
+ val done = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
+ codec.releaseOutputBuffer(index, false)
+ if (done) return
+ } else if (!finish) return
+ }
+ }
+ override fun close() {
+ try { codec.stop() } finally {
+ codec.release()
+ try { if (started) muxer.stop() } finally { muxer.release() }
+ }
+ }
+}
diff --git a/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationModule.kt b/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationModule.kt
new file mode 100644
index 000000000..2cbc64a91
--- /dev/null
+++ b/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationModule.kt
@@ -0,0 +1,263 @@
+package ai.offgridmobile.video
+
+import ai.offgridmobile.BuildConfig
+import android.content.Intent
+import android.view.Window
+import android.view.WindowManager
+import androidx.core.content.ContextCompat
+import com.facebook.react.bridge.*
+import com.facebook.react.common.LifecycleState
+import com.facebook.react.modules.core.DeviceEventManagerModule
+import java.io.File
+import java.util.concurrent.Executors
+import java.util.concurrent.CompletableFuture
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicBoolean
+
+class VideoGenerationModule(private val context: ReactApplicationContext) : ReactContextBaseJavaModule(context), LifecycleEventListener {
+ companion object {
+ init { System.loadLibrary("offgrid_video") }
+ // JNI owns one process-wide runtime, even when React recreates its bridge.
+ private val busy = AtomicBoolean(false)
+ @Volatile private var activeVideo: VideoGenerationModule? = null
+ @Volatile private var videoStatus: Map? = null
+ }
+ private val executor = Executors.newSingleThreadExecutor()
+ private val cancelled = AtomicBoolean(false)
+ private val videoScreenActive = AtomicBoolean(false)
+ private var awakeWindow: Window? = null
+ private var addedScreenFlag = false
+ private var encoder: VideoEncoder? = null
+ private var previewFile: File? = null
+ private var videoSteps = 0
+ init { context.addLifecycleEventListener(this) }
+ private fun releaseScreenFlag() {
+ if (addedScreenFlag) awakeWindow?.clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
+ awakeWindow = null
+ addedScreenFlag = false
+ }
+ private fun updateScreenFlag() {
+ val window = if (videoScreenActive.get() && context.lifecycleState == LifecycleState.RESUMED) context.currentActivity?.window else null
+ if (window === awakeWindow) return
+ releaseScreenFlag()
+ if (window != null) {
+ awakeWindow = window
+ addedScreenFlag = window.attributes.flags and WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON == 0
+ if (addedScreenFlag) window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
+ }
+ }
+ private fun keepVideoScreenAwake(active: Boolean) {
+ videoScreenActive.set(active)
+ UiThreadUtil.runOnUiThread { updateScreenFlag() }
+ }
+ override fun onHostResume() { updateScreenFlag() }
+ override fun onHostPause() { releaseScreenFlag() }
+ override fun onHostDestroy() { releaseScreenFlag() }
+ override fun getName() = "VideoGenerationModule"
+ private external fun nativeGenerate(weight: String, vae: String, encoder: String, prompt: String, negative: String, width: Int, height: Int, frames: Int, fps: Int, steps: Int, guidance: Double, seed: Double, llm: String, embeddings: String, audioVae: String, flowShift: Double, diagnosticBackend: String)
+ private external fun nativeCancel()
+ private external fun nativePrepare()
+ private external fun nativeSetRuntimeDirectory(path: String)
+ private fun prepareHexagonRuntime() {
+ val directory = File(context.filesDir, "video-hexagon").apply { mkdirs() }
+ val names = context.assets.list("video-hexagon") ?: emptyArray()
+ check(names.isNotEmpty()) { "The video NPU runtime is missing from this build." }
+ for (name in names) {
+ check(name.matches(Regex("liboffgrid-video-htp-v[0-9]+\\.so")))
+ val temporary = File(directory, "$name.tmp")
+ context.assets.open("video-hexagon/$name").use { input ->
+ temporary.outputStream().use { output -> input.copyTo(output) }
+ }
+ check(temporary.renameTo(File(directory, name))) { "Could not install the video NPU runtime." }
+ }
+ nativeSetRuntimeDirectory(directory.absolutePath)
+ }
+ @ReactMethod fun addListener(name: String) {}
+ @ReactMethod fun removeListeners(count: Int) {}
+ private fun stop() { cancelled.set(true); nativeCancel() }
+ @ReactMethod fun cancel(promise: Promise) { (activeVideo ?: this).stop(); promise.resolve(null) }
+ @ReactMethod fun getVideoStatus(outputPath: String, promise: Promise) {
+ val status = videoStatus?.takeIf { it["path"] == outputPath }
+ keepVideoScreenAwake(status?.get("phase") == "running")
+ promise.resolve(status?.let { Arguments.makeNativeMap(it) })
+ }
+ // Called synchronously by JNI while its worker owns the runtime.
+ fun conditioning(backend: String) { emit("conditioning", 0, 0, backend) }
+ fun progress(step: Int, total: Int) {
+ // Loading and VAE counters share this callback; preserve the sampling snapshot.
+ if (total != videoSteps || step !in 0..videoSteps) return
+ emit("generating", step, total)
+ }
+ fun decoding(completed: Int, total: Int) { emit("decoding", completed, total) }
+ fun preview(rgb: ByteArray, width: Int, height: Int) {
+ if (cancelled.get()) return
+ val destination = previewFile ?: return
+ runCatching {
+ saveRgbPng(rgb, width, height, destination)
+ emit("encoding", 0, 0, preview = Arguments.createMap().apply {
+ putString("path", destination.path); putInt("width", width); putInt("height", height)
+ })
+ }.onFailure { destination.delete() }
+ }
+ private fun saveRgbPng(rgb: ByteArray, width: Int, height: Int, destination: File, channels: Int = 3) {
+ require(width > 0 && height > 0 && channels in 3..4 && rgb.size.toLong() == width.toLong() * height * channels)
+ val pixels = IntArray(width * height) { i ->
+ val offset = i * channels
+ android.graphics.Color.argb(if (channels == 4) rgb[offset + 3].toInt() and 255 else 255,
+ rgb[offset].toInt() and 255, rgb[offset + 1].toInt() and 255, rgb[offset + 2].toInt() and 255)
+ }
+ val bitmap = android.graphics.Bitmap.createBitmap(pixels, width, height, android.graphics.Bitmap.Config.ARGB_8888)
+ try { destination.outputStream().use { check(bitmap.compress(android.graphics.Bitmap.CompressFormat.PNG, 100, it)) } }
+ finally { bitmap.recycle() }
+ }
+ fun frame(rgb: ByteArray, width: Int, height: Int, channels: Int) {
+ check(!cancelled.get()) { "Video generation stopped." }
+ emit("encoding", 0, 0)
+ checkNotNull(encoder).append(rgb, channels) { cancelled.get() }
+ }
+ private fun emit(stage: String, step: Int, total: Int, backend: String? = null, preview: WritableMap? = null) {
+ videoStatus = videoStatus?.plus(mapOf(
+ "stage" to stage, "step" to step, "total" to total, "backend" to backend,
+ ))?.let { status -> if (preview != null) status + ("preview" to preview.toHashMap()) else status }
+ if (!context.hasActiveReactInstance()) return
+ context.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java)
+ .emit("VideoGenerationProgress", Arguments.createMap().apply { putString("stage", stage); putInt("step", step); putInt("total", total); if (backend != null) putString("backend", backend); if (preview != null) putMap("preview", preview) })
+ }
+ @ReactMethod fun generate(input: ReadableMap, promise: Promise) {
+ if (!busy.compareAndSet(false, true)) { promise.reject("VIDEO_BUSY", "Video generation is already running."); return }
+ activeVideo = this
+ videoStatus = mapOf("path" to input.getString("outputPath"), "phase" to "running", "stage" to "preparing", "step" to 0, "total" to 0)
+ cancelled.set(false); nativePrepare()
+ executor.execute {
+ var output: File? = null
+ var terminal: Map = mapOf("phase" to "failed", "code" to "VIDEO_FAILED", "error" to "Video generation failed.")
+ try {
+ keepVideoScreenAwake(true)
+ prepareHexagonRuntime()
+ val destination = File(checkNotNull(input.getString("outputPath")))
+ output = destination
+ previewFile = File(destination.path + ".preview.png")
+ VideoGenerationService.admission = CompletableFuture()
+ VideoGenerationService.cancel = { stop() }
+ ContextCompat.startForegroundService(context, Intent(context, VideoGenerationService::class.java))
+ VideoGenerationService.admission.get(5, TimeUnit.SECONDS)
+ check(!cancelled.get()) { "Video generation stopped." }
+ videoSteps = input.getInt("steps")
+ emit("preparing", 0, videoSteps)
+ VideoEncoder(destination.path, input.getInt("width"), input.getInt("height"), input.getInt("fps")).use { writer ->
+ encoder = writer
+ nativeGenerate(checkNotNull(input.getString("weight")), checkNotNull(input.getString("vae")), if (input.hasKey("encoder")) input.getString("encoder") ?: "" else "",
+ checkNotNull(input.getString("prompt")), input.getString("negativePrompt") ?: "", input.getInt("width"), input.getInt("height"),
+ input.getInt("frames"), input.getInt("fps"), input.getInt("steps"), input.getDouble("guidance"), input.getDouble("seed"),
+ if (input.hasKey("llm")) input.getString("llm") ?: "" else "",
+ if (input.hasKey("embeddings")) input.getString("embeddings") ?: "" else "",
+ if (input.hasKey("audioVae")) input.getString("audioVae") ?: "" else "",
+ input.getDouble("flowShift"), if (BuildConfig.DEBUG && input.hasKey("diagnosticBackend")) input.getString("diagnosticBackend") ?: "auto" else "auto")
+ writer.finish { cancelled.get() }
+ }
+ check(destination.length() > 0) { "Video encoder produced no file." }
+ terminal = mapOf("phase" to "succeeded")
+ if (context.hasActiveReactInstance())
+ promise.resolve(Arguments.createMap().apply { putString("path", destination.path) })
+ } catch (error: Throwable) {
+ val code = if (cancelled.get()) "VIDEO_CANCELLED" else "VIDEO_FAILED"
+ terminal = mapOf("phase" to "failed", "code" to code, "error" to (error.message ?: "Video generation failed."))
+ output?.delete()
+ if (context.hasActiveReactInstance()) promise.reject(code, error.message, error)
+ } finally {
+ keepVideoScreenAwake(false)
+ encoder = null; previewFile = null; VideoGenerationService.cancel = null
+ context.stopService(Intent(context, VideoGenerationService::class.java))
+ videoStatus = videoStatus?.plus(terminal)
+ activeVideo = null
+ busy.set(false)
+ }
+ }
+ }
+ private external fun nativeLoadImage(path: String, weight: String, vae: String, llm: String, threads: Int, cpuOnly: Boolean, family: String, sampler: String, scheduler: String)
+ private external fun nativeUnloadImage()
+ private external fun nativeImagePath(): String
+ private external fun nativeGenerateImage(prompt: String, negative: String, width: Int, height: Int, steps: Int, guidance: Double, seed: Double, previewInterval: Int): ByteArray
+ private var imagePreviewFile: File? = null
+ private var imageSteps = 0
+ fun imagePreview(pixels: ByteArray, width: Int, height: Int, channels: Int, step: Int) {
+ if (cancelled.get()) return
+ val file = imagePreviewFile ?: return
+ runCatching {
+ saveRgbPng(pixels, width, height, file, channels)
+ if (context.hasActiveReactInstance()) context.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java)
+ .emit("SDImageProgress", Arguments.createMap().apply {
+ putInt("step", step); putInt("totalSteps", imageSteps)
+ putDouble("progress", if (imageSteps > 0) step.toDouble() / imageSteps else 0.0)
+ putString("previewPath", file.path)
+ })
+ }
+ }
+ fun imageProgress(step: Int, total: Int) {
+ if (!context.hasActiveReactInstance()) return
+ context.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java).emit("SDImageProgress", Arguments.createMap().apply {
+ putInt("step", step); putInt("totalSteps", total); putDouble("progress", if (total > 0) step.toDouble() / total else 0.0)
+ })
+ }
+ @ReactMethod fun getLoadedImagePath(promise: Promise) { promise.resolve(nativeImagePath().ifEmpty { null }) }
+ @ReactMethod fun loadImageModel(input: ReadableMap, promise: Promise) {
+ if (!busy.compareAndSet(false, true)) { promise.reject("IMAGE_BUSY", "Image or video generation is running."); return }
+ cancelled.set(false); nativePrepare()
+ executor.execute {
+ try {
+ prepareHexagonRuntime()
+ nativeLoadImage(checkNotNull(input.getString("modelPath")), checkNotNull(input.getString("weight")),
+ if (input.hasKey("vae")) input.getString("vae") ?: "" else "",
+ if (input.hasKey("llm")) input.getString("llm") ?: "" else "",
+ input.getInt("threads"), input.getBoolean("cpuOnly"),
+ if (input.hasKey("family")) input.getString("family") ?: "" else "",
+ if (input.hasKey("sampler")) input.getString("sampler") ?: "" else "",
+ if (input.hasKey("scheduler")) input.getString("scheduler") ?: "" else "")
+ promise.resolve(true)
+ } catch (error: Throwable) { promise.reject("IMAGE_LOAD_FAILED", error.message, error) }
+ finally { busy.set(false) }
+ }
+ }
+ @ReactMethod fun unloadImageModel(promise: Promise) {
+ if (!busy.compareAndSet(false, true)) { promise.reject("IMAGE_BUSY", "Image or video generation is running."); return }
+ executor.execute {
+ try { nativeUnloadImage(); promise.resolve(true) }
+ catch (error: Throwable) { promise.reject("IMAGE_UNLOAD_FAILED", error.message, error) }
+ finally { busy.set(false) }
+ }
+ }
+ @ReactMethod fun generateImage(input: ReadableMap, promise: Promise) {
+ if (!busy.compareAndSet(false, true)) { promise.reject("IMAGE_BUSY", "Image or video generation is running."); return }
+ cancelled.set(false); nativePrepare()
+ executor.execute {
+ var output: File? = null
+ try {
+ val width = input.getInt("width"); val height = input.getInt("height")
+ require(width in 64..2048 && height in 64..2048 && width % 16 == 0 && height % 16 == 0)
+ output = File(checkNotNull(input.getString("outputPath")))
+ VideoGenerationService.admission = CompletableFuture()
+ VideoGenerationService.cancel = { stop() }
+ ContextCompat.startForegroundService(context, Intent(context, VideoGenerationService::class.java).putExtra("modality", "image"))
+ VideoGenerationService.admission.get(5, TimeUnit.SECONDS)
+ imagePreviewFile = File(output.path + ".preview.png")
+ imageSteps = input.getInt("steps")
+ val bytes = nativeGenerateImage(checkNotNull(input.getString("prompt")), input.getString("negativePrompt") ?: "", width, height, input.getInt("steps"), input.getDouble("guidanceScale"), input.getDouble("seed"), if (input.hasKey("previewInterval")) input.getInt("previewInterval").coerceAtLeast(0) else 0)
+ check(!cancelled.get()) { "Image generation stopped." }
+ saveRgbPng(bytes, width, height, output, bytes.size / (width * height))
+ check(!cancelled.get()) { "Image generation stopped." }
+ promise.resolve(Arguments.createMap().apply {
+ putString("imagePath", output.path); putInt("width", width); putInt("height", height); putDouble("seed", input.getDouble("seed")); putString("id", input.getString("id"))
+ })
+ } catch (error: Throwable) { output?.delete(); promise.reject("IMAGE_FAILED", error.message, error) }
+ finally { imagePreviewFile?.delete(); imagePreviewFile = null; VideoGenerationService.cancel = null; context.stopService(Intent(context, VideoGenerationService::class.java)); busy.set(false) }
+ }
+ }
+ override fun invalidate() {
+ keepVideoScreenAwake(false)
+ context.removeLifecycleEventListener(this)
+ // A bridge reload must not cancel a video owned by the foreground service.
+ // The replacement bridge reads its retained status and completion.
+ if (activeVideo == null) stop()
+ executor.shutdown(); super.invalidate()
+ }
+}
diff --git a/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationPackage.kt b/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationPackage.kt
new file mode 100644
index 000000000..9c0b2a1d3
--- /dev/null
+++ b/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationPackage.kt
@@ -0,0 +1,9 @@
+package ai.offgridmobile.video
+import com.facebook.react.ReactPackage
+import com.facebook.react.bridge.NativeModule
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.uimanager.ViewManager
+class VideoGenerationPackage : ReactPackage {
+ override fun createNativeModules(context: ReactApplicationContext): List = listOf(VideoGenerationModule(context))
+ override fun createViewManagers(context: ReactApplicationContext): List> = emptyList()
+}
diff --git a/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationService.kt b/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationService.kt
new file mode 100644
index 000000000..c6301c118
--- /dev/null
+++ b/android/app/src/main/java/ai/offgridmobile/video/VideoGenerationService.kt
@@ -0,0 +1,36 @@
+package ai.offgridmobile.video
+
+import android.app.*
+import android.content.Intent
+import android.content.pm.ServiceInfo
+import android.os.Build
+import android.os.IBinder
+import java.util.concurrent.CompletableFuture
+
+class VideoGenerationService : Service() {
+ companion object {
+ const val CHANNEL = "offgrid-video-generation"
+ var admission = CompletableFuture()
+ var cancel: (() -> Unit)? = null
+ }
+ override fun onBind(intent: Intent?): IBinder? = null
+ override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
+ if (intent?.action == "cancel") { cancel?.invoke(); return START_NOT_STICKY }
+ try {
+ val manager = getSystemService(NotificationManager::class.java)
+ if (Build.VERSION.SDK_INT >= 26) manager.createNotificationChannel(NotificationChannel(CHANNEL, "Media generation", NotificationManager.IMPORTANCE_LOW))
+ val stop = PendingIntent.getService(this, 0, Intent(this, VideoGenerationService::class.java).setAction("cancel"), PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT)
+ val builder = if (Build.VERSION.SDK_INT >= 26) Notification.Builder(this, CHANNEL) else Notification.Builder(this)
+ val notification = builder.setContentTitle(if (intent?.getStringExtra("modality") == "image") "Generating image" else "Generating video").setContentText("Off Grid is using this device.")
+ .setSmallIcon(android.R.drawable.ic_menu_slideshow).setOngoing(true)
+ .addAction(Notification.Action.Builder(null, "Stop", stop).build()).build()
+ if (Build.VERSION.SDK_INT >= 35) startForeground(7413, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROCESSING)
+ else if (Build.VERSION.SDK_INT >= 34) startForeground(7413, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE)
+ else startForeground(7413, notification)
+ admission.complete(Unit)
+ } catch (error: Throwable) { admission.completeExceptionally(error); cancel?.invoke(); stopSelf() }
+ return START_NOT_STICKY
+ }
+ override fun onTimeout(startId: Int, fgsType: Int) { cancel?.invoke(); stopSelf() }
+ override fun onDestroy() { cancel?.invoke(); super.onDestroy() }
+}
diff --git a/docs/VIDEO_GENERATION.md b/docs/VIDEO_GENERATION.md
new file mode 100644
index 000000000..32fef9bfe
--- /dev/null
+++ b/docs/VIDEO_GENERATION.md
@@ -0,0 +1,87 @@
+# Video generation
+
+## Current engine
+
+Local generation uses stable-diffusion.cpp at the revision in
+`scripts/video/revision`. Models uses the shared Hugging Face search, catalog,
+and architecture-specific pack resolver. The catalog currently includes Wan 2.1,
+Wan 2.2, LTX-2.3, LTX-2.5, and HunyuanVideo 1.5. Each pack must include every
+required weight, encoder, VAE, and connector file. Catalog availability is not
+proof that a model fits or has completed generation on a particular phone.
+
+Video request validation and settings keys live in `@offgrid/models`. Mobile's
+video generation service owns local and OGAD jobs, progress, cancellation,
+recovery, and completed chat/gallery records. The native bridges share the C++
+runtime lifecycle. Pro owns sync and model transfer through existing extension
+points.
+
+## Build setup
+
+CocoaPods builds the pinned iOS framework during `pod install`. Gradle prepares the Android runtime before CMake configuration. Both paths reuse their build cache. For explicit setup, run from the mobile repository after installing JavaScript dependencies:
+
+```sh
+bash scripts/prepare-video-runtime.sh
+OFFGRID_IOS_DEVICE_ONLY=1 bash scripts/build-video-ios.sh
+(cd ios && pod install)
+```
+
+The command above builds the physical arm64 iPhone framework only. Pass
+`OFFGRID_IOS_DEVICE_ONLY=1` to the physical Xcode build as well. Building requires
+an Xcode SDK with the iOS 26 background task APIs. Deployment remains iOS 17.
+
+For Android, install CMake, Make, Python 3, and the project's Android SDK/NDK:
+
+```sh
+bash scripts/prepare-video-runtime.sh --android
+(cd android && ./gradlew assembleDebug)
+```
+
+Android preparation builds a pinned host shader compiler. The NDK compiler is
+not sufficient for this engine's Vulkan shaders. Generated sources, libraries,
+and compiler tools stay in `.video-build` and are not committed. The small
+Vulkan patch uses the engine's dynamic dispatcher so linking remains compatible
+with the app's Android API 24 minimum.
+
+## Device limits and recovery
+
+Model download does not guarantee that a device has enough memory to generate.
+The residency manager checks the pack and clip size before loading. Default
+clips are 320 × 192, 17 frames, at 8 FPS. Speed and peak memory still need manual
+checks on physical iOS and Android devices.
+
+On iOS 26, continued background GPU work requires OS support and admission.
+If admission is not available, generation stops when the app enters the
+background. Android uses a foreground service with a Stop action; an OS timeout
+also cancels the job. Partial output is removed on failure or cancellation.
+
+After restart, interrupted jobs appear with Retry. Local Retry restarts the
+clip. Remote Retry uses the same OGAD job ID, so a lost connection does not start
+a second job. Settings and model selection can change while a saved request
+retains its resolved generation parameters.
+
+Only OGAD remote video servers are supported. REST and MCP share OGAD's video
+job owner. Model transfer sends and checks every file in the selected pack
+before registration. Cancel and retry use the shared package transaction and
+rollback. Local download resume preserves complete files; an incomplete file can
+restart after process death.
+
+## Integration audit (2026-09-29)
+
+- Storage reads actual sizes for all installed video pack files, including old
+ records whose catalog size is missing. Download completion stores actual sizes.
+- Auto Configure includes video only when the native runtime, complete catalog
+ pack, RAM minimum, and existing memory budget allow it. Models below these
+ limits remain manually selectable; automatic setup does not use Run Anyway.
+- iOS excludes downloaded/transferred video model folders from device backups,
+ as it does for text and image models. SD image packs use `image_models` and its
+ existing exclusion. Android disables app backup in its manifest. Generated
+ videos and user content are not excluded by this model-cache policy.
+- Remote video requires OGAD. Local and remote labels use the shared catalog,
+ while model IDs and per-model settings keys remain unchanged.
+- Native progress changes and background admission details enter the app debug
+ log. A heartbeat alone is not proof that a sampling step advanced.
+
+Source checks cover these integrations. Device checks for the latest backup
+exclusion and Auto Configure changes remain pending. Complete video quality and
+all-model coverage are not established by successful builds or partial sampling.
+No automated tests were added or run for this audit.
diff --git a/ios/OffgridMobile/Info.plist b/ios/OffgridMobile/Info.plist
index 350fa2054..592d2f8d3 100644
--- a/ios/OffgridMobile/Info.plist
+++ b/ios/OffgridMobile/Info.plist
@@ -108,6 +108,10 @@
UILaunchStoryboardName
LaunchScreen
+ UIBackgroundModes
+
+ audio
+
UIRequiredDeviceCapabilities
arm64
@@ -124,5 +128,9 @@
UIViewControllerBasedStatusBarAppearance
+ BGTaskSchedulerPermittedIdentifiers
+
+ $(PRODUCT_BUNDLE_IDENTIFIER).video.*
+
diff --git a/ios/OffgridMobile/OffgridMobile.entitlements b/ios/OffgridMobile/OffgridMobile.entitlements
index 510563fbc..e480f62a3 100644
--- a/ios/OffgridMobile/OffgridMobile.entitlements
+++ b/ios/OffgridMobile/OffgridMobile.entitlements
@@ -6,5 +6,7 @@
com.apple.developer.kernel.extended-virtual-addressing
+ com.apple.developer.background-tasks.continued-processing.gpu
+
diff --git a/ios/Podfile b/ios/Podfile
index fd0564901..a56c23f49 100644
--- a/ios/Podfile
+++ b/ios/Podfile
@@ -20,6 +20,10 @@ end
target 'OffgridMobile' do
config = use_native_modules!
+ # The script reuses a framework only when its pinned revision and headers match.
+ video_builder = File.expand_path('../scripts/build-video-ios.sh', __dir__)
+ raise 'Video runtime build failed' unless system('bash', video_builder)
+ pod 'OffgridVideo', :path => '../native/video'
use_react_native!(
:path => config[:reactNativePath],
diff --git a/ios/Podfile.lock b/ios/Podfile.lock
index 1bb3fde0a..fe9372a1e 100644
--- a/ios/Podfile.lock
+++ b/ios/Podfile.lock
@@ -42,6 +42,8 @@ PODS:
- MMKV (2.4.0):
- MMKVCore (~> 2.4.0)
- MMKVCore (2.4.0)
+ - OffgridVideo (0.1.0):
+ - React-Core
- op-sqlite (15.2.5):
- boost
- DoubleConversion
@@ -2303,6 +2305,92 @@ PODS:
- react-native-tcp-socket (6.4.1):
- CocoaAsyncSocket
- React-Core
+ - react-native-video (6.19.3):
+ - boost
+ - DoubleConversion
+ - fast_float
+ - fmt
+ - glog
+ - hermes-engine
+ - RCT-Folly
+ - RCT-Folly/Fabric
+ - RCTRequired
+ - RCTTypeSafety
+ - React-Core
+ - React-debug
+ - React-Fabric
+ - React-featureflags
+ - React-graphics
+ - React-ImageManager
+ - React-jsi
+ - react-native-video/Video (= 6.19.3)
+ - React-NativeModulesApple
+ - React-RCTFabric
+ - React-renderercss
+ - React-rendererdebug
+ - React-utils
+ - ReactCodegen
+ - ReactCommon/turbomodule/bridging
+ - ReactCommon/turbomodule/core
+ - SocketRocket
+ - Yoga
+ - react-native-video/Fabric (6.19.3):
+ - boost
+ - DoubleConversion
+ - fast_float
+ - fmt
+ - glog
+ - hermes-engine
+ - RCT-Folly
+ - RCT-Folly/Fabric
+ - RCTRequired
+ - RCTTypeSafety
+ - React-Core
+ - React-debug
+ - React-Fabric
+ - React-featureflags
+ - React-graphics
+ - React-ImageManager
+ - React-jsi
+ - React-NativeModulesApple
+ - React-RCTFabric
+ - React-renderercss
+ - React-rendererdebug
+ - React-utils
+ - ReactCodegen
+ - ReactCommon/turbomodule/bridging
+ - ReactCommon/turbomodule/core
+ - SocketRocket
+ - Yoga
+ - react-native-video/Video (6.19.3):
+ - boost
+ - DoubleConversion
+ - fast_float
+ - fmt
+ - glog
+ - hermes-engine
+ - RCT-Folly
+ - RCT-Folly/Fabric
+ - RCTRequired
+ - RCTTypeSafety
+ - React-Core
+ - React-debug
+ - React-Fabric
+ - React-featureflags
+ - React-graphics
+ - React-ImageManager
+ - React-jsi
+ - react-native-video/Fabric
+ - React-NativeModulesApple
+ - React-RCTFabric
+ - React-renderercss
+ - React-rendererdebug
+ - React-utils
+ - ReactCodegen
+ - ReactCommon/turbomodule/bridging
+ - ReactCommon/turbomodule/core
+ - SocketRocket
+ - Yoga
- react-native-voice (3.2.4):
- React-Core
- react-native-zeroconf (0.14.0):
@@ -3506,6 +3594,7 @@ DEPENDENCIES:
- glog (from `../node_modules/react-native/third-party-podspecs/glog.podspec`)
- hermes-engine (from `../node_modules/react-native/sdks/hermes-engine/hermes-engine.podspec`)
- llama-rn (from `../node_modules/llama.rn`)
+ - OffgridVideo (from `../native/video`)
- "op-sqlite (from `../node_modules/@op-engineering/op-sqlite`)"
- RCT-Folly (from `../node_modules/react-native/third-party-podspecs/RCT-Folly.podspec`)
- RCTDeprecation (from `../node_modules/react-native/ReactApple/Libraries/RCTFoundation/RCTDeprecation`)
@@ -3554,6 +3643,7 @@ DEPENDENCIES:
- react-native-safe-area-context (from `../node_modules/react-native-safe-area-context`)
- "react-native-slider (from `../node_modules/@react-native-community/slider`)"
- react-native-tcp-socket (from `../node_modules/react-native-tcp-socket`)
+ - react-native-video (from `../node_modules/react-native-video`)
- "react-native-voice (from `../node_modules/@react-native-voice/voice`)"
- react-native-zeroconf (from `../node_modules/react-native-zeroconf`)
- React-NativeModulesApple (from `../node_modules/react-native/ReactCommon/react/nativemodule/core/platform/ios`)
@@ -3636,6 +3726,8 @@ EXTERNAL SOURCES:
:tag: hermes-v0.14.0
llama-rn:
:path: "../node_modules/llama.rn"
+ OffgridVideo:
+ :path: "../native/video"
op-sqlite:
:path: "../node_modules/@op-engineering/op-sqlite"
RCT-Folly:
@@ -3730,6 +3822,8 @@ EXTERNAL SOURCES:
:path: "../node_modules/@react-native-community/slider"
react-native-tcp-socket:
:path: "../node_modules/react-native-tcp-socket"
+ react-native-video:
+ :path: "../node_modules/react-native-video"
react-native-voice:
:path: "../node_modules/@react-native-voice/voice"
react-native-zeroconf:
@@ -3849,6 +3943,7 @@ SPEC CHECKSUMS:
llama-rn: fc4d176920fff190ee3a5a4172ce739531150cc4
MMKV: 86859fdfa2b0b21db1fd6e48788474a6416a2c77
MMKVCore: 3d16ce9f7d411e135020915fde98a056859a1efa
+ OffgridVideo: 3a0d4acebd54e2ab315c9ff737aa017ccc3d36b8
op-sqlite: bafff369cecaee4fe65c89eec47deaba26f2db95
opencv-rne: 2305807573b6e29c8c87e3416ab096d09047a7a0
RCT-Folly: 846fda9475e61ec7bcbf8a3fe81edfcaeb090669
@@ -3897,6 +3992,7 @@ SPEC CHECKSUMS:
react-native-safe-area-context: c00143b4823773bba23f2f19f85663ae89ceb460
react-native-slider: 34064ca1a6864d7b263e44dd76a2d794e8d26744
react-native-tcp-socket: 7c7e53a07f122ecf00fb3626684bc0ca82c4f044
+ react-native-video: 61841250a61ed594812b0bd4ce3f03549d574efc
react-native-voice: 908a0eba96c8c3d643e4f98b7232c6557d0a6f9c
react-native-zeroconf: eb2e5584308f20f5fc3eb0cea2ceafbbd345b48b
React-NativeModulesApple: a2c3d2cbec893956a5b3e4060322db2984fff75b
@@ -3953,6 +4049,6 @@ SPEC CHECKSUMS:
whisper-rn: 7566faf9b7d78e39ab9fc634cb90fdee81177793
Yoga: 5456bb010373068fc92221140921b09d126b116e
-PODFILE CHECKSUM: d2e7563af7ca0531bcab79cceff20997d2fbe48c
+PODFILE CHECKSUM: a51a87720248f5187c458d3af5dfc690def5388e
-COCOAPODS: 1.15.2
+COCOAPODS: 1.16.2
diff --git a/ios/StreamingHashModule.m b/ios/StreamingHashModule.m
index b7a3a52e5..67b419c56 100644
--- a/ios/StreamingHashModule.m
+++ b/ios/StreamingHashModule.m
@@ -6,4 +6,8 @@ @interface RCT_EXTERN_MODULE(StreamingHashModule, NSObject)
resolver:(RCTPromiseResolveBlock)resolve
rejecter:(RCTPromiseRejectBlock)reject)
+RCT_EXTERN_METHOD(sha256:(NSString *)path
+ resolver:(RCTPromiseResolveBlock)resolve
+ rejecter:(RCTPromiseRejectBlock)reject)
+
@end
diff --git a/ios/StreamingHashModule.swift b/ios/StreamingHashModule.swift
index 8b97e2363..9d3f76c3d 100644
--- a/ios/StreamingHashModule.swift
+++ b/ios/StreamingHashModule.swift
@@ -8,12 +8,25 @@ enum StreamingFileHasher {
at url: URL,
chunkSize: Int = defaultChunkSize,
didConsumeChunk: (() -> Void)? = nil
+ ) throws -> String {
+ try digestHex(at: url, using: SHA512(), chunkSize: chunkSize, didConsumeChunk: didConsumeChunk)
+ }
+
+ static func sha256Hex(at url: URL) throws -> String {
+ try digestHex(at: url, using: SHA256())
+ }
+
+ private static func digestHex(
+ at url: URL,
+ using initialHasher: H,
+ chunkSize: Int = defaultChunkSize,
+ didConsumeChunk: (() -> Void)? = nil
) throws -> String {
precondition(chunkSize > 0)
let handle = try FileHandle(forReadingFrom: url)
defer { try? handle.close() }
- var hasher = SHA512()
+ var hasher = initialHasher
while true {
// FileHandle can back each Swift Data with an autoreleased NSData. This work runs in one
// long-lived GCD block, so the queue's outer autorelease pool is not drained until the whole
@@ -45,6 +58,24 @@ final class StreamingHashModule: NSObject {
_ path: String,
resolver resolve: @escaping RCTPromiseResolveBlock,
rejecter reject: @escaping RCTPromiseRejectBlock
+ ) {
+ hash(path, digest: { try StreamingFileHasher.sha512Hex(at: $0) }, resolve: resolve, reject: reject)
+ }
+
+ @objc
+ func sha256(
+ _ path: String,
+ resolver resolve: @escaping RCTPromiseResolveBlock,
+ rejecter reject: @escaping RCTPromiseRejectBlock
+ ) {
+ hash(path, digest: StreamingFileHasher.sha256Hex, resolve: resolve, reject: reject)
+ }
+
+ private func hash(
+ _ path: String,
+ digest: @escaping (URL) throws -> String,
+ resolve: @escaping RCTPromiseResolveBlock,
+ reject: @escaping RCTPromiseRejectBlock
) {
DispatchQueue.global(qos: .utility).async {
do {
@@ -54,7 +85,7 @@ final class StreamingHashModule: NSObject {
defer {
if scoped { url.stopAccessingSecurityScopedResource() }
}
- resolve(try StreamingFileHasher.sha512Hex(at: url))
+ resolve(try digest(url))
} catch {
reject("streaming_hash_failed", error.localizedDescription, error)
}
diff --git a/native/video/OffgridVideo.podspec b/native/video/OffgridVideo.podspec
new file mode 100644
index 000000000..2b607031b
--- /dev/null
+++ b/native/video/OffgridVideo.podspec
@@ -0,0 +1,15 @@
+Pod::Spec.new do |s|
+ s.name = 'OffgridVideo'
+ s.version = '0.1.0'
+ s.summary = 'Local Wan video generation and MP4 encoding for Off Grid AI'
+ s.homepage = 'https://github.com/off-grid-ai/OGAM'
+ s.license = { :type => 'AGPL-3.0', :file => '../../LICENSE' }
+ s.author = 'Off Grid AI'
+ s.source = { :git => 'https://github.com/off-grid-ai/OGAM.git' }
+ s.platform = :ios, '17.0'
+ s.source_files = '*.{h,mm}', 'common/*.hpp'
+ s.vendored_frameworks = 'OffgridVideoRuntime.xcframework'
+ s.frameworks = 'BackgroundTasks', 'AVFoundation', 'CoreVideo', 'Metal', 'Accelerate', 'UIKit'
+ s.dependency 'React-Core'
+ s.pod_target_xcconfig = { 'CLANG_CXX_LANGUAGE_STANDARD' => 'c++17' }
+end
diff --git a/native/video/RUNTIME-LICENSE b/native/video/RUNTIME-LICENSE
new file mode 100644
index 000000000..56e1e5a63
--- /dev/null
+++ b/native/video/RUNTIME-LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 leejet
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
\ No newline at end of file
diff --git a/native/video/VideoEncoder.h b/native/video/VideoEncoder.h
new file mode 100644
index 000000000..aa823f535
--- /dev/null
+++ b/native/video/VideoEncoder.h
@@ -0,0 +1,5 @@
+#import
+#import
+#include
+BOOL OGEncodeVideo(sd_image_t *frames, int count, int fps, NSString *path,
+ const std::atomic_bool &cancelled, NSError **error);
diff --git a/native/video/VideoEncoder.mm b/native/video/VideoEncoder.mm
new file mode 100644
index 000000000..09a96b031
--- /dev/null
+++ b/native/video/VideoEncoder.mm
@@ -0,0 +1,64 @@
+#import "VideoEncoder.h"
+#import
+
+BOOL OGEncodeVideo(sd_image_t *frames, int count, int fps, NSString *path,
+ const std::atomic_bool &cancelled, NSError **error) {
+ if (!frames || count < 1 || fps < 1) return NO;
+ AVAssetWriter *writer = [[AVAssetWriter alloc] initWithURL:[NSURL fileURLWithPath:path]
+ fileType:AVFileTypeMPEG4 error:error];
+ if (!writer) return NO;
+ NSDictionary *settings = @{ AVVideoCodecKey: AVVideoCodecTypeH264,
+ AVVideoWidthKey: @(frames[0].width), AVVideoHeightKey: @(frames[0].height) };
+ AVAssetWriterInput *input = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo outputSettings:settings];
+ input.expectsMediaDataInRealTime = NO;
+ NSDictionary *attributes = @{ (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA),
+ (id)kCVPixelBufferWidthKey: @(frames[0].width), (id)kCVPixelBufferHeightKey: @(frames[0].height),
+ (id)kCVPixelBufferIOSurfacePropertiesKey: @{} };
+ AVAssetWriterInputPixelBufferAdaptor *adaptor = [AVAssetWriterInputPixelBufferAdaptor
+ assetWriterInputPixelBufferAdaptorWithAssetWriterInput:input sourcePixelBufferAttributes:attributes];
+ if (![writer canAddInput:input]) return NO;
+ [writer addInput:input];
+ if (![writer startWriting]) { if (error) *error = writer.error; return NO; }
+ [writer startSessionAtSourceTime:kCMTimeZero];
+ BOOL ok = YES;
+ for (int i = 0; i < count && ok; ++i) {
+ @autoreleasepool {
+ // Bound a stalled encoder and permit Stop while waiting for its next buffer.
+ NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:30];
+ while (!input.readyForMoreMediaData && writer.status == AVAssetWriterStatusWriting &&
+ !cancelled.load() && deadline.timeIntervalSinceNow > 0) [NSThread sleepForTimeInterval:0.01];
+ if (cancelled.load() || !input.readyForMoreMediaData || frames[i].channel < 3 || !frames[i].data || frames[i].width != frames[0].width || frames[i].height != frames[0].height) { ok = NO; break; }
+ CVPixelBufferRef buffer = nil;
+ if (CVPixelBufferPoolCreatePixelBuffer(NULL, adaptor.pixelBufferPool, &buffer) != kCVReturnSuccess) { ok = NO; break; }
+ CVPixelBufferLockBaseAddress(buffer, 0);
+ auto *pixels = (uint8_t *)CVPixelBufferGetBaseAddress(buffer);
+ size_t stride = CVPixelBufferGetBytesPerRow(buffer);
+ for (uint32_t y = 0; y < frames[i].height; ++y) {
+ for (uint32_t x = 0; x < frames[i].width; ++x) {
+ const uint8_t *rgb = frames[i].data + (y * frames[i].width + x) * frames[i].channel;
+ uint8_t *bgra = pixels + y * stride + x * 4;
+ bgra[0] = rgb[2]; bgra[1] = rgb[1]; bgra[2] = rgb[0]; bgra[3] = 255;
+ }
+ }
+ CVPixelBufferUnlockBaseAddress(buffer, 0);
+ ok = [adaptor appendPixelBuffer:buffer withPresentationTime:CMTimeMake(i, fps)];
+ CVPixelBufferRelease(buffer);
+ }
+ }
+ if (ok) {
+ [input markAsFinished];
+ dispatch_semaphore_t finished = dispatch_semaphore_create(0);
+ [writer finishWritingWithCompletionHandler:^{ dispatch_semaphore_signal(finished); }];
+ NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:30];
+ while (dispatch_semaphore_wait(finished, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_MSEC * 100)) != 0) {
+ if (cancelled.load() || deadline.timeIntervalSinceNow <= 0) { [writer cancelWriting]; break; }
+ }
+ ok = writer.status == AVAssetWriterStatusCompleted && !cancelled.load();
+ }
+ if (!ok) {
+ if (error) *error = writer.error;
+ [writer cancelWriting];
+ [[NSFileManager defaultManager] removeItemAtPath:path error:nil];
+ }
+ return ok;
+}
diff --git a/native/video/VideoGenerationModule.mm b/native/video/VideoGenerationModule.mm
new file mode 100644
index 000000000..0ce96d363
--- /dev/null
+++ b/native/video/VideoGenerationModule.mm
@@ -0,0 +1,358 @@
+#import
+#import
+#import
+#import
+#import "VideoEncoder.h"
+#include "common/VideoRuntime.hpp"
+#include
+#include
+
+static BOOL OGSaveRgbPng(const sd_image_t &image, NSString *output) {
+ if (!image.data || (image.channel != 3 && image.channel != 4)) return NO;
+ CFDataRef data = CFDataCreate(kCFAllocatorDefault, image.data, image.width * image.height * image.channel);
+ CGDataProviderRef provider = data ? CGDataProviderCreateWithCFData(data) : nullptr;
+ CGColorSpaceRef color = CGColorSpaceCreateDeviceRGB();
+ CGImageRef bitmap = provider ? CGImageCreate(image.width, image.height, 8, 8 * image.channel, image.width * image.channel, color, image.channel == 4 ? kCGImageAlphaLast : kCGImageAlphaNone, provider, nullptr, false, kCGRenderingIntentDefault) : nullptr;
+ NSData *png = bitmap ? UIImagePNGRepresentation([UIImage imageWithCGImage:bitmap]) : nil;
+ if (bitmap) CGImageRelease(bitmap);
+ CGColorSpaceRelease(color); if (provider) CGDataProviderRelease(provider); if (data) CFRelease(data);
+ return png && [png writeToFile:output atomically:YES];
+}
+
+@interface VideoGenerationModule : RCTEventEmitter
+@end
+@implementation VideoGenerationModule {
+ offgrid::VideoRuntime _runtime;
+ dispatch_queue_t _worker;
+ BOOL _busy;
+ BOOL _listeners;
+ BOOL _holdsIdleTimer;
+ BOOL _previousIdleTimerDisabled;
+ NSString *_videoInterruptionReason;
+ NSString *_imageNativeError;
+ BGTask *_continued;
+ NSString *_taskIdentifier;
+ dispatch_block_t _continuedWork;
+}
+RCT_EXPORT_MODULE(VideoGenerationModule)
++ (BOOL)requiresMainQueueSetup { return YES; }
+- (instancetype)init {
+ if ((self = [super init])) {
+ _worker = dispatch_queue_create("ai.offgrid.video", DISPATCH_QUEUE_SERIAL);
+
+ [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(backgrounded)
+ name:UIApplicationDidEnterBackgroundNotification object:nil];
+ }
+ return self;
+}
+- (void)dealloc {
+ [[NSNotificationCenter defaultCenter] removeObserver:self];
+ if (_holdsIdleTimer) {
+ BOOL previous = _previousIdleTimerDisabled;
+ dispatch_async(dispatch_get_main_queue(), ^{ UIApplication.sharedApplication.idleTimerDisabled = previous; });
+ }
+}
+- (void)keepScreenAwake {
+ if (_holdsIdleTimer) return;
+ _previousIdleTimerDisabled = UIApplication.sharedApplication.idleTimerDisabled;
+ _holdsIdleTimer = YES;
+ UIApplication.sharedApplication.idleTimerDisabled = YES;
+}
+- (void)restoreScreenIdleTimer {
+ if (!_holdsIdleTimer) return;
+ UIApplication.sharedApplication.idleTimerDisabled = _previousIdleTimerDisabled;
+ _holdsIdleTimer = NO;
+}
+- (NSArray *)supportedEvents { return @[@"VideoGenerationProgress", @"SDImageProgress"]; }
+- (void)startObserving { _listeners = YES; }
+- (void)stopObserving { _listeners = NO; }
+- (void)logLifecycle:(NSString *)detail {
+ NSLog(@"[VideoLifecycle] %@", detail);
+ if (_listeners) [self sendEventWithName:@"VideoGenerationProgress"
+ body:@{@"lifecycle":detail, @"at":@([[NSDate date] timeIntervalSince1970] * 1000)}];
+}
+- (void)beginImageDiagnostics {
+ @synchronized (self) { _imageNativeError = nil; }
+ sd_set_log_callback([](enum sd_log_level_t level, const char *text, void *data) {
+ if (!text) return;
+ std::fputs(text, stderr);
+ // Retain placement and failure evidence, without prompts or model contents.
+ if (std::strstr(text, "prompt")) return;
+ if (level < SD_LOG_WARN && !std::strstr(text, "auto-fit") &&
+ !std::strstr(text, "backend") && !std::strstr(text, "MiB") &&
+ !std::strstr(text, "sampling completed") && !std::strstr(text, "cancelling")) return;
+ VideoGenerationModule *owner = (__bridge VideoGenerationModule *)data;
+ NSString *detail = [NSString stringWithUTF8String:text];
+ if (!detail) return;
+ if (detail.length > 2048) detail = [detail substringToIndex:2048];
+ if (level == SD_LOG_ERROR) {
+ @synchronized (owner) { owner->_imageNativeError = detail; }
+ }
+ dispatch_async(dispatch_get_main_queue(), ^{
+ if (owner->_listeners) [owner sendEventWithName:@"SDImageProgress" body:@{@"diagnostic":detail}];
+ });
+ }, (__bridge void *)self);
+}
+- (NSString *)finishImageDiagnostics:(NSString *)failure {
+ sd_set_log_callback(nullptr, nullptr);
+ @synchronized (self) {
+ return failure && _imageNativeError.length
+ ? [NSString stringWithFormat:@"%@ %@", failure, _imageNativeError] : failure;
+ }
+}
+- (void)interruptVideo:(NSString *)reason {
+ _videoInterruptionReason = reason;
+ _runtime.cancel();
+ if (_listeners) [self sendEventWithName:@"VideoGenerationProgress"
+ body:@{@"interruption":reason}];
+}
+- (void)backgrounded {
+ // Metal work must stop before the OS removes GPU access. Continued GPU tasks
+ // are admitted separately by the background coordinator on supported systems.
+ [self logLifecycle:[NSString stringWithFormat:@"background busy=%d admittedGPU=%d", _busy, _continued != nil]];
+ if (_busy && !_continued) {
+ [self interruptVideo:@"Video generation stopped when the app went into the background. Keep Off Grid open and try again."];
+ }
+}
+- (void)emitStage:(NSString *)stage step:(int)step total:(int)total {
+ dispatch_async(dispatch_get_main_queue(), ^{
+ if (self->_videoInterruptionReason) return;
+ if (@available(iOS 26.0, *)) {
+ if (self->_continued && total > 0) {
+ BGContinuedProcessingTask *task = (BGContinuedProcessingTask *)self->_continued;
+ task.progress.totalUnitCount = total + 2;
+ task.progress.completedUnitCount = MIN(step + 1, total + 1);
+ [task updateTitle:@"Generating video" subtitle:stage];
+ }
+ }
+ if (self->_listeners) [self sendEventWithName:@"VideoGenerationProgress"
+ body:@{@"stage":stage, @"step":@(step), @"total":@(total)}];
+ });
+}
+RCT_REMAP_METHOD(cancel, cancelWithResolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) {
+ dispatch_async(dispatch_get_main_queue(), ^{
+ self->_runtime.cancel();
+ if (self->_continuedWork) {
+ if (self->_taskIdentifier) [BGTaskScheduler.sharedScheduler cancelTaskRequestWithIdentifier:self->_taskIdentifier];
+ dispatch_block_t pending = self->_continuedWork; self->_continuedWork = nil;
+ pending(); // The worker observes cancellation and settles the JS promise.
+ }
+ resolve(nil);
+ });
+}
+RCT_REMAP_METHOD(generate, generate:(NSDictionary *)input resolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) {
+ dispatch_async(dispatch_get_main_queue(), ^{
+ if (self->_busy) { reject(@"VIDEO_BUSY", @"Video generation is already running.", nil); return; }
+ self->_busy = YES; self->_runtime.cancelled.store(false);
+ self->_videoInterruptionReason = nil;
+ [self keepScreenAwake];
+ [self startContinuedWork:^{
+ dispatch_async(self->_worker, ^{
+ @autoreleasepool {
+ NSString *output = input[@"outputPath"];
+ NSError *failure = nil;
+ // Keep targeted runtime diagnostics in the same persistent lifecycle log.
+ sd_set_log_callback([](enum sd_log_level_t, const char *text, void *data) {
+ if (!text) return;
+ std::fputs(text, stderr);
+ if (!std::strstr(text, "Wan VAE decode backend=")) return;
+ VideoGenerationModule *owner = (__bridge VideoGenerationModule *)data;
+ NSString *detail = [NSString stringWithUTF8String:text];
+ dispatch_async(dispatch_get_main_queue(), ^{ [owner logLifecycle:detail]; });
+ }, (__bridge void *)self);
+ try {
+ offgrid::VideoRequest request{
+ [input[@"weight"] UTF8String], [input[@"vae"] UTF8String], [(input[@"encoder"] ?: @"") UTF8String],
+ [input[@"prompt"] UTF8String], [input[@"negativePrompt"] UTF8String],
+ [input[@"width"] intValue], [input[@"height"] intValue], [input[@"frames"] intValue],
+ [input[@"fps"] intValue], [input[@"steps"] intValue], [input[@"guidance"] floatValue], [input[@"seed"] longLongValue]
+ };
+ request.llm = [(input[@"llm"] ?: @"") UTF8String];
+ request.embeddings = [(input[@"embeddings"] ?: @"") UTF8String];
+ request.audioVae = [(input[@"audioVae"] ?: @"") UTF8String];
+ request.flowShift = [input[@"flowShift"] floatValue];
+ [self emitStage:@"preparing" step:0 total:request.steps];
+ self->_runtime.run(request, [&](int step, int total) {
+ [self emitStage:@"generating" step:step total:total];
+ }, [&](sd_image_t *frames, int count, int fps) {
+ [self emitStage:@"encoding" step:0 total:count];
+ NSError *encodingError = nil;
+ if (!OGEncodeVideo(frames, count, fps, output, self->_runtime.cancelled, &encodingError))
+ throw std::runtime_error(encodingError ? encodingError.localizedDescription.UTF8String : "Video encoding stopped.");
+ }, [&](const char *) { [self emitStage:@"conditioning" step:0 total:0]; },
+ [&](int completed, int total) { [self emitStage:@"decoding" step:completed total:total]; },
+ [&](const sd_image_t &image) {
+ if (self->_runtime.cancelled.load()) return;
+ NSString *path = [output stringByAppendingString:@".preview.png"];
+ @try {
+ if (!OGSaveRgbPng(image, path)) return;
+ } @catch (NSException *exception) {
+ [[NSFileManager defaultManager] removeItemAtPath:path error:nil];
+ return;
+ }
+ NSDictionary *preview = @{@"path":path, @"width":@(image.width), @"height":@(image.height)};
+ dispatch_async(dispatch_get_main_queue(), ^{
+ if (self->_listeners) [self sendEventWithName:@"VideoGenerationProgress"
+ body:@{@"stage":@"encoding", @"step":@0, @"total":@0, @"preview":preview}];
+ });
+ });
+ } catch (const std::exception &error) {
+ failure = [NSError errorWithDomain:@"OffgridVideo" code:1 userInfo:@{NSLocalizedDescriptionKey:@(error.what())}];
+ [[NSFileManager defaultManager] removeItemAtPath:output error:nil];
+ }
+ sd_set_log_callback(nullptr, nullptr);
+ dispatch_async(dispatch_get_main_queue(), ^{
+ self->_busy = NO;
+ [self restoreScreenIdleTimer];
+ if (self->_continued) {
+ [self->_continued setTaskCompletedWithSuccess:failure == nil]; self->_continued = nil;
+ }
+ if (failure && self->_videoInterruptionReason)
+ reject(@"VIDEO_BACKGROUND_INTERRUPTED", self->_videoInterruptionReason, failure);
+ else if (failure) reject(self->_runtime.cancelled.load() ? @"VIDEO_CANCELLED" : @"VIDEO_FAILED", failure.localizedDescription, failure);
+ else resolve(@{@"path":output});
+ });
+ }
+ });
+ }];
+ });
+}
+RCT_REMAP_METHOD(getLoadedImagePath, imagePathWithResolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) {
+ auto path = _runtime.loadedImagePath(); resolve(path.empty() ? nil : @(path.c_str()));
+}
+RCT_REMAP_METHOD(loadImageModel, loadImage:(NSDictionary *)input resolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) {
+ dispatch_async(dispatch_get_main_queue(), ^{
+ if (self->_busy) { reject(@"IMAGE_BUSY", @"Image or video generation is running.", nil); return; }
+ self->_busy = YES; self->_runtime.cancelled.store(false);
+ dispatch_async(self->_worker, ^{
+ NSString *failure = nil;
+ [self beginImageDiagnostics];
+ try {
+ offgrid::VideoRequest request{};
+ request.weight = [input[@"weight"] UTF8String]; request.vae = [(input[@"vae"] ?: @"") UTF8String];
+ request.llm = [(input[@"llm"] ?: @"") UTF8String];
+ request.imageFamily = [(input[@"family"] ?: @"") UTF8String];
+ request.imageSampler = [(input[@"sampler"] ?: @"") UTF8String];
+ request.imageScheduler = [(input[@"scheduler"] ?: @"") UTF8String];
+ request.threads = [input[@"threads"] intValue]; request.cpuOnly = [input[@"cpuOnly"] boolValue];
+ self->_runtime.loadImage(request, [input[@"modelPath"] UTF8String]);
+ } catch (const std::exception &error) { failure = @(error.what()); }
+ failure = [self finishImageDiagnostics:failure];
+ dispatch_async(dispatch_get_main_queue(), ^{
+ self->_busy = NO;
+ if (failure) reject(@"IMAGE_LOAD_FAILED", failure, nil); else resolve(@YES);
+ });
+ });
+ });
+}
+RCT_REMAP_METHOD(unloadImageModel, unloadImageWithResolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) {
+ dispatch_async(dispatch_get_main_queue(), ^{
+ if (self->_busy) { reject(@"IMAGE_BUSY", @"Image or video generation is running.", nil); return; }
+ self->_busy = YES;
+ dispatch_async(self->_worker, ^{
+ NSString *failure = nil;
+ try { self->_runtime.unloadImage(); } catch (const std::exception &error) { failure = @(error.what()); }
+ dispatch_async(dispatch_get_main_queue(), ^{
+ self->_busy = NO;
+ if (failure) reject(@"IMAGE_UNLOAD_FAILED", failure, nil); else resolve(@YES);
+ });
+ });
+ });
+}
+RCT_REMAP_METHOD(generateImage, generateImage:(NSDictionary *)input resolver:(RCTPromiseResolveBlock)resolve rejecter:(RCTPromiseRejectBlock)reject) {
+ dispatch_async(dispatch_get_main_queue(), ^{
+ if (self->_busy) { reject(@"IMAGE_BUSY", @"Image or video generation is running.", nil); return; }
+ self->_busy = YES; self->_runtime.cancelled.store(false);
+ [self keepScreenAwake];
+ dispatch_async(self->_worker, ^{
+ @autoreleasepool {
+ NSString *output = input[@"outputPath"];
+ NSString *failure = nil;
+ [self beginImageDiagnostics];
+ try {
+ offgrid::VideoRequest request{};
+ request.prompt = [input[@"prompt"] UTF8String]; request.negative = [input[@"negativePrompt"] UTF8String];
+ request.width = [input[@"width"] intValue]; request.height = [input[@"height"] intValue];
+ request.steps = [input[@"steps"] intValue]; request.guidance = [input[@"guidanceScale"] floatValue]; request.seed = [input[@"seed"] longLongValue];
+ if (request.width < 64 || request.width > 2048 || request.height < 64 || request.height > 2048 || request.width % 16 || request.height % 16)
+ throw std::runtime_error("Image dimensions are not supported.");
+ self->_runtime.image(request, [&](int step, int total) {
+ dispatch_async(dispatch_get_main_queue(), ^{
+ if (self->_listeners) [self sendEventWithName:@"SDImageProgress" body:@{@"step":@(step), @"totalSteps":@(total), @"progress":@(total > 0 ? double(step) / total : 0)}];
+ });
+ }, [&](const sd_image_t &image) {
+ if (!image.data || (image.channel != 3 && image.channel != 4) || image.width != request.width || image.height != request.height)
+ throw std::runtime_error("The image engine returned invalid pixels.");
+ if (!OGSaveRgbPng(image, output)) throw std::runtime_error("Could not save the image.");
+ }, std::max(0, [input[@"previewInterval"] intValue]), [&](int step, const sd_image_t &image) {
+ NSString *path = [output stringByAppendingString:@".preview.png"];
+ if (!OGSaveRgbPng(image, path)) return;
+ const int total = request.steps;
+ dispatch_async(dispatch_get_main_queue(), ^{
+ if (self->_listeners) [self sendEventWithName:@"SDImageProgress" body:@{
+ @"step":@(step), @"totalSteps":@(total), @"progress":@(total > 0 ? double(step) / total : 0), @"previewPath":path
+ }];
+ });
+ });
+ if (self->_runtime.cancelled.load()) throw std::runtime_error("Image generation stopped.");
+ } catch (const std::exception &error) {
+ failure = @(error.what()); [[NSFileManager defaultManager] removeItemAtPath:output error:nil];
+ }
+ failure = [self finishImageDiagnostics:failure];
+ dispatch_async(dispatch_get_main_queue(), ^{
+ self->_busy = NO;
+ [self restoreScreenIdleTimer];
+ [[NSFileManager defaultManager] removeItemAtPath:[output stringByAppendingString:@".preview.png"] error:nil];
+ if (failure) reject(@"IMAGE_FAILED", failure, nil);
+ else resolve(@{@"id":input[@"id"], @"imagePath":output, @"width":input[@"width"], @"height":input[@"height"], @"seed":input[@"seed"]});
+ });
+ }
+ });
+ });
+}
+- (void)startContinuedWork:(dispatch_block_t)work {
+ _continuedWork = [work copy];
+ if (@available(iOS 26.0, *)) {
+ BGContinuedProcessingTaskRequestResources resources = BGTaskScheduler.supportedResources;
+ [self logLifecycle:[NSString stringWithFormat:@"supportedBackgroundResources=%lu gpu=%d", (unsigned long)resources, (resources & BGContinuedProcessingTaskRequestResourcesGPU) != 0]];
+ if (resources & BGContinuedProcessingTaskRequestResourcesGPU) {
+ if (!_taskIdentifier) {
+ _taskIdentifier = [NSString stringWithFormat:@"%@.video.generation", NSBundle.mainBundle.bundleIdentifier];
+ __weak VideoGenerationModule *weakSelf = self;
+ BOOL registered = [BGTaskScheduler.sharedScheduler registerForTaskWithIdentifier:_taskIdentifier usingQueue:dispatch_get_main_queue() launchHandler:^(BGTask *task) {
+ VideoGenerationModule *owner = weakSelf;
+ if (!owner || !owner->_busy) { [task setTaskCompletedWithSuccess:NO]; return; }
+ owner->_continued = task;
+ [owner logLifecycle:[NSString stringWithFormat:@"admitted continued GPU task %@", task.identifier]];
+ __weak BGTask *expiringTask = task;
+ task.expirationHandler = ^{
+ dispatch_async(dispatch_get_main_queue(), ^{
+ VideoGenerationModule *active = weakSelf;
+ if (active && active->_continued == expiringTask && active->_busy) {
+ [active logLifecycle:[NSString stringWithFormat:@"continued GPU task expired %@", expiringTask.identifier]];
+ [active interruptVideo:@"iOS stopped background video generation. Keep Off Grid open and try again."];
+ }
+ });
+ };
+ dispatch_block_t admitted = owner->_continuedWork; owner->_continuedWork = nil;
+ if (admitted) admitted();
+ }];
+ [self logLifecycle:[NSString stringWithFormat:@"registration %@ success=%d", _taskIdentifier, registered]];
+ if (!registered) { _taskIdentifier = nil; _continuedWork = nil; work(); return; }
+ }
+ BGContinuedProcessingTaskRequest *request = [[BGContinuedProcessingTaskRequest alloc] initWithIdentifier:_taskIdentifier title:@"Generating video" subtitle:@"Loading model"];
+ request.requiredResources = BGContinuedProcessingTaskRequestResourcesGPU;
+ request.strategy = BGContinuedProcessingTaskRequestSubmissionStrategyFail;
+ NSError *submissionError = nil;
+ if ([BGTaskScheduler.sharedScheduler submitTaskRequest:request error:&submissionError]) {
+ [self logLifecycle:[NSString stringWithFormat:@"submitted continued GPU task %@", _taskIdentifier]];
+ return;
+ }
+ [self logLifecycle:[NSString stringWithFormat:@"continued GPU task refused domain=%@ code=%ld detail=%@", submissionError.domain, (long)submissionError.code, submissionError.localizedDescription]];
+ }
+ }
+ [self logLifecycle:[NSString stringWithFormat:@"running in foreground without admitted background GPU access"]];
+ _continuedWork = nil;
+ work();
+}
+@end
diff --git a/native/video/common/VideoRuntime.hpp b/native/video/common/VideoRuntime.hpp
new file mode 100644
index 000000000..0a49101c4
--- /dev/null
+++ b/native/video/common/VideoRuntime.hpp
@@ -0,0 +1,294 @@
+#pragma once
+#ifdef __APPLE__
+#include
+#else
+#include "stable-diffusion.h"
+#include "ggml-backend.h"
+#endif
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace offgrid {
+struct VideoRequest {
+ std::string weight, vae, encoder, prompt, negative;
+ int width, height, frames, fps, steps;
+ float guidance;
+ int64_t seed;
+ std::string llm, embeddings, audioVae;
+ float flowShift = 0;
+ bool cpuOnly = false;
+ bool skipNpu = false;
+ int threads = 4;
+ std::string imageFamily, imageSampler, imageScheduler;
+};
+// One instance per process. Both host bridges use the same native lifecycle.
+class VideoRuntime {
+ std::mutex contextMutex;
+ std::mutex executionMutex;
+ sd_ctx_t *context = nullptr;
+ sd_ctx_t *imageContext = nullptr;
+ std::string imagePath;
+ sample_method_t imageSampler = EULER_SAMPLE_METHOD;
+ scheduler_t imageScheduler = SCHEDULER_COUNT;
+ sd_ctx_t *loadContext(const VideoRequest &request, std::string &preferred) {
+ sd_ctx_params_t config;
+ sd_ctx_params_init(&config);
+ if (request.imageFamily == "checkpoint") config.model_path = request.weight.c_str();
+ else config.diffusion_model_path = request.weight.c_str();
+ config.vae_path = request.vae.c_str();
+ config.t5xxl_path = request.encoder.c_str();
+ config.llm_path = request.llm.c_str();
+ config.embeddings_connectors_path = request.embeddings.c_str();
+ config.audio_vae_path = request.audioVae.c_str();
+ config.enable_mmap = true;
+ config.diffusion_flash_attn = true;
+ config.auto_fit = true;
+ config.eager_load = false;
+#ifdef __ANDROID__
+ // Probe the NPU before choosing it. Devices without a usable Hexagon
+ // runtime retain Vulkan diffusion and the CPU conditioning fallback.
+ sd_list_devices(nullptr, 0);
+
+ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
+ auto device = ggml_backend_dev_get(i);
+ const char *name = ggml_backend_dev_name(device);
+ if (request.cpuOnly || request.skipNpu || std::string(name).rfind("HTP", 0) != 0) continue;
+ try {
+ auto probe = ggml_backend_dev_init(device, nullptr);
+ if (!probe) continue;
+ preferred = name;
+ ggml_backend_free(probe);
+ break;
+ } catch (const std::exception &) {
+ // An installed driver can still reject the DSP session.
+ }
+ }
+ // Wan decode still loses the Adreno device between spatial tiles after
+ // the matvec compiler workaround. Keep HTP diffusion/conditioning, but
+ // route this decoder to CPU before it can invalidate the GPU context.
+ const auto gpuDevice = ggml_backend_dev_by_name("Vulkan0");
+ const std::string gpuDescription = gpuDevice ? ggml_backend_dev_description(gpuDevice) : "";
+ const std::string vaeName = request.vae.substr(request.vae.find_last_of("/\\") + 1);
+ const bool wanVae = vaeName == "wan_2.1_vae.safetensors" || vaeName == "wan2.2_vae.safetensors";
+ const bool adrenoWan = wanVae && gpuDescription.find("Adreno") != std::string::npos;
+ const std::string vaeBackend = gpuDevice && !adrenoWan ? ",vae=Vulkan0" : ",vae=cpu";
+ const std::string npuBackend = preferred + vaeBackend;
+ const std::string gpuBackend = "Vulkan0,te=cpu" + vaeBackend;
+ config.backend = preferred.empty() ? gpuBackend.c_str() : npuBackend.c_str();
+ // Stream NPU weights from disk per graph segment. Keeping a second
+ // complete copy on Vulkan retains the text encoder during diffusion and
+ // can trigger Android's low-memory killer. The graph cap still controls
+ // staging even though explicit disk placement disables auto-fit.
+ if (!preferred.empty()) {
+ config.params_backend = "disk";
+ config.auto_fit = false;
+ }
+ config.max_vram = "HTP0=1.5";
+ config.disable_prefetch = true;
+#endif
+ config.n_threads = request.threads;
+ if (request.cpuOnly) {
+ config.backend = "cpu"; config.params_backend = nullptr; config.auto_fit = true;
+#ifdef __ANDROID__
+ preferred.clear();
+#endif
+ }
+ sd_ctx_t *loaded = nullptr;
+#ifdef __ANDROID__
+ try { loaded = new_sd_ctx(&config); }
+ catch (const std::exception &) {
+ if (request.cpuOnly || cancelled.load()) throw;
+ }
+ if (!loaded && !preferred.empty() && !cancelled.load()) {
+ preferred.clear();
+ config.params_backend = nullptr;
+ config.auto_fit = true;
+ config.backend = gpuBackend.c_str();
+ try { loaded = new_sd_ctx(&config); }
+ catch (const std::exception &) {
+ if (cancelled.load()) throw;
+ }
+ }
+ if (!loaded && !cancelled.load()) {
+ config.backend = "cpu";
+ loaded = new_sd_ctx(&config);
+ }
+#else
+ loaded = new_sd_ctx(&config);
+#endif
+ return loaded;
+ }
+public:
+ ~VideoRuntime() {
+ if (context) free_sd_ctx(context);
+ if (imageContext) free_sd_ctx(imageContext);
+ }
+ std::atomic_bool cancelled{false};
+ void cancel() {
+ cancelled.store(true);
+ std::lock_guard guard(contextMutex);
+ if (context) sd_cancel_generation(context, SD_CANCEL_ALL);
+ if (imageContext) sd_cancel_generation(imageContext, SD_CANCEL_ALL);
+ }
+ void loadImage(const VideoRequest &request, const std::string &path) {
+ std::unique_lock execution(executionMutex, std::try_to_lock);
+ if (!execution.owns_lock()) throw std::runtime_error("Image or video generation is running.");
+ if (!request.imageFamily.empty() && request.imageFamily != "checkpoint" && request.imageFamily != "qwen-image-2.1")
+ throw std::runtime_error("This image model family is not supported.");
+ const auto sampler = request.imageSampler.empty() ? EULER_SAMPLE_METHOD : str_to_sample_method(request.imageSampler.c_str());
+ const auto scheduler = request.imageScheduler.empty() ? SCHEDULER_COUNT : str_to_scheduler(request.imageScheduler.c_str());
+ if (sampler == SAMPLE_METHOD_COUNT || (!request.imageScheduler.empty() && scheduler == SCHEDULER_COUNT))
+ throw std::runtime_error("The image sampler or scheduler is not supported.");
+ {
+ std::lock_guard guard(contextMutex);
+ if (imageContext) free_sd_ctx(imageContext);
+ imageContext = nullptr; imagePath.clear();
+ }
+ std::string preferred;
+ auto loaded = loadContext(request, preferred);
+ if (!loaded) throw std::runtime_error("Could not load the image model pack.");
+ std::lock_guard guard(contextMutex);
+ if (cancelled.load()) { free_sd_ctx(loaded); throw std::runtime_error("Image loading stopped."); }
+ imageContext = loaded; imagePath = path;
+ imageSampler = sampler;
+ imageScheduler = scheduler == SCHEDULER_COUNT ? sd_get_default_scheduler(loaded, sampler) : scheduler;
+ }
+ void unloadImage() {
+ std::unique_lock execution(executionMutex, std::try_to_lock);
+ if (!execution.owns_lock()) throw std::runtime_error("Image or video generation is running.");
+ std::lock_guard guard(contextMutex);
+ if (imageContext) free_sd_ctx(imageContext);
+ imageContext = nullptr; imagePath.clear();
+ }
+ std::string loadedImagePath() {
+ std::lock_guard guard(contextMutex);
+ return imageContext ? imagePath : "";
+ }
+ void image(const VideoRequest &request,
+ const std::function &progress,
+ const std::function &save,
+ int previewInterval = 0,
+ const std::function &preview = {}) {
+ std::unique_lock execution(executionMutex, std::try_to_lock);
+ if (!execution.owns_lock()) throw std::runtime_error("Image or video generation is running.");
+ if (!imageContext) throw std::runtime_error("Image model is unloaded.");
+ sd_image_t *images = nullptr; int count = 0;
+ auto cleanup = [&] {
+ sd_set_progress_callback(nullptr, nullptr);
+ sd_set_preview_callback(nullptr, PREVIEW_NONE, 0, false, false, nullptr);
+ if (images) free_sd_images(images, count);
+ };
+ try {
+ if (cancelled.load()) throw std::runtime_error("Image generation stopped.");
+ sd_set_progress_callback([](int step, int total, float, void *data) {
+ (*static_cast *>(data))(step, total);
+ }, const_cast(static_cast(&progress)));
+ if (preview && previewInterval > 0) {
+ // Project the existing latent tensor. Do not run an extra VAE decode.
+ sd_set_preview_callback([](int step, int count, sd_image_t* frames, bool, void* data) {
+ if (count < 1 || !frames || !frames[0].data) return;
+ try { (*static_cast*>(data))(step, frames[0]); }
+ catch (...) { /* Optional previews must not fail generation. */ }
+ }, PREVIEW_PROJ, previewInterval, true, false,
+ const_cast(static_cast(&preview)));
+ }
+ sd_img_gen_params_t params; sd_img_gen_params_init(¶ms);
+ params.prompt = request.prompt.c_str(); params.negative_prompt = request.negative.c_str();
+ params.width = request.width; params.height = request.height; params.seed = request.seed;
+ params.batch_count = 1;
+ params.sample_params.sample_steps = request.steps;
+ params.sample_params.guidance.txt_cfg = request.guidance;
+ params.sample_params.sample_method = imageSampler;
+ params.sample_params.scheduler = imageScheduler;
+ params.vae_tiling_params.enabled = true;
+ if (!generate_image(imageContext, ¶ms, &images, &count) || !images || count != 1)
+ throw std::runtime_error("The image engine produced no image.");
+ if (cancelled.load()) throw std::runtime_error("Image generation stopped.");
+ save(images[0]);
+ } catch (...) { cleanup(); throw; }
+ cleanup();
+ }
+ void run(const VideoRequest &request,
+ const std::function &progress,
+ const std::function &encode,
+ const std::function &conditioning,
+ const std::function &decodeProgress,
+ const std::function &preview) {
+ std::unique_lock execution(executionMutex, std::try_to_lock);
+ if (!execution.owns_lock()) throw std::runtime_error("Video generation is already running.");
+ sd_image_t *frames = nullptr;
+ int count = 0, fps = request.fps;
+ struct DecodeObserver {
+ const std::function &progress;
+ const std::function &frame;
+ } observer{decodeProgress, preview};
+ auto cleanup = [&] {
+ if (frames) free_sd_images(frames, count);
+ sd_set_video_decode_callback(nullptr, nullptr, nullptr);
+ sd_set_progress_callback(nullptr, nullptr);
+ std::lock_guard guard(contextMutex);
+ if (context) free_sd_ctx(context);
+ context = nullptr;
+ };
+ try {
+ if (cancelled.load()) throw std::runtime_error("Video generation stopped.");
+ std::string preferred;
+ sd_ctx_t *loaded = loadContext(request, preferred);
+ {
+ std::lock_guard guard(contextMutex);
+ context = loaded;
+ }
+ if (!loaded) throw std::runtime_error("Could not load the video model pack.");
+ if (cancelled.load()) throw std::runtime_error("Video generation stopped.");
+ sd_set_progress_callback([](int step, int total, float, void *data) {
+ (*static_cast *>(data))(step, total);
+ }, const_cast(static_cast(&progress)));
+ sd_set_video_decode_callback([](int completed, int total, void *data) {
+ static_cast(data)->progress(completed, total);
+ }, [](const sd_image_t *image, void *data) {
+ try { static_cast(data)->frame(*image); }
+ catch (...) { /* The optional preview must not fail generation. */ }
+ }, &observer);
+ sd_vid_gen_params_t params;
+ sd_vid_gen_params_init(¶ms);
+ params.prompt = request.prompt.c_str(); params.negative_prompt = request.negative.c_str();
+ params.width = request.width; params.height = request.height;
+ params.video_frames = request.frames; params.fps = request.fps; params.seed = request.seed;
+ params.sample_params.sample_steps = request.steps;
+ params.sample_params.guidance.txt_cfg = request.guidance;
+ params.sample_params.flow_shift = request.flowShift > 0 ? request.flowShift : std::numeric_limits::infinity();
+ params.sample_params.sample_method = EULER_SAMPLE_METHOD;
+ params.sample_params.scheduler = sd_get_default_scheduler(loaded, EULER_SAMPLE_METHOD);
+ params.vae_tiling_params.enabled = true;
+#ifdef __ANDROID__
+ // Mobile Vulkan drivers can lose the device on a full temporal decode.
+ // Keep each decode graph small instead of waiting for allocation failure:
+ // a lost GPU cannot recover through the allocator's tiling retry.
+ params.vae_tiling_params.temporal_tiling = true;
+ params.vae_tiling_params.tile_size_w = 128;
+ params.vae_tiling_params.tile_size_h = 128;
+ params.vae_tiling_params.extra_tiling_args = "temporal_tile_frames=1";
+ conditioning(preferred.empty() ? "cpu" : "npu");
+#else
+ // Wan's stateful decoder retains causal feature caches between temporal
+ // chunks. Bound the graph without dropping frames or changing resolution.
+ const std::string vaeName = request.vae.substr(request.vae.find_last_of("/\\") + 1);
+ if (vaeName == "wan_2.1_vae.safetensors" || vaeName == "wan2.2_vae.safetensors") {
+ params.vae_tiling_params.temporal_tiling = true;
+ params.vae_tiling_params.extra_tiling_args = "temporal_tile_frames=1";
+ }
+ conditioning("");
+#endif
+ if (!generate_video(loaded, ¶ms, &frames, &count, nullptr, &fps) || !frames || count == 0)
+ throw std::runtime_error(cancelled.load() ? "Video generation stopped." : "The video engine produced no frames.");
+ if (cancelled.load()) throw std::runtime_error("Video generation stopped.");
+ encode(frames, count, fps);
+ } catch (...) { cleanup(); throw; }
+ cleanup();
+ }
+};
+}
diff --git a/package-lock.json b/package-lock.json
index 23f27de91..063c9a635 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -59,6 +59,8 @@
"react-native-tcp-socket": "^6.4.1",
"react-native-url-polyfill": "^3.0.0",
"react-native-vector-icons": "^10.3.0",
+ "react-native-video": "^6.19.3",
+ "react-native-video-player": "0.16.3",
"react-native-vision-camera": "^4.7.3",
"react-native-worklets": "^0.7.3",
"react-native-zeroconf": "^0.14.0",
@@ -139,6 +141,7 @@
"dependencies": {
"@noble/hashes": "1.8.0",
"@offgrid/automation": "*",
+ "@offgrid/models": "*",
"bonjour-service": "^1.2.1",
"js-sha512": "^0.9.0",
"tweetnacl": "^1.0.3",
@@ -15276,6 +15279,31 @@
"node": ">=10"
}
},
+ "node_modules/react-native-video": {
+ "version": "6.19.3",
+ "resolved": "https://registry.npmjs.org/react-native-video/-/react-native-video-6.19.3.tgz",
+ "integrity": "sha512-hI0YhEH6NdkfbFfMoZTZizEmLIm13rLlMf6KyWlGSe0+6If7/gvhapR2j/BMKBEwmlAIZjU4A6ax4Qn7yssjLQ==",
+ "license": "MIT",
+ "peerDependencies": {
+ "react": "*",
+ "react-native": "*"
+ }
+ },
+ "node_modules/react-native-video-player": {
+ "version": "0.16.3",
+ "resolved": "https://registry.npmjs.org/react-native-video-player/-/react-native-video-player-0.16.3.tgz",
+ "integrity": "sha512-7FMPl2uO9HRkklAaAGqtuAx+8qphbluacWTxyOhV19AOLc0GC4hkD0FrQoj6ksmgrwFJEXIkBlWkkfu1GS1YyA==",
+ "license": "MIT",
+ "workspaces": [
+ "example",
+ "docs"
+ ],
+ "peerDependencies": {
+ "react": "*",
+ "react-native": "*",
+ "react-native-video": "^6.8.0"
+ }
+ },
"node_modules/react-native-vision-camera": {
"version": "4.7.3",
"resolved": "https://registry.npmjs.org/react-native-vision-camera/-/react-native-vision-camera-4.7.3.tgz",
diff --git a/package.json b/package.json
index 864b69f8e..245ab5469 100644
--- a/package.json
+++ b/package.json
@@ -81,6 +81,8 @@
"react-native-tcp-socket": "^6.4.1",
"react-native-url-polyfill": "^3.0.0",
"react-native-vector-icons": "^10.3.0",
+ "react-native-video": "^6.19.3",
+ "react-native-video-player": "0.16.3",
"react-native-vision-camera": "^4.7.3",
"react-native-worklets": "^0.7.3",
"react-native-zeroconf": "^0.14.0",
diff --git a/patches/react-native-video-player+0.16.3.patch b/patches/react-native-video-player+0.16.3.patch
new file mode 100644
index 000000000..f799f32bd
--- /dev/null
+++ b/patches/react-native-video-player+0.16.3.patch
@@ -0,0 +1,448 @@
+diff --git a/node_modules/react-native-video-player/lib/commonjs/Video.js b/node_modules/react-native-video-player/lib/commonjs/Video.js
+index 9ff315d..181c87a 100644
+--- a/node_modules/react-native-video-player/lib/commonjs/Video.js
++++ b/node_modules/react-native-video-player/lib/commonjs/Video.js
+@@ -19,6 +19,8 @@ const RenderVideo = exports.RenderVideo = /*#__PURE__*/(0, _react.memo)(/*#__PUR
+ defaultMuted,
+ disableControlsAutoHide,
+ disableFullscreen,
++ disableMute,
++ additionalControl,
+ disableSeek,
+ fullScreenOnLongPress,
+ hideControlsOnStart = false,
+@@ -123,7 +125,8 @@ const RenderVideo = exports.RenderVideo = /*#__PURE__*/(0, _react.memo)(/*#__PUR
+ const _onLoad = (0, _react.useCallback)(event => {
+ if (onLoad) onLoad(event);
+ setDuration(event.duration);
+- }, [onLoad]);
++ if (!hideControlsOnStart) _showControls();
++ }, [onLoad, hideControlsOnStart, _showControls]);
+ const _onPlayPress = (0, _react.useCallback)(() => {
+ if (onPlayPress) onPlayPress();
+ setIsPlaying(prev => !prev);
+@@ -179,6 +182,8 @@ const RenderVideo = exports.RenderVideo = /*#__PURE__*/(0, _react.memo)(/*#__PUR
+ customStyles: customStyles,
+ showDuration: showDuration,
+ disableFullscreen: disableFullscreen,
++ disableMute: disableMute,
++ additionalControl: additionalControl,
+ duration: props.duration || duration,
+ isPlaying: isPlaying,
+ isMuted: isMuted,
+diff --git a/node_modules/react-native-video-player/lib/commonjs/controls/Controls.js b/node_modules/react-native-video-player/lib/commonjs/controls/Controls.js
+index bb476fb..54ba537 100644
+--- a/node_modules/react-native-video-player/lib/commonjs/controls/Controls.js
++++ b/node_modules/react-native-video-player/lib/commonjs/controls/Controls.js
+@@ -20,6 +20,8 @@ const Controls = exports.Controls = /*#__PURE__*/(0, _react.forwardRef)(({
+ onToggleFullScreen,
+ duration,
+ disableFullscreen,
++ disableMute,
++ additionalControl,
+ onMutePress,
+ isMuted,
+ animationDuration,
+@@ -63,6 +65,7 @@ const Controls = exports.Controls = /*#__PURE__*/(0, _react.forwardRef)(({
+ children: [/*#__PURE__*/(0, _jsxRuntime.jsx)(_PlayButton.PlayButton, {
+ controlButtonCustomStyles: customStyles?.controlButton,
+ playControlCustomStyles: customStyles?.playControl,
++ controlIconCustomStyles: customStyles?.controlIcon,
+ isPlaying: isPlaying,
+ onPlayPress: onPlayPress
+ }), /*#__PURE__*/(0, _jsxRuntime.jsx)(_Seekbar.Seekbar, {
+@@ -83,12 +86,12 @@ const Controls = exports.Controls = /*#__PURE__*/(0, _react.forwardRef)(({
+ ref: durationRef,
+ duration: duration,
+ durationTextCustomStyles: customStyles?.durationText
+- }), /*#__PURE__*/(0, _jsxRuntime.jsx)(_Mute.Mute, {
++ }), !disableMute && /*#__PURE__*/(0, _jsxRuntime.jsx)(_Mute.Mute, {
+ controlButtonCustomStyles: customStyles?.controlButton,
+ controlIconCustomStyles: customStyles?.controlIcon,
+ isMuted: isMuted,
+ onMutePress: onMutePress
+- }), !disableFullscreen && /*#__PURE__*/(0, _jsxRuntime.jsx)(_Fullscreen.Fullscreen, {
++ }), additionalControl, !disableFullscreen && /*#__PURE__*/(0, _jsxRuntime.jsx)(_Fullscreen.Fullscreen, {
+ onToggleFullScreen: onToggleFullScreen,
+ controlButtonCustomStyles: customStyles?.controlButton,
+ controlIconCustomStyles: customStyles?.controlIcon
+diff --git a/node_modules/react-native-video-player/lib/commonjs/controls/PlayButton.js b/node_modules/react-native-video-player/lib/commonjs/controls/PlayButton.js
+index 2c49655..340f714 100644
+--- a/node_modules/react-native-video-player/lib/commonjs/controls/PlayButton.js
++++ b/node_modules/react-native-video-player/lib/commonjs/controls/PlayButton.js
+@@ -9,14 +9,18 @@ var _react = require("react");
+ var _jsxRuntime = require("react/jsx-runtime");
+ const PlayButton = exports.PlayButton = /*#__PURE__*/(0, _react.memo)(({
+ isPlaying,
++ controlIconCustomStyles,
+ onPlayPress,
+ playControlCustomStyles,
+ controlButtonCustomStyles
+ }) => {
+ return /*#__PURE__*/(0, _jsxRuntime.jsx)(_reactNative.TouchableOpacity, {
++ accessibilityRole: 'button',
++ accessibilityLabel: isPlaying ? 'Pause video' : 'Play video',
+ onPress: onPlayPress,
+ style: [styles.playControl, controlButtonCustomStyles, playControlCustomStyles],
+ children: /*#__PURE__*/(0, _jsxRuntime.jsx)(_reactNative.Image, {
++ style: controlIconCustomStyles,
+ source: isPlaying ? require('../img/pause.png') : require('../img/play.png')
+ })
+ });
+diff --git a/node_modules/react-native-video-player/lib/commonjs/index.js b/node_modules/react-native-video-player/lib/commonjs/index.js
+index 02eadd9..b1b30a2 100644
+--- a/node_modules/react-native-video-player/lib/commonjs/index.js
++++ b/node_modules/react-native-video-player/lib/commonjs/index.js
+@@ -12,6 +12,7 @@ var _jsxRuntime = require("react/jsx-runtime");
+ const VideoPlayer = /*#__PURE__*/(0, _react.forwardRef)((props, ref) => {
+ const {
+ autoplay = false,
++ preload = false,
+ customStyles = {},
+ endThumbnail,
+ endWithThumbnail,
+@@ -79,7 +80,7 @@ const VideoPlayer = /*#__PURE__*/(0, _react.forwardRef)((props, ref) => {
+ customStylesPlayButton: customStyles.playButton,
+ customStylesPlayArrow: customStyles.playArrow
+ });
+- if (!isStarted) return /*#__PURE__*/(0, _jsxRuntime.jsx)(_reactNative.View, {
++ if (!isStarted && !preload) return /*#__PURE__*/(0, _jsxRuntime.jsx)(_reactNative.View, {
+ style: [styles.preloadingPlaceholder, sizeStyles, style],
+ children: /*#__PURE__*/(0, _jsxRuntime.jsx)(_Thumbnail.StartButton, {
+ onStart: _onStart,
+@@ -96,7 +97,7 @@ const VideoPlayer = /*#__PURE__*/(0, _react.forwardRef)((props, ref) => {
+ onEnd: _onEnd,
+ sizeStyle: sizeStyles
+ });
+- }, [hasEnded, endThumbnail, isStarted, thumbnail, style, sizeStyles, _onStart, customStyles, rest, autoplay, _onEnd]);
++ }, [preload, hasEnded, endThumbnail, isStarted, thumbnail, style, sizeStyles, _onStart, customStyles, rest, autoplay, _onEnd]);
+ return /*#__PURE__*/(0, _jsxRuntime.jsx)(_reactNative.View, {
+ onLayout: onLayout,
+ style: customStyles.wrapper,
+diff --git a/node_modules/react-native-video-player/lib/module/Video.js b/node_modules/react-native-video-player/lib/module/Video.js
+index d190ce5..733710b 100644
+--- a/node_modules/react-native-video-player/lib/module/Video.js
++++ b/node_modules/react-native-video-player/lib/module/Video.js
+@@ -14,6 +14,8 @@ export const RenderVideo = /*#__PURE__*/memo(/*#__PURE__*/forwardRef((props, ref
+ defaultMuted,
+ disableControlsAutoHide,
+ disableFullscreen,
++ disableMute,
++ additionalControl,
+ disableSeek,
+ fullScreenOnLongPress,
+ hideControlsOnStart = false,
+@@ -118,7 +120,8 @@ export const RenderVideo = /*#__PURE__*/memo(/*#__PURE__*/forwardRef((props, ref
+ const _onLoad = useCallback(event => {
+ if (onLoad) onLoad(event);
+ setDuration(event.duration);
+- }, [onLoad]);
++ if (!hideControlsOnStart) _showControls();
++ }, [onLoad, hideControlsOnStart, _showControls]);
+ const _onPlayPress = useCallback(() => {
+ if (onPlayPress) onPlayPress();
+ setIsPlaying(prev => !prev);
+@@ -174,6 +177,8 @@ export const RenderVideo = /*#__PURE__*/memo(/*#__PURE__*/forwardRef((props, ref
+ customStyles: customStyles,
+ showDuration: showDuration,
+ disableFullscreen: disableFullscreen,
++ disableMute: disableMute,
++ additionalControl: additionalControl,
+ duration: props.duration || duration,
+ isPlaying: isPlaying,
+ isMuted: isMuted,
+diff --git a/node_modules/react-native-video-player/lib/module/controls/Controls.js b/node_modules/react-native-video-player/lib/module/controls/Controls.js
+index 64b64f4..9ed6aa2 100644
+--- a/node_modules/react-native-video-player/lib/module/controls/Controls.js
++++ b/node_modules/react-native-video-player/lib/module/controls/Controls.js
+@@ -16,6 +16,8 @@ export const Controls = /*#__PURE__*/forwardRef(({
+ onToggleFullScreen,
+ duration,
+ disableFullscreen,
++ disableMute,
++ additionalControl,
+ onMutePress,
+ isMuted,
+ animationDuration,
+@@ -59,6 +61,7 @@ export const Controls = /*#__PURE__*/forwardRef(({
+ children: [/*#__PURE__*/_jsx(PlayButton, {
+ controlButtonCustomStyles: customStyles?.controlButton,
+ playControlCustomStyles: customStyles?.playControl,
++ controlIconCustomStyles: customStyles?.controlIcon,
+ isPlaying: isPlaying,
+ onPlayPress: onPlayPress
+ }), /*#__PURE__*/_jsx(Seekbar, {
+@@ -79,12 +82,12 @@ export const Controls = /*#__PURE__*/forwardRef(({
+ ref: durationRef,
+ duration: duration,
+ durationTextCustomStyles: customStyles?.durationText
+- }), /*#__PURE__*/_jsx(Mute, {
++ }), !disableMute && /*#__PURE__*/_jsx(Mute, {
+ controlButtonCustomStyles: customStyles?.controlButton,
+ controlIconCustomStyles: customStyles?.controlIcon,
+ isMuted: isMuted,
+ onMutePress: onMutePress
+- }), !disableFullscreen && /*#__PURE__*/_jsx(Fullscreen, {
++ }), additionalControl, !disableFullscreen && /*#__PURE__*/_jsx(Fullscreen, {
+ onToggleFullScreen: onToggleFullScreen,
+ controlButtonCustomStyles: customStyles?.controlButton,
+ controlIconCustomStyles: customStyles?.controlIcon
+diff --git a/node_modules/react-native-video-player/lib/module/controls/PlayButton.js b/node_modules/react-native-video-player/lib/module/controls/PlayButton.js
+index 1dea3ae..b177c91 100644
+--- a/node_modules/react-native-video-player/lib/module/controls/PlayButton.js
++++ b/node_modules/react-native-video-player/lib/module/controls/PlayButton.js
+@@ -5,14 +5,18 @@ import { memo } from 'react';
+ import { jsx as _jsx } from "react/jsx-runtime";
+ export const PlayButton = /*#__PURE__*/memo(({
+ isPlaying,
++ controlIconCustomStyles,
+ onPlayPress,
+ playControlCustomStyles,
+ controlButtonCustomStyles
+ }) => {
+ return /*#__PURE__*/_jsx(TouchableOpacity, {
++ accessibilityRole: 'button',
++ accessibilityLabel: isPlaying ? 'Pause video' : 'Play video',
+ onPress: onPlayPress,
+ style: [styles.playControl, controlButtonCustomStyles, playControlCustomStyles],
+ children: /*#__PURE__*/_jsx(Image, {
++ style: controlIconCustomStyles,
+ source: isPlaying ? require('../img/pause.png') : require('../img/play.png')
+ })
+ });
+diff --git a/node_modules/react-native-video-player/lib/module/index.js b/node_modules/react-native-video-player/lib/module/index.js
+index 5fc06ec..83e65b1 100644
+--- a/node_modules/react-native-video-player/lib/module/index.js
++++ b/node_modules/react-native-video-player/lib/module/index.js
+@@ -8,6 +8,7 @@ import { jsx as _jsx } from "react/jsx-runtime";
+ const VideoPlayer = /*#__PURE__*/forwardRef((props, ref) => {
+ const {
+ autoplay = false,
++ preload = false,
+ customStyles = {},
+ endThumbnail,
+ endWithThumbnail,
+@@ -75,7 +76,7 @@ const VideoPlayer = /*#__PURE__*/forwardRef((props, ref) => {
+ customStylesPlayButton: customStyles.playButton,
+ customStylesPlayArrow: customStyles.playArrow
+ });
+- if (!isStarted) return /*#__PURE__*/_jsx(View, {
++ if (!isStarted && !preload) return /*#__PURE__*/_jsx(View, {
+ style: [styles.preloadingPlaceholder, sizeStyles, style],
+ children: /*#__PURE__*/_jsx(StartButton, {
+ onStart: _onStart,
+@@ -92,7 +93,7 @@ const VideoPlayer = /*#__PURE__*/forwardRef((props, ref) => {
+ onEnd: _onEnd,
+ sizeStyle: sizeStyles
+ });
+- }, [hasEnded, endThumbnail, isStarted, thumbnail, style, sizeStyles, _onStart, customStyles, rest, autoplay, _onEnd]);
++ }, [preload, hasEnded, endThumbnail, isStarted, thumbnail, style, sizeStyles, _onStart, customStyles, rest, autoplay, _onEnd]);
+ return /*#__PURE__*/_jsx(View, {
+ onLayout: onLayout,
+ style: customStyles.wrapper,
+diff --git a/node_modules/react-native-video-player/lib/typescript/commonjs/src/index.d.ts b/node_modules/react-native-video-player/lib/typescript/commonjs/src/index.d.ts
+index a5eb66e..8b79e74 100644
+--- a/node_modules/react-native-video-player/lib/typescript/commonjs/src/index.d.ts
++++ b/node_modules/react-native-video-player/lib/typescript/commonjs/src/index.d.ts
+@@ -27,11 +27,14 @@ export interface CustomStyles {
+ export interface VideoPlayerProps extends ReactVideoProps {
+ animationDuration?: number;
+ autoplay?: boolean;
++ preload?: boolean;
+ controlsTimeout?: number;
+ customStyles?: CustomStyles;
+ defaultMuted?: boolean;
+ disableControlsAutoHide?: boolean;
+ disableFullscreen?: boolean;
++ disableMute?: boolean;
++ additionalControl?: React.ReactNode;
+ disableSeek?: boolean;
+ duration?: number;
+ endThumbnail?: ImageSourcePropType;
+diff --git a/node_modules/react-native-video-player/lib/typescript/module/src/index.d.ts b/node_modules/react-native-video-player/lib/typescript/module/src/index.d.ts
+index a5eb66e..8b79e74 100644
+--- a/node_modules/react-native-video-player/lib/typescript/module/src/index.d.ts
++++ b/node_modules/react-native-video-player/lib/typescript/module/src/index.d.ts
+@@ -27,11 +27,14 @@ export interface CustomStyles {
+ export interface VideoPlayerProps extends ReactVideoProps {
+ animationDuration?: number;
+ autoplay?: boolean;
++ preload?: boolean;
+ controlsTimeout?: number;
+ customStyles?: CustomStyles;
+ defaultMuted?: boolean;
+ disableControlsAutoHide?: boolean;
+ disableFullscreen?: boolean;
++ disableMute?: boolean;
++ additionalControl?: React.ReactNode;
+ disableSeek?: boolean;
+ duration?: number;
+ endThumbnail?: ImageSourcePropType;
+diff --git a/node_modules/react-native-video-player/src/Video.tsx b/node_modules/react-native-video-player/src/Video.tsx
+index 191e724..1d0484b 100644
+--- a/node_modules/react-native-video-player/src/Video.tsx
++++ b/node_modules/react-native-video-player/src/Video.tsx
+@@ -31,6 +31,8 @@ type RenderVideoProps = Pick<
+ | 'defaultMuted'
+ | 'disableControlsAutoHide'
+ | 'disableFullscreen'
++ | 'disableMute'
++ | 'additionalControl'
+ | 'disableSeek'
+ | 'duration'
+ | 'fullScreenOnLongPress'
+@@ -63,6 +65,8 @@ export const RenderVideo = memo(
+ defaultMuted,
+ disableControlsAutoHide,
+ disableFullscreen,
++ disableMute,
++ additionalControl,
+ disableSeek,
+ fullScreenOnLongPress,
+ hideControlsOnStart = false,
+@@ -190,8 +194,9 @@ export const RenderVideo = memo(
+ (event: OnLoadData) => {
+ if (onLoad) onLoad(event);
+ setDuration(event.duration);
++ if (!hideControlsOnStart) _showControls();
+ },
+- [onLoad]
++ [onLoad, hideControlsOnStart, _showControls]
+ );
+
+ const _onPlayPress = useCallback(() => {
+@@ -262,6 +267,8 @@ export const RenderVideo = memo(
+ customStyles={customStyles}
+ showDuration={showDuration}
+ disableFullscreen={disableFullscreen}
++ disableMute={disableMute}
++ additionalControl={additionalControl}
+ duration={props.duration || duration}
+ isPlaying={isPlaying}
+ isMuted={isMuted}
+diff --git a/node_modules/react-native-video-player/src/controls/Controls.tsx b/node_modules/react-native-video-player/src/controls/Controls.tsx
+index 7ac7e72..eca3620 100644
+--- a/node_modules/react-native-video-player/src/controls/Controls.tsx
++++ b/node_modules/react-native-video-player/src/controls/Controls.tsx
+@@ -11,6 +11,8 @@ interface ControlsProps
+ extends Omit {
+ customStyles: VideoPlayerProps['customStyles'];
+ showDuration: VideoPlayerProps['showDuration'];
++ disableMute?: boolean;
++ additionalControl?: React.ReactNode;
+ disableFullscreen: VideoPlayerProps['disableFullscreen'];
+ animationDuration: number;
+ duration: number;
+@@ -36,6 +38,8 @@ export const Controls = forwardRef(
+ onToggleFullScreen,
+ duration,
+ disableFullscreen,
++ disableMute,
++ additionalControl,
+ onMutePress,
+ isMuted,
+ animationDuration,
+@@ -88,6 +92,7 @@ export const Controls = forwardRef(
+
+@@ -113,12 +118,13 @@ export const Controls = forwardRef(
+ durationTextCustomStyles={customStyles?.durationText}
+ />
+ )}
+-
++ />}
++ {additionalControl}
+ {!disableFullscreen && (
+ void;
+ }
+@@ -12,12 +13,15 @@ interface PlayButtonProps {
+ export const PlayButton = memo(
+ ({
+ isPlaying,
++ controlIconCustomStyles,
+ onPlayPress,
+ playControlCustomStyles,
+ controlButtonCustomStyles,
+ }: PlayButtonProps) => {
+ return (
+
+ (
+ (props, ref) => {
+ const {
+ autoplay = false,
++ preload = false,
+ customStyles = {},
+ endThumbnail,
+ endWithThumbnail,
+@@ -167,7 +171,7 @@ const VideoPlayer = forwardRef(
+ customStylesPlayArrow={customStyles.playArrow}
+ />
+ );
+- if (!isStarted)
++ if (!isStarted && !preload)
+ return (
+
+ (
+ />
+ );
+ }, [
++ preload,
+ hasEnded,
+ endThumbnail,
+ isStarted,
diff --git a/pro b/pro
index 035e38bd2..c5553c251 160000
--- a/pro
+++ b/pro
@@ -1 +1 @@
-Subproject commit 035e38bd2345569d82377809cd8a582701592cfd
+Subproject commit c5553c25162a3f792cd344cbe495a1d3c272efa5
diff --git a/scripts/build-video-ios.sh b/scripts/build-video-ios.sh
new file mode 100755
index 000000000..3670806ba
--- /dev/null
+++ b/scripts/build-video-ios.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+set -euo pipefail
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+REVISION="$(cat "$ROOT/scripts/video/revision")"
+CACHE="${OFFGRID_VIDEO_BUILD_DIR:-$ROOT/.video-build}"
+SOURCE="${OFFGRID_SD_SOURCE:-$CACHE/source}"
+OUTPUT="$ROOT/native/video/OffgridVideoRuntime.xcframework"
+BUILD_KEY="${OFFGRID_IOS_DEVICE_ONLY:-0}-$(cat "$ROOT/scripts/video/revision" "$ROOT/scripts/video/exports.txt" "$ROOT/scripts/video/CMakeLists.txt" "$ROOT/scripts/video/conditioning-errors.patch" "$ROOT/scripts/video/video-decode-observer.patch" | shasum -a 256 | cut -d' ' -f1)"
+if [ -f "$OUTPUT/ios-arm64/OffgridVideoRuntime.framework/Headers/stable-diffusion.h" ] && [ -f "$OUTPUT/revision" ] && [ "$(cat "$OUTPUT/revision")" = "$BUILD_KEY" ]; then exit 0; fi
+mkdir -p "$CACHE"
+if [ ! -d "$SOURCE/.git" ]; then
+ git clone --filter=blob:none --no-checkout https://github.com/leejet/stable-diffusion.cpp.git "$SOURCE"
+fi
+if [ "$(git -C "$SOURCE" rev-parse HEAD)" != "$REVISION" ]; then
+ git -C "$SOURCE" fetch origin "$REVISION"
+ git -C "$SOURCE" checkout --detach "$REVISION"
+fi
+git -C "$SOURCE" submodule update --init --depth 1 ggml
+for PATCH in "$ROOT/scripts/video/conditioning-errors.patch" "$ROOT/scripts/video/video-decode-observer.patch"; do
+ if ! git -C "$SOURCE" apply --reverse --check "$PATCH" 2>/dev/null; then
+ git -C "$SOURCE" apply "$PATCH"
+ fi
+done
+SDKS=(iphoneos)
+if [ "${OFFGRID_IOS_DEVICE_ONLY:-0}" != "1" ]; then SDKS+=(iphonesimulator); fi
+for SDK in "${SDKS[@]}"; do
+ cmake -S "$ROOT/scripts/video" -B "$CACHE/$SDK" -G Xcode \
+ -DSD_SOURCE="$SOURCE" -DCMAKE_SYSTEM_NAME=iOS -DCMAKE_OSX_SYSROOT="$SDK" \
+ -DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_OSX_DEPLOYMENT_TARGET=17.0
+ cmake --build "$CACHE/$SDK" --config Release --target stable-diffusion -- -quiet CODE_SIGNING_ALLOWED=NO
+ done
+# Replace only this generated build artifact.
+rm -rf "$OUTPUT"
+FRAMEWORKS=()
+for SDK in "${SDKS[@]}"; do FRAMEWORKS+=(-framework "$CACHE/$SDK/bin/Release/OffgridVideoRuntime.framework"); done
+xcodebuild -create-xcframework "${FRAMEWORKS[@]}" -output "$OUTPUT"
+printf '%s\n' "$BUILD_KEY" > "$OUTPUT/revision"
+cp "$SOURCE/LICENSE" "$ROOT/native/video/RUNTIME-LICENSE"
diff --git a/scripts/prepare-video-runtime.sh b/scripts/prepare-video-runtime.sh
new file mode 100644
index 000000000..326708667
--- /dev/null
+++ b/scripts/prepare-video-runtime.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+set -euo pipefail
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+SOURCE="$ROOT/.video-build/source"
+REVISION="$(cat "$ROOT/scripts/video/revision")"
+mkdir -p "$ROOT/.video-build"
+if [ ! -d "$SOURCE/.git" ]; then
+ git clone --filter=blob:none --no-checkout https://github.com/leejet/stable-diffusion.cpp.git "$SOURCE"
+fi
+git -C "$SOURCE" fetch origin "$REVISION"
+git -C "$SOURCE" checkout --detach "$REVISION"
+git -C "$SOURCE" submodule update --init --depth 1 ggml
+# Use the initialized Vulkan dispatcher for Vulkan 1.1 features. Direct symbols
+# are absent from Android API 24's loader even when the device supports them.
+PATCH="$ROOT/scripts/video/android-vulkan-dispatch.patch"
+if ! git -C "$SOURCE/ggml" apply --reverse --check "$PATCH" 2>/dev/null; then
+ git -C "$SOURCE/ggml" apply "$PATCH"
+fi
+# This overlay extends the scheduler inserted by android-hardware-fallback.patch.
+# Remove it before checking the base patch, then apply it again in order below.
+# Do not reset the source tree: other platform patches must remain intact.
+ROUTING_PATCH="$ROOT/scripts/video/android-attention-query-routing.patch"
+if git -C "$SOURCE" apply --reverse --check "$ROUTING_PATCH" 2>/dev/null; then
+ git -C "$SOURCE" apply --reverse "$ROUTING_PATCH"
+fi
+for entry in "ggml:android-vulkan-device-fault.patch" "ggml:android-vulkan-pipeline-diagnostics.patch" "ggml:android-vulkan-matvec-fallback.patch" "ggml:android-vulkan-attention-fallback.patch" "ggml:android-hexagon.patch" "ggml:android-hexagon-buffer-validation.patch" "ggml:android-hexagon-precision.patch" ".:android-hardware-fallback.patch" ".:conditioning-errors.patch" ".:android-attention-buffer-ownership.patch" ".:android-attention-query-chunks.patch" ".:android-attention-query-routing.patch" ".:video-decode-observer.patch" ".:android-video-numerics.patch"; do
+ target="${entry%%:*}"
+ patch="$ROOT/scripts/video/${entry#*:}"
+ if ! git -C "$SOURCE/$target" apply --reverse --check "$patch" 2>/dev/null; then
+ git -C "$SOURCE/$target" apply "$patch"
+ fi
+done
+# Match the header versions used by the runtime's Vulkan build. These are build
+# inputs only; the device still supplies its Vulkan driver.
+prepare_headers() {
+ local project="$1" revision="$2" directory="$ROOT/.video-build/$1-source"
+ if [ ! -d "$directory/.git" ]; then git clone --filter=blob:none --no-checkout "https://github.com/KhronosGroup/$project.git" "$directory"; fi
+ git -C "$directory" fetch origin "$revision"
+ git -C "$directory" checkout --detach "$revision"
+ cmake -S "$directory" -B "$directory/build" -DCMAKE_INSTALL_PREFIX="$ROOT/.video-build/vulkan-header-prefix"
+ cmake --install "$directory/build"
+}
+prepare_headers SPIRV-Headers 01e0577914a75a2569c846778c2f93aa8e6feddd
+prepare_headers Vulkan-Headers 19725e4d48082fe78e26622b15d3080ccd54112b
+
+if [ "${1:-}" = "--android" ]; then
+ bash "$ROOT/scripts/video/build-shaderc.sh"
+ bash "$ROOT/scripts/video/build-hexagon.sh"
+fi
+
+touch "$ROOT/.video-build/runtime-ready"
diff --git a/scripts/rag/__tests__/embeddingCatalog.test.cjs b/scripts/rag/__tests__/embeddingCatalog.test.cjs
new file mode 100644
index 000000000..8df15a99d
--- /dev/null
+++ b/scripts/rag/__tests__/embeddingCatalog.test.cjs
@@ -0,0 +1,67 @@
+/* eslint-env node, es2022 */
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const { readFileSync } = require('node:fs');
+const path = require('node:path');
+const vm = require('node:vm');
+const ts = require('typescript');
+
+// HTTP is the external boundary. Repository and file responses use the Hub API contract.
+function browser(fetchBoundary) {
+ const filename = path.resolve(__dirname, '../../../src/services/huggingFaceModelBrowser.ts');
+ const source = ts.transpileModule(readFileSync(filename, 'utf8'), {
+ compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 },
+ }).outputText;
+ const module = { exports: {} };
+ const nativeRequire = name => name === 'react-native-fs' ? require('node:fs/promises') : require(name);
+ vm.runInThisContext(`(function(require,module,exports,fetch){${source}\n})`, { filename })(nativeRequire, module, module.exports, fetchBoundary);
+ return module.exports;
+}
+const revision = 'a'.repeat(40);
+const response = data => new Response(JSON.stringify(data));
+
+test('search combines embedding tasks and returns pinned single-file candidates only', async () => {
+ const { searchEmbeddingModels } = browser(async address => {
+ const url = new URL(address);
+ if (url.pathname === '/api/models') {
+ return response(url.searchParams.get('pipeline_tag') === 'feature-extraction'
+ ? [{ id: 'org/encoder' }, { id: 'org/decoder' }]
+ : [{ id: 'org/encoder' }]);
+ }
+ if (url.pathname === '/api/models/org/decoder') return response({ sha: revision, gguf: { architecture: 'llama' }, siblings: [] });
+ if (url.pathname === '/api/models/org/encoder') return response({
+ sha: revision, gguf: { architecture: 'bert' }, siblings: [
+ { rfilename: 'nested/encoder Q8.gguf', lfs: { size: 40000000, sha256: 'b'.repeat(64) } },
+ { rfilename: 'encoder-00001-of-00002.gguf', size: 30 },
+ { rfilename: 'mmproj.gguf', size: 30 },
+ { rfilename: 'encoder.onnx', size: 30 },
+ { rfilename: 'unknown.gguf' },
+ ],
+ });
+ throw new Error(`Unexpected API resource: ${url.pathname}`);
+ });
+ const found = await searchEmbeddingModels('encoder');
+ assert.equal(found.length, 1);
+ assert.equal(found[0].id, `org/encoder@${revision}/nested/encoder Q8.gguf`);
+ assert.equal(found[0].downloadUrl, `https://huggingface.co/org/encoder/resolve/${revision}/nested/encoder%20Q8.gguf`);
+ assert.equal(found[0].size, 40000000);
+ assert.equal(found[0].sha256, 'b'.repeat(64));
+});
+
+test('search reports a failed file listing instead of claiming no compatible models exist', async () => {
+ const { searchEmbeddingModels } = browser(async address => new URL(address).pathname === '/api/models'
+ ? response([{ id: 'org/encoder' }]) : new Response('', { status: 503 }));
+ await assert.rejects(searchEmbeddingModels('encoder'), /Could not read embedding model files/);
+});
+
+test('recommendations have pinned identities, checksums, and explicit download bytes', () => {
+ const { RECOMMENDED_EMBEDDING_MODELS, BUNDLED_EMBEDDING_MODEL } = browser(fetch);
+ assert.equal(BUNDLED_EMBEDDING_MODEL.size, 0);
+ assert.equal(BUNDLED_EMBEDDING_MODEL.downloadUrl, undefined);
+ for (const candidate of RECOMMENDED_EMBEDDING_MODELS) {
+ assert.match(candidate.id, /@[a-f0-9]{40}\/.+\.gguf$/);
+ assert.match(candidate.downloadUrl, /\/resolve\/[a-f0-9]{40}\/.+\.gguf$/);
+ assert.match(candidate.sha256, /^[a-f0-9]{64}$/);
+ assert.ok(candidate.size > 0);
+ }
+});
diff --git a/scripts/rag/__tests__/embeddingIndex.test.cjs b/scripts/rag/__tests__/embeddingIndex.test.cjs
new file mode 100644
index 000000000..db1c3526d
--- /dev/null
+++ b/scripts/rag/__tests__/embeddingIndex.test.cjs
@@ -0,0 +1,121 @@
+/* eslint-env node, es2022 */
+/* Real SQLite at the native op-sqlite boundary. No application services are replaced. */
+const { test } = require('node:test');
+const assert = require('node:assert/strict');
+const { DatabaseSync } = require('node:sqlite');
+const { readFileSync, mkdtempSync, rmSync } = require('node:fs');
+const { tmpdir } = require('node:os');
+const path = require('node:path');
+const vm = require('node:vm');
+const ts = require('typescript');
+
+function database(t) {
+ const directory = mkdtempSync(path.join(tmpdir(), 'embedding-index-'));
+ const connections = [];
+ const sqlite = {
+ open({ name }) {
+ const db = new DatabaseSync(path.join(directory, name));
+ connections.push(db);
+ db.exec('PRAGMA max_page_count = 100');
+ return {
+ executeSync(sql, params = []) {
+ const statement = db.prepare(sql);
+ const bindings = params.map(p => p instanceof ArrayBuffer ? new Uint8Array(p) : p);
+ if (statement.columns().length) {
+ return { rows: statement.all(...bindings).map(row => Object.fromEntries(
+ Object.entries(row).map(([key, value]) => [key, value instanceof Uint8Array
+ ? value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength) : value]),
+ )) };
+ }
+ const result = statement.run(...bindings);
+ return { rows: [], insertId: Number(result.lastInsertRowid), rowsAffected: result.changes };
+ },
+ };
+ },
+ };
+ t.after(() => { connections.forEach(db => db.close()); rmSync(directory, { recursive: true }); });
+ function reopen() {
+ const cache = new Map();
+ function load(filename) {
+ if (cache.has(filename)) return cache.get(filename).exports;
+ const module = { exports: {} };
+ cache.set(filename, module);
+ const source = ts.transpileModule(readFileSync(filename, 'utf8'), {
+ compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 },
+ }).outputText;
+ const boundaryRequire = name => {
+ if (name === '@op-engineering/op-sqlite') return sqlite;
+ if (name.startsWith('.')) return load(path.resolve(path.dirname(filename), `${name}.ts`));
+ return require(name);
+ };
+ vm.runInThisContext(`(function(require,module,exports,__DEV__){${source}\n})`, { filename })(boundaryRequire, module, module.exports, false);
+ return module.exports;
+ }
+ return load(path.resolve(__dirname, '../../../src/services/rag/database.ts')).ragDatabase;
+ }
+ return reopen;
+}
+
+async function seed(db) {
+ await db.ensureReady();
+ const docId = db.insertDocument({ projectId: 'project', name: 'note.txt', path: '/note.txt', size: 12 });
+ const [chunkRowid] = db.insertChunks(docId, [{ content: 'Searchable note', position: 0 }]);
+ db.insertEmbeddingsBatch([{ docId, chunkRowid, embedding: [1, 0] }]);
+ return { docId, chunkRowid };
+}
+
+function replace(db, model, entries) {
+ db.beginEmbeddingRebuild();
+ for (const entry of entries) db.stageEmbedding(entry);
+ db.commitEmbeddingRebuild(model);
+ db.discardEmbeddingRebuild();
+}
+
+const model = { id: 'repo@revision/encoder.gguf', name: 'Encoder', filePath: '/encoder.gguf', size: 42 };
+
+test('switch persists the model and replacement vectors together; original documents remain', async t => {
+ const reopen = database(t);
+ const db = reopen();
+ const entry = await seed(db);
+ assert.equal(db.getEmbeddingModel(), null);
+ replace(db, model, [{ ...entry, embedding: [0, 1, 0] }]);
+ const restarted = reopen();
+ await restarted.ensureReady();
+ assert.deepEqual(restarted.getEmbeddingModel(), model);
+ assert.deepEqual(restarted.getEmbeddingsByProject('project').map(row => row.embedding), [[0, 1, 0]]);
+ assert.equal(restarted.getDocumentsByProject('project')[0].name, 'note.txt');
+ assert.equal(restarted.getChunksByDocument(entry.docId)[0].content, 'Searchable note');
+ replace(restarted, null, [{ ...entry, embedding: [1, 0] }]);
+ assert.equal(restarted.getEmbeddingModel(), null);
+ assert.deepEqual(restarted.getEmbeddingsByProject('project').map(row => row.embedding), [[1, 0]]);
+});
+
+test('a storage failure preserves both the prior model and searchable vectors', async t => {
+ const reopen = database(t);
+ const db = reopen();
+ const entry = await seed(db);
+ replace(db, model, [{ ...entry, embedding: [0, 1] }]);
+ assert.throws(() => replace(db, { ...model, id: 'another' }, [
+ { ...entry, embedding: new Array(200000).fill(1) },
+ ]));
+ const restarted = reopen();
+ await restarted.ensureReady();
+ assert.deepEqual(restarted.getEmbeddingModel(), model);
+ assert.deepEqual(restarted.getEmbeddingsByProject('project').map(row => row.embedding), [[0, 1]]);
+});
+
+test('uncommitted or cancelled staging never changes the saved model or same-size vectors', async t => {
+ const reopen = database(t);
+ const db = reopen();
+ const entry = await seed(db);
+ db.beginEmbeddingRebuild();
+ db.stageEmbedding({ ...entry, embedding: [0, 1] });
+ const restarted = reopen();
+ await restarted.ensureReady();
+ assert.equal(restarted.getEmbeddingModel(), null);
+ assert.deepEqual(restarted.getEmbeddingsByProject('project').map(row => row.embedding), [[1, 0]]);
+ db.discardEmbeddingRebuild();
+ assert.deepEqual(db.getEmbeddingsByProject('project').map(row => row.embedding), [[1, 0]]);
+ replace(db, model, [{ ...entry, embedding: [0, 1] }]);
+ assert.deepEqual(db.getEmbeddingsByProject('project').map(row => row.embedding), [[0, 1]]);
+});
diff --git a/scripts/video/CMakeLists.txt b/scripts/video/CMakeLists.txt
new file mode 100644
index 000000000..8b38e0bf6
--- /dev/null
+++ b/scripts/video/CMakeLists.txt
@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.24)
+project(OffgridVideoRuntime LANGUAGES C CXX OBJC OBJCXX)
+set(SD_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
+set(SD_BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE)
+set(SD_BUILD_SHARED_GGML_LIB OFF CACHE BOOL "" FORCE)
+set(SD_METAL ON CACHE BOOL "" FORCE)
+set(SD_WEBP OFF CACHE BOOL "" FORCE)
+set(SD_WEBM OFF CACHE BOOL "" FORCE)
+set(GGML_METAL_EMBED_LIBRARY ON CACHE BOOL "" FORCE)
+set(GGML_NATIVE OFF CACHE BOOL "" FORCE)
+set(GGML_OPENMP OFF CACHE BOOL "" FORCE)
+set(CMAKE_C_VISIBILITY_PRESET hidden)
+set(CMAKE_CXX_VISIBILITY_PRESET hidden)
+add_subdirectory("${SD_SOURCE}" runtime)
+target_sources(stable-diffusion PRIVATE "${SD_SOURCE}/include/stable-diffusion.h")
+set_target_properties(stable-diffusion PROPERTIES
+ FRAMEWORK TRUE FRAMEWORK_VERSION A
+ OUTPUT_NAME OffgridVideoRuntime
+ MACOSX_FRAMEWORK_IDENTIFIER ai.offgrid.video.runtime
+ XCODE_ATTRIBUTE_PRODUCT_BUNDLE_IDENTIFIER ai.offgrid.video.runtime
+ PUBLIC_HEADER "${SD_SOURCE}/include/stable-diffusion.h"
+ XCODE_ATTRIBUTE_CODE_SIGNING_ALLOWED NO
+ INSTALL_NAME_DIR "@rpath")
+target_link_options(stable-diffusion PRIVATE "-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/exports.txt")
diff --git a/scripts/video/android-attention-buffer-ownership.patch b/scripts/video/android-attention-buffer-ownership.patch
new file mode 100644
index 000000000..00447423e
--- /dev/null
+++ b/scripts/video/android-attention-buffer-ownership.patch
@@ -0,0 +1,24 @@
+--- a/src/core/ggml_extend.cpp
++++ b/src/core/ggml_extend.cpp
+@@ -802,11 +802,21 @@
+
+ auto kq = ggml_mul_mat(ctx, k, q); // [N * n_head, L_q, L_k]
+ ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
++#ifdef __ANDROID__
++ // F32 attention matmul can fall back from HTP to Vulkan. In-place
++ // outputs would retain that device's buffer across scheduler splits.
++ kq = ggml_scale(ctx, kq, scale);
++ if (mask) {
++ kq = ggml_add(ctx, kq, mask);
++ }
++ kq = ggml_soft_max(ctx, kq);
++#else
+ kq = ggml_scale_inplace(ctx, kq, scale);
+ if (mask) {
+ kq = ggml_add_inplace(ctx, kq, mask);
+ }
+ kq = ggml_soft_max_inplace(ctx, kq);
++#endif
+
+ kqv = ggml_mul_mat(ctx, v, kq); // [N * n_head, L_q, d_head]
+
diff --git a/scripts/video/android-attention-query-chunks.patch b/scripts/video/android-attention-query-chunks.patch
new file mode 100644
index 000000000..33c4af2b5
--- /dev/null
+++ b/scripts/video/android-attention-query-chunks.patch
@@ -0,0 +1,78 @@
+--- a/src/core/ggml_extend.cpp
++++ b/src/core/ggml_extend.cpp
+@@ -721,6 +721,46 @@
+ mask_in = ggml_cast(ctx, mask_in, GGML_TYPE_F16);
+ }
+
++#ifdef __ANDROID__
++ // Adreno can lose the device on a single long video attention dispatch.
++ // Split only queries: each query still attends to every original key/value.
++ auto is_adreno = [](ggml_backend_dev_t device) {
++ const char* name = device ? ggml_backend_dev_name(device) : nullptr;
++ const char* description = device ? ggml_backend_dev_description(device) : nullptr;
++ return name && description && std::string(name).find("Vulkan") != std::string::npos &&
++ std::string(description).find("Adreno") != std::string::npos;
++ };
++ bool adreno_attention = backend && is_adreno(ggml_backend_get_device(backend));
++ // HTP may accept the initial default-precision node, then delegate its
++ // FP32 execution to Vulkan through the scheduler's fallback backend.
++ if (!adreno_attention && sd_backend_is(backend, "HTP")) {
++ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
++ if (is_adreno(ggml_backend_dev_get(i))) { adreno_attention = true; break; }
++ }
++ }
++ if (mask_in == nullptr && adreno_attention && q_in->ne[1] > 128) {
++ // One owned full-Q buffer lets all query views stay on the GPU.
++ // RoPE may otherwise leave Q aliased to an in-place HTP result.
++ q_in = ggml_cont(ctx, q_in);
++ const int64_t work_per_query = 2 * k_in->ne[1] *
++ (q_in->ne[0] + v_in->ne[0]) * q_in->ne[2] * q_in->ne[3];
++ const int64_t chunk_rows = std::max(8, std::min(128,
++ (4000000000LL / std::max(1, work_per_query) / 8) * 8));
++ ggml_tensor* joined = nullptr;
++ for (int64_t start = 0; start < q_in->ne[1]; start += chunk_rows) {
++ const int64_t rows = std::min(chunk_rows, q_in->ne[1] - start);
++ auto query = ggml_cont(ctx, ggml_view_4d(ctx, q_in,
++ q_in->ne[0], rows, q_in->ne[2], q_in->ne[3],
++ q_in->nb[1], q_in->nb[2], q_in->nb[3], start * q_in->nb[1]));
++ auto part = ggml_flash_attn_ext(ctx, query, k_in, v_in, nullptr, scale / kv_scale, 0, 0);
++ if (!ggml_backend_supports_op(backend, part)) return nullptr;
++ ggml_flash_attn_ext_set_prec(part, GGML_PREC_F32);
++ joined = joined ? ggml_concat(ctx, joined, part, 2) : part;
++ }
++ if (kv_scale != 1.0f) joined = ggml_ext_scale(ctx, joined, 1.0f / kv_scale);
++ return joined;
++ }
++#endif
+ auto out = ggml_flash_attn_ext(ctx, q_in, k_in, v_in, mask_in, scale / kv_scale, 0, 0);
+ if (!ggml_backend_supports_op(backend, out)) {
+ return nullptr;
+--- a/src/model/diffusion/wan.hpp
++++ b/src/model/diffusion/wan.hpp
+@@ -988,7 +988,25 @@
+ float vace_strength = 1.f,
+ const sd::Tensor& audio_embed_tensor = {},
+ const sd::Tensor& ref_latent_tensor = {}) {
+- ggml_cgraph* gf = new_graph_custom(WAN_GRAPH_SIZE);
++ size_t graph_size = WAN_GRAPH_SIZE;
++#ifdef __ANDROID__
++ // Query chunks keep full attention but add view/copy/concat nodes.
++ // Reserve graph metadata for that path inside the existing 327680-node arena.
++ if (sd_backend_is(runtime_backend, "Vulkan") || sd_backend_is(runtime_backend, "HTP")) {
++ for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
++ auto device = ggml_backend_dev_get(i);
++ const char* name = ggml_backend_dev_name(device);
++ const char* description = ggml_backend_dev_description(device);
++ if (name && description && std::string(name).find("Vulkan") != std::string::npos &&
++ std::string(description).find("Adreno") != std::string::npos) {
++ graph_size = 131072;
++ LOG_DEBUG("Adreno chunked Wan graph capacity: %zu", graph_size);
++ break;
++ }
++ }
++ }
++#endif
++ ggml_cgraph* gf = new_graph_custom(graph_size);
+
+ ggml_tensor* x = make_input(x_tensor);
+ ggml_tensor* timesteps = make_input(timesteps_tensor);
diff --git a/scripts/video/android-attention-query-routing.patch b/scripts/video/android-attention-query-routing.patch
new file mode 100644
index 000000000..5e6ddb3e5
--- /dev/null
+++ b/scripts/video/android-attention-query-routing.patch
@@ -0,0 +1,71 @@
+--- a/src/core/ggml_runner.cpp
++++ b/src/core/ggml_runner.cpp
+@@ -210,6 +210,29 @@
+ if (sched == nullptr || gf == nullptr) {
+ return;
+ }
++#ifdef __ANDROID__
++ ggml_backend_t attention_backend = nullptr;
++ int attention_nodes_pinned = 0;
++ const int backend_count = ggml_backend_sched_get_n_backends(sched);
++ for (int index = 0; index < backend_count; ++index) {
++ auto candidate = ggml_backend_sched_get_backend(sched, index);
++ auto device = candidate ? ggml_backend_get_device(candidate) : nullptr;
++ const char* description = device ? ggml_backend_dev_description(device) : nullptr;
++ if (sd_backend_is(candidate, "Vulkan") && description &&
++ std::string(description).find("Adreno") != std::string::npos) {
++ attention_backend = candidate;
++ break;
++ }
++ }
++ auto is_query_chunk = [](const ggml_tensor* node) {
++ if (!node || node->op != GGML_OP_FLASH_ATTN_EXT || node->src[3] ||
++ node->op_params[3] != GGML_PREC_F32) return false;
++ auto query = node->src[0];
++ auto view = query && query->op == GGML_OP_CONT ? query->src[0] : nullptr;
++ return view && view->op == GGML_OP_VIEW && view->src[0] &&
++ query->ne[1] <= 128 && view->ne[1] < view->src[0]->ne[1];
++ };
++#endif
+ ggml_backend_t current = runtime_backend;
+ const int n_nodes = ggml_graph_n_nodes(gf);
+ for (int i = 0; i < n_nodes; i++) {
+@@ -230,6 +253,28 @@
+ node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE) {
+ continue;
+ }
++#ifdef __ANDROID__
++ // Keep query copies and their joined attention output on the same GPU.
++ // Pinning these to HTP would copy each chunk across both devices.
++ if (attention_backend && (is_query_chunk(node) ||
++ (node->op == GGML_OP_CONCAT && is_query_chunk(node->src[1]))) &&
++ ggml_backend_supports_op(attention_backend, node)) {
++ ggml_backend_sched_set_tensor_backend(sched, node, attention_backend);
++ ++attention_nodes_pinned;
++ if (is_query_chunk(node) && ggml_backend_supports_op(attention_backend, node->src[0])) {
++ ggml_backend_sched_set_tensor_backend(sched, node->src[0], attention_backend);
++ // Query views share this full-Q contiguous producer. Keep it
++ // on GPU too, so the scheduler does not recopy it for each view.
++ auto query_view = node->src[0]->src[0];
++ auto query_storage = query_view ? query_view->view_src : nullptr;
++ if (query_storage && query_storage->op == GGML_OP_CONT &&
++ ggml_backend_supports_op(attention_backend, query_storage)) {
++ ggml_backend_sched_set_tensor_backend(sched, query_storage, attention_backend);
++ }
++ }
++ continue;
++ }
++#endif
+ // In-place operations must follow the device chosen for their source.
+ // Its buffer may not exist yet when this graph is assigned.
+ if (node->view_src != nullptr) continue;
+@@ -239,6 +284,9 @@
+ ggml_backend_sched_set_tensor_backend(sched, node, current);
+ }
+ }
++#ifdef __ANDROID__
++ if (attention_backend) LOG_DEBUG("Adreno query-chunk GPU nodes pinned: %d", attention_nodes_pinned);
++#endif
+ }
+
+ size_t GGMLRunner::retained_runtime_buffer_bytes(ggml_backend_t backend) const {
diff --git a/scripts/video/android-hardware-fallback.patch b/scripts/video/android-hardware-fallback.patch
new file mode 100644
index 000000000..5e2411a4d
--- /dev/null
+++ b/scripts/video/android-hardware-fallback.patch
@@ -0,0 +1,111 @@
+diff --git a/src/core/compute_workspace.cpp b/src/core/compute_workspace.cpp
+index 54f0538..9f20b7e 100644
+--- a/src/core/compute_workspace.cpp
++++ b/src/core/compute_workspace.cpp
+@@ -18,6 +18,7 @@ namespace sd {
+ segment_end();
+ release();
+ ggml_backend_free(cpu_backend_);
++ ggml_backend_free(gpu_fallback_);
+ }
+
+ void ComputeWorkspace::set_extra_backends(const std::vector& backends) {
+@@ -43,6 +44,19 @@ namespace sd {
+ ggml_backend_sched_t ComputeWorkspace::make_scheduler(size_t graph_size) {
+ std::vector backends{backend_};
+ backends.insert(backends.end(), extra_backends_.begin(), extra_backends_.end());
++#ifdef __ANDROID__
++ // Hexagon is preferred; unsupported operations go to Vulkan, then CPU.
++ if (sd_backend_is(backend_, "HTP")) {
++ if (gpu_fallback_ == nullptr) {
++ auto device = ggml_backend_dev_by_name("Vulkan0");
++ if (device) gpu_fallback_ = ggml_backend_dev_init(device, nullptr);
++ }
++ if (gpu_fallback_ != nullptr &&
++ std::find(backends.begin(), backends.end(), gpu_fallback_) == backends.end()) {
++ backends.push_back(gpu_fallback_);
++ }
++ }
++#endif
+ if (!sd_backend_is_cpu(backend_)) {
+ if (cpu_backend_ == nullptr) {
+ cpu_backend_ = sd_backend_cpu_init();
+@@ -207,11 +221,12 @@ namespace sd {
+ auto scheduler = make_scheduler(graph_size);
+ if (scheduler != nullptr) {
+ assign_nodes(scheduler, copy_graph);
+- std::vector sizes(extra_backends_.size() + 2);
++ const int count = ggml_backend_sched_get_n_backends(scheduler);
++ std::vector sizes(count);
+ ggml_backend_sched_reserve_size(scheduler, copy_graph, sizes.data());
+- result.buffers.push_back({backend_, sizes[0]});
+- for (size_t i = 0; i < extra_backends_.size(); ++i) {
+- result.buffers.push_back({extra_backends_[i], sizes[i + 1]});
++ for (int i = 0; i < count; ++i) {
++ auto backend = ggml_backend_sched_get_backend(scheduler, i);
++ if (backend != cpu_backend_) result.buffers.push_back({backend, sizes[i]});
+ }
+ ggml_backend_sched_free(scheduler);
+ }
+diff --git a/src/core/compute_workspace.h b/src/core/compute_workspace.h
+index fcf14b8..203d62c 100644
+--- a/src/core/compute_workspace.h
++++ b/src/core/compute_workspace.h
+@@ -17,6 +17,7 @@ namespace sd {
+ ggml_backend_t backend_;
+ std::vector extra_backends_;
+ ggml_backend_t cpu_backend_ = nullptr;
++ ggml_backend_t gpu_fallback_ = nullptr;
+ ggml_gallocr_t allocator_ = nullptr;
+ ggml_backend_sched_t scheduler_ = nullptr;
+ size_t scheduler_capacity_ = 0;
+diff --git a/src/core/ggml_extend.cpp b/src/core/ggml_extend.cpp
+index d87353d..e491eff 100644
+--- a/src/core/ggml_extend.cpp
++++ b/src/core/ggml_extend.cpp
+@@ -517,7 +517,11 @@ ggml_tensor* ggml_ext_conv_3d(ggml_context* ctx,
+ }
+ if (b != nullptr) {
+ b = ggml_reshape_4d(ctx, b, 1, 1, 1, b->ne[0]); // [OC, 1, 1, 1]
+- x = ggml_add_inplace(ctx, x, b);
++ // The convolution may fall back to CPU while ADD runs on an accelerator.
++ // Give ADD its own output so the scheduler can transfer the input.
++ x = backend != nullptr && !ggml_backend_supports_op(backend, x)
++ ? ggml_add(ctx, x, b)
++ : ggml_add_inplace(ctx, x, b);
+ }
+ return x;
+ }
+diff --git a/src/core/ggml_runner.cpp b/src/core/ggml_runner.cpp
+index 4c156ed..c01c8d0 100644
+--- a/src/core/ggml_runner.cpp
++++ b/src/core/ggml_runner.cpp
+@@ -230,7 +230,12 @@ void GGMLRunner::pin_multi_device_nodes(ggml_backend_sched_t sched, ggml_cgraph*
+ node->op == GGML_OP_PERMUTE || node->op == GGML_OP_TRANSPOSE) {
+ continue;
+ }
+- if (ggml_backend_supports_op(current, node)) {
++ // In-place operations must follow the device chosen for their source.
++ // Its buffer may not exist yet when this graph is assigned.
++ if (node->view_src != nullptr) continue;
++ auto buffer = node->buffer;
++ if (ggml_backend_supports_op(current, node) &&
++ (buffer == nullptr || ggml_backend_supports_buft(current, ggml_backend_buffer_get_type(buffer)))) {
+ ggml_backend_sched_set_tensor_backend(sched, node, current);
+ }
+ }
+@@ -728,8 +733,12 @@ std::vector GGMLRunner::memory_requests(
+ const size_t reusable = workspace_.bytes(size.backend);
+ const size_t cache_bytes = size.backend == runtime_backend ? pending_cache_bytes : 0;
+ const size_t pending = add_bytes(size.bytes > reusable ? size.bytes - reusable : 0, cache_bytes);
+- size_t limit = max_graph_vram_bytes;
+- if (is_multi_device()) {
++ // A fallback device has its own physical memory checks. Do not apply
++ // the primary device's address-space cap (for example Hexagon's) to it.
++ const bool configured_backend = size.backend == runtime_backend ||
++ std::find(extra_runtime_backends.begin(), extra_runtime_backends.end(), size.backend) != extra_runtime_backends.end();
++ size_t limit = configured_backend ? max_graph_vram_bytes : 0;
++ if (configured_backend && is_multi_device()) {
+ size_t index = 0;
+ if (size.backend != runtime_backend) {
+ auto position = std::find(extra_runtime_backends.begin(), extra_runtime_backends.end(), size.backend);
diff --git a/scripts/video/android-hexagon-buffer-validation.patch b/scripts/video/android-hexagon-buffer-validation.patch
new file mode 100644
index 000000000..d2f7a4024
--- /dev/null
+++ b/scripts/video/android-hexagon-buffer-validation.patch
@@ -0,0 +1,29 @@
+--- a/src/ggml-hexagon/ggml-hexagon.cpp
++++ b/src/ggml-hexagon/ggml-hexagon.cpp
+@@ -6474,6 +6474,26 @@
+ }
+ }
+
++ // Mixed-backend graphs can retain tensors backed by another device.
++ // Validate the full split before queuing any DSP work: buffer contexts
++ // from CPU/Vulkan are not ggml_hexagon_shared_buffer objects.
++ for (const auto & node : *nodes_ptr) {
++ auto valid_buffer = [&](const ggml_tensor * tensor) {
++ if (!tensor) return true;
++ if (tensor->buffer && ggml_backend_buffer_is_hexagon(tensor->buffer)) return true;
++ GGML_LOG_ERROR("ggml-hex: %s cannot execute %s: tensor %s uses %s buffer\n",
++ sess->c_name(), node.op_name().c_str(), tensor->name,
++ tensor->buffer ? ggml_backend_buffer_name(tensor->buffer) : "missing");
++ return false;
++ };
++ for (const auto * tensor : node.get_inputs()) {
++ if (!valid_buffer(tensor)) return GGML_STATUS_FAILED;
++ }
++ for (const auto * tensor : node.get_outputs()) {
++ if (!valid_buffer(tensor)) return GGML_STATUS_FAILED;
++ }
++ }
++
+ // Queue and execute
+ for (const auto & node : *nodes_ptr) {
+ sess->enqueue_op(node);
diff --git a/scripts/video/android-hexagon-precision.patch b/scripts/video/android-hexagon-precision.patch
new file mode 100644
index 000000000..fc525fc7d
--- /dev/null
+++ b/scripts/video/android-hexagon-precision.patch
@@ -0,0 +1,30 @@
+--- a/src/ggml-hexagon/ggml-hexagon.cpp
++++ b/src/ggml-hexagon/ggml-hexagon.cpp
+@@ -4287,6 +4287,8 @@
+ }
+
+ static bool ggml_hexagon_supported_flash_attn_ext(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
++ // HMX keeps attention scores and recurrent output in F16 tiles.
++ if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) return false;
+ const struct ggml_tensor * src0 = op->src[0];
+ const struct ggml_tensor * src1 = op->src[1];
+ const struct ggml_tensor * src2 = op->src[2];
+@@ -5391,6 +5393,9 @@
+ }
+
+ static bool ggml_hexagon_supported_mul_mat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * dst) {
++ // HMX can lower activations to F16. Honor explicit F32 accumulation
++ // requests (for example video attention) through another backend.
++ if (ggml_get_op_params_i32(dst, 0) == GGML_PREC_F32) return false;
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+@@ -5481,7 +5486,7 @@
+ return false;
+ }
+
+- if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32) {
++ if (ggml_get_op_params_i32(op, 0) == GGML_PREC_F32) {
+ return false;
+ }
+
diff --git a/scripts/video/android-hexagon.patch b/scripts/video/android-hexagon.patch
new file mode 100644
index 000000000..5c0e8b2b5
--- /dev/null
+++ b/scripts/video/android-hexagon.patch
@@ -0,0 +1,32 @@
+diff --git a/src/ggml-hexagon/CMakeLists.txt b/src/ggml-hexagon/CMakeLists.txt
+index 845b922..0a9f237 100644
+--- a/src/ggml-hexagon/CMakeLists.txt
++++ b/src/ggml-hexagon/CMakeLists.txt
+@@ -1,3 +1,9 @@
++# Off Grid builds DSP libraries separately so Android hosts can use macOS.
++if(OFFGRID_VIDEO_HEXAGON_CMAKE)
++ include("${OFFGRID_VIDEO_HEXAGON_CMAKE}")
++ return()
++endif()
++
+ file(TO_CMAKE_PATH "${HEXAGON_SDK_ROOT}" HEXAGON_SDK_ROOT)
+ file(TO_CMAKE_PATH "${HEXAGON_TOOLS_ROOT}" HEXAGON_TOOLS_ROOT)
+
+diff --git a/src/ggml-hexagon/ggml-hexagon.cpp b/src/ggml-hexagon/ggml-hexagon.cpp
+index 9a35002..0ca4222 100644
+--- a/src/ggml-hexagon/ggml-hexagon.cpp
++++ b/src/ggml-hexagon/ggml-hexagon.cpp
+@@ -3887,7 +3887,12 @@ void ggml_hexagon_session::allocate(const ggml_hexagon_device_config & config) n
+ char session_uri[256];
+ {
+ char htp_uri[256];
+- snprintf(htp_uri, sizeof(htp_uri), "file:///libggml-htp-v%u.so?htp_iface_skel_handle_invoke&_modver=1.0", opt_arch);
++ const char * video_htp_dir = getenv("OFFGRID_VIDEO_HTP_DIR");
++ if (video_htp_dir && *video_htp_dir) {
++ snprintf(htp_uri, sizeof(htp_uri), "file:///liboffgrid-video-htp-v%u.so?htp_iface_skel_handle_invoke&_modver=1.0", opt_arch);
++ } else {
++ throw std::runtime_error("Video Hexagon runtime directory is not configured");
++ }
+
+ struct remote_rpc_get_uri u = {};
+ u.session_id = this->session_id;
diff --git a/scripts/video/android-video-numerics.patch b/scripts/video/android-video-numerics.patch
new file mode 100644
index 000000000..e65cb8500
--- /dev/null
+++ b/scripts/video/android-video-numerics.patch
@@ -0,0 +1,55 @@
+--- a/src/pipeline/video.cpp
++++ b/src/pipeline/video.cpp
+@@ -1214,6 +1214,32 @@
+ return embeds;
+ }
+
++#if defined(__ANDROID__)
++ // These tensors are already in host memory. Scan once without copying them.
++ static void log_video_tensor_summary(const char* stage, const sd::Tensor& tensor) {
++ int64_t finite = 0;
++ int64_t zero = 0;
++ double minimum = INFINITY;
++ double maximum = -INFINITY;
++ double sum = 0;
++ double squares = 0;
++ for (const float value : tensor.values()) {
++ if (!std::isfinite(value)) continue;
++ ++finite;
++ if (value == 0.0f) ++zero;
++ minimum = std::min(minimum, static_cast(value));
++ maximum = std::max(maximum, static_cast(value));
++ sum += value;
++ squares += static_cast(value) * value;
++ }
++ LOG_INFO("OFFGRID_VIDEO_NUMERICS stage=%s values=%lld finite=%lld nonfinite=%lld zero=%lld min=%.9g max=%.9g mean=%.9g rms=%.9g",
++ stage, (long long)tensor.numel(), (long long)finite,
++ (long long)(tensor.numel() - finite), (long long)zero,
++ minimum, maximum, finite ? sum / finite : 0.0,
++ finite ? std::sqrt(squares / finite) : 0.0);
++ }
++#endif
++
+ static sd_image_t* decode_video_outputs(StableDiffusionGGML* sd,
+ const GenerationRequest& request,
+ const sd::Tensor& final_latent,
+@@ -1226,6 +1252,9 @@
+ LOG_ERROR("cancelling video decode");
+ return nullptr;
+ }
++#if defined(__ANDROID__)
++ log_video_tensor_summary("diffusion_latent", final_latent);
++#endif
+ sd::Tensor video_latent = final_latent;
+ if ((sd_version_is_ltxav(sd->version) || sd_version_is_minimax_h3(sd->version)) &&
+ video_latent.shape()[3] > sd->get_latent_channel()) {
+@@ -1254,6 +1283,9 @@
+ (int)vid.shape()[1],
+ (int)vid.shape()[2],
+ (int)vid.shape()[3]);
++#if defined(__ANDROID__)
++ log_video_tensor_summary("decoded_rgb", vid);
++#endif
+ if (request.frames > 0 &&
+ vid.shape()[2] > request.frames) {
+ vid = sd::ops::slice(vid, 2, 0, request.frames);
diff --git a/scripts/video/android-vulkan-attention-fallback.patch b/scripts/video/android-vulkan-attention-fallback.patch
new file mode 100644
index 000000000..5b6a0ea52
--- /dev/null
+++ b/scripts/video/android-vulkan-attention-fallback.patch
@@ -0,0 +1,74 @@
+--- a/src/ggml-vulkan/ggml-vulkan.cpp
++++ b/src/ggml-vulkan/ggml-vulkan.cpp
+@@ -1259,6 +1259,16 @@
+ const uint32_t D_lsb = D ^ (D & (D-1)); // extract lowest set bit
+
+ result.d_split = std::min(std::min(result.subgroup_size, 8u), D_lsb / 4);
++
++#ifdef __ANDROID__
++ if (device->vendor_id == VK_VENDOR_ID_QUALCOMM && f32acc &&
++ k_type == GGML_TYPE_F16 && v_type == GGML_TYPE_F16) {
++ // Use the existing shared-memory reduction path instead of the
++ // subgroup shuffle pipeline that loses the Adreno device in Wan.
++ result.disable_subgroups = true;
++ result.block_rows = std::min(result.block_rows, 8u);
++ }
++#endif
+
+ result.shmem_staging = (device->vendor_id == VK_VENDOR_ID_NVIDIA && hsk < 256 && hsv < 256) ? 1 : 0;
+
+@@ -1982,6 +1992,16 @@
+ }
+ name = aligned ? "flash_attn_f32_f16_aligned" : "flash_attn_f32_f16";
+ }
++#ifdef __ANDROID__
++ if (device->vendor_id == VK_VENDOR_ID_QUALCOMM && f32acc &&
++ fa.first.k_type == GGML_TYPE_F16 && fa.first.v_type == GGML_TYPE_F16) {
++ // Preserve FP32 accumulation and avoid the mobile FP16 arithmetic
++ // path. K/V storage stays F16; no extra attention matrix is stored.
++ spv_data = flash_attn_f32_f16_fp32_data;
++ spv_size = flash_attn_f32_f16_fp32_len;
++ name = "flash_attn_adreno_f32_shared";
++ }
++#endif
+ ggml_vk_create_pipeline(device, fa.second, name, spv_size, spv_data, "main", 8,
+ sizeof(vk_flash_attn_push_constants), {Br, 1, 1},
+ get_fa_spec_constants(fa.first), aligned ? Bc : 1, true,
+@@ -14226,6 +14246,13 @@
+ uint64_t batch_flops = 0;
+ uint64_t total_flops = 0;
+ uint64_t flops_cap = 200'000'000'000ULL;
++#ifdef __ANDROID__
++ // Keep query chunks in separate submissions on mobile Adreno. A full
++ // Wan attention dispatch exceeds 130 GFLOP and can hit the driver limit.
++ if (ctx->device->vendor_id == VK_VENDOR_ID_QUALCOMM) {
++ flops_cap = 8'000'000'000ULL;
++ }
++#endif
+
+ // On weaker AMD GPUs larger submissions can hit a driver timeout, submit more often to avoid this
+ if (ctx->device->vendor_id == VK_VENDOR_ID_AMD && ctx->device->shader_core_count > 0) {
+@@ -14236,6 +14263,11 @@
+ }
+ }
+ uint64_t flops_per_submit = std::min(flops_cap, ctx->last_total_flops / 40u);
++#ifdef __ANDROID__
++ if (ctx->device->vendor_id == VK_VENDOR_ID_QUALCOMM) {
++ flops_per_submit = flops_cap;
++ }
++#endif
+
+ auto const submit_after = [&](int start, int end) {
+ if (ctx->device->serialize_submissions) {
+@@ -14263,6 +14295,11 @@
+ batch_flops = 0;
+ if (submit_count < 3) {
+ flops_per_submit *= 2;
++#ifdef __ANDROID__
++ if (ctx->device->vendor_id == VK_VENDOR_ID_QUALCOMM) {
++ flops_per_submit = std::min(flops_cap, flops_per_submit);
++ }
++#endif
+ }
+ submit_count++;
+ };
diff --git a/scripts/video/android-vulkan-device-fault.patch b/scripts/video/android-vulkan-device-fault.patch
new file mode 100644
index 000000000..9762ca261
--- /dev/null
+++ b/scripts/video/android-vulkan-device-fault.patch
@@ -0,0 +1,18 @@
+--- a/src/ggml-vulkan/ggml-vulkan.cpp
++++ b/src/ggml-vulkan/ggml-vulkan.cpp
+@@ -146,6 +146,15 @@
+ return std::find(lightning_indexer_k_types.begin(), lightning_indexer_k_types.end(), type) != lightning_indexer_k_types.end();
+ }
+ void ggml_vk_print_device_fault_info(const vk_device& device) {
++#ifdef __ANDROID__
++ // Adreno's fault-info callback can dereference null after device loss
++ // (observed on SM8635). Keep normal error reporting and exception handling
++ // alive instead of crashing while asking the driver for extra diagnostics.
++ if (device->vendor_id == VK_VENDOR_ID_QUALCOMM) {
++ GGML_LOG_ERROR("ggml_vulkan: skipping unsafe Adreno device-fault query\n");
++ return;
++ }
++#endif
+ if (!device->device_fault || !device->pfn_vkGetDeviceFaultInfoEXT) {
+ return;
+ }
diff --git a/scripts/video/android-vulkan-dispatch.patch b/scripts/video/android-vulkan-dispatch.patch
new file mode 100644
index 000000000..41a93caff
--- /dev/null
+++ b/scripts/video/android-vulkan-dispatch.patch
@@ -0,0 +1,29 @@
+--- a/src/ggml-vulkan/ggml-vulkan.cpp
++++ b/src/ggml-vulkan/ggml-vulkan.cpp
+@@ -4306,7 +4306,7 @@
+ device_extensions.push_back("VK_EXT_device_fault");
+ }
+
+- vkGetPhysicalDeviceFeatures2(device->physical_device, &device_features2);
++ VULKAN_HPP_DEFAULT_DISPATCHER.vkGetPhysicalDeviceFeatures2(device->physical_device, &device_features2);
+
+ device->device_fault = device->device_fault && fault_features.deviceFault;
+
+@@ -4930,7 +4930,7 @@
+ }
+ #endif
+
+- vkGetPhysicalDeviceFeatures2(physical_device, &device_features2);
++ VULKAN_HPP_DEFAULT_DISPATCHER.vkGetPhysicalDeviceFeatures2(physical_device, &device_features2);
+
+ fp16 = fp16 && vk12_features.shaderFloat16;
+
+@@ -16170,7 +16170,7 @@
+ vk11_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES;
+ device_features2.pNext = &vk11_features;
+
+- vkGetPhysicalDeviceFeatures2(vkdev, &device_features2);
++ VULKAN_HPP_DEFAULT_DISPATCHER.vkGetPhysicalDeviceFeatures2(vkdev, &device_features2);
+
+ return vk11_features.storageBuffer16BitAccess;
+ }
diff --git a/scripts/video/android-vulkan-matvec-fallback.patch b/scripts/video/android-vulkan-matvec-fallback.patch
new file mode 100644
index 000000000..feaa46db6
--- /dev/null
+++ b/scripts/video/android-vulkan-matvec-fallback.patch
@@ -0,0 +1,36 @@
+--- a/src/ggml-vulkan/ggml-vulkan.cpp
++++ b/src/ggml-vulkan/ggml-vulkan.cpp
+@@ -15333,6 +15333,33 @@
+ case GGML_OP_MUL_MAT_ID:
+ {
+ ggml_type src0_type = op->src[0]->type;
++#ifdef __ANDROID__
++ // SM8635's proprietary Adreno compiler crashes while compiling
++ // a floating-point dmmv shader in Wan VAE decode. Reject only
++ // float vector/small-column MUL_MAT shapes so the scheduler can
++ // use another backend. Keep quantized and regular GPU matrices.
++ const bool float_weight = src0_type == GGML_TYPE_F32 ||
++ src0_type == GGML_TYPE_F16 ||
++ src0_type == GGML_TYPE_BF16;
++ const bool float_input = op->src[1]->type == GGML_TYPE_F32 ||
++ op->src[1]->type == GGML_TYPE_F16;
++ const bool vector_shape = op->ne[1] == 1 ||
++ (op->ne[1] <= mul_mat_vec_max_cols &&
++ op->src[1]->ne[2] * op->src[1]->ne[3] == 1);
++ // The dispatcher can swap a single output row into matvec too.
++ const bool swapped_vector = op->ne[0] == 1 &&
++ op->src[0]->ne[2] == 1 && op->src[0]->ne[3] == 1 &&
++ op->src[1]->ne[2] == 1 && op->src[1]->ne[3] == 1;
++ if (device->vendor_id == VK_VENDOR_ID_QUALCOMM &&
++ op->op == GGML_OP_MUL_MAT && float_weight && float_input &&
++ (vector_shape || swapped_vector)) {
++ static std::atomic_flag reported = ATOMIC_FLAG_INIT;
++ if (!reported.test_and_set()) {
++ GGML_LOG_WARN("ggml_vulkan: routing Adreno float matvec to another backend to avoid shader compiler crash\n");
++ }
++ return false;
++ }
++#endif
+ if (src0_type == GGML_TYPE_I8) {
+ return op->op == GGML_OP_MUL_MAT &&
+ ggml_vk_can_use_mul_mat_i8_tensorwise(device, op);
diff --git a/scripts/video/android-vulkan-pipeline-diagnostics.patch b/scripts/video/android-vulkan-pipeline-diagnostics.patch
new file mode 100644
index 000000000..5c7c1410b
--- /dev/null
+++ b/scripts/video/android-vulkan-pipeline-diagnostics.patch
@@ -0,0 +1,24 @@
+--- a/src/ggml-vulkan/ggml-vulkan.cpp
++++ b/src/ggml-vulkan/ggml-vulkan.cpp
+@@ -586,6 +586,21 @@
+ GGML_ASSERT(parameter_count > 0);
+ GGML_ASSERT(parameter_count <= MAX_PARAMETER_COUNT);
+ GGML_ASSERT(wg_denoms[0] > 0 && wg_denoms[1] > 0 && wg_denoms[2] > 0); // NOLINT
++
++#ifdef __ANDROID__
++ // The Adreno compiler can terminate the process inside pipeline creation,
++ // before Vulkan can return an error. Record the lazy shader variant first.
++ if (device->vendor_id == VK_VENDOR_ID_QUALCOMM) {
++ std::string constants;
++ for (const auto value : specialization_constants) {
++ if (!constants.empty()) constants += ",";
++ constants += std::to_string(value);
++ }
++ GGML_LOG_INFO("ggml_vulkan: Adreno compiling %s (specialization=[%s], subgroup=%u, full_subgroups=%d, spirv_bytes=%zu)\n",
++ pipeline->name.c_str(), constants.c_str(), required_subgroup_size,
++ require_full_subgroups ? 1 : 0, spv_size);
++ }
++#endif
+
+ vk::ShaderModuleCreateInfo shader_module_create_info({}, spv_size, reinterpret_cast(spv_data));
+
diff --git a/scripts/video/build-hexagon.sh b/scripts/video/build-hexagon.sh
new file mode 100755
index 000000000..438ee5b49
--- /dev/null
+++ b/scripts/video/build-hexagon.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+set -euo pipefail
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+BUILD="$ROOT/.video-build"
+SDK_VERSION=6.4.0.2
+SDK_SHA256=b4a57a774795cf12da19a777a5d306e970905bf9758a4c4765e5e4593428ae0b
+SDK="$BUILD/hexagon-sdk/$SDK_VERSION"
+# Reuse the DSP artifacts when their source revision and SDK match.
+REVISION="$(cat "$ROOT/scripts/video/revision")"
+READY="$BUILD/hexagon-runtime/ready"
+COMPLETE=true
+for file in htp_iface_stub.c htp_iface.h; do
+ [ -s "$BUILD/hexagon-runtime/$file" ] || COMPLETE=false
+done
+for arch in v73 v75 v79 v81; do
+ [ -s "$BUILD/hexagon-assets/video-hexagon/liboffgrid-video-htp-$arch.so" ] || COMPLETE=false
+done
+if $COMPLETE && [ "$(cat "$READY" 2>/dev/null || true)" = "$REVISION:$SDK_VERSION" ]; then exit 0; fi
+mkdir -p "$BUILD/hexagon-sdk" "$BUILD/hexagon-runtime" "$BUILD/hexagon-assets/video-hexagon"
+if [ ! -f "$SDK/hexagon_sdk.json" ]; then
+ curl -fL --retry 3 -C - "https://github.com/snapdragon-toolchain/hexagon-sdk/releases/download/v$SDK_VERSION/hexagon-sdk-v$SDK_VERSION-amd64-lnx.tar.xz" -o "$BUILD/hexagon-sdk.tar.xz"
+ printf '%s %s\n' "$SDK_SHA256" "$BUILD/hexagon-sdk.tar.xz" | shasum -a 256 -c -
+ tar -xJf "$BUILD/hexagon-sdk.tar.xz" -C "$BUILD/hexagon-sdk"
+fi
+# The official Hexagon tools run on Linux. Only this build directory is mounted.
+docker run --rm --platform linux/amd64 -v "$BUILD:/work" -w /work ubuntu:22.04 bash -euc '
+ apt-get update -qq
+ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ninja-build python3 python3-pip build-essential libncurses5 libtinfo5
+ python3 -m pip install --no-cache-dir cmake==3.31.6
+ export HEXAGON_SDK_ROOT=/work/hexagon-sdk/6.4.0.2
+ export HEXAGON_TOOLS_ROOT=$HEXAGON_SDK_ROOT/tools/HEXAGON_Tools/19.0.04
+ export DEFAULT_HLOS_ARCH=64 DEFAULT_TOOLS_VARIANT=toolv19 DEFAULT_NO_QURT_INC=0
+ for arch in v73 v75 v79 v81; do
+ cmake -S source/ggml/src/ggml-hexagon/htp -B hexagon-build/$arch -G Ninja \
+ -DCMAKE_TOOLCHAIN_FILE=/work/source/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake \
+ -DCMAKE_BUILD_TYPE=Release -DHEXAGON_SDK_ROOT=$HEXAGON_SDK_ROOT \
+ -DHEXAGON_TOOLS_ROOT=$HEXAGON_TOOLS_ROOT -DDSP_VERSION=$arch \
+ -DPREBUILT_LIB_DIR=toolv19_$arch -DHEXAGON_HTP_DEBUG=OFF
+ cmake --build hexagon-build/$arch --parallel 2
+ cp hexagon-build/$arch/libggml-htp-$arch.so hexagon-assets/video-hexagon/liboffgrid-video-htp-$arch.so
+ done
+ cp hexagon-build/v73/htp_iface_stub.c hexagon-build/v73/htp_iface.h hexagon-runtime/
+'
+cp "$ROOT/scripts/video/revision" "$BUILD/hexagon-runtime/revision"
+printf '%s:%s\n' "$REVISION" "$SDK_VERSION" > "$READY"
diff --git a/scripts/video/build-shaderc.sh b/scripts/video/build-shaderc.sh
new file mode 100644
index 000000000..f0664dbc4
--- /dev/null
+++ b/scripts/video/build-shaderc.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+set -euo pipefail
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+REVISION=d393a813c9b67c709d37b329203d72beb69d719e
+SOURCE="$ROOT/.video-build/shaderc-source"
+PREFIX="$ROOT/.video-build/shaderc"
+if [ -x "$PREFIX/bin/glslc" ] && [ "$(cat "$PREFIX/revision" 2>/dev/null || true)" = "$REVISION" ]; then exit 0; fi
+# A host package of this exact release is also suitable for shader compilation.
+if command -v glslc >/dev/null && [[ "$(glslc --version)" == "shaderc v2026.4 "* ]]; then
+ mkdir -p "$PREFIX/bin"
+ ln -sf "$(command -v glslc)" "$PREFIX/bin/glslc"
+ printf '%s\n' "$REVISION" > "$PREFIX/revision"
+ exit 0
+fi
+mkdir -p "$ROOT/.video-build"
+if [ ! -d "$SOURCE/.git" ]; then
+ git clone --filter=blob:none --no-checkout https://github.com/google/shaderc.git "$SOURCE"
+fi
+git -C "$SOURCE" fetch origin "$REVISION"
+git -C "$SOURCE" checkout --detach "$REVISION"
+(cd "$SOURCE" && python3 utils/git-sync-deps)
+cmake -S "$SOURCE" -B "$SOURCE/build-host" -G "Unix Makefiles" \
+ -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$PREFIX" \
+ -DSHADERC_SKIP_TESTS=ON -DSHADERC_SKIP_EXAMPLES=ON -DSHADERC_SKIP_COPYRIGHT_CHECK=ON
+cmake --build "$SOURCE/build-host" --target glslc --parallel "${VIDEO_BUILD_JOBS:-4}"
+mkdir -p "$PREFIX/bin"
+cp "$SOURCE/build-host/glslc/glslc" "$PREFIX/bin/glslc"
+printf '%s\n' "$REVISION" > "$PREFIX/revision"
diff --git a/scripts/video/conditioning-errors.patch b/scripts/video/conditioning-errors.patch
new file mode 100644
index 000000000..15b07dea3
--- /dev/null
+++ b/scripts/video/conditioning-errors.patch
@@ -0,0 +1,16 @@
+diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp
+index 7796a40..33ec41e 100644
+--- a/src/conditioning/conditioner.hpp
++++ b/src/conditioning/conditioner.hpp
+@@ -1570,7 +1570,10 @@ struct T5CLIPEmbedder : public Conditioner {
+ input_ids,
+ t5_attn_mask_chunk,
+ false);
+- GGML_ASSERT(!chunk_hidden_states.empty());
++ if (chunk_hidden_states.empty()) {
++ LOG_ERROR("T5 encoder returned no output; the backend failed or generation was cancelled");
++ return {};
++ }
+ chunk_hidden_states = apply_token_weights(std::move(chunk_hidden_states), chunk_weights);
+
+ if (zero_out_masked) {
diff --git a/scripts/video/exports.txt b/scripts/video/exports.txt
new file mode 100644
index 000000000..25fb6eb21
--- /dev/null
+++ b/scripts/video/exports.txt
@@ -0,0 +1,8 @@
+_sd_*
+_new_sd_ctx
+_free_sd_ctx
+_free_sd_images
+_generate_video
+_generate_image
+_str_to_sample_method
+_str_to_scheduler
diff --git a/scripts/video/hexagon-prebuilt.cmake b/scripts/video/hexagon-prebuilt.cmake
new file mode 100644
index 000000000..ebb950c18
--- /dev/null
+++ b/scripts/video/hexagon-prebuilt.cmake
@@ -0,0 +1,18 @@
+# Host-side Hexagon backend. DSP code is built by build-hexagon.sh from the
+# same pinned GGML source; do not reuse llama.rn's different DSP ABI.
+set(VIDEO_HEXAGON_ROOT "${SD_SOURCE}/../hexagon-runtime")
+set(VIDEO_HEXAGON_SDK "${SD_SOURCE}/../hexagon-sdk/6.4.0.2")
+if(NOT EXISTS "${VIDEO_HEXAGON_ROOT}/htp_iface_stub.c")
+ message(FATAL_ERROR "Run scripts/video/build-hexagon.sh before building Android video.")
+endif()
+add_library(htp_iface OBJECT "${VIDEO_HEXAGON_ROOT}/htp_iface_stub.c")
+set_target_properties(htp_iface PROPERTIES POSITION_INDEPENDENT_CODE ON)
+target_include_directories(htp_iface PUBLIC
+ "${VIDEO_HEXAGON_SDK}/incs" "${VIDEO_HEXAGON_SDK}/incs/stddef"
+ "${VIDEO_HEXAGON_SDK}/ipc/fastrpc/rpcmem/inc"
+ "${VIDEO_HEXAGON_SDK}/utils/examples" "${CMAKE_CURRENT_SOURCE_DIR}/htp"
+ "${VIDEO_HEXAGON_ROOT}")
+target_link_options(htp_iface PUBLIC -llog -ldl)
+ggml_add_backend_library(ggml-hexagon ggml-hexagon.cpp htp-drv.cpp htp-drv.h libdl.h ../../include/ggml-hexagon.h)
+target_link_libraries(ggml-hexagon PRIVATE htp_iface)
+target_include_directories(ggml-hexagon PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/htp" "${VIDEO_HEXAGON_ROOT}")
diff --git a/scripts/video/revision b/scripts/video/revision
new file mode 100644
index 000000000..8e5956cae
--- /dev/null
+++ b/scripts/video/revision
@@ -0,0 +1 @@
+3f8527a46c54ecf4cb4ed6003da8e8982283c73c
diff --git a/scripts/video/video-decode-observer.patch b/scripts/video/video-decode-observer.patch
new file mode 100644
index 000000000..c1ffcafee
--- /dev/null
+++ b/scripts/video/video-decode-observer.patch
@@ -0,0 +1,186 @@
+diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h
+index 8e5577d..8713281 100644
+--- a/include/stable-diffusion.h
++++ b/include/stable-diffusion.h
+@@ -461,10 +461,16 @@ struct ggml_tensor;
+ typedef void (*sd_log_cb_t)(enum sd_log_level_t level, const char* text, void* data);
+ typedef void (*sd_progress_cb_t)(int step, int steps, float time, void* data);
+ typedef void (*sd_preview_cb_t)(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data);
++// The decoded frame is borrowed for the duration of this callback only.
++typedef void (*sd_video_decode_progress_cb_t)(int completed, int total, void* data);
++typedef void (*sd_video_decoded_frame_cb_t)(const sd_image_t* frame, void* data);
+ typedef bool (*sd_graph_eval_callback_t)(struct ggml_tensor* t, bool ask, void* user_data);
+
+ SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data);
+ SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data);
++// Reports final video decode sections and the first complete frame, without an extra decode.
++// total == 0 means the section count is not yet known. Does not report sampling previews.
++SD_API void sd_set_video_decode_callback(sd_video_decode_progress_cb_t progress, sd_video_decoded_frame_cb_t frame, void* data);
+ // In each sampling pass, a positive interval previews every Nth denoiser step, while a
+ // negative interval previews only completed logical step -interval. Zero previews the final
+ // completed step of the first sampling pass (base-resolution or high-noise).
+diff --git a/src/core/util.cpp b/src/core/util.cpp
+index b90f70e..0fc1aed 100644
+--- a/src/core/util.cpp
++++ b/src/core/util.cpp
+@@ -357,6 +357,46 @@ int32_t sd_get_num_physical_cores() {
+ return n_threads > 0 ? (n_threads <= 4 ? n_threads : n_threads / 2) : 4;
+ }
+
++static sd_video_decode_progress_cb_t sd_decode_progress_cb = nullptr;
++static sd_video_decoded_frame_cb_t sd_decoded_frame_cb = nullptr;
++static void* sd_decode_cb_data = nullptr;
++static thread_local bool sd_video_decode_active = false;
++static thread_local int sd_decode_spatial_completed = 0;
++static thread_local int sd_decode_spatial_total = 0;
++static thread_local int sd_decode_temporal_total = 1;
++
++SDVideoDecodeScope::SDVideoDecodeScope() : previous(sd_video_decode_active) {
++ sd_video_decode_active = true;
++ sd_video_decode_progress(0, 0);
++}
++SDVideoDecodeScope::~SDVideoDecodeScope() { sd_video_decode_active = previous; }
++void sd_video_decode_progress(int completed, int total) {
++ if (!sd_video_decode_active) return;
++ // A fresh spatial pass also resets the multiplier on an allocation retry.
++ if (completed == 0) sd_decode_temporal_total = 1;
++ sd_decode_spatial_completed = completed;
++ sd_decode_spatial_total = total;
++ if (sd_decode_progress_cb)
++ sd_decode_progress_cb(completed * sd_decode_temporal_total,
++ total * sd_decode_temporal_total, sd_decode_cb_data);
++}
++void sd_video_decode_temporal_progress(int completed, int total) {
++ if (!sd_video_decode_active || total <= 0) return;
++ sd_decode_temporal_total = total;
++ if (sd_decode_progress_cb)
++ sd_decode_progress_cb(sd_decode_spatial_completed * total + completed,
++ std::max(1, sd_decode_spatial_total) * total, sd_decode_cb_data);
++}
++void sd_video_decoded_frame(const sd_image_t* frame) {
++ if (frame && frame->data && sd_decoded_frame_cb)
++ sd_decoded_frame_cb(frame, sd_decode_cb_data);
++}
++void sd_set_video_decode_callback(sd_video_decode_progress_cb_t progress, sd_video_decoded_frame_cb_t frame, void* data) {
++ sd_decode_progress_cb = progress;
++ sd_decoded_frame_cb = frame;
++ sd_decode_cb_data = data;
++}
++
+ static sd_progress_cb_t sd_progress_cb = nullptr;
+ void* sd_progress_cb_data = nullptr;
+
+@@ -581,6 +621,8 @@ static void print_progress_line(int step, int steps, const std::string& speed_te
+ }
+
+ void pretty_progress(int step, int steps, float time) {
++ // Video decode has its own section callback; keep it out of sampling progress.
++ if (sd_video_decode_active && sd_decode_progress_cb) return;
+ if (sd_progress_cb) {
+ sd_progress_cb(step, steps, time, sd_progress_cb_data);
+ return;
+@@ -600,6 +642,8 @@
+ void pretty_bytes_progress(int step, int steps, uint64_t bytes_processed, float elapsed_seconds) {
++ // Loading counts are not sampling steps, even when their totals happen to match.
++ if (sd_decode_progress_cb) return;
+ if (sd_progress_cb) {
+ float time = elapsed_seconds / (step + 1e-6f);
+ sd_progress_cb(step, steps, time, sd_progress_cb_data);
+ return;
+ }
+diff --git a/src/core/util.h b/src/core/util.h
+index 334224d..64f3351 100644
+--- a/src/core/util.h
++++ b/src/core/util.h
+@@ -98,6 +98,19 @@ std::vector> parse_prompt_attention(const std::str
+ std::vector> split_quotation_attention(
+ const std::vector>& parsed_attention);
+
++class SDVideoDecodeScope {
++public:
++ SDVideoDecodeScope();
++ ~SDVideoDecodeScope();
++ SDVideoDecodeScope(const SDVideoDecodeScope&) = delete;
++ SDVideoDecodeScope& operator=(const SDVideoDecodeScope&) = delete;
++private:
++ bool previous;
++};
++void sd_video_decode_progress(int completed, int total);
++void sd_video_decode_temporal_progress(int completed, int total);
++void sd_video_decoded_frame(const sd_image_t* frame);
++
+ sd_progress_cb_t sd_get_progress_callback();
+ void* sd_get_progress_callback_data();
+
+diff --git a/src/model/vae/wan_vae.hpp b/src/model/vae/wan_vae.hpp
+index 9dd30fd..e28887b 100644
+--- a/src/model/vae/wan_vae.hpp
++++ b/src/model/vae/wan_vae.hpp
+@@ -1500,6 +1500,9 @@ namespace WAN {
+ free_cache_ctx_and_buffer();
+ ae.clear_cache();
+
++ LOG_INFO("Wan VAE decode backend=%s temporal_chunks=%d latent_frames=%lld",
++ ggml_backend_name(runtime_backend), (int)plan.tiles.size(), (long long)input.shape()[2]);
++ sd_video_decode_temporal_progress(0, (int)plan.tiles.size());
+ auto output = process_vae_temporal_tiles(input, plan, [&](const sd::Tensor& input_tile, const VAETemporalTile& tile) {
+ LOG_VERBOSE("Wan VAE temporal tile %d/%d: latent frames [%lld, %lld)",
+ tile.index + 1,
+@@ -1509,9 +1512,11 @@ namespace WAN {
+ auto get_graph = [&]() -> ggml_cgraph* {
+ return build_temporal_tile_graph(input_tile, static_cast(tile.start));
+ };
+- return restore_trailing_singleton_dims(
++ auto decoded = restore_trailing_singleton_dims(
+ GGMLRunner::compute(get_graph, n_threads, false),
+ static_cast(input.dim()));
++ if (!decoded.empty()) sd_video_decode_temporal_progress(tile.index + 1, (int)plan.tiles.size());
++ return decoded;
+ });
+
+ free_cache_ctx_and_buffer();
+diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp
+index 2fcff4c..dc0f30f 100644
+--- a/src/pipeline/video.cpp
++++ b/src/pipeline/video.cpp
+@@ -1238,7 +1238,11 @@ namespace sd::pipeline {
+ (int)video_latent.shape()[3]);
+ // auto z = sd::load_tensor_from_file_as_tensor("ltx_vae_z.bin");
+ int64_t t4 = ggml_time_ms();
+- sd::Tensor vid = sd->decode_first_stage(video_latent, true);
++ sd::Tensor vid;
++ {
++ SDVideoDecodeScope decode_scope;
++ vid = sd->decode_first_stage(video_latent, true);
++ }
+ int64_t t5 = ggml_time_ms();
+ LOG_INFO("decode_first_stage completed, taking %.2fs", (t5 - t4) * 1.0f / 1000);
+ if (vid.empty()) {
+@@ -1265,6 +1269,7 @@ namespace sd::pipeline {
+
+ for (int64_t i = 0; i < vid.shape()[2]; i++) {
+ result_images[i] = tensor_to_sd_image(vid, static_cast(i));
++ if (i == 0) sd_video_decoded_frame(&result_images[i]);
+ }
+
+ return result_images;
+diff --git a/src/runtime/tiling.cpp b/src/runtime/tiling.cpp
+index 38ea4c2..771968c 100644
+--- a/src/runtime/tiling.cpp
++++ b/src/runtime/tiling.cpp
+@@ -193,6 +193,7 @@ sd::Tensor process_tiles_2d(const sd::Tensor& input,
+ }
+
+ int num_tiles = num_tiles_x * num_tiles_y;
++ sd_video_decode_progress(0, num_tiles);
+ int tile_count = 1;
+ bool last_y = false;
+ bool last_x = false;
+@@ -254,6 +255,7 @@ sd::Tensor process_tiles_2d(const sd::Tensor& input,
+ last_time = (t2 - t1) / 1000.0f;
+ pretty_progress(tile_count, num_tiles, last_time);
+ }
++ sd_video_decode_progress(tile_count, num_tiles);
+ tile_count++;
+ }
+ last_x = false;
diff --git a/src/components/Button.tsx b/src/components/Button.tsx
index 47517b4bd..e4c2d3037 100644
--- a/src/components/Button.tsx
+++ b/src/components/Button.tsx
@@ -1,6 +1,6 @@
import React from 'react';
import { TouchableOpacity, Text, ViewStyle, TextStyle } from 'react-native';
-import type { AccessibilityState } from 'react-native';
+import type { AccessibilityRole, AccessibilityState } from 'react-native';
import { useTheme, useThemedStyles } from '../theme';
import type { ThemeColors, ThemeShadows } from '../theme';
import { SPACING, TYPOGRAPHY } from '../constants';
@@ -24,6 +24,7 @@ interface ButtonProps {
testID?: string;
accessibilityLabel?: string;
accessibilityState?: AccessibilityState;
+ accessibilityRole?: AccessibilityRole;
}
export const Button: React.FC = ({
@@ -40,6 +41,7 @@ export const Button: React.FC = ({
testID,
accessibilityLabel,
accessibilityState,
+ accessibilityRole = 'button',
}) => {
const { colors } = useTheme();
const styles = useThemedStyles(createStyles);
@@ -69,7 +71,7 @@ export const Button: React.FC = ({
disabled={disabled || loading}
activeOpacity={0.7}
testID={testID}
- accessibilityRole="button"
+ accessibilityRole={accessibilityRole}
accessibilityLabel={accessibilityLabel ?? title}
accessibilityState={accessibilityState}
>
diff --git a/src/components/ChatInput/Popovers.tsx b/src/components/ChatInput/Popovers.tsx
index 8673e95e4..d7125f2e7 100644
--- a/src/components/ChatInput/Popovers.tsx
+++ b/src/components/ChatInput/Popovers.tsx
@@ -77,6 +77,7 @@ interface QuickSettingsPopoverProps {
anchorX: number;
imageMode: ImageModeState;
onImageModeToggle: () => void;
+ onVideoModeToggle?: () => void;
imageModelLoaded: boolean;
supportsThinking: boolean;
supportsToolCalling: boolean;
@@ -88,7 +89,8 @@ interface QuickSettingsPopoverProps {
function getImageModeBadge(mode: ImageModeState, colors: any) {
if (mode === 'force') return { label: 'ON', bg: colors.primary };
- if (mode === 'disabled') return { label: 'OFF', bg: colors.textMuted };
+ if (mode === 'disabled' || mode === 'video')
+ return { label: 'OFF', bg: colors.textMuted };
return { label: 'Auto', bg: `${colors.textMuted}80` };
}
@@ -111,10 +113,20 @@ function getToolsStyle(supported: boolean, count: number, colors: any) {
}
export const QuickSettingsPopover: React.FC = ({
- visible, onClose, anchorY, anchorX,
- imageMode, onImageModeToggle, imageModelLoaded, supportsThinking,
- supportsToolCalling, enabledToolCount, onToolsPress,
- mcpToolCount = 0, onMcpPress,
+ visible,
+ onClose,
+ anchorY,
+ anchorX,
+ imageMode,
+ onImageModeToggle,
+ onVideoModeToggle,
+ imageModelLoaded,
+ supportsThinking,
+ supportsToolCalling,
+ enabledToolCount,
+ onToolsPress,
+ mcpToolCount = 0,
+ onMcpPress,
}) => {
const { colors } = useTheme();
const thinkingEnabled = useAppStore(state => state.settings.thinkingEnabled);
@@ -139,7 +151,12 @@ export const QuickSettingsPopover: React.FC = ({
const mcpBadgeBg = showMcpWarning ? TOOL_WARNING_COLOR : mcpDefaultBg;
return (
-
+
{/* accessible={false} on the SCRIM too. The inner wrapper was fixed first, but this outer
dismiss layer wraps the whole popover and merges it just the same - iOS reported one
control named ", Image Gen, Auto, , Thinking, ON, , Voice, Chat, , Tools, 1, Pro Tools, 6"
@@ -155,24 +172,72 @@ export const QuickSettingsPopover: React.FC = ({
reach one setting. Android exposes the rows individually; this is what made iOS
differ. */}
-
+
{ triggerHaptic('impactLight'); onImageModeToggle(); }}
+ onPress={() => {
+ triggerHaptic('impactLight');
+ onImageModeToggle();
+ }}
>
-
- Image Gen
-
- {imgBadge.label}
+
+
+ Image Gen
+
+
+
+ {imgBadge.label}
+
+
+
+
+ Video Gen
+
+
+ {imageMode === 'video' ? 'ON' : 'OFF'}
+
+
+
{supportsThinking && (
= ({
updateSettings({ thinkingEnabled: !thinkingEnabled });
}}
>
-
- Thinking
-
-
+
+
+ Thinking
+
+
+
{thinkingEnabled ? 'ON' : 'OFF'}
@@ -206,9 +291,25 @@ export const QuickSettingsPopover: React.FC = ({
}}
>
- Tools
-
- {tools.badgeLabel}
+
+ Tools
+
+
+
+ {tools.badgeLabel}
+
@@ -221,11 +322,29 @@ export const QuickSettingsPopover: React.FC = ({
onMcpPress?.();
}}
>
-
- Pro Tools
+
+
+ Pro Tools
+
{mcpToolCount > 0 && (
-
- {mcpToolCount}
+
+
+ {mcpToolCount}
+
)}
@@ -250,15 +369,25 @@ interface AttachPickerPopoverProps {
}
export const AttachPickerPopover: React.FC = ({
- visible, onClose, anchorY, anchorX,
- supportsVision, onPhoto, onDocument,
+ visible,
+ onClose,
+ anchorY,
+ anchorX,
+ supportsVision,
+ onPhoto,
+ onDocument,
}) => {
const { colors } = useTheme();
if (!visible) return null;
return (
-
+
{/* accessible={false} on the SCRIM too. The inner wrapper was fixed first, but this outer
dismiss layer wraps the whole popover and merges it just the same - iOS reported one
control named ", Image Gen, Auto, , Thinking, ON, , Voice, Chat, , Tools, 1, Pro Tools, 6"
@@ -274,24 +403,41 @@ export const AttachPickerPopover: React.FC = ({
reach one setting. Android exposes the rows individually; this is what made iOS
differ. */}
-
+
{ onClose(); onPhoto(); }}
+ onPress={() => {
+ onClose();
+ onPhoto();
+ }}
>
-
- Photo
+
+
+ Photo
+
{ onClose(); onDocument(); }}
+ onPress={() => {
+ onClose();
+ onDocument();
+ }}
>
Document
diff --git a/src/components/ChatInput/index.tsx b/src/components/ChatInput/index.tsx
index 215f1c5c7..5baee16e7 100644
--- a/src/components/ChatInput/index.tsx
+++ b/src/components/ChatInput/index.tsx
@@ -26,7 +26,12 @@ import { Button } from '../Button';
type AssistantAvailability = 'no-pro' | 'needs-sync' | 'unavailable' | 'ready';
interface ChatInputProps {
- onSend: (message: string, attachments?: MediaAttachment[], imageMode?: ImageModeState, assistantEnabled?: boolean) => void;
+ onSend: (
+ message: string,
+ attachments?: MediaAttachment[],
+ imageMode?: ImageModeState,
+ assistantEnabled?: boolean,
+ ) => void;
onStop?: () => void;
disabled?: boolean;
isGenerating?: boolean;
@@ -120,7 +125,15 @@ const buildNoVisionAlert = (opts: {
[
{ text: 'Cancel', onPress: opts.dismiss },
...(opts.onRepairVision
- ? [{ text: 'Go to Download Manager', onPress: () => { opts.dismiss(); opts.onRepairVision!(); } }]
+ ? [
+ {
+ text: 'Go to Download Manager',
+ onPress: () => {
+ opts.dismiss();
+ opts.onRepairVision!();
+ },
+ },
+ ]
: [{ text: 'OK' }]),
],
);
@@ -223,13 +236,26 @@ export const ChatInput: React.FC = ({
addAudioAttachment,
clearAttachments,
onHaptic: () => triggerHaptic('impactMedium'),
- appendTranscript: (text) => setMessage(prev => {
- const prefix = prev.trim() ? `${prev.trim()} ` : '';
- return prefix + text;
- }),
+ appendTranscript: text =>
+ setMessage(prev => {
+ const prefix = prev.trim() ? `${prev.trim()} ` : '';
+ return prefix + text;
+ }),
});
- const { isRecording, isModelLoading, isStartingRecording, isTranscribing, partialResult, error, voiceAvailable, isAwaitingSpeech, startRecording, stopRecording, cancelRecording } = useVoiceInput({
+ const {
+ isRecording,
+ isModelLoading,
+ isStartingRecording,
+ isTranscribing,
+ partialResult,
+ error,
+ voiceAvailable,
+ isAwaitingSpeech,
+ startRecording,
+ stopRecording,
+ cancelRecording,
+ } = useVoiceInput({
conversationId,
interfaceMode,
onTranscript: voiceHandlers.onTranscript,
@@ -279,7 +305,7 @@ export const ChatInput: React.FC = ({
setMessage('');
clearAttachments();
inputRef.current?.focus();
- if (imageMode === 'force') {
+ if (imageMode === 'force' || imageMode === 'video') {
setImageMode('auto');
onImageModeChange?.('auto');
}
@@ -341,26 +367,27 @@ export const ChatInput: React.FC = ({
}, ANIM_DURATION_IN);
};
- const assistantGate = assistantAvailability === 'no-pro'
- ? {
- title: 'Assistant requires Pro',
- body: 'Assistant uses Web Use and Computer Use on a connected Desktop.',
- action: 'View Pro',
- onPress: onAssistantUpgrade,
- }
- : assistantAvailability === 'unavailable'
+ const assistantGate =
+ assistantAvailability === 'no-pro'
? {
- title: 'Desktop tasks unavailable',
- body: 'This Desktop is connected, but Web Use and Computer Use are not available. Allow remote tasks on the Desktop and try again.',
- action: 'OK',
- onPress: undefined,
+ title: 'Assistant requires Pro',
+ body: 'Assistant uses Web Use and Computer Use on a connected Desktop.',
+ action: 'View Pro',
+ onPress: onAssistantUpgrade,
}
- : {
- title: 'Connect a Desktop',
- body: 'Set up Sync and connect Off Grid AI Desktop to use Web Use and Computer Use.',
- action: 'Set up Sync',
- onPress: onAssistantSetupSync,
- };
+ : assistantAvailability === 'unavailable'
+ ? {
+ title: 'Desktop tasks unavailable',
+ body: 'This Desktop is connected, but Web Use and Computer Use are not available. Allow remote tasks on the Desktop and try again.',
+ action: 'OK',
+ onPress: undefined,
+ }
+ : {
+ title: 'Connect a Desktop',
+ body: 'Set up Sync and connect Off Grid AI Desktop to use Web Use and Computer Use.',
+ action: 'Set up Sync',
+ onPress: onAssistantSetupSync,
+ };
const handleAttachPress = () => {
logger.log(`[COMPOSER-SM] attach pressed platform=${Platform.OS} supportsVision=${supportsVision}`);
@@ -370,7 +397,7 @@ export const ChatInput: React.FC = ({
: ['Document', 'Cancel'];
ActionSheetIOS.showActionSheetWithOptions(
{ options, cancelButtonIndex: options.length - 1 },
- (index) => {
+ index => {
if (supportsVision) {
if (index === 0) handleVisionPress();
else if (index === 1) handlePickDocument();
@@ -391,64 +418,70 @@ export const ChatInput: React.FC = ({
if (isAudioMode && AudioInput) {
return (
<>
-
- setAssistantGateOpen(false)}
- title={assistantGate.title}
- enableDynamicSizing
- >
-
- {assistantGate.body}
-
-
+
+ setImageMode(imageMode === 'video' ? 'auto' : 'video')
+ }
+ onThinkingToggle={handleThinkingToggle}
+ onToolsPress={onToolsPress}
+ onMcpPress={onMcpPress}
+ mcpToolCount={mcpToolCount}
+ onVisionPress={handleVisionPress}
+ onPickDocument={handlePickDocument}
+ onAttachPress={handleAttachPress}
+ attachPicker={attachPicker}
+ voicePicker={voicePicker}
+ quickSettings={quickSettings}
+ supportsVision={supportsVision}
+ alertState={alertState}
+ setAlertState={setAlertState}
+ assistantSelected={assistantEnabled}
+ onAssistantPress={handleAssistantPress}
+ />
+ setAssistantGateOpen(false)}
+ title={assistantGate.title}
+ enableDynamicSizing
+ >
+
+ {assistantGate.body}
+
+
);
}
@@ -513,16 +546,23 @@ export const ChatInput: React.FC = ({
size="small"
testID="assistant-toggle"
accessibilityLabel="Assistant"
- accessibilityState={{ selected: assistantEnabled, busy: assistantTransitioning }}
+ accessibilityState={{
+ selected: assistantEnabled,
+ busy: assistantTransitioning,
+ }}
onPress={handleAssistantPress}
active={assistantEnabled}
loading={assistantTransitioning}
style={styles.assistantButton}
- icon={}
+ icon={
+
+ }
/>
= ({
style={styles.pillInput}
value={message}
onChangeText={setMessage}
- placeholder={placeholder}
+ placeholder={
+ imageMode === 'video' ? 'Describe a video' : placeholder
+ }
placeholderTextColor={colors.textMuted}
multiline
scrollEnabled
@@ -575,6 +617,9 @@ export const ChatInput: React.FC = ({
anchorX={quickSettings.anchor.x}
imageMode={imageMode}
onImageModeToggle={handleImageModeToggle}
+ onVideoModeToggle={() =>
+ setImageMode(imageMode === 'video' ? 'auto' : 'video')
+ }
imageModelLoaded={imageModelLoaded}
supportsThinking={supportsThinking}
supportsToolCalling={supportsToolCalling}
diff --git a/src/components/ChatMessage/components/GenerationMeta.tsx b/src/components/ChatMessage/components/GenerationMeta.tsx
index 6abd19f30..bf044ebd2 100644
--- a/src/components/ChatMessage/components/GenerationMeta.tsx
+++ b/src/components/ChatMessage/components/GenerationMeta.tsx
@@ -33,6 +33,9 @@ function formatOptionalMeta(meta: NonNullable, tps: n
['steps', m.steps == null ? undefined : `${m.steps} steps`],
['cfg', m.guidanceScale == null ? undefined : `cfg ${m.guidanceScale}`],
['res', m.resolution],
+ ['frames', m.frames == null ? undefined : `${m.frames} frames`],
+ ['fps', m.fps == null ? undefined : `${m.fps} fps`],
+ ['seed', m.seed == null ? undefined : `Seed: ${m.seed}`],
['cache', m.cacheType ? `KV ${m.cacheType}` : undefined],
];
return entries
diff --git a/src/components/ChatMessage/components/MessageAttachments.tsx b/src/components/ChatMessage/components/MessageAttachments.tsx
index 619639f62..a4e8de921 100644
--- a/src/components/ChatMessage/components/MessageAttachments.tsx
+++ b/src/components/ChatMessage/components/MessageAttachments.tsx
@@ -10,8 +10,14 @@ import Animated, {
useSharedValue,
useAnimatedStyle,
withTiming,
+ useReducedMotion,
} from 'react-native-reanimated';
import Icon from 'react-native-vector-icons/Feather';
+import VideoPlayer, { type VideoPlayerRef } from 'react-native-video-player';
+import { ResizeMode } from 'react-native-video';
+import { useTheme } from '../../../theme';
+import { SPACING, TYPOGRAPHY } from '../../../constants';
+import { Button } from '../../Button';
// Imported directly, not through the barrel: a component that reaches its sibling via the index
// resolves undefined at render time.
import { LoadingDots } from '../../LoadingDots';
@@ -26,13 +32,14 @@ interface FadeInImageProps {
testID?: string;
wrapperTestID?: string;
onPress?: () => void;
+ accessibilityLabel?: string;
}
function resolveMediaUri(uri: string): string {
return uri.includes('/Documents/') ? `file://${resolveDocumentPath(uri)}` : uri;
}
-function FadeInImage({ uri, imageStyle, testID, wrapperTestID, onPress }: FadeInImageProps) {
+export function FadeInImage({ uri, imageStyle, testID, wrapperTestID, onPress, accessibilityLabel }: FadeInImageProps) {
const displayUri = resolveMediaUri(uri);
const opacity = useSharedValue(0);
const [loaded, setLoaded] = React.useState(false);
@@ -44,10 +51,11 @@ function FadeInImage({ uri, imageStyle, testID, wrapperTestID, onPress }: FadeIn
testID={wrapperTestID}
style={fadeInImageStyles.wrapper}
onPress={onPress}
+ disabled={!onPress}
activeOpacity={0.8}
- accessibilityRole="button"
+ accessibilityRole={onPress ? 'button' : 'image'}
accessibilityLabel={
- isGeneratedImage ? `Generated image ${loaded ? 'loaded' : 'loading'}` : undefined
+ accessibilityLabel ?? (isGeneratedImage ? `Generated image ${loaded ? 'loaded' : 'loading'}` : undefined)
}
>
void;
}
+/** The library owns playback and controls; this adapter supplies attachment data and theme. */
+function VideoAttachment({ attachment }: { attachment: MediaAttachment }) {
+ const { colors } = useTheme();
+ const player = React.useRef(null);
+ const reducedMotion = useReducedMotion();
+ const [size, setSize] = React.useState({
+ width: attachment.width || 16,
+ height: attachment.height || 9,
+ });
+ const [loaded, setLoaded] = React.useState(false);
+ const [error, setError] = React.useState(null);
+ return (
+
+ }
+ style={{ width: SPACING.xl * 2, height: SPACING.xl * 2, paddingHorizontal: 0 }}
+ onPress={async () => {
+ try {
+ await player.current?.enterPictureInPicture();
+ } catch {
+ setError('Picture in picture is not available on this device.');
+ }
+ }}
+ />
+ }
+ pauseOnPress
+ animationDuration={reducedMotion ? 0 : 150}
+ playInBackground
+ playWhenInactive
+ ignoreSilentSwitch="ignore"
+ enterPictureInPictureOnLeave
+ onRestoreUserInterfaceForPictureInPictureStop={() => {
+ player.current?.restoreUserInterfaceForPictureInPictureStopCompleted(true);
+ }}
+ onLoad={({ naturalSize }) => {
+ setLoaded(true);
+ setError(null);
+ if (naturalSize.width > 0 && naturalSize.height > 0) {
+ setSize({ width: naturalSize.width, height: naturalSize.height });
+ }
+ }}
+ onError={() => setError('This video could not be played.')}
+ customStyles={{
+ wrapper: { width: '100%', borderRadius: SPACING.sm, overflow: 'hidden' },
+ controls: { backgroundColor: colors.surface, height: SPACING.xl * 2, marginTop: 0 },
+ controlButton: { width: SPACING.xl * 2, height: SPACING.xl * 2, padding: SPACING.sm, alignItems: 'center', justifyContent: 'center' },
+ controlIcon: { tintColor: colors.text, width: SPACING.lg, height: SPACING.lg },
+ playArrow: { tintColor: colors.text, width: SPACING.lg, height: SPACING.lg, marginLeft: 0 },
+ playButton: { backgroundColor: colors.surface, width: SPACING.xl * 2, height: SPACING.xl * 2, borderRadius: SPACING.sm },
+ seekBar: { flex: 1, minWidth: 0, paddingHorizontal: SPACING.xs, marginLeft: 0, marginRight: SPACING.sm },
+ seekBarProgress: { backgroundColor: colors.primary },
+ seekBarKnob: { backgroundColor: colors.primary, width: SPACING.sm, height: SPACING.sm, marginHorizontal: -SPACING.xs, marginVertical: 0 },
+ seekBarBackground: { backgroundColor: colors.border },
+ durationText: { ...TYPOGRAPHY.meta, color: colors.textSecondary, flexShrink: 0 },
+ }}
+ />
+ {error && {error}}
+
+
+ );
+}
+
function AudioAttachment({
index,
isUser,
@@ -197,6 +283,8 @@ export function MessageAttachments({
styles={styles}
colors={colors}
/>
+ ) : attachment.type === 'video' ? (
+
) : attachment.type === 'audio' ? (
= ({
onLongPress,
onMenuOpen,
}) => {
+ const savedVideo = useAppStore(state =>
+ message.role === 'assistant'
+ ? state.generatedVideos.find(video =>
+ message.attachments?.some(attachment =>
+ attachment.type === 'video' && attachment.id === video.id,
+ ),
+ )
+ : undefined,
+ );
+ const generationMeta = savedVideo
+ ? { ...videoGenerationMeta(savedVideo), ...message.generationMeta }
+ : message.generationMeta;
const timelineHasThinking = Boolean(
message.timeline?.some(entry => entry.kind === 'thinking'),
);
@@ -227,6 +241,12 @@ const MessageBubble: React.FC = ({
const answerParsedContent = hasAssistantWork
? { ...parsedContent, thinking: '' }
: parsedContent;
+ const isGeneratedVideo = Boolean(
+ !isUser &&
+ message.turnKind === 'video' &&
+ message.attachments?.length &&
+ message.attachments.every(attachment => attachment.type === 'video'),
+ );
const hasVisibleAnswer = Boolean(
hasAttachments || answerParsedContent.response.trim(),
);
@@ -313,7 +333,13 @@ const MessageBubble: React.FC = ({
{(!hideProse || hasAttachments) && showAnswerBubble && (
{hasAttachments && (
= ({
/>
)}
-
+ {!isGeneratedVideo && (
+
+ )}
)}
@@ -368,10 +396,10 @@ const MessageBubble: React.FC = ({
)}
- {showTurnFooter && showGenerationDetails && message.generationMeta && (
+ {showTurnFooter && showGenerationDetails && generationMeta && (
diff --git a/src/components/ChatMessage/styles.ts b/src/components/ChatMessage/styles.ts
index 8d6742ee7..b233011cd 100644
--- a/src/components/ChatMessage/styles.ts
+++ b/src/components/ChatMessage/styles.ts
@@ -38,6 +38,10 @@ const createBubbleStyles = (colors: ThemeColors, shadows: ThemeShadows) => ({
width: MESSAGE_MAX_WIDTH,
alignSelf: 'flex-start' as const,
},
+ videoMessage: {
+ width: MESSAGE_MAX_WIDTH,
+ alignSelf: 'flex-start' as const,
+ },
toolCallPreText: {
alignSelf: 'flex-start' as const,
paddingBottom: 6,
diff --git a/src/components/GenerationSettingsModal/ImageQualitySliders.tsx b/src/components/GenerationSettingsModal/ImageQualitySliders.tsx
index f0c6607e9..fa252900b 100644
--- a/src/components/GenerationSettingsModal/ImageQualitySliders.tsx
+++ b/src/components/GenerationSettingsModal/ImageQualitySliders.tsx
@@ -1,12 +1,11 @@
import React from 'react';
import { View, Text, Switch, Platform, TouchableOpacity } from 'react-native';
+import { Button } from '../Button';
import { SliderSetting } from '../SliderSetting';
import { useTheme, useThemedStyles } from '../../theme';
import { useAppStore } from '../../stores';
-import { DEFAULT_SETTINGS } from '../../stores/appStore';
-import { useClearGpuCache } from '../../hooks/useImageGenerationSettings';
+import { useClearGpuCache, useImageParameterSettings } from '../../hooks/useImageGenerationSettings';
import {
- defaultImageSteps,
MAX_IMAGE_STEPS,
SWEET_SPOT_SIZE,
} from '../../utils/imageGenAdvice';
@@ -32,6 +31,7 @@ const ClearGPUCacheButton: React.FC = () => {
/** Basic controls: Image Steps + Image Size */
export const ImageQualityBasicSliders: React.FC = () => {
+ const { parameters, defaults, maxSize, applyDefaults } = useImageParameterSettings();
const { settings, updateSettings } = useAppStore();
return (
@@ -41,7 +41,7 @@ export const ImageQualityBasicSliders: React.FC = () => {
compact
label="Image Steps"
description="4-8 steps for speed, 20-50 for quality"
- value={settings.imageSteps || defaultImageSteps(Platform.OS)}
+ value={parameters.steps}
min={4} max={MAX_IMAGE_STEPS} step={1}
onChange={(value) => updateSettings({ imageSteps: value })}
/>
@@ -50,18 +50,20 @@ export const ImageQualityBasicSliders: React.FC = () => {
testID="image-size"
compact
label="Image Size"
- description="Output resolution. Default: 512x512."
- value={Math.max(SWEET_SPOT_SIZE, settings.imageWidth ?? DEFAULT_SETTINGS.imageWidth)}
- min={SWEET_SPOT_SIZE} max={512} step={64}
+ description={`Output resolution. Recommended: ${defaults.size}x${defaults.size}.`}
+ value={parameters.size}
+ min={SWEET_SPOT_SIZE} max={maxSize} step={64}
formatValue={(v) => `${v}x${v}`}
onChange={(value) => updateSettings({ imageWidth: value, imageHeight: value })}
/>
+
);
};
/** Advanced controls: Guidance Scale, Image Threads, GPU Acceleration */
export const ImageQualityAdvancedSliders: React.FC = () => {
+ const { parameters } = useImageParameterSettings();
const { colors } = useTheme();
const styles = useThemedStyles(createStyles);
const { settings, updateSettings } = useAppStore();
@@ -73,7 +75,7 @@ export const ImageQualityAdvancedSliders: React.FC = () => {
compact
label="Guidance Scale"
description="Higher = follows prompt more strictly (5-15 range)"
- value={settings.imageGuidanceScale || 7.5}
+ value={parameters.guidanceScale}
min={1} max={20} step={0.5} decimals={1}
onChange={(value) => updateSettings({ imageGuidanceScale: value })}
/>
diff --git a/src/components/GenerationSettingsModal/TextGenerationSection.tsx b/src/components/GenerationSettingsModal/TextGenerationSection.tsx
index 9f959a900..65d6a00b4 100644
--- a/src/components/GenerationSettingsModal/TextGenerationSection.tsx
+++ b/src/components/GenerationSettingsModal/TextGenerationSection.tsx
@@ -1,22 +1,19 @@
import React, { useState } from 'react';
-import { View } from 'react-native';
+import { Text, View } from 'react-native';
import { AdvancedToggle } from '../AdvancedToggle';
import { SliderSetting } from '../SliderSetting';
import { useThemedStyles } from '../../theme';
import { createStyles } from './styles';
+import { SystemPromptSection } from '../../screens/ModelSettingsScreen/SystemPromptSection';
import {
type NumericSettingModel,
useTextGenerationSettings,
} from '../../hooks/useTextGenerationSettings';
import {
- BackendSelector,
BatchSizeSlider,
CpuThreadsSlider,
FlashAttentionToggle,
KvCacheTypeToggle,
- LiteRTBackendSelector,
- ModelLoadingModeSelector,
- ShowGenerationDetailsToggle,
SpeculativeDecodingToggle,
ThinkingBudgetSelector,
} from '../settings/textGenAdvancedSections';
@@ -38,11 +35,12 @@ export const TextGenerationSection: React.FC = () => {
return (
+ Default System Prompt
+
{basicSettings.map(setting => (
))}
{!isLiteRT && }
-
setShowAdvanced(current => !current)}
@@ -53,20 +51,13 @@ export const TextGenerationSection: React.FC = () => {
{advancedSettings.map(setting => (
))}
- {isLiteRT ? (
- <>
-
-
-
- ) : (
+ {!isLiteRT && (
<>
-
-
)}
diff --git a/src/components/GenerationSettingsModal/VideoGenerationSection.tsx b/src/components/GenerationSettingsModal/VideoGenerationSection.tsx
new file mode 100644
index 000000000..7cb08a297
--- /dev/null
+++ b/src/components/GenerationSettingsModal/VideoGenerationSection.tsx
@@ -0,0 +1,147 @@
+import { videoModelDisplayName } from '../../utils/modelHelpers';
+import { Accordion } from '../Accordion';
+import { useRemoteServerStore } from '../../stores/remoteServerStore';
+import React, { useEffect, useState } from 'react';
+import { Text, TextInput, View, Switch } from 'react-native';
+import { videoModelDefaults, videoModelLimits } from '@offgrid/models';
+import { useAppStore } from '../../stores';
+import { useTheme, useThemedStyles } from '../../theme';
+import { SliderSetting } from '../SliderSetting';
+import { createStyles } from './styles';
+
+const CONTROLS = [
+ { key: 'width', label: 'Width', min: 256, max: 832, step: 16 },
+ { key: 'height', label: 'Height', min: 192, max: 480, step: 16 },
+ { key: 'frames', label: 'Frames', min: 9, max: 81, step: 4 },
+ { key: 'fps', label: 'Frames per second', min: 4, max: 24, step: 1 },
+ { key: 'steps', label: 'Steps', min: 4, max: 50, step: 1 },
+ { key: 'guidance', label: 'Guidance', min: 0, max: 20, step: 0.5 },
+] as const;
+export const VideoGenerationSection: React.FC = () => {
+ const {
+ settings,
+ updateSettings,
+ activeVideoModelId,
+ downloadedVideoModels,
+ } = useAppStore();
+ const { colors } = useTheme(),
+ styles = useThemedStyles(createStyles);
+ const model = downloadedVideoModels.find(m => m.id === activeVideoModelId);
+ const remoteName = useRemoteServerStore(
+ s =>
+ s.servers.find(server => server.id === s.activeRemoteMediaServerIds.video)
+ ?.mediaModels?.video,
+ );
+ const key =
+ remoteName ??
+ model?.files.find(f => f.role === 'primary')?.name ??
+ 'default';
+ const saved =
+ settings.videoParams?.[key] ?? settings.videoParams?.default ?? {};
+ const limits = videoModelLimits(key);
+ const values = { ...videoModelDefaults(key), ...saved };
+ const controls = CONTROLS.map(control => ({
+ ...control,
+ ...(control.key === 'width'
+ ? { max: limits.widthMax, step: limits.sizeStep }
+ : {}),
+ ...(control.key === 'height'
+ ? { max: limits.heightMax, step: limits.sizeStep }
+ : {}),
+ ...(control.key === 'frames'
+ ? { max: limits.framesMax, step: limits.frameStep }
+ : {}),
+ }));
+ const [seedText, setSeedText] = useState(
+ settings.videoSeed === -1 ? '' : String(settings.videoSeed ?? ''),
+ );
+ useEffect(
+ () =>
+ setSeedText(
+ settings.videoSeed === -1 ? '' : String(settings.videoSeed ?? ''),
+ ),
+ [settings.videoSeed],
+ );
+ const saveSeed = () => {
+ const seed = seedText.trim() === '' ? -1 : Number(seedText);
+ if (Number.isInteger(seed) && seed >= -1 && seed <= 2147483647)
+ updateSettings({ videoSeed: seed });
+ else
+ setSeedText(
+ settings.videoSeed === -1 ? '' : String(settings.videoSeed ?? ''),
+ );
+ };
+ return (
+
+
+ {remoteName
+ ? videoModelDisplayName(remoteName)
+ : model
+ ? videoModelDisplayName(model.id, model.name)
+ : 'Default video settings'}
+
+ {controls.map(control => (
+
+ updateSettings({
+ videoParams: {
+ ...settings.videoParams,
+ [key]: {
+ ...saved,
+ [control.key]:
+ control.key === 'frames'
+ ? 1 +
+ Math.round((value - 1) / limits.frameStep) *
+ limits.frameStep
+ : control.key === 'width' || control.key === 'height'
+ ? Math.round(value / limits.sizeStep) * limits.sizeStep
+ : value,
+ },
+ },
+ })
+ }
+ />
+ ))}
+ Seed
+
+
+ Leave blank for a new seed with each video.
+
+ Negative prompt
+ updateSettings({ videoNegative })}
+ />
+
+ Enhance video prompts
+
+ updateSettings({ enhanceVideoPrompts })
+ }
+ />
+
+
+ );
+};
diff --git a/src/components/GenerationSettingsModal/index.tsx b/src/components/GenerationSettingsModal/index.tsx
index a295a26f6..4ae63eea9 100644
--- a/src/components/GenerationSettingsModal/index.tsx
+++ b/src/components/GenerationSettingsModal/index.tsx
@@ -1,3 +1,4 @@
+import { VideoGenerationSection } from './VideoGenerationSection';
import React, { useState, useEffect } from 'react';
import { View, Text, ScrollView, TouchableOpacity } from 'react-native';
import Icon from 'react-native-vector-icons/Feather';
@@ -7,6 +8,7 @@ import { useAppStore } from '../../stores';
import { llmService } from '../../services';
import { createStyles } from './styles';
import { VoiceTurnSettings } from '../settings/voiceSections';
+import { GeneralGenerationControls } from '../settings/GeneralGenerationControls';
import { ConversationActionsSection } from './ConversationActionsSection';
import { ImageGenerationSection } from './ImageGenerationSection';
import { TextGenerationSection } from './TextGenerationSection';
@@ -25,7 +27,9 @@ interface GenerationSettingsModalProps {
isRemote?: boolean;
}
-export const GenerationSettingsModal: React.FC = ({
+export const GenerationSettingsModal: React.FC<
+ GenerationSettingsModalProps
+> = ({
visible,
onClose,
onOpenProject,
@@ -42,6 +46,7 @@ export const GenerationSettingsModal: React.FC = (
const [performanceStats, setPerformanceStats] = useState(llmService.getPerformanceStats());
const [imageSettingsOpen, setImageSettingsOpen] = useState(false);
+ const [generalSettingsOpen, setGeneralSettingsOpen] = useState(false);
const [textSettingsOpen, setTextSettingsOpen] = useState(false);
const [sttSettingsOpen, setSttSettingsOpen] = useState(false);
const [ttsSettingsOpen, setTtsSettingsOpen] = useState(false);
@@ -95,12 +100,22 @@ export const GenerationSettingsModal: React.FC = (
activeProjectName={activeProjectName}
/>
+ setGeneralSettingsOpen(!generalSettingsOpen)}
+ activeOpacity={0.7}
+ testID="modal-general-accordion"
+ accessibilityRole="button"
+ accessibilityState={{ expanded: generalSettingsOpen }}
+ >
+ GENERAL
+
+
+ {generalSettingsOpen && }
+
{/* IMAGE GENERATION SETTINGS */}
setImageSettingsOpen(!imageSettingsOpen)}
activeOpacity={0.7}
testID="modal-image-accordion"
@@ -113,6 +128,7 @@ export const GenerationSettingsModal: React.FC = (
/>
{imageSettingsOpen && }
+
{/* TEXT GENERATION SETTINGS */}
= (
- These settings only apply to local models and won't affect the current remote session.
+ These settings only apply to local models and won't affect the
+ current remote session.
)}
diff --git a/src/components/ModelSelectorModal/VideoTab.tsx b/src/components/ModelSelectorModal/VideoTab.tsx
new file mode 100644
index 000000000..61bfaff4c
--- /dev/null
+++ b/src/components/ModelSelectorModal/VideoTab.tsx
@@ -0,0 +1,48 @@
+import { videoModelDisplayName } from '../../utils/modelHelpers';
+import { RemoteModelOptionsSection } from '../models/RemoteModelOptionsSection';
+import { remoteServerManager } from '../../services/remoteServerManager';
+import React from 'react';
+import { Text } from 'react-native';
+import { useAppStore } from '../../stores';
+import { ModelCard } from '../ModelCard';
+import { Button } from '../Button';
+import { useTheme } from '../../theme';
+import { videoGenerationService } from '../../services/videoGenerationService';
+export function VideoTab({ onSelect }: { onSelect: () => void }) {
+ const models = useAppStore(s => s.downloadedVideoModels),
+ selected = useAppStore(s => s.activeVideoModelId);
+ const { colors } = useTheme();
+ return (
+ <>
+
+ {!models.length && (
+
+ Download a video model from Models.
+
+ )}
+ {models.map(model => (
+ {
+ remoteServerManager.clearActiveRemoteMediaModel('video');
+ useAppStore.getState().setActiveVideoModelId(model.id);
+ onSelect();
+ }}
+ />
+ ))}
+ {selected && (
+
{downloadedImageModels.length}
+
+
+
+ Video Models
+
+ {downloadedVideoModels.length}
+
@@ -189,15 +220,22 @@ export const StorageSettingsScreen: React.FC = () => {
Clear All
-
- These download entries have invalid or missing data and can be safely cleared.
+
+ These download entries have invalid or missing data and can be
+ safely cleared.
{staleDownloads.map(entry => (
Download #{entry.downloadId}
- {entry.fileName || 'Unknown file'} • {entry.modelId || 'Unknown model'}
+ {entry.fileName || 'Unknown file'} •{' '}
+ {entry.modelId || 'Unknown model'}
!onnxImageGeneratorService.isGenerating(), sizeMB: Math.round((hardwareService.estimateImageModelRam(model) || 0) / (1024 * 1024)) },
() => this.doUnloadImageModelLocked(true), // eviction keeps the selection
);
},
@@ -376,7 +376,9 @@ class ActiveModelService {
this.notifyListeners();
try {
if (isNativeLoaded) {
- await onnxImageGeneratorService.unloadModel();
+ if (!(await onnxImageGeneratorService.unloadModel())) {
+ throw new Error('The image model is still in use and could not be unloaded.');
+ }
}
this.loadedImageModelId = null;
this.loadedImageModelThreads = null;
diff --git a/src/services/activeModelService/loaders.ts b/src/services/activeModelService/loaders.ts
index 294c7f724..aa5cc80f1 100644
--- a/src/services/activeModelService/loaders.ts
+++ b/src/services/activeModelService/loaders.ts
@@ -281,27 +281,45 @@ export async function doLoadImageModel(ctx: ImageLoadContext): Promise {
}
let imgTimeoutId: ReturnType | null = null;
+ let loadTimedOut = false;
const timeoutPromise = new Promise((_, reject) => {
imgTimeoutId = setTimeout(
- () => reject(new Error('Image model loading timed out')),
+ () => {
+ loadTimedOut = true;
+ reject(new Error('Image model loading timed out'));
+ },
ctx.timeoutMs,
);
});
-
+ const nativeLoad = onnxImageGeneratorService.loadModel(
+ ctx.model.modelPath,
+ ctx.imageThreads,
+ {
+ backend: ctx.model.backend === 'sd' ? 'sd' : 'auto',
+ modelId: ctx.model.id,
+ cpuOnly: ctx.cpuOnly,
+ attentionVariant: ctx.model.attentionVariant,
+ preferGpu: ctx.preferGpu,
+ },
+ );
try {
- await Promise.race([
- onnxImageGeneratorService.loadModel(
- ctx.model.modelPath,
- ctx.imageThreads,
- {
- backend: 'auto',
- cpuOnly: ctx.cpuOnly,
- attentionVariant: ctx.model.attentionVariant,
- preferGpu: ctx.preferGpu,
- },
- ),
- timeoutPromise,
- ]);
+ await Promise.race([nativeLoad, timeoutPromise]);
+ } catch (error) {
+ if (loadTimedOut && ctx.model.backend === 'sd') {
+ // A JS timeout cannot stop native model allocation. Retain the residency
+ // lock until it settles, then remove any late context before admitting
+ // another modality. Otherwise that context is resident but unregistered.
+ logger.warn('[Image] SD load timed out; waiting for native cleanup');
+ await nativeLoad.catch(() => {});
+ if (!(await onnxImageGeneratorService.unloadModel())) {
+ // Cleanup failure is not proof that native memory was released. Keep
+ // the existing residency registration/eviction path responsible for it.
+ ctx.onLoaded(ctx.modelId, ctx.imageThreads);
+ throw new Error('The timed-out image model could not be unloaded.');
+ }
+ modelResidencyManager.release('image');
+ }
+ throw error;
} finally {
if (imgTimeoutId !== null) clearTimeout(imgTimeoutId);
}
diff --git a/src/services/autoSetupCatalog.ts b/src/services/autoSetupCatalog.ts
index 0670bef13..dbb253f04 100644
--- a/src/services/autoSetupCatalog.ts
+++ b/src/services/autoSetupCatalog.ts
@@ -1,3 +1,4 @@
+import { CATALOG, videoPackError } from '@offgrid/models';
import { recommendedModelsForDevice, ramFitScore } from '../utils/recommendedModels';
import { fileExceedsBudget } from './memoryBudget';
import { fetchModelFiles } from './modelCatalogFiles';
@@ -5,20 +6,24 @@ import { hardwareService } from './hardware';
import { WHISPER_MODELS } from './whisperModels';
import type { AutoSetupCompatibleCatalog } from './autoSetupPlan';
import { autoSetupImageCatalogProvider } from './autoSetupImageCatalogProvider';
+import { BUNDLED_EMBEDDING_MODEL, RECOMMENDED_EMBEDDING_MODELS } from './huggingFaceModelBrowser';
const MB = 1024 * 1024;
type CompatibleTextModel = ReturnType[number];
export interface AutoSetupCatalogBoundaries {
- totalMemoryGB: () => number;
+ totalMemoryGB: () => number | Promise;
fetchTextFiles: typeof fetchModelFiles;
imageRecommendation: typeof hardwareService.getImageModelRecommendation;
imageModels: typeof autoSetupImageCatalogProvider.load;
}
const productionCatalogBoundaries: AutoSetupCatalogBoundaries = {
- totalMemoryGB: () => hardwareService.getTotalMemoryGB(),
+ totalMemoryGB: async () => {
+ await hardwareService.getDeviceInfo();
+ return hardwareService.getTotalMemoryGB();
+ },
fetchTextFiles: fetchModelFiles,
imageRecommendation: () => hardwareService.getImageModelRecommendation(),
imageModels: () => autoSetupImageCatalogProvider.load(),
@@ -49,7 +54,7 @@ export function buildAutoSetupTextCandidates(
export async function loadAutoSetupCompatibleCatalog(
boundaries: AutoSetupCatalogBoundaries = productionCatalogBoundaries,
): Promise {
- const ramGB = boundaries.totalMemoryGB();
+ const ramGB = await boundaries.totalMemoryGB();
const textModels = recommendedModelsForDevice(ramGB).filter(model => model.type === 'vision');
const files = await boundaries.fetchTextFiles(textModels);
const text = buildAutoSetupTextCandidates(textModels, files, ramGB);
@@ -87,5 +92,28 @@ export async function loadAutoSetupCompatibleCatalog(
payload: { modelId: model.id },
}));
- return { text, image, stt };
+ const completeVideoPacks = CATALOG.filter(model =>
+ model.kind === 'video' && model.availability !== 'coming_soon' &&
+ !!model.minRamGb && !videoPackError(model.files) &&
+ model.files.every(file => !!file.sizeBytes),
+ );
+ const video = completeVideoPacks.map(model => {
+ const sizeBytes = model.files.reduce((sum, file) => sum + (file.sizeBytes ?? 0), 0);
+ return {
+ id: model.id, name: model.name, kind: 'video' as const, sizeBytes,
+ fitScore: ramFitScore(model.minRamGb!, ramGB), payload: model,
+ };
+ });
+ const videoExclusionReason = video.length ? undefined
+ : 'Auto Setup has no complete video model to download.';
+ const embedding = [BUNDLED_EMBEDDING_MODEL, ...RECOMMENDED_EMBEDDING_MODELS]
+ .map((model, index) => ({
+ id: model.id,
+ name: model.name,
+ kind: 'embedding' as const,
+ sizeBytes: model.size,
+ fitScore: index === 1 ? 0 : index,
+ payload: model,
+ }));
+ return { text, image, stt, video, videoExclusionReason, embedding };
}
diff --git a/src/services/autoSetupPlan.ts b/src/services/autoSetupPlan.ts
index 199fe1984..3c47c9a6f 100644
--- a/src/services/autoSetupPlan.ts
+++ b/src/services/autoSetupPlan.ts
@@ -1,8 +1,18 @@
+import type { ModelEntry } from '@offgrid/models';
import type { ModelFile } from '../types';
import type { ImageModelDescriptor } from './imageModelDownloadTypes';
export type AutoSetupTier = 'lean' | 'balanced' | 'extreme';
-type AutoSetupModelKind = 'text' | 'image' | 'stt';
+export type AutoSetupModelKind = 'text' | 'image' | 'stt' | 'video' | 'embedding';
+
+export interface AutoSetupEmbeddingModel {
+ id: string;
+ name: string;
+ size: number;
+ downloadUrl?: string;
+ sha256?: string;
+ description?: string;
+}
interface AutoSetupCandidate {
id: string;
@@ -22,11 +32,19 @@ export interface AutoSetupPlan {
AutoSetupCandidate<{ modelId: string; file: ModelFile }>,
AutoSetupCandidate,
AutoSetupCandidate<{ modelId: string }>,
+ ...AutoSetupCandidate[],
];
totalBytes: number;
+ videoExclusionReason?: string;
+ embedding?: AutoSetupCandidate;
}
+export type AutoSetupItem = AutoSetupPlan['items'][number] | NonNullable;
+
export interface AutoSetupCompatibleCatalog {
+ video?: AutoSetupCandidate[];
+ videoExclusionReason?: string;
+ embedding?: AutoSetupCandidate[];
text: AutoSetupCandidate<{ modelId: string; file: ModelFile }>[];
image: AutoSetupCandidate[];
stt: AutoSetupCandidate<{ modelId: string }>[];
@@ -36,11 +54,11 @@ const PLAN_COPY: Record<
AutoSetupTier,
Pick
> = {
- lean: { title: 'Lean', summary: 'Small downloads with lower memory use.' },
- balanced: { title: 'Balanced', summary: 'The best balance for this device.' },
+ lean: { title: 'Lean', summary: 'Smaller model downloads.' },
+ balanced: { title: 'Balanced', summary: 'Models near the middle of each list.' },
extreme: {
title: 'Extreme',
- summary: 'The largest safe models for this device.',
+ summary: 'Largest model downloads.',
},
};
@@ -111,12 +129,16 @@ function selectAutoSetupPlan(
: selectedImage);
const stt = choose(tier, catalog.stt);
if (!text || !image || !stt) return null;
- const items: AutoSetupPlan['items'] = [text, image, stt];
+ const video = choose(tier, catalog.video ?? []);
+ const embedding = choose(tier, catalog.embedding ?? []);
+ const items: AutoSetupPlan['items'] = [text, image, stt, ...(video ? [video] : [])];
return {
tier,
...PLAN_COPY[tier],
items,
- totalBytes: items.reduce((total, item) => total + item.sizeBytes, 0),
+ totalBytes: items.reduce((total, item) => total + item.sizeBytes, embedding?.sizeBytes ?? 0),
+ ...(embedding ? { embedding } : {}),
+ ...(video ? {} : { videoExclusionReason: catalog.videoExclusionReason }),
};
}
diff --git a/src/services/autoSetupService.ts b/src/services/autoSetupService.ts
index f880be1f7..18ab8d2d4 100644
--- a/src/services/autoSetupService.ts
+++ b/src/services/autoSetupService.ts
@@ -3,12 +3,16 @@ import type { ModelDownload, ModelDownloadStartRequest } from './modelDownloadSe
import { useAppStore } from '../stores';
import { activeModelService } from './activeModelService';
import { uniformDownloadId } from './modelDownloadService/uniformId';
+import { embeddingService } from './rag/embedding';
import {
loadAutoSetupCompatibleCatalog,
type AutoSetupCatalogBoundaries,
} from './autoSetupCatalog';
import {
selectAutoSetupPlans,
+ type AutoSetupEmbeddingModel,
+ type AutoSetupItem,
+ type AutoSetupModelKind,
type AutoSetupPlan,
type AutoSetupTier,
} from './autoSetupPlan';
@@ -20,6 +24,11 @@ export interface AutoSetupDownloadBoundaries {
subscribe: (listener: () => void) => () => void;
}
+export interface AutoSetupEmbeddingBoundaries {
+ isDownloaded: (model: AutoSetupEmbeddingModel) => Promise;
+ start: (model: AutoSetupEmbeddingModel, progress: (fraction: number) => void, signal: AbortSignal) => Promise;
+}
+
const productionDownloadBoundaries: AutoSetupDownloadBoundaries = {
start: request => modelDownloadService.start(request),
list: () => modelDownloadService.list(),
@@ -27,6 +36,21 @@ const productionDownloadBoundaries: AutoSetupDownloadBoundaries = {
subscribe: listener => modelDownloadService.subscribe(listener),
};
+const productionEmbeddingBoundaries: AutoSetupEmbeddingBoundaries = {
+ isDownloaded: model => model.size === 0
+ ? Promise.resolve(true)
+ : embeddingService.isModelDownloaded(model),
+ async start(model, progress, signal) {
+ if (!model.downloadUrl) throw new Error('This embedding model has no download file.');
+ await embeddingService.downloadModel(
+ { ...model, downloadUrl: model.downloadUrl },
+ () => undefined,
+ signal,
+ progress,
+ );
+ },
+};
+
type AutoSetupItemPhase =
| 'waiting'
| 'starting'
@@ -46,6 +70,8 @@ interface AutoSetupSnapshot {
phase: 'loading_catalog' | 'ready' | 'downloading' | 'completed' | 'failed';
plans: AutoSetupPlan[];
selectedTier: AutoSetupTier;
+ selectedKinds: AutoSetupModelKind[];
+ installedIds: string[];
outcomes: Record;
error: string | null;
}
@@ -55,7 +81,9 @@ export interface AutoSetupSession {
subscribe(listener: () => void): () => void;
load(): Promise;
selectTier(tier: AutoSetupTier): void;
+ toggleKind(kind: AutoSetupModelKind): void;
start(): Promise;
+ cancel(): Promise;
complete(): void;
dispose(): void;
}
@@ -63,16 +91,11 @@ export interface AutoSetupSession {
export interface AutoSetupSessionBoundaries {
catalog?: AutoSetupCatalogBoundaries;
downloads?: AutoSetupDownloadBoundaries;
+ embedding?: AutoSetupEmbeddingBoundaries;
catalogDeadlineMs?: number;
}
const DEFAULT_CATALOG_DEADLINE_MS = 15_000;
-const TIER_POLICY = {
- lean: 'conservative',
- balanced: 'balanced',
- extreme: 'aggressive',
-} as const;
-
function tierFromPersistedIntent(): AutoSetupTier {
const mode = useAppStore.getState().settings.modelLoadingMode;
if (mode === 'conservative') return 'lean';
@@ -80,18 +103,22 @@ function tierFromPersistedIntent(): AutoSetupTier {
return 'balanced';
}
-function persistTierIntent(tier: AutoSetupTier): void {
- useAppStore
- .getState()
- .updateSettings({ modelLoadingMode: TIER_POLICY[tier] });
-}
-
export function autoSetupDownloadId(
- item: AutoSetupPlan['items'][number],
+ item: AutoSetupItem,
): string {
+ if (item.kind === 'embedding') return `embedding:${item.id}`;
return uniformDownloadId(item.kind, item.id);
}
+export function autoSetupItemNeedsAction(
+ item: AutoSetupItem,
+ installedIds: readonly string[],
+): boolean {
+ return item.kind !== 'embedding' || (
+ item.sizeBytes > 0 && !installedIds.includes(autoSetupDownloadId(item))
+ );
+}
+
function message(error: unknown): string {
if (error instanceof Error) return error.message;
if (typeof error === 'string') return error;
@@ -107,10 +134,10 @@ function message(error: unknown): string {
}
function initialOutcomes(
- plan: AutoSetupPlan,
+ items: AutoSetupItem[],
): Record {
return Object.fromEntries(
- plan.items.map(item => {
+ items.map(item => {
const id = autoSetupDownloadId(item);
return [id, { id, phase: 'waiting', progress: 0 }];
}),
@@ -164,8 +191,11 @@ export function createAutoSetupSession(
boundaries: AutoSetupSessionBoundaries = {},
): AutoSetupSession {
const downloads = boundaries.downloads ?? productionDownloadBoundaries;
+ const embedding = boundaries.embedding ?? productionEmbeddingBoundaries;
const listeners = new Set<() => void>();
const activeIds = new Set();
+ let pendingEmbeddingId: string | null = null;
+ let embeddingAbort: AbortController | null = null;
let disposed = false;
let operation = 0;
let refreshInFlight = false;
@@ -173,6 +203,8 @@ export function createAutoSetupSession(
phase: 'loading_catalog',
plans: [],
selectedTier: tierFromPersistedIntent(),
+ selectedKinds: ['text'],
+ installedIds: [],
outcomes: {},
error: null,
};
@@ -186,16 +218,23 @@ export function createAutoSetupSession(
const selectedPlan = (): AutoSetupPlan | undefined =>
state.plans.find(plan => plan.tier === state.selectedTier) ??
state.plans[0];
+ const selectedItems = (plan: AutoSetupPlan) =>
+ [...plan.items, ...(plan.embedding ? [plan.embedding] : [])]
+ .filter(item => state.selectedKinds.includes(item.kind) &&
+ autoSetupItemNeedsAction(item, state.installedIds));
const stopActive = async (cancelled: boolean): Promise => {
- const ids = [...activeIds];
+ const ids = [...activeIds, ...(pendingEmbeddingId ? [pendingEmbeddingId] : [])];
activeIds.clear();
- await Promise.allSettled(ids.map(id => downloads.cancel(id)));
+ pendingEmbeddingId = null;
+ embeddingAbort?.abort();
+ embeddingAbort = null;
+ await Promise.allSettled(ids.filter(id => !id.startsWith('embedding:')).map(id => downloads.cancel(id)));
if (cancelled && !disposed) {
const outcomes = { ...state.outcomes };
for (const id of ids) {
const current = outcomes[id];
- if (current && current.phase !== 'completed') {
+ if (current && current.phase !== 'completed' && current.phase !== 'failed') {
outcomes[id] = { ...current, phase: 'cancelled' };
}
}
@@ -205,26 +244,39 @@ export function createAutoSetupSession(
const refreshDownloads = async (): Promise => {
if (disposed || refreshInFlight || activeIds.size === 0) return;
+ const token = operation;
refreshInFlight = true;
try {
const listed = await downloads.list();
- if (disposed) return;
+ if (disposed || token !== operation) return;
const { outcomes, failure, allCompleted } = projectActiveDownloads(
activeIds,
listed,
state.outcomes,
);
- publish({ outcomes });
+ publish({
+ outcomes,
+ installedIds: [
+ ...state.installedIds.filter(id => id.startsWith('embedding:')),
+ ...listed.filter(download => download.status === 'completed').map(download => download.id),
+ ],
+ });
if (failure) {
operation += 1;
- await stopActive(false);
+ await stopActive(true);
publish({
phase: 'failed',
error: failure.error ?? 'A model download failed. Try again.',
});
} else if (allCompleted) {
activeIds.clear();
- publish({ phase: 'completed', error: null });
+ if (!pendingEmbeddingId) publish({ phase: 'completed', error: null });
+ }
+ } catch (error) {
+ if (!disposed && token === operation) {
+ operation += 1;
+ await stopActive(true);
+ publish({ phase: 'failed', error: message(error) });
}
} finally {
refreshInFlight = false;
@@ -251,9 +303,23 @@ export function createAutoSetupSession(
}),
]);
if (disposed || token !== operation) return;
+ const plans = selectAutoSetupPlans(catalog);
+ const installed = await downloads.list().catch(() => []);
+ const embeddingItems = [...new Map(plans.flatMap(plan =>
+ plan.embedding ? [[plan.embedding.id, plan.embedding] as const] : [],
+ )).values()];
+ const installedEmbedding = await Promise.all(embeddingItems.map(async item =>
+ await embedding.isDownloaded(item.payload).catch(() => false)
+ ? autoSetupDownloadId(item) : null,
+ ));
+ if (disposed || token !== operation) return;
publish({
phase: 'ready',
- plans: selectAutoSetupPlans(catalog),
+ plans,
+ installedIds: [
+ ...installed.filter(download => download.status === 'completed').map(download => download.id),
+ ...installedEmbedding.filter((id): id is string => id !== null),
+ ],
error: null,
});
} catch (error) {
@@ -267,36 +333,106 @@ export function createAutoSetupSession(
}
};
- const start = async (): Promise => {
- const plan = selectedPlan();
- if (!plan || disposed) return;
- const token = ++operation;
- await stopActive(false);
- const existing = await downloads.list();
- if (disposed || token !== operation) return;
+ const startEmbeddingDownload = (
+ item: NonNullable,
+ token: number,
+ ): void => {
+ const id = autoSetupDownloadId(item);
+ const controller = new AbortController();
+ embeddingAbort = controller;
+ embedding.start(item.payload, fraction => {
+ if (disposed || token !== operation) return;
+ publish({ outcomes: {
+ ...state.outcomes,
+ [id]: { id, phase: 'downloading', progress: Math.max(0, Math.min(1, fraction)) },
+ } });
+ }, controller.signal).then(() => {
+ if (disposed || token !== operation) return;
+ pendingEmbeddingId = null;
+ embeddingAbort = null;
+ publish({
+ outcomes: { ...state.outcomes, [id]: { id, phase: 'completed', progress: 1 } },
+ installedIds: [...new Set([...state.installedIds, id])],
+ ...(activeIds.size === 0 ? { phase: 'completed' as const } : {}),
+ });
+ }).catch(async error => {
+ if (disposed || token !== operation) return;
+ operation += 1;
+ await stopActive(true);
+ publish({
+ phase: 'failed',
+ outcomes: { ...state.outcomes, [id]: { id, phase: 'failed', progress: 0, error: message(error) } },
+ error: message(error),
+ });
+ });
+ };
+
+ const prepareSelectedDownloads = async (
+ plan: AutoSetupPlan,
+ items: AutoSetupItem[],
+ token: number,
+ ): Promise => {
+ let existing: ModelDownload[];
+ try {
+ existing = await downloads.list();
+ } catch (error) {
+ if (!disposed && token === operation)
+ publish({ phase: 'failed', error: message(error) });
+ return false;
+ }
+ if (disposed || token !== operation) return false;
const completedIds = new Set(
existing
.filter(download => download.status === 'completed')
.map(download => download.id),
);
- const outcomes = initialOutcomes(plan);
- for (const item of plan.items) {
+ const embeddingItem = plan.embedding && state.selectedKinds.includes('embedding')
+ ? plan.embedding : undefined;
+ const selectedEmbeddingId = embeddingItem ? autoSetupDownloadId(embeddingItem) : null;
+ if (embeddingItem && await embedding.isDownloaded(embeddingItem.payload).catch(() => false))
+ completedIds.add(autoSetupDownloadId(embeddingItem));
+ if (disposed || token !== operation) return false;
+ publish({ installedIds: [
+ ...state.installedIds.filter(id => id.startsWith('embedding:') && id !== selectedEmbeddingId),
+ ...completedIds,
+ ] });
+ const outcomes = initialOutcomes(items);
+ for (const item of items) {
const id = autoSetupDownloadId(item);
if (completedIds.has(id))
outcomes[id] = { id, phase: 'completed', progress: 1 };
else {
outcomes[id] = { id, phase: 'starting', progress: 0 };
- activeIds.add(id);
+ if (item.kind === 'embedding') pendingEmbeddingId = id;
+ else activeIds.add(id);
}
}
publish({
- phase: activeIds.size ? 'downloading' : 'completed',
+ phase: activeIds.size || pendingEmbeddingId ? 'downloading' : 'completed',
outcomes,
error: null,
});
+ return true;
+ };
+
+ const start = async (): Promise => {
+ const plan = selectedPlan();
+ if (!plan || disposed) return;
+ const items = selectedItems(plan);
+ if (items.length === 0) return;
+ const token = ++operation;
+ publish({ phase: 'downloading', error: null });
+ await stopActive(false);
+ if (!await prepareSelectedDownloads(plan, items, token)) return;
+ if (!activeIds.size && !pendingEmbeddingId) return;
+
+ const embeddingItem = plan.embedding && state.selectedKinds.includes('embedding')
+ ? plan.embedding : undefined;
+ if (pendingEmbeddingId && embeddingItem) startEmbeddingDownload(embeddingItem, token);
+
if (activeIds.size === 0) return;
- const [text, image, stt] = plan.items;
+ const [text, image, stt, video] = plan.items;
const jobs = [
{
id: autoSetupDownloadId(text),
@@ -310,6 +446,10 @@ export function createAutoSetupSession(
id: autoSetupDownloadId(stt),
run: () => downloads.start({ modelType: 'stt', modelId: stt.payload.modelId }),
},
+ ...(video ? [{
+ id: autoSetupDownloadId(video),
+ run: () => downloads.start({ modelType: 'video', model: video.payload }),
+ }] : []),
].filter(job => activeIds.has(job.id));
const starts = await Promise.allSettled(jobs.map(job => job.run()));
if (disposed || token !== operation) return;
@@ -319,19 +459,13 @@ export function createAutoSetupSession(
if (failedIndex >= 0) {
const id = jobs[failedIndex].id;
const failure = starts[failedIndex] as PromiseRejectedResult;
- const next = {
- ...state.outcomes,
- [id]: {
- id,
- phase: 'failed' as const,
- progress: 0,
- error: message(failure.reason),
- },
- };
- await stopActive(false);
+ await stopActive(true);
publish({
phase: 'failed',
- outcomes: next,
+ outcomes: {
+ ...state.outcomes,
+ [id]: { id, phase: 'failed', progress: 0, error: message(failure.reason) },
+ },
error: message(failure.reason),
});
return;
@@ -347,11 +481,9 @@ export function createAutoSetupSession(
},
load,
selectTier(tier) {
- // The selected plan is immutable once its download session starts. If a
- // completed or active session could switch tiers, its outcomes would
- // describe the old plan while complete() activated the new plan.
- if (state.phase === 'downloading' || state.phase === 'completed') return;
- persistTierIntent(tier);
+ // These tiers only preview download choices. They never change the live
+ // model loading policy or the user's saved active model selections.
+ if (state.phase === 'downloading') return;
publish({
phase: 'ready',
selectedTier: tier,
@@ -359,17 +491,37 @@ export function createAutoSetupSession(
error: null,
});
},
+ toggleKind(kind) {
+ if (state.phase === 'downloading') return;
+ const plan = selectedPlan();
+ const item = plan && [...plan.items, ...(plan.embedding ? [plan.embedding] : [])]
+ .find(candidate => candidate.kind === kind);
+ if (!item || !autoSetupItemNeedsAction(item, state.installedIds)) return;
+ const selectedKinds = state.selectedKinds.includes(kind)
+ ? state.selectedKinds.filter(selected => selected !== kind)
+ : [...state.selectedKinds, kind];
+ publish({ selectedKinds, outcomes: {}, error: null, phase: 'ready' });
+ },
start,
+ async cancel() {
+ if (state.phase !== 'downloading') return;
+ operation += 1;
+ await stopActive(true);
+ publish({ phase: 'ready', error: null });
+ },
complete() {
const plan = selectedPlan();
if (plan && state.phase === 'completed') {
const app = useAppStore.getState();
- if (app.activeModelId === null) {
+ if (state.selectedKinds.includes('text') && app.activeModelId === null) {
activeModelService.selectTextModel(plan.items[0].id);
}
- if (app.activeImageModelId === null) {
+ if (state.selectedKinds.includes('image') && app.activeImageModelId === null) {
app.setActiveImageModelId(plan.items[1].id);
}
+ if (state.selectedKinds.includes('video') && app.activeVideoModelId === null && plan.items[3]) {
+ app.setActiveVideoModelId(plan.items[3].id);
+ }
}
},
dispose() {
diff --git a/src/services/backgroundDownloadService.ts b/src/services/backgroundDownloadService.ts
index d92c88e6d..bae25774f 100644
--- a/src/services/backgroundDownloadService.ts
+++ b/src/services/backgroundDownloadService.ts
@@ -5,8 +5,12 @@ import logger from '../utils/logger';
import { serializeQueue, saveQueuedDownloads } from './queuedDownloadPersistence';
import type {
DownloadParams,
- DownloadProgressEvent, DownloadCompleteEvent, DownloadErrorEvent,
- DownloadProgressCallback, DownloadCompleteCallback, DownloadErrorCallback,
+ DownloadProgressEvent,
+ DownloadCompleteEvent,
+ DownloadErrorEvent,
+ DownloadProgressCallback,
+ DownloadCompleteCallback,
+ DownloadErrorCallback,
} from './backgroundDownloadTypes';
const { DownloadManagerModule } = NativeModules;
@@ -105,7 +109,10 @@ class BackgroundDownloadService {
/** Actually start a native download. When `counted` (the default) it occupies one
* concurrency slot; an uncounted start (a sidecar) begins immediately and never
* touches activeIds, so it neither consumes a slot nor blocks the queue. */
- private async beginDownload(params: DownloadParams, counted = true): Promise {
+ private async beginDownload(
+ params: DownloadParams,
+ counted = true,
+ ): Promise {
if (!counted) return this.startNativeDownload(params); // sidecar: no slot bookkeeping
// Reserve the slot synchronously (before the first await) so a burst of
// startDownload() calls in the same tick can't all pass the size check and
@@ -130,11 +137,20 @@ class BackgroundDownloadService {
/** The raw native start + BackgroundDownloadInfo mapping — no concurrency accounting.
* Sidecars call this directly; the counted path wraps it in slot bookkeeping. */
- private async startNativeDownload(params: DownloadParams): Promise {
+ private async startNativeDownload(
+ params: DownloadParams,
+ ): Promise {
// Android 13+: prompt for notification permission so the foreground-service download
// notification is visible (the download still runs as an FGS if denied). Best-effort.
- if (Platform.OS === 'android' && typeof DownloadManagerModule.requestNotificationPermission === 'function') {
- try { DownloadManagerModule.requestNotificationPermission(); } catch { /* non-fatal */ }
+ if (
+ Platform.OS === 'android' &&
+ typeof DownloadManagerModule.requestNotificationPermission === 'function'
+ ) {
+ try {
+ DownloadManagerModule.requestNotificationPermission();
+ } catch {
+ /* non-fatal */
+ }
}
const result = await DownloadManagerModule.startDownload({
url: params.url,
@@ -170,7 +186,10 @@ class BackgroundDownloadService {
private pump(): void {
let admitted = false;
- while (this.activeIds.size < MAX_CONCURRENT_DOWNLOADS && this.startQueue.length > 0) {
+ while (
+ this.activeIds.size < MAX_CONCURRENT_DOWNLOADS &&
+ this.startQueue.length > 0
+ ) {
const next = this.startQueue.shift()!;
admitted = true;
// beginDownload reserves the slot synchronously, so the loop condition sees the
@@ -239,8 +258,14 @@ class BackgroundDownloadService {
* them as "Queued". These have no native downloadId yet (they haven't started), so
* they live only here — the queue's owner is the single source of truth for them.
*/
- getQueuedItems(): Array<{ modelKey: string; modelId: string; fileName: string; modelType: string; totalBytes: number }> {
- return this.startQueue.map((q) => ({
+ getQueuedItems(): Array<{
+ modelKey: string;
+ modelId: string;
+ fileName: string;
+ modelType: string;
+ totalBytes: number;
+ }> {
+ return this.startQueue.map(q => ({
modelKey: q.key,
modelId: q.params.modelId,
fileName: q.params.fileName,
@@ -275,9 +300,15 @@ class BackgroundDownloadService {
}
async resumeDownload(downloadId: string): Promise {
- if (!this.isAvailable()) throw new Error('Background downloads not available');
- if (!this.activeIds.has(downloadId) && this.activeIds.size >= MAX_CONCURRENT_DOWNLOADS) {
- throw new Error('Three downloads are already running. Try again when one finishes.');
+ if (!this.isAvailable())
+ throw new Error('Background downloads not available');
+ if (
+ !this.activeIds.has(downloadId) &&
+ this.activeIds.size >= MAX_CONCURRENT_DOWNLOADS
+ ) {
+ throw new Error(
+ 'Three downloads are already running. Try again when one finishes.',
+ );
}
this.activeIds.add(downloadId);
try {
@@ -386,23 +417,44 @@ class BackgroundDownloadService {
}));
}
- async moveCompletedDownload(downloadId: string, targetPath: string): Promise {
+ async moveCompletedDownload(
+ downloadId: string,
+ targetPath: string,
+ ): Promise {
if (!this.isAvailable()) {
throw new Error('Background downloads not available on this platform');
}
return DownloadManagerModule.moveCompletedDownload(downloadId, targetPath);
}
- private registerListener(listeners: Map, key: string, callback: T): () => void {
+ private registerListener(
+ listeners: Map,
+ key: string,
+ callback: T,
+ ): () => void {
listeners.set(key, callback);
return () => listeners.delete(key);
}
- onProgress(downloadId: string, callback: DownloadProgressCallback): () => void {
- return this.registerListener(this.progressListeners, `progress_${downloadId}`, callback);
- }
- onComplete(downloadId: string, callback: DownloadCompleteCallback): () => void {
- return this.registerListener(this.completeListeners, `complete_${downloadId}`, callback);
+ onProgress(
+ downloadId: string,
+ callback: DownloadProgressCallback,
+ ): () => void {
+ return this.registerListener(
+ this.progressListeners,
+ `progress_${downloadId}`,
+ callback,
+ );
+ }
+ onComplete(
+ downloadId: string,
+ callback: DownloadCompleteCallback,
+ ): () => void {
+ return this.registerListener(
+ this.completeListeners,
+ `complete_${downloadId}`,
+ callback,
+ );
}
onError(downloadId: string, callback: DownloadErrorCallback): () => void {
return this.registerListener(this.errorListeners, `error_${downloadId}`, callback);
@@ -471,7 +523,17 @@ class BackgroundDownloadService {
}
downloadFileTo(opts: {
- params: Pick;
+ params: Pick<
+ DownloadParams,
+ | 'url'
+ | 'fileName'
+ | 'modelId'
+ | 'totalBytes'
+ | 'modelType'
+ | 'metadataJson'
+ | 'modelKey'
+ | 'sha256'
+ >;
destPath: string;
onProgress?: (bytesDownloaded: number, totalBytes: number) => void;
silent?: boolean;
@@ -488,7 +550,7 @@ class BackgroundDownloadService {
});
resolveId(info.downloadId);
await new Promise((resolve, reject) => {
- const removeProgress = this.onProgress(info.downloadId, (event) => {
+ const removeProgress = this.onProgress(info.downloadId, event => {
opts.onProgress?.(event.bytesDownloaded, event.totalBytes);
});
const done = () => { removeProgress(); removeComplete(); removeError(); };
@@ -497,7 +559,7 @@ class BackgroundDownloadService {
try { await this.moveCompletedDownload(info.downloadId, opts.destPath); } catch { /* may already be moved */ }
resolve();
});
- const removeError = this.onError(info.downloadId, (err) => {
+ const removeError = this.onError(info.downloadId, err => {
done();
const error = new Error(err.reason || 'Download failed') as Error & { cancelled?: boolean };
// Let callers distinguish a user cancel from a real failure so they can
diff --git a/src/services/backgroundDownloadTypes.ts b/src/services/backgroundDownloadTypes.ts
index adc3fefc3..ab5ad33aa 100644
--- a/src/services/backgroundDownloadTypes.ts
+++ b/src/services/backgroundDownloadTypes.ts
@@ -5,7 +5,7 @@ export interface DownloadParams {
fileName: string;
modelId: string;
modelKey?: string;
- modelType?: 'text' | 'image' | 'stt' | 'tts';
+ modelType?: 'text' | 'image' | 'video' | 'stt' | 'tts';
quantization?: string;
combinedTotalBytes?: number;
mmProjDownloadId?: string;
diff --git a/src/services/exportGeneratedMedia.ts b/src/services/exportGeneratedMedia.ts
new file mode 100644
index 000000000..9b92008fa
--- /dev/null
+++ b/src/services/exportGeneratedMedia.ts
@@ -0,0 +1,42 @@
+import {
+ NativeModules,
+ PermissionsAndroid,
+ Platform,
+ Share,
+} from 'react-native';
+import RNFS from 'react-native-fs';
+import { resolveDocumentPath } from '../utils/resolveDocumentPath';
+
+/** One export boundary for app-owned generated media. */
+export async function exportGeneratedMedia(
+ path: string,
+ fileName: string,
+ mime: string,
+): Promise {
+ const source = resolveDocumentPath(path);
+ if (Platform.OS === 'ios') {
+ await Share.share({ url: `file://${source}` });
+ return;
+ }
+ if (Number(Platform.Version) >= 29) {
+ await NativeModules.SyncDownloadsModule.saveFileToDownloads(
+ source,
+ fileName,
+ mime,
+ );
+ return;
+ }
+ const permission = await PermissionsAndroid.request(
+ PermissionsAndroid.PERMISSIONS.WRITE_EXTERNAL_STORAGE,
+ );
+ if (permission !== PermissionsAndroid.RESULTS.GRANTED)
+ throw new Error('Storage permission is required to save the file.');
+ if (
+ !source.startsWith(RNFS.DocumentDirectoryPath + '/') ||
+ source.split('/').includes('..') ||
+ /[/\\]/.test(fileName)
+ ) {
+ throw new Error('Invalid media export path.');
+ }
+ await RNFS.copyFile(source, `${RNFS.DownloadDirectoryPath}/${fileName}`);
+}
diff --git a/src/services/generationRemoteHelpers.ts b/src/services/generationRemoteHelpers.ts
index df3882d09..9b4e96951 100644
--- a/src/services/generationRemoteHelpers.ts
+++ b/src/services/generationRemoteHelpers.ts
@@ -1,3 +1,4 @@
+import { videoGenerationService } from './videoGenerationService';
// Remote (OpenAI-compatible server) generation paths for GenerationService.
//
// Split from generationServiceHelpers.ts, which owns the LOCAL engines (llama.rn and LiteRT). The two
@@ -151,7 +152,7 @@ export async function generateRemoteWithToolsImpl(
try {
// Use the same tool loop but with remote provider
- await runToolLoop({
+ const outcome = await runToolLoop({
conversationId,
messages,
enabledToolIds,
@@ -185,8 +186,14 @@ export async function generateRemoteWithToolsImpl(
);
svc.checkSharePrompt();
svc.resetState();
+ await videoGenerationService.finishDeferred(
+ conversationId,
+ outcome.interrupted,
+ );
+ svc.drainQueue();
}
} catch (error) {
+ videoGenerationService.discardDeferred(conversationId);
if (svc.abortRequested) return;
logger.error('[GenerationService] Remote tool generation error:', error);
// Reset generating state on error, else isGenerating stays stuck → red stop, next send blocked (2026-07-14).
diff --git a/src/services/generationService.ts b/src/services/generationService.ts
index f2d2e9b5a..a5879226c 100644
--- a/src/services/generationService.ts
+++ b/src/services/generationService.ts
@@ -1,3 +1,4 @@
+import { videoGenerationService } from './videoGenerationService';
/** GenerationService - Handles LLM generation independently of UI lifecycle */
import { llmService } from './llm';
import { getActiveEngineService, prepareActiveConversation, stopAllTextEngines } from './engines';
@@ -33,12 +34,15 @@ type FallbackRoute =
| { kind: 'local'; id: string; name: string };
export interface QueuedMessage {
- id: string; conversationId: string; text: string;
- attachments?: MediaAttachment[]; messageText: string;
+ id: string;
+ conversationId: string;
+ text: string;
+ attachments?: MediaAttachment[];
+ messageText: string;
/** The modality the user forced for THIS send (force/disabled/auto). Carried through the queue so a
* message the user explicitly forced to image mode is dispatched as image on drain — never re-decided
* at 'auto' by resolveTurnKind (#510: a queued force-image send generated as text). */
- imageMode?: 'auto' | 'force' | 'disabled';
+ imageMode?: 'auto' | 'force' | 'disabled' | 'video';
assistantEnabled?: boolean;
}
@@ -56,8 +60,12 @@ type QueueProcessor = (item: QueuedMessage) => Promise;
class GenerationService {
private state: GenerationState = {
- isGenerating: false, isThinking: false, conversationId: null,
- streamingContent: '', startTime: null, queuedMessages: [],
+ isGenerating: false,
+ isThinking: false,
+ conversationId: null,
+ streamingContent: '',
+ startTime: null,
+ queuedMessages: [],
};
private listeners: Set = new Set();
@@ -66,12 +74,19 @@ class GenerationService {
private generationAttempt: number = 0;
/** Whether the last/active generation was stopped by the user — lets callers skip a
* "no response" retry prompt when the empty result was an intentional abort. */
- wasAborted(): boolean { return this.abortRequested; }
+ wasAborted(): boolean {
+ return this.abortRequested;
+ }
private pendingStop: Promise | null = null;
private queueProcessor: QueueProcessor | null = null;
private currentRemoteAbortController: AbortController | null = null;
private remoteTimeToFirstToken: number | undefined;
- private contextUsage: Pick | undefined;
+ private contextUsage:
+ | Pick<
+ GenerationMeta,
+ 'contextPromptTokens' | 'contextWindowTokens' | 'contextEstimate'
+ >
+ | undefined;
// Token batching — collect tokens and flush to UI at a controlled rate
private tokenBuffer: string = '';
@@ -121,11 +136,16 @@ class GenerationService {
this.flushTokenBuffer();
}
- private normalizeStreamChunk(data: StreamChunk): { content?: string; reasoningContent?: string } {
+ private normalizeStreamChunk(data: StreamChunk): {
+ content?: string;
+ reasoningContent?: string;
+ } {
return typeof data === 'string' ? { content: data } : data;
}
- getState(): GenerationState { return { ...this.state }; }
+ getState(): GenerationState {
+ return { ...this.state };
+ }
isGeneratingFor(conversationId: string): boolean {
return this.state.isGenerating && this.state.conversationId === conversationId;
@@ -135,7 +155,9 @@ class GenerationService {
this.listeners.add(listener); listener(this.getState()); return () => this.listeners.delete(listener);
}
- private notifyListeners(): void { this.listeners.forEach(l => l(this.getState())); }
+ private notifyListeners(): void {
+ this.listeners.forEach(l => l(this.getState()));
+ }
private updateState(partial: Partial): void {
this.state = { ...this.state, ...partial };
@@ -149,8 +171,12 @@ class GenerationService {
checkProPromptForText(delayMs);
}
- private buildToolLoopHandlers() { return buildToolLoopHandlersImpl(this); }
- private buildGenerationMeta(): GenerationMeta { return buildGenerationMetaImpl(this); }
+ private buildToolLoopHandlers() {
+ return buildToolLoopHandlersImpl(this);
+ }
+ private buildGenerationMeta(): GenerationMeta {
+ return buildGenerationMetaImpl(this);
+ }
private async prepareGeneration(conversationId: string): Promise {
return prepareGenerationImpl(this, conversationId);
}
@@ -180,30 +206,51 @@ class GenerationService {
const selectedId = startedRemote ? remote.activeRemoteTextModelId : local.activeModelId;
const selectedName = startedRemote
? remote.getActiveRemoteTextModel()?.name || selectedId || 'Remote model'
- : local.downloadedModels.find(model => model.id === selectedId)?.name || 'Local model';
+ : local.downloadedModels.find(model => model.id === selectedId)?.name ||
+ 'Local model';
const needsVision = messages.some(message =>
message.attachments?.some(attachment => attachment.type === 'image'),
);
const remoteRoutes = startedRemote
? remote.servers.flatMap(server =>
(remote.discoveredModels[server.id] || [])
- .filter(model =>
- (server.id !== remote.activeServerId || model.id !== selectedId) &&
- (!needsVision || model.capabilities.supportsVision),
+ .filter(
+ model =>
+ (server.id !== remote.activeServerId ||
+ model.id !== selectedId) &&
+ (!needsVision || model.capabilities.supportsVision),
)
- .map(model => ({ kind: 'remote' as const, serverId: server.id, id: model.id, name: model.name })),
+ .map(model => ({
+ kind: 'remote' as const,
+ serverId: server.id,
+ id: model.id,
+ name: model.name,
+ })),
)
: [];
const localRoutes = local.downloadedModels
- .filter(model =>
- model.id !== selectedId &&
- (!needsVision || (model.engine === 'litert' ? model.liteRTVision : model.isVisionModel)),
+ .filter(
+ model =>
+ model.id !== selectedId &&
+ (!needsVision ||
+ (model.engine === 'litert'
+ ? model.liteRTVision
+ : model.isVisionModel)),
)
.sort((a, b) => a.fileSize - b.fileSize)
- .map(model => ({ kind: 'local' as const, id: model.id, name: model.name }));
+ .map(model => ({
+ kind: 'local' as const,
+ id: model.id,
+ name: model.name,
+ }));
return [
startedRemote
- ? { kind: 'remote', serverId: remote.activeServerId || '', id: selectedId || '', name: selectedName }
+ ? {
+ kind: 'remote',
+ serverId: remote.activeServerId || '',
+ id: selectedId || '',
+ name: selectedName,
+ }
: { kind: 'local', id: selectedId || '', name: selectedName },
...remoteRoutes,
...localRoutes,
@@ -250,7 +297,8 @@ class GenerationService {
this.totalReasoningLength = 0;
this.remoteTimeToFirstToken = undefined;
useChatStore.getState().addMessage(conversationId, {
- role: 'tool', toolName: 'model_fallback',
+ role: 'tool',
+ toolName: 'model_fallback',
content: `${failedName} could not answer. Trying ${route.name}.`,
});
prepared = this.state.isGenerating;
@@ -301,46 +349,81 @@ class GenerationService {
options.onToolCallComplete?.(name, result);
},
};
- return this.withModelFallback(conversationId, messages, async (route, prepared) => {
- if (route.kind === 'remote') {
- return generateRemoteWithToolsImpl(this, {
- conversationId, messages,
- options: { ...trackedOptions, prepared, preservePartialOnError: false },
- });
- }
- const { enabledToolIds, projectId, contextUsage, assistantEnabled, ...callbacks } = trackedOptions;
- if (!prepared && !(await this.prepareGeneration(conversationId))) return;
- this.contextUsage = contextUsage;
- try {
- const outcome = await runToolLoop({
- conversationId,
- messages,
- enabledToolIds,
- projectId,
- assistantEnabled,
- callbacks,
- ...this.buildToolLoopHandlers(),
- });
-
- // If aborted, stopGeneration() already handled cleanup.
- logger.log(`[GenService][ToolLoop] runToolLoop done — aborted=${this.abortRequested}, streamingContent=${this.state.streamingContent?.length ?? 0}ch, tokenBuffer=${this.tokenBuffer?.length ?? 0}ch`);
- if (!this.abortRequested) {
- this.forceFlushTokens();
- const store = useChatStore.getState();
- logger.log(`[GenService][ToolLoop] pre-finalize — streamingForConvId=${store.streamingForConversationId}, targetConvId=${conversationId}, streamingMsg=${store.streamingMessage?.length ?? 0}ch`);
- const generationTime = this.state.startTime ? Date.now() - this.state.startTime : undefined;
- store.finalizeStreamingMessage(conversationId, generationTime, this.buildGenerationMeta());
- logger.log(`[GenService][ToolLoop] finalizeStreamingMessage called — convId=${conversationId}`);
- this.checkSharePrompt();
- this.resetState();
- }
- return outcome;
- } catch (error) {
- if (this.abortRequested) return;
- logger.error('[GenerationService] Tool generation error:', error);
- throw error;
- }
- }, () => !toolStarted);
+ return this.withModelFallback(
+ conversationId,
+ messages,
+ async (route, prepared) => {
+ if (route.kind === 'remote') {
+ return generateRemoteWithToolsImpl(this, {
+ conversationId,
+ messages,
+ options: {
+ ...trackedOptions,
+ prepared,
+ preservePartialOnError: false,
+ },
+ });
+ }
+ const {
+ enabledToolIds,
+ projectId,
+ contextUsage,
+ assistantEnabled,
+ ...callbacks
+ } = trackedOptions;
+ if (!prepared && !(await this.prepareGeneration(conversationId)))
+ return;
+ this.contextUsage = contextUsage;
+ try {
+ const outcome = await runToolLoop({
+ conversationId,
+ messages,
+ enabledToolIds,
+ projectId,
+ assistantEnabled,
+ callbacks,
+ ...this.buildToolLoopHandlers(),
+ });
+
+ // If aborted, stopGeneration() already handled cleanup.
+ logger.log(
+ `[GenService][ToolLoop] runToolLoop done — aborted=${this.abortRequested}, streamingContent=${this.state.streamingContent?.length ?? 0}ch, tokenBuffer=${this.tokenBuffer?.length ?? 0}ch`,
+ );
+ if (!this.abortRequested) {
+ this.forceFlushTokens();
+ const store = useChatStore.getState();
+ logger.log(
+ `[GenService][ToolLoop] pre-finalize — streamingForConvId=${store.streamingForConversationId}, targetConvId=${conversationId}, streamingMsg=${store.streamingMessage?.length ?? 0}ch`,
+ );
+ const generationTime = this.state.startTime
+ ? Date.now() - this.state.startTime
+ : undefined;
+ store.finalizeStreamingMessage(
+ conversationId,
+ generationTime,
+ this.buildGenerationMeta(),
+ );
+ logger.log(
+ `[GenService][ToolLoop] finalizeStreamingMessage called — convId=${conversationId}`,
+ );
+ this.checkSharePrompt();
+ this.resetState();
+ }
+ await videoGenerationService.finishDeferred(
+ conversationId,
+ outcome.interrupted,
+ );
+ this.drainQueue();
+ return outcome;
+ } catch (error) {
+ videoGenerationService.discardDeferred(conversationId);
+ if (this.abortRequested) return;
+ logger.error('[GenerationService] Tool generation error:', error);
+ throw error;
+ }
+ },
+ () => !toolStarted,
+ );
}
/**
@@ -424,8 +507,10 @@ class GenerationService {
// generations can drain it before starting.
const engine = getActiveEngineService();
this.pendingStop = (engine?.stopGeneration() ?? Promise.resolve())
- .catch(() => { })
- .finally(() => { this.pendingStop = null; });
+ .catch(() => {})
+ .finally(() => {
+ this.pendingStop = null;
+ });
return partialContent;
}
@@ -460,9 +545,14 @@ class GenerationService {
this.notifyListeners();
}
- clearQueue(): void { this.state = { ...this.state, queuedMessages: [] }; this.notifyListeners(); }
+ clearQueue(): void {
+ this.state = { ...this.state, queuedMessages: [] };
+ this.notifyListeners();
+ }
- setQueueProcessor(processor: QueueProcessor | null): void { this.queueProcessor = processor; }
+ setQueueProcessor(processor: QueueProcessor | null): void {
+ this.queueProcessor = processor;
+ }
/**
* Process queued messages now. Text generation drains its own queue on
@@ -471,26 +561,23 @@ class GenerationService {
* text generation is currently running.
*/
drainQueue(): void {
- if (this.state.isGenerating) return;
+ if (this.state.isGenerating || videoGenerationService.hasPending()) return;
this.processNextInQueue();
}
private processNextInQueue(): void {
- if (this.state.queuedMessages.length === 0 || !this.queueProcessor) return;
- const all = this.state.queuedMessages;
- this.state = { ...this.state, queuedMessages: [] };
+ if (
+ this.state.queuedMessages.length === 0 ||
+ !this.queueProcessor ||
+ videoGenerationService.hasPending()
+ )
+ return;
+ const [combined, ...remaining] = this.state.queuedMessages;
+ this.state = { ...this.state, queuedMessages: remaining };
this.notifyListeners();
- const combined: QueuedMessage = all.length === 1 ? all[0] : {
- id: all[0].id, conversationId: all[0].conversationId,
- text: all.map(m => m.text).join('\n\n'),
- attachments: all.flatMap(m => m.attachments || []),
- messageText: all.map(m => m.messageText).join('\n\n'),
- // If ANY coalesced send forced image mode, the combined dispatch must force image too — the
- // user's explicit force must never be dropped by the merge (mirror of the single-message carry).
- imageMode: all.some(m => m.imageMode === 'force') ? 'force' : all[0].imageMode,
- assistantEnabled: all.some(m => m.assistantEnabled),
- };
- this.queueProcessor(combined).catch(e => { logger.error('[GenerationService] Queue processor error:', e); });
+ this.queueProcessor(combined).catch(e => {
+ logger.error('[GenerationService] Queue processor error:', e);
+ });
}
private resetState(): void {
diff --git a/src/services/hardware.ts b/src/services/hardware.ts
index efa6d5779..42df86444 100644
--- a/src/services/hardware.ts
+++ b/src/services/hardware.ts
@@ -296,8 +296,8 @@ class HardwareService {
* so the gate doesn't refuse an ANE load that fits (nor admit a GPU load that
* OOMs). Android (ONNX/QNN reserves accelerator memory up front) keeps 2.5×.
*/
- estimateImageModelRam(model: { fileSize?: number; size?: number; mmProjFileSize?: number }): number {
- const multiplier = Platform.OS === 'ios' && !this.preferGpuForImageGen() ? 1.8 : 2.5;
+ estimateImageModelRam(model: { fileSize?: number; size?: number; mmProjFileSize?: number; backend?: string }): number {
+ const multiplier = model.backend !== 'sd' && Platform.OS === 'ios' && !this.preferGpuForImageGen() ? 1.8 : 2.5;
return this.estimateModelRam(model, multiplier);
}
formatModelRam(model: { fileSize?: number; size?: number; mmProjFileSize?: number }, multiplier = 1.5): string {
diff --git a/src/services/huggingFaceModelBrowser.ts b/src/services/huggingFaceModelBrowser.ts
index 9ff485a65..f6bb98d70 100644
--- a/src/services/huggingFaceModelBrowser.ts
+++ b/src/services/huggingFaceModelBrowser.ts
@@ -1,8 +1,14 @@
+import { Buffer } from 'buffer';
+import { CATALOG, searchHuggingFace, extractQuantization, standardImageModelDefaults } from '@offgrid/models';
+import RNFS from 'react-native-fs';
+import type { ImageModelDescriptor } from './imageModelDownloadTypes';
+
export interface HFImageModel {
id: string;
name: string;
displayName: string;
- backend: 'mnn' | 'qnn';
+ backend: ImageModelDescriptor['backend'];
+ huggingFaceFiles?: ImageModelDescriptor['huggingFaceFiles'];
variant?: string;
downloadUrl: string;
fileName: string;
@@ -141,3 +147,242 @@ export function guessStyle(name: string): string {
}
return 'anime';
}
+
+const QWEN_IMAGE_WEIGHT = /^qwen_image_2\.1-.*\.gguf$/i;
+const IMAGE_HEADER_BYTES = 1024 * 1024;
+
+/** Bundled complete checkpoints share the Desktop catalog. Split diffusion packs
+ * need an explicit companion contract before they can be offered on mobile. */
+export function getSDImageModels(): HFImageModel[] {
+ return CATALOG.filter(model => model.id === 'leejet/Qwen-Image-2.1-GGUF' ||
+ (model.kind === 'image' && model.files.length === 1 &&
+ /^(offgrid-ai\/|mzwing\/SDXL-Lightning|OlegSkutte\/sdxl-turbo)/i.test(model.id)))
+ .map(model => ({
+ id: `sd-${model.id.replaceAll('/', '--')}`,
+ name: model.name, displayName: model.name, backend: 'sd' as const,
+ repo: model.id, fileName: model.files[0].name, downloadUrl: model.files[0].url,
+ size: model.files.reduce((sum, file) => sum + (file.sizeBytes ?? 0), 0),
+ huggingFaceFiles: model.files.map(file => ({ path: file.name, size: file.sizeBytes ?? 0, downloadUrl: file.url, sha256: file.sha256 })),
+ }));
+}
+
+/** Filename candidates only. New checkpoint candidates must pass tensor validation. */
+export const isSDImageWeight = (name: string): boolean => QWEN_IMAGE_WEIGHT.test(name) ||
+ (/\.(gguf|safetensors)$/i.test(name) && !/[/\\]|(?:vae|clip|t5|encoder|mmproj|lora|adapter|qwen|flux|wan|hunyuan|ltx|z_image|sd3|\d{5}-of-\d{5})/i.test(name));
+
+/** Read tensor names, never model data or metadata strings, to reject text models,
+ * adapters, and UNet-only exports. A bounded or malformed header fails closed. */
+function isCompleteSDCheckpoint(bytes: Buffer): boolean {
+ try {
+ let offset = 0;
+ const take = (size: number) => {
+ if (!Number.isSafeInteger(size) || size < 0 || offset + size > bytes.length) throw new Error('Incomplete header');
+ const start = offset; offset += size; return start;
+ };
+ const u32 = () => bytes.readUInt32LE(take(4));
+ const u64 = () => { const pos = take(8); return bytes.readUInt32LE(pos) + bytes.readUInt32LE(pos + 4) * 4294967296; };
+ const string = () => { const length = u64(); return bytes.toString('utf8', take(length), offset); };
+ let names: string[];
+ if (bytes.toString('ascii', 0, 4) === 'GGUF') {
+ take(4);
+ const version = u32();
+ if (version !== 2 && version !== 3) return false;
+ const tensors = u64(), metadata = u64();
+ if (tensors > 20000 || metadata > 10000) return false;
+ const skipValue = (type: number, depth = 0): void => {
+ if (depth > 1) throw new Error('Invalid metadata');
+ const sizes: Record = { 0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 10: 8, 11: 8, 12: 8 };
+ if (type === 8) { string(); return; }
+ if (type === 9) {
+ const subtype = u32(), count = u64();
+ if (count > IMAGE_HEADER_BYTES) throw new Error('Large metadata');
+ for (let i = 0; i < count; i++) skipValue(subtype, depth + 1);
+ return;
+ }
+ if (!sizes[type]) throw new Error('Unknown metadata');
+ take(sizes[type]);
+ };
+ for (let i = 0; i < metadata; i++) { string(); skipValue(u32()); }
+ names = [];
+ for (let i = 0; i < tensors; i++) {
+ names.push(string());
+ const dimensions = u32();
+ if (dimensions > 4) return false;
+ take(dimensions * 8 + 4 + 8);
+ }
+ } else {
+ const length = u64();
+ names = Object.keys(JSON.parse(bytes.toString('utf8', take(length), offset)));
+ }
+ return names.some(name => /^model\.diffusion_model\.input_blocks\./.test(name)) &&
+ names.some(name => /^first_stage_model\.decoder\./.test(name)) &&
+ names.some(name => /^(cond_stage_model\.|conditioner\.embedders\.)/.test(name));
+ } catch { return false; }
+}
+
+export async function validateSDCheckpointFile(path: string): Promise {
+ const size = Number((await RNFS.stat(path)).size);
+ return isCompleteSDCheckpoint(Buffer.from(await RNFS.read(path, Math.min(size, IMAGE_HEADER_BYTES), 0, 'base64'), 'base64'));
+}
+
+/** The same required-file list is used by loading, recovery, and model transfer. */
+export function getSDImagePackFiles(names: string[], modelId?: string): NonNullable | null {
+ const weights = names.filter(isSDImageWeight);
+ if (weights.length !== 1) return null;
+ const templates = getSDImageModels();
+ if (!QWEN_IMAGE_WEIGHT.test(weights[0])) {
+ const template = templates.find(model => model.id === modelId && model.fileName === weights[0]);
+ return [{ path: weights[0], size: 0, sha256: template?.huggingFaceFiles?.[0].sha256 }];
+ }
+ const template = templates.find(model => model.repo === 'leejet/Qwen-Image-2.1-GGUF');
+ if (!template?.huggingFaceFiles) return null;
+ return template.huggingFaceFiles.map(file => file.path === template.fileName
+ ? { path: weights[0], size: 0, sha256: modelId === template.id && weights[0] === file.path ? file.sha256 : undefined }
+ : file);
+}
+
+/** Replace catalog size estimates with immutable Hugging Face file metadata. */
+export async function resolveSDImageDownloadFiles(repo: string, files: NonNullable) {
+ if (files.every(file => file.sha256 && /\/resolve\/[a-f0-9]{40}\//i.test(file.downloadUrl ?? ''))) return files;
+ const response = await fetch(`https://huggingface.co/api/models/${repo}?blobs=true`);
+ if (!response.ok) throw new Error('Could not verify the image download files. Try again.');
+ const data = await response.json() as { sha: string; siblings?: { rfilename: string; size?: number; lfs?: { size: number; sha256?: string } }[] };
+ if (!/^[a-f0-9]{40}$/i.test(data.sha)) throw new Error('The model repository has no stable revision.');
+ return files.map(file => {
+ if (file.sha256 && /\/resolve\/[a-f0-9]{40}\//i.test(file.downloadUrl ?? '')) return file;
+ const source = data.siblings?.find(candidate => candidate.rfilename === file.path);
+ const size = source?.lfs?.size ?? source?.size ?? 0;
+ if (!source || size <= 0) throw new Error(`Required image file is missing: ${file.path}`);
+ return { path: file.path, size, sha256: source.lfs?.sha256,
+ downloadUrl: `https://huggingface.co/${repo}/resolve/${data.sha}/${file.path.split('/').map(encodeURIComponent).join('/')}` };
+ });
+}
+
+export async function searchSDImageModels(query: string, signal?: AbortSignal): Promise {
+ const params = new URLSearchParams({ search: query, pipeline_tag: 'text-to-image', sort: 'downloads', direction: '-1', limit: '10' });
+ const [ggufRepos, checkpointResponse] = await Promise.all([
+ searchHuggingFace(query, { kind: 'image', limit: 10, fetchImpl: (url, init) => fetch(url, { ...init, signal }) }),
+ fetch(`https://huggingface.co/api/models?${params}`, { signal }),
+ ]);
+ if (!checkpointResponse.ok) throw new Error('Could not search image models on Hugging Face. Try again.');
+ const checkpointRepos = await checkpointResponse.json() as { id: string }[];
+ const repositories = [...new Map([...ggufRepos, ...checkpointRepos].map(repo => [repo.id, repo])).values()];
+ const catalogModels = getSDImageModels();
+ const template = catalogModels.find(model => model.repo === 'leejet/Qwen-Image-2.1-GGUF')!;
+ const companion = template.huggingFaceFiles!.filter(file => file.path !== template.fileName);
+ const models: HFImageModel[] = [];
+ let readableRepos = 0;
+ for (const repo of repositories) {
+ if (signal?.aborted) throw new Error('Image search cancelled.');
+ try {
+ const response = await fetch(`https://huggingface.co/api/models/${repo.id}?blobs=true`, { signal });
+ if (!response.ok) continue;
+ const data = await response.json() as { sha: string; siblings?: { rfilename: string; size?: number; lfs?: { size: number; sha256?: string } }[] };
+ if (!/^[a-f0-9]{40}$/i.test(data.sha)) continue;
+ readableRepos++;
+ // Bound header work per repo. Unsupported files remain absent from downloads.
+ const candidates = (data.siblings ?? []).filter(file => isSDImageWeight(file.rfilename) && (file.lfs?.size ?? file.size ?? 0) > 0).slice(0, 8);
+ for (const file of candidates) {
+ const downloadUrl = `https://huggingface.co/${repo.id}/resolve/${data.sha}/${encodeURIComponent(file.rfilename)}`;
+ const qwen = QWEN_IMAGE_WEIGHT.test(file.rfilename);
+ if (!qwen) {
+ const header = await fetch(downloadUrl, { signal, headers: { Range: `bytes=0-${IMAGE_HEADER_BYTES - 1}` } });
+ if (header.status !== 206 || !/^bytes 0-\d+\//.test(header.headers.get('content-range') ?? '') ||
+ Number(header.headers.get('content-length')) > IMAGE_HEADER_BYTES) continue;
+ if (!isCompleteSDCheckpoint(Buffer.from(await header.arrayBuffer()))) continue;
+ }
+ const size = file.lfs?.size ?? file.size ?? 0;
+ const canonical = catalogModels.find(model => model.repo === repo.id && model.fileName === file.rfilename);
+ const id = canonical?.id ?? `sd-${repo.id.replaceAll('/', '--')}--${file.rfilename}`;
+ if (id.length > 160) continue;
+ const displayName = canonical?.displayName ?? (qwen ? `Qwen Image 2.1 ${extractQuantization(file.rfilename)}` : file.rfilename.replace(/\.(gguf|safetensors)$/i, ''));
+ const parts = [{ path: file.rfilename, size, downloadUrl, sha256: file.lfs?.sha256 }, ...(qwen ? companion : [])];
+ models.push({ id, name: displayName, displayName, backend: 'sd', repo: repo.id, fileName: file.rfilename,
+ downloadUrl, size: parts.reduce((sum, part) => sum + part.size, 0), huggingFaceFiles: parts });
+ }
+ } catch { /* One inaccessible repository must not hide other supported results. */ }
+ }
+ if (repositories.length && !readableRepos) throw new Error('Could not read image model files from Hugging Face. Try again.');
+ if (repositories.length && !models.length) throw new Error('No complete supported image checkpoints found. Use SD 1.x, SD 2.x, SDXL, or Qwen Image 2.1. Split models and adapters need other files.');
+ return models;
+}
+
+export async function resolveSDImagePack(_modelId: string, modelPath: string) {
+ const files = getSDImagePackFiles((await RNFS.readDir(modelPath)).filter(file => file.isFile()).map(file => file.name));
+ if (!files) throw new Error('Select one complete SD checkpoint or a Qwen Image 2.1 pack.');
+ const weight = files[0].path;
+ const defaults = standardImageModelDefaults(weight);
+ if (!QWEN_IMAGE_WEIGHT.test(weight)) {
+ if (!await validateSDCheckpointFile(`${modelPath}/${weight}`)) throw new Error('This checkpoint is unsupported or incomplete. It must include the SD image model, text encoder, and VAE.');
+ return { family: 'checkpoint', weight: `${modelPath}/${weight}`, vae: '', llm: '', sampler: defaults.sampler, scheduler: defaults.scheduler };
+ }
+ const required = (pattern: RegExp) => {
+ const file = files.find(part => pattern.test(part.path));
+ if (!file) throw new Error('The image model pack is incomplete.');
+ return `${modelPath}/${file.path}`;
+ };
+ return { family: 'qwen-image-2.1', weight: `${modelPath}/${weight}`,
+ vae: required(/vae.*\.safetensors$/i), llm: required(/^Qwen3VL-.*\.gguf$/i),
+ sampler: defaults.sampler, scheduler: defaults.scheduler };
+}
+
+export const BUNDLED_EMBEDDING_MODEL = {
+ id: 'bundled:all-MiniLM-L6-v2-Q8_0', name: 'MiniLM L6 (built-in)',
+ description: 'English text search. Included with the app.', size: 0, downloadUrl: undefined,
+} as const;
+
+/** Pinned Hugging Face files; runtime validation is still required on each phone. */
+export const RECOMMENDED_EMBEDDING_MODELS = [
+ {
+ id: 'leliuga/all-MiniLM-L12-v2-GGUF@f048c4f3577816f9825989a59a7eed3c9afa3f1d/all-MiniLM-L12-v2.Q8_0.gguf',
+ name: 'MiniLM L12', description: 'English text search, 12-layer encoder.', size: 36413728,
+ downloadUrl: 'https://huggingface.co/leliuga/all-MiniLM-L12-v2-GGUF/resolve/f048c4f3577816f9825989a59a7eed3c9afa3f1d/all-MiniLM-L12-v2.Q8_0.gguf',
+ sha256: '161d07a32057e754e1fe82e30547c736032ab255a6719890b7e76414c565b748',
+ },
+ {
+ id: 'armand01/paraphrase-multilingual-MiniLM-L12-v2-Q6_K-GGUF@34b69e1683fccf80bbbe7255b8651cd7a76e8891/paraphrase-multilingual-minilm-l12-v2.Q6_K.gguf',
+ name: 'Multilingual MiniLM L12', description: 'Text search across multiple languages.', size: 130844160,
+ downloadUrl: 'https://huggingface.co/armand01/paraphrase-multilingual-MiniLM-L12-v2-Q6_K-GGUF/resolve/34b69e1683fccf80bbbe7255b8651cd7a76e8891/paraphrase-multilingual-minilm-l12-v2.Q6_K.gguf',
+ sha256: 'b5780f54a02b2e9a1cded186d349800aedfd6fb38635c41f532a9385fed34d32',
+ },
+] as const;
+
+/** GGUF text encoders are candidates until the local runtime validates the file. */
+export async function searchEmbeddingModels(query: string, signal?: AbortSignal) {
+ const params = new URLSearchParams({
+ search: query.trim(), filter: 'gguf',
+ sort: 'downloads', direction: '-1', limit: '20',
+ });
+ const searches = await Promise.all(['sentence-similarity', 'feature-extraction'].map(async tag => {
+ const response = await fetch(`https://huggingface.co/api/models?${params}&pipeline_tag=${tag}`, { signal });
+ if (!response.ok) throw new Error(`Embedding search failed: HTTP ${response.status}`);
+ return await response.json() as { id: string }[];
+ }));
+ const repos = [...new Map(searches.flat().map(repo => [repo.id, repo])).values()];
+ const listings = await Promise.allSettled(repos.map(async repo => {
+ const result = await fetch(`https://huggingface.co/api/models/${repo.id}?blobs=true`, { signal });
+ if (!result.ok) throw new Error(`Could not read ${repo.id}`);
+ const data = await result.json() as {
+ sha: string;
+ gguf?: { architecture?: string };
+ siblings?: { rfilename: string; size?: number; lfs?: { size: number; sha256?: string } }[];
+ };
+ if (!/^[a-f0-9]{40}$/i.test(data.sha) || (data.gguf?.architecture && data.gguf.architecture !== 'bert')) return [];
+ return (data.siblings ?? []).filter(file =>
+ /\.gguf$/i.test(file.rfilename) &&
+ !/mmproj|(?:-\d{5}-of-\d{5})/i.test(file.rfilename) &&
+ (file.lfs?.size ?? file.size ?? 0) > 0,
+ ).map(file => ({
+ id: `${repo.id}@${data.sha}/${file.rfilename}`,
+ name: `${repo.id} / ${file.rfilename}`,
+ size: file.lfs?.size ?? file.size ?? 0,
+ sha256: file.lfs?.sha256,
+ downloadUrl: `https://huggingface.co/${repo.id}/resolve/${data.sha}/${file.rfilename.split('/').map(encodeURIComponent).join('/')}`,
+ }));
+ }));
+ if (signal?.aborted) throw new Error('Embedding search cancelled');
+ if (listings.length && listings.every(result => result.status === 'rejected')) {
+ throw new Error('Could not read embedding model files. Try again.');
+ }
+ return listings.flatMap(result => result.status === 'fulfilled' ? result.value : []);
+}
diff --git a/src/services/imageDownloadActions.ts b/src/services/imageDownloadActions.ts
index 0c83ce33a..45c9afcfe 100644
--- a/src/services/imageDownloadActions.ts
+++ b/src/services/imageDownloadActions.ts
@@ -1,3 +1,4 @@
+import { resolveSDImagePack, resolveSDImageDownloadFiles } from './huggingFaceModelBrowser';
/** Standalone async image download handlers - no hooks. All download state flows through
* useDownloadStore via the stable image: modelKey (single source of truth). */
import { Platform } from 'react-native';
@@ -13,7 +14,8 @@ import { useDownloadStore, isActiveStatus } from '../stores/downloadStore';
import { makeImageModelKey } from '../utils/modelKey';
import { ImageModelDescriptor, ImageDownloadDeps } from './imageModelDownloadTypes';
import { getQnnWarningMessage, showQnnWarningAlert } from './imageDownloadQnn';
-import { ensureImageExtractionComplete } from '../utils/imageModelIntegrity';
+import { ensureImageExtractionComplete, validateMultifileComplete } from '../utils/imageModelIntegrity';
+export { validateMultifileComplete } from '../utils/imageModelIntegrity';
import logger from '../utils/logger';
// ImageDownloadDeps now lives in ./types (so imageDownloadQnn can import it without cycling back
@@ -26,11 +28,11 @@ interface ImageMetadata {
imageModelDescription: string;
imageModelSize: number;
imageModelStyle?: string;
- imageModelBackend?: 'mnn' | 'qnn' | 'coreml';
+ imageModelBackend?: 'mnn' | 'qnn' | 'coreml' | 'sd';
imageModelRepo?: string;
imageModelAttentionVariant?: string;
imageModelDownloadUrl?: string;
- imageModelHuggingFaceFiles?: { path: string; size: number }[];
+ imageModelHuggingFaceFiles?: ImageModelDescriptor['huggingFaceFiles'];
imageModelCoremlFiles?: { path: string; relativePath: string; size: number; downloadUrl: string }[];
}
@@ -44,6 +46,10 @@ type MultifileRuntime = {
const activeMultifileDownloads = new Map();
const USER_CANCELLED_ERROR = 'user_cancelled';
+export function isMultifileImageDownloadActive(modelId: string): boolean {
+ return activeMultifileDownloads.has(modelId);
+}
+
/** Build a synthetic downloadId for multi-file flows that don't go through WorkManager. */
function makeMultifileId(modelId: string): string {
return `image-multi:${modelId}`;
@@ -157,6 +163,7 @@ type MultifileDownloadSpec = {
relativePath: string;
size: number;
url: string;
+ sha256?: string;
};
async function downloadSequentialFiles(opts: {
@@ -176,6 +183,15 @@ async function downloadSequentialFiles(opts: {
const filePath = `${modelDir}/${file.relativePath}`;
const fileDir = filePath.substring(0, filePath.lastIndexOf('/'));
await ensureDirectory(fileDir);
+ // Reuse a complete pinned part after a restart or device-to-device copy.
+ // Size alone cannot distinguish a valid model from corrupted bytes.
+ if (file.sha256 && (await statFile(filePath))?.size === file.size &&
+ (await RNFS.hash(filePath, 'sha256')).toLowerCase() === file.sha256.toLowerCase()) {
+ assertNotCancelled(modelInfo.id, runtime);
+ downloadedSize += file.size;
+ useDownloadStore.getState().updateProgress(syntheticId, downloadedSize, totalSize);
+ continue;
+ }
const tempFileName = `${modelInfo.id}_${file.relativePath.replaceAll('/', '_')}`;
const capturedDownloadedSize = downloadedSize;
@@ -198,18 +214,6 @@ async function downloadSequentialFiles(opts: {
}
}
-/** Verify every part is present and non-empty before registering — a download can
- * resolve "successfully" yet write a 0-byte file (200 with no body). Existence +
- * non-empty only (NOT exact size: descriptor sizes drift from real bytes). Throws so
- * the caller's catch fails it (retry-able) instead of registering garbage. */
-async function validateMultifileComplete(modelDir: string, files: MultifileDownloadSpec[]): Promise {
- for (const file of files) {
- const filePath = `${modelDir}/${file.relativePath}`;
- const size = (await statFile(filePath))?.size ?? -1;
- if (size <= 0) throw new Error(`Downloaded file missing or empty: ${file.relativePath} — tap retry`);
- }
-}
-
/** Remove the entry from the store. Use after register-and-notify or on error. */
function removeStoreEntry(modelId: string) {
useDownloadStore.getState().remove(makeImageModelKey(modelId));
@@ -305,6 +309,15 @@ export async function downloadHuggingFaceModel(
deps.setAlertState(showAlert('Error', 'Invalid HuggingFace model configuration'));
return;
}
+ if (modelInfo.backend === 'sd') {
+ try {
+ const files = await resolveSDImageDownloadFiles(modelInfo.huggingFaceRepo, modelInfo.huggingFaceFiles);
+ modelInfo = { ...modelInfo, huggingFaceFiles: files, size: files.reduce((sum, file) => sum + file.size, 0) };
+ } catch (error) {
+ deps.setAlertState(showAlert('Download unavailable', error instanceof Error ? error.message : 'Could not verify the image files.'));
+ return;
+ }
+ }
const syntheticId = makeMultifileId(modelInfo.id);
const created = addImageEntry({
modelId: modelInfo.id,
@@ -330,14 +343,17 @@ export async function downloadHuggingFaceModel(
await ensureDirectory(imageModelsDir);
await ensureDirectory(modelDir);
- const files = modelInfo.huggingFaceFiles.map((file) => ({
+ const files = modelInfo.huggingFaceFiles!.map((file) => ({
relativePath: file.path,
size: file.size,
- url: `https://huggingface.co/${modelInfo.huggingFaceRepo}/resolve/main/${file.path}`,
+ expectedSize: modelInfo.backend === 'sd' ? file.size : undefined,
+ url: file.downloadUrl ?? `https://huggingface.co/${modelInfo.huggingFaceRepo}/resolve/main/${file.path}`,
+ sha256: file.sha256,
}));
await downloadSequentialFiles({ modelInfo, runtime, syntheticId, modelDir, files });
assertNotCancelled(modelInfo.id, runtime);
await validateMultifileComplete(modelDir, files); // reject a silently-truncated part before registering
+ if (modelInfo.backend === 'sd') await resolveSDImagePack(modelInfo.id, modelDir);
useDownloadStore.getState().setProcessing(syntheticId);
assertNotCancelled(modelInfo.id, runtime);
await RNFS.writeFile(`${modelDir}/_ready`, '', 'utf8').catch(() => {});
diff --git a/src/services/imageGenerationHelpers.ts b/src/services/imageGenerationHelpers.ts
index e7247e7e3..c3fdc87f7 100644
--- a/src/services/imageGenerationHelpers.ts
+++ b/src/services/imageGenerationHelpers.ts
@@ -186,20 +186,25 @@ export function buildImageGenMeta(
},
): GenerationMeta {
const backend = model.backend ?? 'mnn';
+ // SD selects placement in native code. Its result does not report hardware,
+ // so platform and OpenCL settings cannot establish whether GPU was used.
+ const isSD = backend === 'sd';
const isGpu =
Platform.OS === 'ios' ||
backend === 'qnn' ||
(backend === 'mnn' && opts.useOpenCL);
const gpuBackend =
- Platform.OS === 'ios'
- ? 'Core ML (ANE)'
+ isSD
+ ? 'SD runtime'
+ : Platform.OS === 'ios'
+ ? 'Core ML'
: backend === 'qnn'
? 'QNN (NPU)'
: isGpu
? 'MNN (GPU)'
: 'MNN (CPU)';
return {
- gpu: isGpu,
+ gpu: isSD ? undefined : isGpu,
gpuBackend,
modelName: model.name,
steps: opts.steps,
diff --git a/src/services/imageGenerationService.ts b/src/services/imageGenerationService.ts
index f11ef5843..1fc8e803d 100644
--- a/src/services/imageGenerationService.ts
+++ b/src/services/imageGenerationService.ts
@@ -52,6 +52,9 @@ class ImageGenerationService {
private readonly listeners: Set = new Set();
private cancelRequested: boolean = false;
+ // UI cancellation can precede native load/enhancement completion. This promise
+ // owns admission until that work settles; it is not another UI phase.
+ private pendingGeneration: Promise | null = null;
private remoteRequest: AbortController | null = null;
/** Last generate request, so a failure card's Retry button can re-run it. */
private _lastParams: GenerateImageParams | null = null;
@@ -195,6 +198,10 @@ class ImageGenerationService {
);
return true;
} catch (error: any) {
+ if (this.cancelRequested) {
+ this.resetState();
+ return false;
+ }
// Pass the TYPED error as `cause` — an OverridableMemoryError here is what lets
// the failure card offer "Load Anyway". Stringifying it (as before) hid it.
this._fail(
@@ -225,10 +232,10 @@ class ImageGenerationService {
// the single cross-platform signal (so the notice shows once on every device);
// the OpenCL kernel-cache check is an extra Android signal in case the cache was
// cleared after the flag was set.
- let isFirstRun = !useAppStore
+ let isFirstRun = activeImageModel.backend !== 'sd' && !useAppStore
.getState()
.warmedImageModels.includes(activeImageModel.id);
- if (useOpenCL) {
+ if (useOpenCL && activeImageModel.backend !== 'sd') {
try {
const hasCache = await onnxImageGeneratorService.hasKernelCache(
activeImageModel.modelPath,
@@ -242,7 +249,7 @@ class ImageGenerationService {
this.updateState({
phase: 'generating',
- status: isFirstRun
+ status: activeImageModel.backend === 'sd' ? 'Processing image prompt...' : isFirstRun
? 'Optimizing GPU for your device (~120s, one-time)...'
: 'Starting image generation...',
});
@@ -301,7 +308,7 @@ class ImageGenerationService {
});
} catch (error: any) {
const errorMsg = error?.message || 'Image generation failed';
- if (errorMsg.includes('cancelled')) {
+ if (this.cancelRequested || errorMsg.includes('cancelled')) {
this.resetState();
} else {
logger.error('[ImageGenerationService] Generation error:', error);
@@ -327,16 +334,28 @@ class ImageGenerationService {
* Generate an image. Runs independently of UI lifecycle.
* If conversationId is provided, the result will be added as a chat message.
*/
- async generateImage(
+ generateImage(
params: GenerateImageParams,
opts?: { override?: boolean },
): Promise {
- if (isInFlight(this.state.phase)) {
+ // Native cancellation can finish after the UI has cleared its progress.
+ // Keep admission closed until the engine's generation promise settles.
+ if (this.pendingGeneration || isInFlight(this.state.phase) || onnxImageGeneratorService.isGenerating()) {
logger.log(
'[ImageGenerationService] Already generating, ignoring request',
);
- return null;
+ return Promise.resolve(null);
}
+ this.pendingGeneration = this.runImageGeneration(params, opts).finally(() => {
+ this.pendingGeneration = null;
+ });
+ return this.pendingGeneration;
+ }
+
+ private async runImageGeneration(
+ params: GenerateImageParams,
+ opts?: { override?: boolean },
+ ): Promise {
this.cancelRequested = false;
this._lastParams = params; // so a failure card's Retry can re-run this exact request
const remoteServer = useRemoteServerStore
diff --git a/src/services/imageModelDownloadTypes.ts b/src/services/imageModelDownloadTypes.ts
index f08044ef7..5e1040a0e 100644
--- a/src/services/imageModelDownloadTypes.ts
+++ b/src/services/imageModelDownloadTypes.ts
@@ -16,10 +16,10 @@ export interface ImageModelDescriptor {
downloadUrl: string;
size: number;
style: string;
- backend: 'mnn' | 'qnn' | 'coreml';
+ backend: 'mnn' | 'qnn' | 'coreml' | 'sd';
variant?: string;
huggingFaceRepo?: string;
- huggingFaceFiles?: { path: string; size: number }[];
+ huggingFaceFiles?: { path: string; size: number; downloadUrl?: string; sha256?: string }[];
coremlFiles?: { path: string; relativePath: string; size: number; downloadUrl: string }[];
repo?: string;
attentionVariant?: 'split_einsum' | 'original';
diff --git a/src/services/imageParameterPolicy.ts b/src/services/imageParameterPolicy.ts
index 9b770e938..26794ac3c 100644
--- a/src/services/imageParameterPolicy.ts
+++ b/src/services/imageParameterPolicy.ts
@@ -1,9 +1,24 @@
+import { Platform } from 'react-native';
import {
effectiveImageParameter,
resolveImageParameters,
type ImageParameterStore,
} from '@offgrid/models';
-import { SWEET_SPOT_SIZE } from '../utils/imageGenAdvice';
+import { defaultImageSteps, SWEET_SPOT_SIZE } from '../utils/imageGenAdvice';
+
+type ImageParameterModel = { id: string; name?: string; backend?: string };
+
+/** Native legacy packs keep their fixed size limit; SD uses the shared model policy. */
+export function mobileImageParameterDefaults(model: ImageParameterModel, platform: string) {
+ const defaults = resolveImageParameters(model, undefined);
+ return model.backend === 'sd'
+ ? { steps: defaults.steps, guidanceScale: defaults.cfgScale, size: defaults.size }
+ : { steps: defaultImageSteps(platform), guidanceScale: 7.5, size: 512 };
+}
+
+export function mobileImageSizeLimit(model: ImageParameterModel): number {
+ return model.backend === 'sd' ? resolveImageParameters(model, undefined).size : 512;
+}
export interface MobileImageParameterSettings {
imageSteps?: number | null;
@@ -23,7 +38,7 @@ const positiveFinite = (value: number | null | undefined): number | undefined =>
/** Adapt Mobile's current settings shape into the shared model-specific policy. */
export function resolveMobileImageParameters(
- model: { id: string; name?: string },
+ model: ImageParameterModel,
settings: MobileImageParameterSettings,
request: MobileImageParameterRequest = {},
): { steps: number; guidanceScale: number; size: number } {
@@ -35,6 +50,12 @@ export function resolveMobileImageParameters(
},
};
const resolved = resolveImageParameters(model, store);
+ if (model.backend !== 'sd') {
+ const defaults = mobileImageParameterDefaults(model, Platform.OS);
+ resolved.steps = positiveFinite(settings.imageSteps) ?? defaults.steps;
+ resolved.cfgScale = positiveFinite(settings.imageGuidanceScale) ?? defaults.guidanceScale;
+ resolved.size = positiveFinite(settings.imageWidth) ?? defaults.size;
+ }
return {
steps: effectiveImageParameter(
positiveFinite(request.steps),
@@ -44,6 +65,6 @@ export function resolveMobileImageParameters(
positiveFinite(request.guidanceScale),
resolved.cfgScale,
),
- size: Math.max(SWEET_SPOT_SIZE, resolved.size),
+ size: Math.min(mobileImageSizeLimit(model), Math.max(SWEET_SPOT_SIZE, resolved.size)),
};
}
diff --git a/src/services/litert.ts b/src/services/litert.ts
index 4bb6a21ec..d7478e566 100644
--- a/src/services/litert.ts
+++ b/src/services/litert.ts
@@ -100,9 +100,13 @@ class LiteRTService {
async loadModel(modelPath: string, preferredBackend: LiteRTBackend, opts: { supportsVision?: boolean; supportsAudio?: boolean; maxNumTokens?: number } = {}): Promise {
if (!this.isAvailable()) throw new Error('LiteRT is not available on this platform');
+ const requestedBackend = preferredBackend;
// NPU was exposed in a test build, but is not part of this release.
// Keep saved selections usable without attempting an unsupported native load.
- if (preferredBackend === 'npu') preferredBackend = 'gpu';
+ if (preferredBackend === 'npu') {
+ logger.warn(TAG, 'LiteRT NPU is disabled in this release; trying GPU, then CPU.');
+ preferredBackend = 'gpu';
+ }
const { supportsVision = false, supportsAudio = false, maxNumTokens = 4096 } = opts;
this.configuredMaxTokens = maxNumTokens;
logger.log(TAG, `loadModel — path=${modelPath} backend=${preferredBackend} supportsVision=${supportsVision} supportsAudio=${supportsAudio} maxNumTokens=${maxNumTokens}`);
@@ -114,7 +118,7 @@ class LiteRTService {
// older native build (backward-compatible).
const res: string | { backend: string; maxNumTokens?: number } =
await LiteRTModule.loadModel(modelPath, preferredBackend, supportsVision, supportsAudio, maxNumTokens);
- logger.log(`[WIRE-LITERT-LOAD] ${JSON.stringify({ requested: preferredBackend, supportsVision, supportsAudio, maxNumTokens, res })}`); // [WIRE]
+ logger.log(`[WIRE-LITERT-LOAD] ${JSON.stringify({ requested: requestedBackend, attempted: preferredBackend, supportsVision, supportsAudio, maxNumTokens, res })}`); // [WIRE]
const actualBackend = typeof res === 'string' ? res : res.backend;
if (typeof res === 'object' && typeof res.maxNumTokens === 'number' && res.maxNumTokens > 0) {
if (res.maxNumTokens !== this.configuredMaxTokens) {
diff --git a/src/services/llm.ts b/src/services/llm.ts
index 3a4adf0f7..ce419b524 100644
--- a/src/services/llm.ts
+++ b/src/services/llm.ts
@@ -168,11 +168,14 @@ class LLMService {
availableBytes: deviceInfo.availableMemory,
});
if (safeGpuLayers !== params.nGpuLayers) logger.log(`[LLM] GPU layers capped (${(deviceInfo.totalMemory / BYTES_PER_GB).toFixed(1)}GB RAM, ${Platform.OS}): ${params.nGpuLayers} → ${safeGpuLayers}`);
+ let fallbackGpuLayers = 0;
let resolvedBaseParams: object = params.baseParams;
if (Platform.OS === 'android') {
const settings = useAppStore.getState().settings;
const backend = settings?.inferenceBackend ?? INFERENCE_BACKENDS.CPU;
if (backend === INFERENCE_BACKENDS.HTP) {
+ const gpuCapability = await hardwareService.getOpenCLCapability();
+ fallbackGpuLayers = gpuCapability.supported ? safeGpuLayers : 0;
// HTP routes to the Hexagon NPU — not subject to Adreno GPU layer caps,
// but we still respect the RAM-based safeGpuLayers floor (0 on ≤4GB devices).
safeGpuLayers = safeGpuLayers > 0 ? (settings?.gpuLayers ?? 99) : 0;
@@ -196,10 +199,7 @@ class LLMService {
// The model metadata and the user's setting own context length. Do not impose a
// second RAM-tier ceiling here. validateAndPrepareModel checks this exact
// model + cache + selected context against live memory without changing it.
- return {
- ...await initContextWithFallback(resolvedBaseParams, params.ctxLen, safeGpuLayers),
- attemptedGpuLayers: safeGpuLayers,
- };
+ return initContextWithFallback(resolvedBaseParams, params.ctxLen, safeGpuLayers, fallbackGpuLayers);
}
/** Multimodal init on a NOT-YET-PUBLISHED context (the load pipeline) — no instance-state writes. */
private async deriveMultimodalFromProjector(context: LlamaContext, modelPath: string, mmProjPath: string): Promise<{ initialized: boolean; support: MultimodalSupport }> {
diff --git a/src/services/llmHelpers.ts b/src/services/llmHelpers.ts
index 214df1ae2..f2b241882 100644
--- a/src/services/llmHelpers.ts
+++ b/src/services/llmHelpers.ts
@@ -123,6 +123,7 @@ export interface ContextInitResult {
context: LlamaContext;
gpuAttemptFailed: boolean;
actualLength: number;
+ attemptedGpuLayers: number;
}
/** Timeout for Adreno GPU context init on Android. 8s proved too tight on-device: Adreno 735
* first-load OpenCL kernel compilation exceeded it (2026-07-13 20:11 log: "timed out after
@@ -174,11 +175,12 @@ async function tryGpuInit(promise: Promise, nGpuLayers: number, is
catch (e) { timedOut = true; throw e; }
}
-/** Init llama with GPU/HTP, then retry on CPU at the selected context. */
+/** Try the selected accelerator, a supported GPU fallback, then CPU. */
export async function initContextWithFallback(
params: object,
contextLength: number,
nGpuLayers: number,
+ fallbackGpuLayers: number = 0,
): Promise {
const modelPath = (params as any).model || 'unknown';
const isHtp = HTP_ENABLED && Array.isArray((params as any).devices) && (params as any).devices.some((d: string) => d.startsWith('HTP'));
@@ -195,7 +197,7 @@ export async function initContextWithFallback(
const gpuInitPromise = initLlama({ ...params, n_ctx: contextLength, n_gpu_layers: nGpuLayers } as any);
const context = await tryGpuInit(gpuInitPromise, nGpuLayers, isHtp);
logger.log('[LLM] GPU init succeeded');
- return { context, gpuAttemptFailed, actualLength: contextLength };
+ return { context, gpuAttemptFailed, actualLength: contextLength, attemptedGpuLayers: nGpuLayers };
} catch (gpuError: any) {
const gpuMsg = gpuError?.message || String(gpuError);
if (nGpuLayers > 0) {
@@ -204,6 +206,19 @@ export async function initContextWithFallback(
} else {
logger.warn(`[LLM] Attempt 1/2 failed (no GPU requested): ${gpuMsg}`);
}
+ if (isHtp && nGpuLayers > 0 && fallbackGpuLayers > 0) {
+ const gpuParams = { ...(params as Record) };
+ delete gpuParams.devices;
+ delete gpuParams.cache_type_k;
+ delete gpuParams.cache_type_v;
+ try {
+ logger.warn('[LLM] NPU initialization failed; trying the supported GPU backend');
+ const context = await tryGpuInit(initLlama({ ...gpuParams, n_ctx: contextLength, n_gpu_layers: fallbackGpuLayers } as any), fallbackGpuLayers);
+ return { context, gpuAttemptFailed: false, actualLength: contextLength, attemptedGpuLayers: fallbackGpuLayers };
+ } catch (fallbackError) {
+ logger.warn('[LLM] GPU fallback failed:', fallbackError);
+ }
+ }
try {
logger.log(`[LLM] Attempt 2/2: CPU init (ctx=${contextLength}, gpu_layers=0)`);
// Strip devices — HTP requires n_gpu_layers > 0; CPU fallback must not request it
@@ -211,7 +226,7 @@ export async function initContextWithFallback(
delete cpuParams.devices;
const context = await initLlama({ ...cpuParams, n_ctx: contextLength, n_gpu_layers: 0 } as any);
logger.log('[LLM] CPU init succeeded');
- return { context, gpuAttemptFailed, actualLength: contextLength };
+ return { context, gpuAttemptFailed, actualLength: contextLength, attemptedGpuLayers: 0 };
} catch (cpuError: any) {
const cpuMsg = cpuError?.message || String(cpuError);
logger.warn(`[LLM] Attempt 2/2 failed (CPU, ctx=${contextLength}): ${cpuMsg}`);
diff --git a/src/services/localDreamGenerator.ts b/src/services/localDreamGenerator.ts
index ce69749df..d33368b26 100644
--- a/src/services/localDreamGenerator.ts
+++ b/src/services/localDreamGenerator.ts
@@ -5,11 +5,13 @@ import {
ImageGenerationProgress,
GeneratedImage,
} from '../types';
-import { generateRandomSeed } from '../utils/generateId';
+import { resolveSDImagePack } from './huggingFaceModelBrowser';
+import { validateImageModelDir } from '../utils/imageModelIntegrity';
+import { generateId, generateRandomSeed } from '../utils/generateId';
import logger from '../utils/logger';
import { resolveOwnedDocumentPath } from '../utils/resolveDocumentPath';
-const { LocalDreamModule, CoreMLDiffusionModule } = NativeModules;
+const { LocalDreamModule, CoreMLDiffusionModule, VideoGenerationModule: SDModule } = NativeModules;
const PROGRESS_LOG_SAMPLE_STEPS = 5;
// Pick the right native module per platform
@@ -34,25 +36,27 @@ type PreviewCallback = (preview: { previewPath: string; step: number; totalSteps
* Progress events are emitted via NativeEventEmitter from the native side.
*/
class LocalDreamGeneratorService {
+ private usingSD = false;
private loadedThreads: number | null = null;
private generating = false;
private eventEmitter: NativeEventEmitter | null = null;
private getEmitter(): NativeEventEmitter {
if (!this.eventEmitter) {
- this.eventEmitter = new NativeEventEmitter(DiffusionModule);
+ this.eventEmitter = new NativeEventEmitter(this.usingSD ? SDModule : DiffusionModule);
}
return this.eventEmitter;
}
isAvailable(): boolean {
- return DiffusionModule != null;
+ return DiffusionModule != null || SDModule?.generateImage != null;
}
async isModelLoaded(): Promise {
if (!this.isAvailable()) return false;
try {
- return await DiffusionModule.isModelLoaded();
+ if (SDModule?.getLoadedImagePath && await SDModule.getLoadedImagePath()) { this.usingSD = true; return true; }
+ return await DiffusionModule?.isModelLoaded() ?? false;
} catch {
return false;
}
@@ -61,18 +65,38 @@ class LocalDreamGeneratorService {
async getLoadedModelPath(): Promise {
if (!this.isAvailable()) return null;
try {
- return await DiffusionModule.getLoadedModelPath();
+ const sdPath = await SDModule?.getLoadedImagePath?.();
+ if (sdPath) { this.usingSD = true; return sdPath; }
+ return await DiffusionModule?.getLoadedModelPath() ?? null;
} catch {
return null;
}
}
- async loadModel(modelPath: string, threads?: number, opts: { backend?: 'mnn' | 'qnn' | 'auto'; cpuOnly?: boolean; attentionVariant?: 'split_einsum' | 'original'; preferGpu?: boolean } = {}): Promise {
+ async loadModel(modelPath: string, threads?: number, opts: { backend?: 'mnn' | 'qnn' | 'auto' | 'sd'; modelId?: string; cpuOnly?: boolean; attentionVariant?: 'split_einsum' | 'original'; preferGpu?: boolean } = {}): Promise {
if (!this.isAvailable()) {
throw new Error('LocalDream image generation is not available on this platform');
}
const backend = opts.backend ?? 'auto';
+ if (this.generating) throw new Error('Image generation is running.');
+ if (backend === 'sd') {
+ if (!SDModule?.loadImageModel) throw new Error('This build does not include the SD image runtime.');
+ const integrity = await validateImageModelDir(modelPath, 'sd');
+ if (!integrity.complete) throw new Error(`The image model pack is incomplete: ${integrity.missing.join(', ')}`);
+ const pack = await resolveSDImagePack(opts.modelId ?? modelPath.split('/').pop() ?? '', modelPath);
+ if (await DiffusionModule?.isModelLoaded()) await DiffusionModule.unloadModel();
+ const diagnostics = new NativeEventEmitter(SDModule).addListener('SDImageProgress', (event: { diagnostic?: string }) => {
+ if (event.diagnostic) logger.log(`[SD-IMAGE-NATIVE] ${event.diagnostic}`);
+ });
+ try {
+ await SDModule.loadImageModel({ modelPath, ...pack, threads: threads ?? 4, cpuOnly: opts.cpuOnly ?? false });
+ } finally { diagnostics.remove(); }
+ this.usingSD = true; this.eventEmitter = null; this.loadedThreads = threads ?? 4;
+ return true;
+ }
+ if (SDModule?.getLoadedImagePath && await SDModule.getLoadedImagePath()) await SDModule.unloadImageModel();
+ this.usingSD = false; this.eventEmitter = null;
const params: { modelPath: string; threads?: number; backend: string; cpuOnly?: boolean; attentionVariant?: string; preferGpu?: boolean } = {
modelPath,
backend,
@@ -104,7 +128,9 @@ class LocalDreamGeneratorService {
async unloadModel(): Promise {
if (!this.isAvailable()) return true;
try {
- const result = await DiffusionModule.unloadModel();
+ if (SDModule?.getLoadedImagePath && await SDModule.getLoadedImagePath()) await SDModule.unloadImageModel();
+ const result = await DiffusionModule?.unloadModel() ?? true;
+ this.usingSD = false; this.eventEmitter = null;
this.loadedThreads = null;
return result;
} catch (e) {
@@ -116,8 +142,12 @@ class LocalDreamGeneratorService {
private subscribeToProgress(onProgress?: ProgressCallback, onPreview?: PreviewCallback): any {
return this.getEmitter().addListener(
- 'LocalDreamProgress',
- (event: { step: number; totalSteps: number; progress: number; previewPath?: string }) => {
+ this.usingSD ? 'SDImageProgress' : 'LocalDreamProgress',
+ (event: { step: number; totalSteps: number; progress: number; previewPath?: string; diagnostic?: string }) => {
+ if (event.diagnostic) {
+ logger.log(`[SD-IMAGE-NATIVE] ${event.diagnostic}`);
+ return;
+ }
if (
!Number.isInteger(event.step) ||
event.step < 1 ||
@@ -191,13 +221,29 @@ class LocalDreamGeneratorService {
}
this.generating = true;
- const progressSubscription = this.subscribeToProgress(onProgress, onPreview);
+ let lastStep = -1;
+ const progressSubscription = this.subscribeToProgress(progress => {
+ if (this.usingSD) {
+ if (progress.totalSteps !== (params.steps || 8) || progress.step <= lastStep || progress.step > progress.totalSteps) return;
+ lastStep = progress.step;
+ }
+ onProgress?.(progress);
+ }, onPreview);
try {
- const result = await DiffusionModule.generateImage(this.buildNativeParams(params, trimmedPrompt));
+ const nativeParams = this.buildNativeParams(params, trimmedPrompt);
+ let result;
+ if (this.usingSD) {
+ const id = generateId();
+ const directory = `${RNFS.DocumentDirectoryPath}/generated_images`;
+ await RNFS.mkdir(directory);
+ result = await SDModule.generateImage({ ...nativeParams, id, outputPath: `${directory}/${id}.png` });
+ } else {
+ result = await DiffusionModule.generateImage(nativeParams);
+ }
// Native side releases the CoreML pipeline after generation to free
// memory, so clear TS-side state so the next request triggers a reload.
- this.loadedThreads = null;
+ if (!this.usingSD) this.loadedThreads = null;
return this.buildResult(params, result);
} catch (error: any) {
const msg = error?.message || '';
@@ -213,11 +259,11 @@ class LocalDreamGeneratorService {
async cancelGeneration(): Promise {
if (!this.isAvailable()) return true;
- this.generating = false;
+ if (this.usingSD) { await SDModule.cancel(); return true; }
return await DiffusionModule.cancelGeneration();
}
- async isGenerating(): Promise {
+ isGenerating(): boolean {
return this.generating;
}
@@ -272,12 +318,12 @@ class LocalDreamGeneratorService {
}
async clearOpenCLCache(modelPath: string): Promise {
- if (Platform.OS !== 'android' || !this.isAvailable()) return 0;
+ if (this.usingSD || Platform.OS !== 'android' || !this.isAvailable()) return 0;
return await DiffusionModule.clearOpenCLCache(modelPath);
}
async hasKernelCache(modelPath: string): Promise {
- if (Platform.OS !== 'android' || !this.isAvailable()) return true;
+ if (this.usingSD || Platform.OS !== 'android' || !this.isAvailable()) return true;
return await DiffusionModule.hasOpenCLCache(modelPath);
}
diff --git a/src/services/modelDownloadService/providers/videoProvider.ts b/src/services/modelDownloadService/providers/videoProvider.ts
new file mode 100644
index 000000000..12e1434f1
--- /dev/null
+++ b/src/services/modelDownloadService/providers/videoProvider.ts
@@ -0,0 +1,288 @@
+import { videoModelDisplayName } from '../../../utils/modelHelpers';
+import { mapStoreStatus } from '../storeStatus';
+import { modelResidencyManager } from '../../modelResidency';
+import RNFS from 'react-native-fs';
+import type { ModelEntry } from '@offgrid/models';
+import { useAppStore } from '../../../stores/appStore';
+import { useDownloadStore } from '../../../stores/downloadStore';
+import { backgroundDownloadService } from '../../backgroundDownloadService';
+import {
+ resolveVideoPack,
+ validateVideoPack,
+ videoModelDirectory,
+} from '../../videoModelFiles';
+import type {
+ DownloadProvider,
+ ModelDownload,
+ ModelDownloadStatus,
+} from '../types';
+
+interface Transfer {
+ cancelled: boolean;
+ paused: boolean;
+ nativeId?: string;
+ wake?: () => void;
+}
+const transfers = new Map();
+const key = (id: string) => `video:${id}`;
+const bare = (id: string) => id.replace(/^video:/, '');
+function addRow(model: ModelEntry, status: 'pending' | 'paused') {
+ useDownloadStore
+ .getState()
+ .add({
+ modelKey: key(model.id),
+ modelId: model.id,
+ downloadId: key(model.id),
+ modelType: 'video',
+ fileName: videoModelDisplayName(model.id, model.name),
+ totalBytes: model.files.reduce((sum, f) => sum + (f.sizeBytes ?? 0), 0),
+ bytesDownloaded: 0,
+ progress: 0,
+ status,
+ quantization: '',
+ combinedTotalBytes: model.files.reduce(
+ (sum, f) => sum + (f.sizeBytes ?? 0),
+ 0,
+ ),
+ createdAt: Date.now(),
+ });
+}
+async function start(model: ModelEntry): Promise {
+ validateVideoPack(model);
+ if (
+ useAppStore
+ .getState()
+ .downloadedVideoModels.some(item => item.id === model.id)
+ )
+ throw new Error(
+ 'Remove the installed video pack before downloading another variant.',
+ );
+ if (transfers.has(model.id)) return;
+ const transfer: Transfer = { cancelled: false, paused: false };
+ transfers.set(model.id, transfer);
+ useAppStore.getState().setVideoDownload(model.id, model);
+ addRow(model, 'pending');
+ const directory = videoModelDirectory(model.id);
+ let bytes = 0;
+ const installedFiles: ModelEntry['files'] = [];
+ const total = model.files.reduce(
+ (sum, file) => sum + (file.sizeBytes ?? 0),
+ 0,
+ );
+ try {
+ await RNFS.mkdir(directory);
+ await backgroundDownloadService.excludeFromBackup(directory);
+ for (const file of model.files) {
+ if (transfer.paused)
+ await new Promise(resolve => {
+ transfer.wake = resolve;
+ });
+ if (transfer.cancelled) throw new Error('Download cancelled.');
+ const destination = `${directory}/${file.name}`;
+ const existing = await RNFS.stat(destination).catch(() => null);
+ if (
+ existing &&
+ file.sizeBytes &&
+ Number(existing.size) === file.sizeBytes &&
+ (!file.sha256 || (await RNFS.hash(destination, 'sha256')).toLowerCase() === file.sha256.toLowerCase())
+ ) {
+ installedFiles.push({ ...file, sizeBytes: Number(existing.size) });
+ bytes += file.sizeBytes;
+ useDownloadStore.getState().updateProgress(key(model.id), bytes, total);
+ continue;
+ }
+ useDownloadStore.getState().setStatus(key(model.id), 'running');
+ const task = backgroundDownloadService.downloadFileTo({
+ params: {
+ url: file.url,
+ fileName: `${encodeURIComponent(model.id)}_${file.name}`,
+ modelId: key(model.id),
+ modelKey: key(model.id),
+ modelType: 'video',
+ totalBytes: file.sizeBytes,
+ sha256: file.sha256,
+ },
+ destPath: destination,
+ onProgress: received =>
+ useDownloadStore
+ .getState()
+ .updateProgress(key(model.id), bytes + received, total),
+ });
+ task.downloadIdPromise
+ ?.then(async id => {
+ transfer.nativeId = id;
+ if (transfer.cancelled)
+ await backgroundDownloadService.cancelDownload(id);
+ else if (transfer.paused)
+ await backgroundDownloadService.pauseDownload(id);
+ })
+ .catch(() => {});
+ await task.promise;
+ transfer.nativeId = undefined;
+ const sizeBytes = Number((await RNFS.stat(destination)).size);
+ installedFiles.push({ ...file, sizeBytes });
+ bytes += sizeBytes;
+ }
+ if (transfer.cancelled) throw new Error('Download cancelled.');
+ useDownloadStore.getState().setProcessing(key(model.id));
+ await resolveVideoPack(model, true);
+ if (transfer.cancelled) throw new Error('Download cancelled.');
+ const app = useAppStore.getState();
+ app.addDownloadedVideoModel({
+ ...model,
+ kind: 'video',
+ files: installedFiles,
+ downloadedAt: new Date().toISOString(),
+ });
+ app.setVideoDownload(model.id, null);
+ useDownloadStore.getState().remove(key(model.id));
+ } catch (error) {
+ if (!transfer.cancelled)
+ useDownloadStore
+ .getState()
+ .setStatus(key(model.id), 'failed', {
+ message:
+ error instanceof Error ? error.message : 'Video download failed.',
+ });
+ else useDownloadStore.getState().remove(key(model.id));
+ } finally {
+ if (transfer.cancelled) await RNFS.unlink(directory).catch(() => {});
+ transfers.delete(model.id);
+ }
+}
+export const videoProvider: DownloadProvider = {
+ modelType: 'video',
+ async start(request) {
+ if (request.modelType === 'video') await start(request.model);
+ },
+ async list() {
+ const result: ModelDownload[] = [];
+ for (const model of Object.values(useAppStore.getState().videoDownloads)) {
+ const row = useDownloadStore.getState().downloads[key(model.id)];
+ const status: ModelDownloadStatus = row ? mapStoreStatus(row.status) : 'paused';
+ result.push({
+ id: key(model.id),
+ modelType: 'video',
+ name: videoModelDisplayName(model.id, model.name),
+ sizeBytes: row?.totalBytes ?? 0,
+ bytesDownloaded: row?.bytesDownloaded ?? 0,
+ progress: row?.progress ?? 0,
+ status,
+ error: row?.errorMessage,
+ capabilities: {
+ cancel: true,
+ retry: true,
+ remove: true,
+ pause: transfers.has(model.id) && status === 'downloading',
+ resume: status === 'paused',
+ resumable: false,
+ determinateProgress: true,
+ },
+ });
+ }
+ for (const model of useAppStore.getState().downloadedVideoModels) {
+ if (result.some(row => row.id === key(model.id))) continue;
+ const size = model.files.reduce(
+ (sum, file) => sum + (file.sizeBytes ?? 0),
+ 0,
+ );
+ result.push({
+ id: key(model.id),
+ modelType: 'video',
+ name: videoModelDisplayName(model.id, model.name),
+ sizeBytes: size,
+ bytesDownloaded: size,
+ progress: 1,
+ status: 'completed',
+ filePath: videoModelDirectory(model.id),
+ capabilities: {
+ cancel: false,
+ retry: false,
+ remove: true,
+ resumable: false,
+ determinateProgress: true,
+ },
+ });
+ }
+ return result;
+ },
+ async pause(id) {
+ const transfer = transfers.get(bare(id));
+ if (!transfer) return;
+ transfer.paused = true;
+ try {
+ if (transfer.nativeId)
+ await backgroundDownloadService.pauseDownload(transfer.nativeId);
+ } catch (error) {
+ transfer.paused = false;
+ throw error;
+ }
+ useDownloadStore.getState().setStatus(id, 'paused');
+ },
+ async resume(id) {
+ const transfer = transfers.get(bare(id));
+ if (!transfer) {
+ await videoProvider.retry(id);
+ return;
+ }
+ if (transfer.nativeId)
+ await backgroundDownloadService.resumeDownload(transfer.nativeId);
+ transfer.paused = false;
+ transfer.wake?.();
+ transfer.wake = undefined;
+ useDownloadStore.getState().setStatus(id, 'running');
+ },
+ async cancel(id) {
+ const modelId = bare(id),
+ transfer = transfers.get(modelId);
+ if (transfer) {
+ transfer.cancelled = true;
+ transfer.wake?.();
+ backgroundDownloadService.cancelQueued(key(modelId));
+ if (transfer.nativeId)
+ await backgroundDownloadService.cancelDownload(transfer.nativeId);
+ }
+ if (!transfer) await RNFS.unlink(videoModelDirectory(modelId)).catch(() => {});
+ useAppStore.getState().setVideoDownload(modelId, null);
+ useDownloadStore.getState().remove(key(modelId));
+ },
+ async retry(id) {
+ const model = useAppStore.getState().videoDownloads[bare(id)];
+ if (model) await start(model);
+ },
+ async remove(id) {
+ const modelId = bare(id);
+ if (
+ modelResidencyManager
+ .getResidents()
+ .some(resident => resident.type === 'video')
+ )
+ throw new Error('Stop video generation before removing this model.');
+ if (transfers.has(modelId))
+ throw new Error('Cancel the download before removing its files.');
+ const directory = videoModelDirectory(modelId);
+ if (await RNFS.exists(directory)) await RNFS.unlink(directory);
+ useAppStore.getState().removeDownloadedVideoModel(modelId);
+ useAppStore.getState().setVideoDownload(modelId, null);
+ useDownloadStore.getState().remove(key(modelId));
+ },
+ subscribe(listener) {
+ const a = useAppStore.subscribe(listener),
+ b = useDownloadStore.subscribe(listener);
+ return () => {
+ a();
+ b();
+ };
+ },
+ async reconcile() {
+ // An old native request has no pack continuation after process death. Preserve
+ // completed pack files, stop the orphan request, and offer an explicit resume.
+ for (const row of await backgroundDownloadService.getActiveDownloads()) {
+ if (row.modelId.startsWith('video:') && !transfers.has(bare(row.modelId)))
+ await backgroundDownloadService.cancelDownload(row.downloadId);
+ }
+ for (const model of Object.values(useAppStore.getState().videoDownloads)) {
+ if (!transfers.has(model.id)) addRow(model, 'paused');
+ }
+ },
+};
diff --git a/src/services/modelDownloadService/registerProviders.ts b/src/services/modelDownloadService/registerProviders.ts
index 05796c8bd..78eb479ab 100644
--- a/src/services/modelDownloadService/registerProviders.ts
+++ b/src/services/modelDownloadService/registerProviders.ts
@@ -1,3 +1,4 @@
+import { videoProvider } from './providers/videoProvider';
/**
* Register the core download providers (text / image / stt) with the single
* ModelDownloadService. Called once at app boot. The tts provider lives in pro and
@@ -15,5 +16,6 @@ export function registerCoreDownloadProviders(): void {
registered = true;
modelDownloadService.register(textProvider);
modelDownloadService.register(imageProvider);
+ modelDownloadService.register(videoProvider);
modelDownloadService.register(sttProvider);
}
diff --git a/src/services/modelDownloadService/types.ts b/src/services/modelDownloadService/types.ts
index ef10ef30a..ee16859dd 100644
--- a/src/services/modelDownloadService/types.ts
+++ b/src/services/modelDownloadService/types.ts
@@ -16,18 +16,26 @@
* See docs/design/MODEL_DOWNLOAD_SERVICE.md.
*/
-export type ModelDownloadType = 'text' | 'image' | 'stt' | 'tts';
+export type ModelDownloadType = 'text' | 'image' | 'video' | 'stt' | 'tts';
export type ModelDownloadStatus =
- | 'queued' // accepted, not yet transferring
- | 'downloading' // bytes moving
- | 'paused' // interrupted (e.g. waiting for network / app was killed) — resumable
- | 'completed' // on disk + registered in its domain store
- | 'error'; // failed; retryable
+ | 'queued' // accepted, not yet transferring
+ | 'downloading' // bytes moving
+ | 'paused' // interrupted (e.g. waiting for network / app was killed) — resumable
+ | 'completed' // on disk + registered in its domain store
+ | 'error'; // failed; retryable
export type ModelDownloadStartRequest =
- | { modelType: 'text'; modelId: string; file: import('../../types').ModelFile }
- | { modelType: 'image'; model: import('../imageModelDownloadTypes').ImageModelDescriptor }
+ | { modelType: 'video'; model: import('@offgrid/models').ModelEntry }
+ | {
+ modelType: 'text';
+ modelId: string;
+ file: import('../../types').ModelFile;
+ }
+ | {
+ modelType: 'image';
+ model: import('../imageModelDownloadTypes').ImageModelDescriptor;
+ }
| { modelType: 'stt'; modelId: string };
/**
@@ -46,12 +54,12 @@ export type ModelDownloadStartRequest =
* download surfaces as needing a manual retry, not a phantom "resuming".
*/
interface DownloadCapabilities {
- pause?: boolean; // can an in-progress transfer stop without removing its bytes?
- resume?: boolean; // can the paused transfer continue?
- cancel: boolean; // can an in-progress transfer be aborted + cleaned up?
- retry: boolean; // can a failed/stuck download be restarted?
- remove: boolean; // can the on-disk model be deleted?
- resumable: boolean; // does it survive an app kill and continue on its own?
+ pause?: boolean; // can an in-progress transfer stop without removing its bytes?
+ resume?: boolean; // can the paused transfer continue?
+ cancel: boolean; // can an in-progress transfer be aborted + cleaned up?
+ retry: boolean; // can a failed/stuck download be restarted?
+ remove: boolean; // can the on-disk model be deleted?
+ resumable: boolean; // does it survive an app kill and continue on its own?
determinateProgress: boolean; // are byteDownloaded/sizeBytes real (vs fraction-only)?
}
@@ -136,5 +144,7 @@ export interface DownloadProvider {
* never admission-controlled/queued can omit it. `params` is passed as the shared shape so the
* service never branches on the concrete type.
*/
- reissue?(params: import('../backgroundDownloadTypes').DownloadParams): Promise;
+ reissue?(
+ params: import('../backgroundDownloadTypes').DownloadParams,
+ ): Promise;
}
diff --git a/src/services/modelDownloadService/uniformId.ts b/src/services/modelDownloadService/uniformId.ts
index 252cdbf66..00d7e1e1a 100644
--- a/src/services/modelDownloadService/uniformId.ts
+++ b/src/services/modelDownloadService/uniformId.ts
@@ -15,13 +15,17 @@ import type { ModelType } from '../../stores/downloadStore';
* (`[DL-SM] … REFUSED: not found`). Every other type passes the modelId through
* unchanged; only STT normalizes, and it normalizes HERE, once.
*/
-export function uniformDownloadId(modelType: ModelType, modelId: string): string {
+export function uniformDownloadId(
+ modelType: ModelType,
+ modelId: string,
+): string {
// Per-type canonicalization, owned HERE so the providers' list() and the View's
// dispatch can't drift. Both are idempotent (safe whether given the bare id or the
// prefixed store id): STT store rows are `whisper-` but whisperService keys by
// the bare id; image store rows carry an `image:` prefix the provider strips.
let canonical = modelId;
if (modelType === 'stt') canonical = modelId.replace(/^whisper-/, '');
+ else if (modelType === 'video') canonical = modelId.replace(/^video:/, '');
else if (modelType === 'image') canonical = modelId.replace(/^image:/, '');
return `${modelType}:${canonical}`;
}
diff --git a/src/services/modelFailureHandler.ts b/src/services/modelFailureHandler.ts
index ae8180047..c06ce69cd 100644
--- a/src/services/modelFailureHandler.ts
+++ b/src/services/modelFailureHandler.ts
@@ -32,6 +32,7 @@ import logger from '../utils/logger';
const TYPE_LABEL: Record = {
text: 'Text model',
image: 'Image model',
+ video: 'Video model',
tts: 'Voice',
stt: 'Transcription',
embedding: 'Memory',
diff --git a/src/services/modelManager/imageTransfer.ts b/src/services/modelManager/imageTransfer.ts
index 495589e94..e69924094 100644
--- a/src/services/modelManager/imageTransfer.ts
+++ b/src/services/modelManager/imageTransfer.ts
@@ -6,7 +6,7 @@ const IMAGE_TRANSFER_ARCHIVE_SUFFIX = '.offgrid-image.zip';
type MobileImagePlatform = Extract;
type TransferableImageBackend = Extract<
NonNullable,
- 'coreml' | 'mnn'
+ 'coreml' | 'mnn' | 'sd'
>;
export interface TransferredImageDescriptor {
@@ -20,7 +20,7 @@ export interface TransferredImageDescriptor {
export interface TransferredImageManifest extends TransferredModelManifest {
kind: 'image';
- engine: 'coreml' | 'localdream-mnn';
+ engine: 'coreml' | 'localdream-mnn' | 'mobile-sd-gguf';
platform: MobileImagePlatform;
image: TransferredImageDescriptor;
}
@@ -36,10 +36,11 @@ function isSafeImageModelId(id: string): boolean {
);
}
-function expectedImageRuntime(platform: MobileImagePlatform): {
+function expectedImageRuntime(platform: MobileImagePlatform, backend?: ONNXImageModel['backend']): {
backend: TransferableImageBackend;
engine: TransferredImageManifest['engine'];
} {
+ if (backend === 'sd') return { backend: 'sd', engine: 'mobile-sd-gguf' };
return platform === 'ios'
? { backend: 'coreml', engine: 'coreml' }
: { backend: 'mnn', engine: 'localdream-mnn' };
@@ -48,8 +49,8 @@ function expectedImageRuntime(platform: MobileImagePlatform): {
/**
* Why an installed image model may or may not move from this phone.
*
- * The generated images are portable. The model packages are not: iOS loads Core ML bundles and
- * Android LocalDream loads an MNN package. QNN is intentionally excluded because an Android label
+ * SD image packs work on both mobile platforms. Core ML and LocalDream MNN packages
+ * stay on their original platform. QNN is intentionally excluded because an Android label
* does not prove that the receiving device has the matching Qualcomm target.
*/
export function imageModelTransferBlocker(
@@ -59,7 +60,7 @@ export function imageModelTransferBlocker(
if (!isSafeImageModelId(model.id)) {
return 'the image model identity is not safe to transfer';
}
- const expected = expectedImageRuntime(platform);
+ const expected = expectedImageRuntime(platform, model.backend);
if (model.backend !== expected.backend) {
if (model.backend === 'qnn') {
return 'QNN image models are tied to a specific Qualcomm target and cannot be sent safely';
@@ -68,6 +69,7 @@ export function imageModelTransferBlocker(
? 'iPhone and iPad can send only Core ML image models'
: 'Android can send only LocalDream MNN image models';
}
+ if (model.backend === 'sd' && !model.id.startsWith('sd-')) return 'this image model pack is not supported';
if (!Number.isFinite(model.size) || model.size <= 0) {
return 'the image model size is not valid';
}
@@ -114,7 +116,7 @@ export function transferredImageManifest(
if (!Number.isSafeInteger(archiveSizeBytes) || archiveSizeBytes <= 0) {
throw new Error('the image model archive size is not valid');
}
- const runtime = expectedImageRuntime(platform);
+ const runtime = expectedImageRuntime(platform, model.backend);
return {
id: model.id,
name: model.name,
@@ -161,9 +163,11 @@ export function transferredImageDescriptor(
if (manifest.kind !== 'image') {
throw new Error('this is not an image model package');
}
- const expected = expectedImageRuntime(receiverPlatform);
+ const sdPack = manifest.engine === 'mobile-sd-gguf';
+ if (sdPack && !manifest.id.startsWith('sd-')) throw new Error('This image model pack is not supported.');
+ const expected = expectedImageRuntime(receiverPlatform, sdPack ? 'sd' : undefined);
if (
- manifest.platform !== receiverPlatform ||
+ (!sdPack && manifest.platform !== receiverPlatform) ||
manifest.engine !== expected.engine
) {
throw new Error(
diff --git a/src/services/modelManager/index.ts b/src/services/modelManager/index.ts
index a0683b6f8..895e9f31a 100644
--- a/src/services/modelManager/index.ts
+++ b/src/services/modelManager/index.ts
@@ -1,3 +1,4 @@
+import { videoModelsDirectory } from '../videoModelFiles';
import RNFS from 'react-native-fs';
import logger from '../../utils/logger';
import { getMmProjFileSize } from '../../utils/modelHelpers';
@@ -67,6 +68,7 @@ class ModelManager {
if (!(await RNFS.exists(this.imageModelsDir))) await RNFS.mkdir(this.imageModelsDir);
const exclude = (p: string) => backgroundDownloadService.excludeFromBackup(p);
await Promise.all([exclude(this.modelsDir), exclude(this.imageModelsDir),
+ exclude(videoModelsDirectory()),
exclude(`${RNFS.DocumentDirectoryPath}/${APP_CONFIG.whisperStorageDir}`)]);
}
diff --git a/src/services/modelManager/restore.ts b/src/services/modelManager/restore.ts
index 898ade3ee..0eed5f03c 100644
--- a/src/services/modelManager/restore.ts
+++ b/src/services/modelManager/restore.ts
@@ -104,12 +104,21 @@ interface RestoreEntryOpts {
onProgress?: DownloadProgressCallback;
}
-function buildMetadataFromActiveDownload(download: RestorableDownloadInfo, modelsDir: string): PersistedDownloadInfo | null {
+function buildMetadataFromActiveDownload(
+ download: RestorableDownloadInfo,
+ modelsDir: string,
+): PersistedDownloadInfo | null {
// image: (image models) and whisper- (STT models) are owned by their own
// managers, not the text model manager. Recovering them here registered them
// as text models, so they showed up under Text in the model selector and the
// Download Manager's downloaded list.
- if (!download.modelId || download.modelId.startsWith('image:') || download.modelId.startsWith('whisper-')) return null;
+ if (
+ !download.modelId ||
+ download.modelId.startsWith('video:') ||
+ download.modelId.startsWith('image:') ||
+ download.modelId.startsWith('whisper-')
+ )
+ return null;
const mainFileSize = download.totalBytes;
const combinedTotal = download.combinedTotalBytes || download.totalBytes;
const mmProjFileSize = Math.max(combinedTotal - mainFileSize, 0);
@@ -126,7 +135,9 @@ function buildMetadataFromActiveDownload(download: RestorableDownloadInfo, model
if (typeof parsed.mmProjFileName === 'string' && parsed.mmProjFileName) {
derivedMmProjFileName = parsed.mmProjFileName;
}
- } catch { /* non-fatal: fall through to heuristic */ }
+ } catch {
+ /* non-fatal: fall through to heuristic */
+ }
}
if (!derivedMmProjFileName && hasMmProj) {
derivedMmProjFileName = mmProjLocalName(download.fileName);
@@ -141,15 +152,22 @@ function buildMetadataFromActiveDownload(download: RestorableDownloadInfo, model
mainFileSize,
mmProjFileName: derivedMmProjFileName,
mmProjFileSize: derivedMmProjFileName ? mmProjFileSize : undefined,
- mmProjLocalPath: derivedMmProjFileName ? `${modelsDir}/${derivedMmProjFileName}` : null,
+ mmProjLocalPath: derivedMmProjFileName
+ ? `${modelsDir}/${derivedMmProjFileName}`
+ : null,
mmProjDownloadId: download.mmProjDownloadId,
};
}
async function restoreDownloadEntry(opts: RestoreEntryOpts): Promise {
const {
- download, metadata, modelsDir, activeDownloads,
- backgroundDownloadContext, backgroundDownloadMetadataCallback, onProgress,
+ download,
+ metadata,
+ modelsDir,
+ activeDownloads,
+ backgroundDownloadContext,
+ backgroundDownloadMetadataCallback,
+ onProgress,
} = opts;
const localPath = `${modelsDir}/${metadata.fileName}`;
@@ -184,36 +202,49 @@ async function restoreDownloadEntry(opts: RestoreEntryOpts): Promise {
let mainBytesDownloaded = download.bytesDownloaded;
let mmProjBytesDownloaded = mmProjCompleted
? mmProjFileSize
- : (mmProjDownload?.bytesDownloaded || 0);
+ : mmProjDownload?.bytesDownloaded || 0;
const reportProgress = () => {
const combinedDownloaded = mainBytesDownloaded + mmProjBytesDownloaded;
onProgress?.({
downloadId: download.downloadId,
- modelId: metadata.modelId, fileName: metadata.fileName,
- bytesDownloaded: combinedDownloaded, totalBytes: combinedTotalBytes,
- progress: combinedTotalBytes > 0 ? combinedDownloaded / combinedTotalBytes : 0,
+ modelId: metadata.modelId,
+ fileName: metadata.fileName,
+ bytesDownloaded: combinedDownloaded,
+ totalBytes: combinedTotalBytes,
+ progress:
+ combinedTotalBytes > 0 ? combinedDownloaded / combinedTotalBytes : 0,
});
};
const removeProgressListener = backgroundDownloadService.onProgress(
- download.downloadId, (event) => {
- mainBytesDownloaded = event.bytesDownloaded; reportProgress();
+ download.downloadId,
+ event => {
+ mainBytesDownloaded = event.bytesDownloaded;
+ reportProgress();
},
);
let removeMmProjProgressListener: (() => void) | undefined;
if (mmProjDownloadId && !mmProjCompleted) {
removeMmProjProgressListener = backgroundDownloadService.onProgress(
- mmProjDownloadId, (event) => {
- mmProjBytesDownloaded = event.bytesDownloaded; reportProgress();
+ mmProjDownloadId,
+ event => {
+ mmProjBytesDownloaded = event.bytesDownloaded;
+ reportProgress();
},
);
}
backgroundDownloadContext.set(download.downloadId, {
- modelId: metadata.modelId, file: fileInfo, localPath, mmProjLocalPath,
- removeProgressListener, mmProjDownloadId, mmProjCompleted, mainCompleted: download.status === 'completed',
+ modelId: metadata.modelId,
+ file: fileInfo,
+ localPath,
+ mmProjLocalPath,
+ removeProgressListener,
+ mmProjDownloadId,
+ mmProjCompleted,
+ mainCompleted: download.status === 'completed',
removeMmProjProgressListener,
});
backgroundDownloadMetadataCallback?.(download.downloadId, { ...metadata, mmProjLocalPath });
@@ -234,8 +265,16 @@ function collectMmProjIds(
return ids;
}
-export async function restoreInProgressDownloads(opts: RestoreDownloadsOpts): Promise {
- const { modelsDir, backgroundDownloadContext, backgroundDownloadMetadataCallback, onProgress, persistedDownloads } = opts;
+export async function restoreInProgressDownloads(
+ opts: RestoreDownloadsOpts,
+): Promise {
+ const {
+ modelsDir,
+ backgroundDownloadContext,
+ backgroundDownloadMetadataCallback,
+ onProgress,
+ persistedDownloads,
+ } = opts;
if (!backgroundDownloadService.isAvailable()) return [];
@@ -267,8 +306,13 @@ export async function restoreInProgressDownloads(opts: RestoreDownloadsOpts): Pr
if (!metadata || backgroundDownloadContext.has(download.downloadId)) continue;
try {
await restoreDownloadEntry({
- download, metadata, modelsDir, activeDownloads,
- backgroundDownloadContext, backgroundDownloadMetadataCallback, onProgress,
+ download,
+ metadata,
+ modelsDir,
+ activeDownloads,
+ backgroundDownloadContext,
+ backgroundDownloadMetadataCallback,
+ onProgress,
});
restoredDownloadIds.push(download.downloadId);
if (isInFlight(download.status)) adoptableIds.push(download.downloadId);
diff --git a/src/services/modelManager/scan.ts b/src/services/modelManager/scan.ts
index 3a22ccb28..6cfed33fc 100644
--- a/src/services/modelManager/scan.ts
+++ b/src/services/modelManager/scan.ts
@@ -1,3 +1,4 @@
+import { getSDImageModels, getSDImagePackFiles, isSDImageWeight } from '../huggingFaceModelBrowser';
import RNFS from 'react-native-fs';
import { statFile } from '../../utils/fileStat';
import { unzip } from 'react-native-zip-archive';
@@ -5,7 +6,7 @@ import { DownloadedModel, LlamaDownloadedModel, ONNXImageModel } from '../../typ
import { loadDownloadedModels, saveModelsList } from './storage';
import { basenameOf } from './reconcileStoredPaths';
import { resolveCoreMLModelDir } from '../../utils/coreMLModelUtils';
-import { ensureImageExtractionComplete } from '../../utils/imageModelIntegrity';
+import { ensureImageExtractionComplete, validateMultifileComplete, validateImageModelDir } from '../../utils/imageModelIntegrity';
// Single source of truth for projector detection + model↔projector matching (see src/services/mmproj.ts).
import { isMMProjFile, pickMmProjForModel } from '../mmproj';
@@ -84,7 +85,8 @@ export async function cleanupMMProjEntries(modelsDir: string): Promise {
return removedCount;
}
-function detectBackend(dirName: string): 'mnn' | 'qnn' | 'coreml' {
+function detectBackend(dirName: string): 'mnn' | 'qnn' | 'coreml' | 'sd' {
+ if (dirName.startsWith('sd-')) return 'sd';
if (dirName.includes('qnn') || dirName.includes('8gen') || dirName.includes('npu')) return 'qnn';
if (dirName.includes('coreml')) return 'coreml';
return 'mnn';
@@ -138,14 +140,15 @@ async function isValidZip(zipPath: string): Promise {
/** Build the ONNXImageModel record for a recovered on-disk dir (coreml resolves its inner model dir). */
async function buildRecoveredImageModel(
item: { name: string; path: string },
- backend: 'mnn' | 'qnn' | 'coreml',
+ backend: 'mnn' | 'qnn' | 'coreml' | 'sd',
): Promise {
let modelPath = item.path;
if (backend === 'coreml') modelPath = await resolveCoreMLModelDir(item.path).catch(() => item.path);
const totalSize = await getDirSize(item.path);
+ const sdWeight = backend === 'sd' ? (await RNFS.readDir(item.path)).find(file => file.isFile() && isSDImageWeight(file.name))?.name : undefined;
return {
id: item.name,
- name: item.name.replaceAll('_', ' '),
+ name: getSDImageModels().find(model => model.id === item.name)?.displayName ?? (sdWeight ? sdWeight.replace(/\.(gguf|safetensors)$/i, '') : item.name.replaceAll('_', ' ')),
description: '',
modelPath,
size: totalSize,
@@ -240,6 +243,7 @@ export async function reconcileFinishedImageDownloads(opts: ReconcileImageModels
const hasReady = await RNFS.exists(readyPath);
if (hasReady) {
+ if (detectBackend(item.name) === 'sd' && !(await validateImageModelDir(item.path, 'sd')).complete) continue;
// Unzip completed but registerAndNotify was killed — register now.
const newModel = await buildRecoveredImageModel(item, detectBackend(item.name));
await addImageModel(newModel);
@@ -247,6 +251,28 @@ export async function reconcileFinishedImageDownloads(opts: ReconcileImageModels
continue;
}
+ // A complete SD pack can survive a stop between its last file and _ready.
+ // Keep partial known packs so the normal download can reuse valid parts.
+ const sdFiles = item.name.startsWith('sd-') ? getSDImagePackFiles((await RNFS.readDir(item.path)).filter(file => file.isFile()).map(file => file.name), item.name) : null;
+ if (item.name.startsWith('sd-')) {
+ try {
+ // A remote variant's expected weight bytes/hash live in its download
+ // metadata, not the catalog. Let Resume validate that descriptor; a
+ // nonempty partial GGUF alone cannot prove this pack is complete.
+ if (!sdFiles || sdFiles.some(file => !file.sha256)) continue;
+ await validateMultifileComplete(item.path, sdFiles.map(file => ({
+ relativePath: file.path, sha256: file.sha256,
+ })));
+ await RNFS.writeFile(readyPath, '', 'utf8');
+ const model = await buildRecoveredImageModel(item, 'sd');
+ await addImageModel(model);
+ recovered.push(model);
+ } catch {
+ // Incomplete or damaged parts stay unregistered until Download repairs them.
+ }
+ continue;
+ }
+
// No _ready — check if a zip exists to re-unzip (mid-unzip kill).
if (await RNFS.exists(`${item.path}/_zip_name`)) {
// Non-fatal on unexpected error: leave the dir for the next startup attempt.
diff --git a/src/services/modelPreloader.ts b/src/services/modelPreloader.ts
index c7e862e19..989039559 100644
--- a/src/services/modelPreloader.ts
+++ b/src/services/modelPreloader.ts
@@ -1,3 +1,4 @@
+import { videoGenerationService } from './videoGenerationService';
/**
* Boot-time model preloader.
*
@@ -36,6 +37,7 @@ export function abortPreload(): void {
function isGenerationActive(): boolean {
return (
generationService.getState().isGenerating ||
+ videoGenerationService.getState().phase === 'running' ||
imageGenerationService.getState().isGenerating
);
}
diff --git a/src/services/modelResidency/index.ts b/src/services/modelResidency/index.ts
index fe1e05585..d7dfb04a0 100644
--- a/src/services/modelResidency/index.ts
+++ b/src/services/modelResidency/index.ts
@@ -433,7 +433,13 @@ class ModelResidencyManager {
return this.runExclusive(`evict:${key}`, async () => {
const r = this.residents.get(key);
if (!r) return false;
- await r.unload().catch(err => logger.log(`[ModelResidency] evict ${key} unload failed:`, err));
+ if (r.canEvict && !r.canEvict()) return false;
+ try {
+ await r.unload();
+ } catch (err) {
+ logger.log(`[ModelResidency] evict ${key} unload failed:`, err);
+ return false;
+ }
this.residents.delete(key);
logger.log(`[ModelResidency] evicted ${r.type} (${key}) by user request`);
return true;
diff --git a/src/services/modelResidency/policy.ts b/src/services/modelResidency/policy.ts
index a2e188304..9e409caa4 100644
--- a/src/services/modelResidency/policy.ts
+++ b/src/services/modelResidency/policy.ts
@@ -19,7 +19,14 @@
import { modelMemoryBudgetMB, LoadPolicy } from '../memoryBudget';
-export type ResidentType = 'text' | 'image' | 'whisper' | 'tts' | 'classifier' | 'embedding';
+export type ResidentType =
+ | 'text'
+ | 'image'
+ | 'video'
+ | 'whisper'
+ | 'tts'
+ | 'classifier'
+ | 'embedding';
export interface Resident {
/** Unique model id. */
@@ -60,7 +67,13 @@ const SIDECAR_TYPES = new Set(['whisper', 'tts', 'embedding']);
// Priority (what to KEEP): text is highest, then image, then the STT/TTS/embedding
// sidecars (equal, lowest), then pinned helpers. Eviction takes the lowest first.
const PRIORITY: Record = {
- text: 3, image: 2, whisper: 1, tts: 1, embedding: 1, classifier: 0,
+ text: 3,
+ image: 2,
+ video: 2,
+ whisper: 1,
+ tts: 1,
+ embedding: 1,
+ classifier: 0,
};
/**
@@ -77,11 +90,14 @@ function selectEvictionVictim(
): Resident | undefined {
const incomingIsSidecar = SIDECAR_TYPES.has(incoming.type);
return current
- .filter(r =>
- !r.pinned && r.key !== incoming.key && !isEvicted(r) &&
- // A sidecar incoming may only reclaim from peer sidecars (never a
- // generation model); a generation incoming may evict anything non-pinned.
- (!incomingIsSidecar || SIDECAR_TYPES.has(r.type)),
+ .filter(
+ r =>
+ !r.pinned &&
+ r.key !== incoming.key &&
+ !isEvicted(r) &&
+ // A sidecar incoming may only reclaim from peer sidecars (never a
+ // generation model); a generation incoming may evict anything non-pinned.
+ (!incomingIsSidecar || SIDECAR_TYPES.has(r.type)),
)
.sort((a, b) => {
const pa = PRIORITY[a.type] ?? 0;
@@ -145,9 +161,11 @@ export function planEviction(
// Extreme mode: evict everything evictable (no co-residency). selectEvictionVictim
// still skips pinned + in-use (canEvict veto) residents, so the classifier and a
// playing TTS survive; every other model is unloaded to free the most RAM.
- for (let victim = selectEvictionVictim(current, incoming, isEvicted);
+ for (
+ let victim = selectEvictionVictim(current, incoming, isEvicted);
victim;
- victim = selectEvictionVictim(current, incoming, isEvicted)) {
+ victim = selectEvictionVictim(current, incoming, isEvicted)
+ ) {
evict.push(victim);
}
return {
diff --git a/src/services/offGridDesktopModels.ts b/src/services/offGridDesktopModels.ts
index 638990f91..2ad11e2e8 100644
--- a/src/services/offGridDesktopModels.ts
+++ b/src/services/offGridDesktopModels.ts
@@ -44,6 +44,7 @@ function record(value: unknown): Record | null {
function categoryForKind(kind: string): GatewayCategory {
if (kind === 'text' || kind === 'vision' || kind === 'chat') return 'text';
+ if (kind === 'video') return 'video';
if (kind === 'image') return 'image';
if (kind === 'voice' || kind === 'speech') return 'voice';
if (kind === 'transcription') return 'transcription';
@@ -63,7 +64,11 @@ function modelFiles(value: unknown): string[] {
function parseCatalog(value: unknown): GatewayCatalogModel[] | null {
const payload = record(value);
- if (!payload || !Array.isArray(payload.models) || !Array.isArray(payload.kinds)) {
+ if (
+ !payload ||
+ !Array.isArray(payload.models) ||
+ !Array.isArray(payload.kinds)
+ ) {
return null;
}
const models: GatewayCatalogModel[] = [];
@@ -191,6 +196,7 @@ function projectActive(
active: Record,
): RemoteMediaModelIds {
const text = activeOptionId(catalog, 'text', active.text);
+ const video = activeOptionId(catalog, 'video', active.video);
const image = activeOptionId(catalog, 'image', active.image);
const transcription = activeOptionId(
catalog,
@@ -205,6 +211,7 @@ function projectActive(
return {
...(text ? { text } : {}),
...(image ? { image } : {}),
+ ...(video ? { video } : {}),
...(transcription ? { transcription } : {}),
...(voice ? { voice } : {}),
};
@@ -224,7 +231,8 @@ function textModels(
const predicted = predictGgufCapabilities(model);
const hasLiveCapabilities = live?.capabilities !== undefined && live.capabilities !== null;
const supportsThinking = hasLiveCapabilities
- ? live?.capabilities?.includes('reasoning') === true || live?.reasoningMandatory === true
+ ? live?.capabilities?.includes('reasoning') === true ||
+ live?.reasoningMandatory === true
: predicted.thinking;
return [
{
@@ -232,17 +240,18 @@ function textModels(
name: model.name,
serverId,
capabilities: {
- supportsVision: model.kind === 'vision' || live?.capabilities?.includes('vision') === true,
+ supportsVision:
+ model.kind === 'vision' ||
+ live?.capabilities?.includes('vision') === true,
supportsToolCalling: hasLiveCapabilities
? live?.capabilities?.includes('tools') === true
- // Desktop offloads idle models, so /v1/models can omit capabilities.
- // Keep tools usable until the runtime makes an authoritative claim.
- : true,
+ : // Desktop offloads idle models, so /v1/models can omit capabilities.
+ // Keep tools usable until the runtime makes an authoritative claim.
+ true,
supportsThinking,
- acceptsThinkingKwarg: model.id.startsWith('remote-vision:') && supportsThinking,
- ...(live?.reasoningMandatory
- ? { thinkingLevelsOnly: true }
- : {}),
+ acceptsThinkingKwarg:
+ model.id.startsWith('remote-vision:') && supportsThinking,
+ ...(live?.reasoningMandatory ? { thinkingLevelsOnly: true } : {}),
},
lastUpdated: new Date().toISOString(),
},
@@ -262,11 +271,7 @@ export async function readOffGridDesktopModelState(
gatewayFetch(server, '/v1/models/active'),
gatewayFetch(server, '/v1/models'),
]);
- if (
- !catalogResponse.ok ||
- !installedResponse.ok ||
- !activeResponse.ok
- ) {
+ if (!catalogResponse.ok || !installedResponse.ok || !activeResponse.ok) {
return null;
}
const [catalogPayload, installedPayload, activePayload] =
diff --git a/src/services/rag/database.ts b/src/services/rag/database.ts
index 2d3e08088..6c9f9307e 100644
--- a/src/services/rag/database.ts
+++ b/src/services/rag/database.ts
@@ -23,6 +23,13 @@ export interface RagSearchResult {
score: number;
}
+export interface EmbeddingModelSelection {
+ id: string;
+ name: string;
+ filePath: string;
+ size: number;
+}
+
interface StoredEmbedding {
chunk_rowid: number;
doc_id: number;
@@ -96,6 +103,9 @@ class RagDatabase {
FOREIGN KEY (doc_id) REFERENCES rag_documents(id)
)`,
);
+ this.db.executeSync(
+ 'CREATE TABLE IF NOT EXISTS rag_embedding_model (id INTEGER PRIMARY KEY CHECK (id = 1), selection TEXT NOT NULL)',
+ );
this.ready = true;
} catch (error) {
logger.error('[RagDB] Failed to initialize:', error);
@@ -162,6 +172,42 @@ class RagDatabase {
return rowIds;
}
+ getEmbeddingModel(): EmbeddingModelSelection | null {
+ const row = this.getDb().executeSync('SELECT selection FROM rag_embedding_model WHERE id = 1').rows?.[0];
+ return row ? JSON.parse(row.selection as string) as EmbeddingModelSelection : null;
+ }
+
+ beginEmbeddingRebuild(): void {
+ const db = this.getDb();
+ db.executeSync('CREATE TEMP TABLE IF NOT EXISTS rag_embedding_rebuild (chunk_rowid INTEGER, doc_id INTEGER, embedding BLOB)');
+ db.executeSync('DELETE FROM rag_embedding_rebuild');
+ }
+
+ stageEmbedding(entry: { chunkRowid: number; docId: number; embedding: number[] }): void {
+ this.getDb().executeSync('INSERT INTO rag_embedding_rebuild (chunk_rowid, doc_id, embedding) VALUES (?, ?, ?)',
+ [entry.chunkRowid, entry.docId, this.embeddingToBlob(entry.embedding)]);
+ }
+
+ discardEmbeddingRebuild(): void {
+ this.getDb().executeSync('DROP TABLE IF EXISTS temp.rag_embedding_rebuild');
+ }
+
+ /** The model identity and all vectors change in one durable SQLite transaction. */
+ commitEmbeddingRebuild(model: EmbeddingModelSelection | null): void {
+ const db = this.getDb();
+ db.executeSync('BEGIN');
+ try {
+ db.executeSync('DELETE FROM rag_embeddings');
+ db.executeSync('INSERT INTO rag_embeddings (chunk_rowid, doc_id, embedding) SELECT chunk_rowid, doc_id, embedding FROM rag_embedding_rebuild');
+ db.executeSync('DELETE FROM rag_embedding_model');
+ if (model) db.executeSync('INSERT INTO rag_embedding_model (id, selection) VALUES (1, ?)', [JSON.stringify(model)]);
+ db.executeSync('COMMIT');
+ } catch (error) {
+ db.executeSync('ROLLBACK');
+ throw error;
+ }
+ }
+
private embeddingToBlob(embedding: number[]): ArrayBuffer {
return new Float32Array(embedding).buffer;
}
diff --git a/src/services/rag/embedding.ts b/src/services/rag/embedding.ts
index 6fdf942c0..c5f8cb7ce 100644
--- a/src/services/rag/embedding.ts
+++ b/src/services/rag/embedding.ts
@@ -1,7 +1,9 @@
-import { initLlama, LlamaContext } from 'llama.rn';
+import { initLlama, loadLlamaModelInfo, LlamaContext } from 'llama.rn';
import { Platform } from 'react-native';
+import { sha256 } from 'js-sha256';
import RNFS from 'react-native-fs';
import logger from '../../utils/logger';
+import { ragDatabase, type EmbeddingModelSelection } from './database';
import { modelResidencyManager } from '../modelResidency';
const EMBEDDING_MODEL_FILENAME = 'all-MiniLM-L6-v2-Q8_0.gguf';
@@ -37,6 +39,117 @@ function withTimeout(promise: Promise, opts: { ms: number; message: string
class EmbeddingService {
private context: LlamaContext | null = null;
private loading: Promise | null = null;
+ private operations: Promise = Promise.resolve();
+ private downloads: Promise = Promise.resolve();
+ private operationActive = false;
+ private pendingModel: EmbeddingModelSelection | null | undefined;
+ private dimension = EMBEDDING_DIMENSION;
+ private modelId = 'bundled:all-MiniLM-L6-v2-Q8_0';
+
+ /** Hold this across vector creation AND index/cache reads or writes. */
+ runExclusive(operation: () => Promise): Promise {
+ const next = this.operations.then(async () => {
+ this.operationActive = true;
+ try { return await operation(); } finally { this.operationActive = false; }
+ });
+ this.operations = next.catch(() => {});
+ return next;
+ }
+
+ getModelId(): string { return this.modelId; }
+
+ /** Caller holds the operation lock; failure leaves the stored model and index intact. */
+ async withRebuildModel(model: EmbeddingModelSelection | null, rebuild: () => Promise): Promise {
+ if (this.loading) await this.loading;
+ await this.unload();
+ this.pendingModel = model;
+ try {
+ await this.load();
+ await this.embed('Embedding compatibility test.');
+ return await rebuild();
+ } catch (error) {
+ await this.unload();
+ throw error;
+ } finally {
+ this.pendingModel = undefined;
+ }
+ }
+
+ async isModelDownloaded(candidate: { id: string; size: number; sha256?: string }): Promise {
+ const filePath = `${RNFS.DocumentDirectoryPath}/embedding-${sha256(candidate.id)}.gguf`;
+ if (!await RNFS.exists(filePath)) return false;
+ return Number((await RNFS.stat(filePath)).size) === candidate.size &&
+ (!candidate.sha256 || await RNFS.hash(filePath, 'sha256') === candidate.sha256);
+ }
+
+ /** Download only. Activation and rebuilding require a separate confirmed RAG operation. */
+ async downloadModel(
+ candidate: { id: string; name: string; size: number; downloadUrl: string; sha256?: string },
+ onProgress: (message: string) => void,
+ signal: AbortSignal,
+ onDownloadProgress?: (fraction: number) => void,
+ ): Promise {
+ const next = this.downloads.then(() => this.downloadModelFile(candidate, onProgress, signal, onDownloadProgress));
+ this.downloads = next.catch(() => {});
+ return next;
+ }
+
+ private async downloadModelFile(
+ candidate: { id: string; name: string; size: number; downloadUrl: string; sha256?: string },
+ onProgress: (message: string) => void,
+ signal: AbortSignal,
+ onDownloadProgress?: (fraction: number) => void,
+ ): Promise {
+ if (signal.aborted) throw new Error('Model download cancelled');
+ if (!Number.isFinite(candidate.size) || candidate.size <= 0) {
+ throw new Error('The embedding model has no valid file size.');
+ }
+ const filePath = `${RNFS.DocumentDirectoryPath}/embedding-${sha256(candidate.id)}.gguf`;
+ const selection = { id: candidate.id, name: candidate.name, size: candidate.size, filePath };
+ if (await RNFS.exists(filePath)) {
+ if (await this.isModelDownloaded(candidate)) {
+ if (signal.aborted) throw new Error('Model download cancelled');
+ onDownloadProgress?.(1);
+ return selection;
+ }
+ await ragDatabase.ensureReady();
+ if (ragDatabase.getEmbeddingModel()?.id === candidate.id) {
+ throw new Error('The active model file is damaged. Select the built-in model, then download this model again.');
+ }
+ await RNFS.unlink(filePath);
+ }
+ const partial = `${filePath}.part`;
+ const download = RNFS.downloadFile({
+ fromUrl: candidate.downloadUrl, toFile: partial,
+ progressInterval: 250,
+ begin: () => { onProgress('Downloading embedding model...'); onDownloadProgress?.(0); },
+ progress: progress => {
+ const fraction = Math.min(1, progress.bytesWritten / candidate.size);
+ onDownloadProgress?.(fraction);
+ onProgress(`Downloading embedding model: ${Math.round(fraction * 100)}%`);
+ },
+ });
+ const stop = () => RNFS.stopDownload(download.jobId);
+ signal.addEventListener('abort', stop);
+ try {
+ if (signal.aborted) { stop(); throw new Error('Model change cancelled'); }
+ const result = await download.promise;
+ if (signal.aborted) throw new Error('Model change cancelled');
+ if (result.statusCode !== 200 || Number((await RNFS.stat(partial)).size) !== candidate.size) {
+ throw new Error('The model download is incomplete. Try again.');
+ }
+ if (candidate.sha256 && await RNFS.hash(partial, 'sha256') !== candidate.sha256) {
+ throw new Error('The model file failed its integrity check. Try again.');
+ }
+ if (signal.aborted) throw new Error('Model download cancelled');
+ await RNFS.moveFile(partial, filePath);
+ onDownloadProgress?.(1);
+ return selection;
+ } finally {
+ signal.removeEventListener('abort', stop);
+ if (await RNFS.exists(partial)) await RNFS.unlink(partial);
+ }
+ }
async load(): Promise {
if (this.context) return;
@@ -51,7 +164,30 @@ class EmbeddingService {
}
private async doLoad(): Promise {
- const modelPath = await this.ensureModelCopied();
+ await ragDatabase.ensureReady();
+ const selected = this.pendingModel !== undefined ? this.pendingModel : ragDatabase.getEmbeddingModel();
+ // iOS can move the app container after a restore; derive its current path from the stable identity.
+ const modelPath = selected
+ ? `${RNFS.DocumentDirectoryPath}/embedding-${sha256(selected.id)}.gguf`
+ : await this.ensureModelCopied();
+ this.dimension = EMBEDDING_DIMENSION;
+ if (selected) {
+ const info = await loadLlamaModelInfo(modelPath) as Record;
+ // Only bidirectional BERT encoders with sentence pooling fit this runtime contract.
+ // Decoder models and rerankers need different prompting/output handling.
+ const contextLength = Number(info['bert.context_length']);
+ if (info['general.architecture'] !== 'bert' ||
+ ![1, 2].includes(Number(info['bert.pooling_type'])) ||
+ !Number.isInteger(contextLength) || contextLength < EMBEDDING_CTX_SIZE) {
+ throw new Error('Unsupported embedding model. Use a BERT GGUF text encoder with mean or CLS pooling and at least 512 tokens.');
+ }
+ const dimension = Number(info['bert.embedding_length']);
+ if (!Number.isInteger(dimension) || dimension < 1) {
+ throw new Error('This model has an unsupported embedding size.');
+ }
+ this.dimension = dimension;
+ }
+ this.modelId = selected?.id ?? 'bundled:all-MiniLM-L6-v2-Q8_0';
logger.log('[Embedding] Loading embedding model...');
// Load through the residency manager's global lock so this small RAG model
// never initializes alongside another model load (the single load gateway).
@@ -59,6 +195,14 @@ class EmbeddingService {
// ThreadPool::startWorkers hang) releases the lock instead of wedging a
// concurrent chat-model load and tripping the OS watchdog.
this.context = await modelResidencyManager.runExclusive('load:embedding', async () => {
+ const spec = {
+ key: EMBEDDING_RESIDENT_KEY, type: 'embedding' as const,
+ sizeMB: selected ? Math.ceil(selected.size / (1024 * 1024)) + 128 : EMBEDDING_RESIDENT_MB,
+ canEvict: () => !this.operationActive,
+ };
+ if (selected && !(await modelResidencyManager.makeRoomFor(spec)).fits) {
+ throw new Error('Not enough memory for this embedding model. Choose a smaller model.');
+ }
const ctx = await withTimeout(
initLlama({
model: modelPath,
@@ -83,7 +227,7 @@ class EmbeddingService {
// against stale free-RAM and OOM. It loads on the tiny MiniLM context and can be
// evicted as a last-resort sidecar; it never evicts the active generation model.
modelResidencyManager.register(
- { key: EMBEDDING_RESIDENT_KEY, type: 'embedding', sizeMB: EMBEDDING_RESIDENT_MB },
+ spec,
() => this.unload(),
);
return ctx;
@@ -112,7 +256,13 @@ class EmbeddingService {
const result = await (this.context as any).embedding(text);
// [WIRE] embedding dim + a sample (not the whole vector) so fixtures match the real model's dimensionality.
logger.log(`[WIRE-EMBED] ${JSON.stringify({ dim: result?.embedding?.length, sample: result?.embedding?.slice?.(0, 8) })}`);
- return result.embedding;
+ const vector: unknown = result.embedding;
+ if (!Array.isArray(vector) || vector.length !== this.dimension ||
+ !vector.every(value => typeof value === 'number' && Number.isFinite(value)) ||
+ !vector.some(value => value !== 0)) {
+ throw new Error('The model did not return a valid sentence embedding.');
+ }
+ return vector;
} catch (error: any) {
const msg = error?.message || String(error) || '';
logger.error('[Embedding] Native error during embedding:', msg);
@@ -156,7 +306,7 @@ class EmbeddingService {
}
getDimension(): number {
- return EMBEDDING_DIMENSION;
+ return this.dimension;
}
}
diff --git a/src/services/rag/index.ts b/src/services/rag/index.ts
index 37e13ea1c..4c33762b9 100644
--- a/src/services/rag/index.ts
+++ b/src/services/rag/index.ts
@@ -1,4 +1,4 @@
-import { ragDatabase } from './database';
+import { ragDatabase, type EmbeddingModelSelection } from './database';
import { chunkDocument } from './chunking';
import { retrievalService } from './retrieval';
import { embeddingService } from './embedding';
@@ -30,11 +30,53 @@ interface IndexDocumentParams {
}
class RagService {
+ private embeddingChange = { busy: false, message: '', error: '' };
+ private embeddingListeners = new Set<() => void>();
+ private embeddingAbort: AbortController | null = null;
+ getEmbeddingChange = () => this.embeddingChange;
+ subscribeEmbeddingChange = (listener: () => void) => {
+ this.embeddingListeners.add(listener);
+ return () => { this.embeddingListeners.delete(listener); };
+ };
+ private reportEmbeddingChange(busy: boolean, message: string, error = '') {
+ this.embeddingChange = { busy, message, error };
+ this.embeddingListeners.forEach(listener => listener());
+ }
+ cancelEmbeddingChange = () => { this.embeddingAbort?.abort(); };
+
+ async installEmbeddingModel(candidate: {
+ id: string; name: string; size: number; downloadUrl: string; sha256?: string;
+ } | null): Promise {
+ if (this.embeddingChange.busy) throw new Error('An embedding model change is already in progress.');
+ const controller = new AbortController();
+ this.embeddingAbort = controller;
+ const progress = (message: string) => this.reportEmbeddingChange(true, message);
+ progress('Preparing model change...');
+ let model: EmbeddingModelSelection | null = null;
+ try {
+ await this.ensureReady();
+ model = candidate ? await embeddingService.downloadModel(candidate, progress, controller.signal) : null;
+ await this.switchEmbeddingModel(model, progress, controller.signal);
+ this.reportEmbeddingChange(false, 'Indexing complete');
+ } catch (error) {
+ this.reportEmbeddingChange(false, '', controller.signal.aborted
+ ? 'Model change cancelled. The previous model and indexes are still active.'
+ : `${error instanceof Error ? error.message : String(error)} The previous model and indexes are still active.`);
+ throw error;
+ } finally {
+ this.embeddingAbort = null;
+ }
+ }
+
async ensureReady(): Promise {
await ragDatabase.ensureReady();
}
async indexDocument(params: IndexDocumentParams): Promise {
+ return embeddingService.runExclusive(() => this.indexDocumentUnlocked(params));
+ }
+
+ private async indexDocumentUnlocked(params: IndexDocumentParams): Promise {
const { projectId, filePath, fileName, fileSize, onProgress } = params;
await this.ensureReady();
@@ -90,7 +132,11 @@ class RagService {
try {
await embeddingService.load();
const texts = chunks.map(c => c.content);
- const embeddings = await embeddingService.embedBatch(texts);
+ const embeddings: number[][] = [];
+ for (const text of texts) {
+ embeddings.push(await embeddingService.embed(text));
+ onProgress?.({ stage: 'embedding', message: `Indexing ${embeddings.length} of ${texts.length} chunks...` });
+ }
const entries = rowIds.map((rowId, i) => ({
chunkRowid: rowId,
docId,
@@ -152,6 +198,10 @@ class RagService {
}
async backfillEmbeddings(projectId: string): Promise {
+ return embeddingService.runExclusive(() => this.backfillEmbeddingsUnlocked(projectId));
+ }
+
+ private async backfillEmbeddingsUnlocked(projectId: string): Promise {
await this.ensureReady();
const docs = ragDatabase.getDocumentsByProject(projectId);
let total = 0;
@@ -184,7 +234,46 @@ class RagService {
return total;
}
+ async switchEmbeddingModel(
+ model: EmbeddingModelSelection | null,
+ onProgress: (message: string) => void,
+ signal?: AbortSignal,
+ ): Promise {
+ return embeddingService.runExclusive(async () => {
+ await this.ensureReady();
+ if (signal?.aborted) throw new Error('Model change cancelled');
+ onProgress('Checking model compatibility...');
+ await embeddingService.withRebuildModel(model, async () => {
+ const documents = ragDatabase.getAllDocuments();
+ const total = documents.reduce((count, doc) => count + ragDatabase.getChunksByDocument(doc.id).length, 0);
+ let completed = 0;
+ ragDatabase.beginEmbeddingRebuild();
+ try {
+ for (const doc of documents) {
+ for (const chunk of ragDatabase.getChunksByDocument(doc.id)) {
+ if (signal?.aborted) throw new Error('Model change cancelled');
+ onProgress(`Rebuilding search and project knowledge: ${completed} of ${total} chunks`);
+ ragDatabase.stageEmbedding({ chunkRowid: chunk.id, docId: doc.id, embedding: await embeddingService.embed(chunk.content) });
+ completed += 1;
+ }
+ }
+ onProgress(`Saving ${completed} chunks...`);
+ if (signal?.aborted) throw new Error('Model change cancelled');
+ ragDatabase.commitEmbeddingRebuild(model);
+ } finally {
+ // Cleanup must not turn a committed switch into an apparent failed switch.
+ try { ragDatabase.discardEmbeddingRebuild(); } catch (error) { logger.warn('Could not clear temporary embedding index', error); }
+ }
+ });
+ onProgress('Indexing complete');
+ });
+ }
+
async deleteDocument(docId: number): Promise {
+ return embeddingService.runExclusive(() => this.deleteDocumentUnlocked(docId));
+ }
+
+ private async deleteDocumentUnlocked(docId: number): Promise {
await this.ensureReady();
const document = ragDatabase.getDocument(docId);
ragDatabase.deleteDocument(docId);
@@ -231,6 +320,10 @@ class RagService {
}
async deleteProjectDocuments(projectId: string): Promise {
+ return embeddingService.runExclusive(() => this.deleteProjectDocumentsUnlocked(projectId));
+ }
+
+ private async deleteProjectDocumentsUnlocked(projectId: string): Promise {
await this.ensureReady();
const documents = ragDatabase.getDocumentsByProject(projectId);
ragDatabase.deleteDocumentsByProject(projectId);
@@ -283,6 +376,10 @@ class RagService {
}
async deleteSyncedDocument(syncId: string): Promise {
+ return embeddingService.runExclusive(() => this.deleteSyncedDocumentUnlocked(syncId));
+ }
+
+ private async deleteSyncedDocumentUnlocked(syncId: string): Promise {
await this.ensureReady();
const document = ragDatabase.getDocumentBySyncId(syncId);
if (document) ragDatabase.deleteDocument(document.id);
diff --git a/src/services/rag/retrieval.ts b/src/services/rag/retrieval.ts
index fd18db059..6b16ae141 100644
--- a/src/services/rag/retrieval.ts
+++ b/src/services/rag/retrieval.ts
@@ -22,7 +22,10 @@ interface SearchResult {
class RetrievalService {
async search(projectId: string, query: string, topK: number = 5): Promise {
- const chunks = await this.searchSemantic(projectId, query, topK);
+ const chunks = await embeddingService.runExclusive(async () => {
+ await ragDatabase.ensureReady();
+ return this.searchSemantic(projectId, query, topK);
+ });
return { chunks, truncated: false };
}
diff --git a/src/services/remoteMediaRuntime.ts b/src/services/remoteMediaRuntime.ts
index 996d2e531..7c0534f91 100644
--- a/src/services/remoteMediaRuntime.ts
+++ b/src/services/remoteMediaRuntime.ts
@@ -1,3 +1,14 @@
+import { isRecordProvenance, type RecordProvenance } from '@offgrid/sync';
+import { generateId } from '../utils/generateId';
+import RNFS from 'react-native-fs';
+import { Buffer } from 'buffer';
+import type {
+ ResolvedVideoRequest,
+ VideoGenerationResultContract,
+ VideoGenerationProgressContract,
+ VideoGenerationStage,
+ VideoGenerationUpdateContract,
+} from '@offgrid/models';
import { remoteServerManager } from './remoteServerManager';
import type { RemoteMediaModelIds, RemoteServer } from '../types';
import { REMOTE_FETCH_REDIRECT_POLICY, remoteAuthorizationHeaders } from './remoteTransportPolicy';
@@ -41,6 +52,7 @@ async function request(
signal?.addEventListener('abort', abort, { once: true });
try {
const apiKey = await remoteServerManager.getApiKey(server.id);
+ if (signal?.aborted) controller.abort();
if (controller.signal.aborted) throw new Error('Remote request cancelled');
const response = await fetch(endpoint(server, path), {
...init,
@@ -58,8 +70,14 @@ async function request(
try {
const body = JSON.parse(detail) as { error?: { code?: unknown }; code?: unknown };
const marker = 'OFFGRID_IMAGE_MEMORY_LIMIT:';
- if ((body.error?.code ?? body.code) === 'OFFGRID_IMAGE_MEMORY_LIMIT' || message.includes(marker)) {
- throw Object.assign(new OverridableMemoryError(message.replace(marker, '').trim()), { remote: true });
+ if (
+ (body.error?.code ?? body.code) === 'OFFGRID_IMAGE_MEMORY_LIMIT' ||
+ message.includes(marker)
+ ) {
+ throw Object.assign(
+ new OverridableMemoryError(message.replace(marker, '').trim()),
+ { remote: true },
+ );
}
} catch (error) {
if (error instanceof OverridableMemoryError) throw error;
@@ -88,6 +106,186 @@ function requiredModel(
/** Thin OpenAI-compatible adapters. The server record owns every endpoint and model choice. */
export const remoteMediaRuntime = {
+ async generateVideo(
+ server: RemoteServer,
+ input: ResolvedVideoRequest,
+ outputPath: string,
+ options: {
+ signal: AbortSignal;
+ model?: string;
+ jobId?: string;
+ onJobStarted: (id: string) => Promise;
+ onProgress: (
+ progress: VideoGenerationProgressContract | null,
+ stage?: VideoGenerationStage,
+ ) => void;
+ onPreview?: (preview: NonNullable) => void;
+ },
+ ): Promise<
+ VideoGenerationResultContract & { provenance?: RecordProvenance }
+ > {
+ if (server.modelManagement !== 'offgrid-desktop-v1')
+ throw new Error('Remote video requires an OGAD server.');
+ const clientJobId = options.jobId ?? generateId();
+ await options.onJobStarted(clientJobId);
+ const job = await request(
+ {
+ server,
+ path: '/v1/videos',
+ signal: options.signal,
+ init: {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ ...input,
+ model: options.model ?? requiredModel(server, 'video'),
+ enhancePrompt: false,
+ client_job_id: clientJobId,
+ }),
+ },
+ },
+ response => response.json() as Promise<{ request_id: string }>,
+ );
+ if (!job.request_id) throw new Error('The server returned no video job.');
+ await options.onJobStarted(job.request_id);
+ const cancel = () => {
+ void request(
+ {
+ server,
+ path: `/v1/videos/${encodeURIComponent(job.request_id)}/cancel`,
+ init: { method: 'POST' },
+ },
+ async response => {
+ await response.text();
+ },
+ ).catch(() => {});
+ };
+ options.signal.addEventListener('abort', cancel, { once: true });
+ const previewController = new AbortController();
+ const stopPreview = () => previewController.abort();
+ options.signal.addEventListener('abort', stopPreview, { once: true });
+ let previewRequest: Promise | undefined;
+ try {
+ if (options.signal.aborted) {
+ cancel();
+ throw new Error('Video generation stopped.');
+ }
+ while (true) {
+ const state = await request(
+ {
+ server,
+ path: `/v1/videos/${encodeURIComponent(job.request_id)}`,
+ signal: options.signal,
+ init: { method: 'GET' },
+ },
+ response =>
+ response.json() as Promise<{
+ status: string;
+ stage?: VideoGenerationStage;
+ progress?: { step: number; total: number };
+ preview?: { width: number; height: number };
+ error?: { message: string };
+ result?: VideoGenerationResultContract & {
+ provenance?: RecordProvenance;
+ };
+ }>,
+ );
+ const progress =
+ state.progress && Number.isFinite(state.progress.step) && Number.isFinite(state.progress.total)
+ ? state.progress
+ : null;
+ const stage =
+ state.stage && ['enhancing', 'preparing', 'conditioning', 'generating', 'decoding', 'encoding'].includes(state.stage)
+ ? state.stage
+ : undefined;
+ if (progress || stage) options.onProgress(progress, stage);
+ if (!previewRequest && options.onPreview && state.preview &&
+ Number.isFinite(state.preview.width) && state.preview.width > 0 &&
+ Number.isFinite(state.preview.height) && state.preview.height > 0) {
+ const dimensions = state.preview;
+ // Fetch only this job's authenticated route, never a URL from server metadata.
+ // Preview transfer must not delay job polling or final video delivery.
+ const timeout = setTimeout(stopPreview, 5000);
+ previewRequest = request({
+ server,
+ path: `/v1/videos/${encodeURIComponent(job.request_id)}/preview`,
+ signal: previewController.signal,
+ init: { method: 'GET', headers: { Accept: 'image/png' } },
+ }, async response => {
+ const bytes = await response.arrayBuffer();
+ if (bytes.byteLength === 0 || bytes.byteLength > 8 * 1024 * 1024) return;
+ const path = `${outputPath}.preview.png`;
+ await RNFS.writeFile(path, Buffer.from(bytes).toString('base64'), 'base64');
+ if (!previewController.signal.aborted) options.onPreview?.({ path, width: dimensions.width, height: dimensions.height });
+ }).catch(() => {
+ // A missing or expired preview must not fail video generation.
+ }).finally(() => clearTimeout(timeout));
+ }
+ if (state.status === 'failed' || state.status === 'cancelled')
+ throw Object.assign(
+ new Error(
+ state.error?.message ?? 'Remote video generation failed.',
+ ),
+ { code: 'VIDEO_REMOTE_FAILED' },
+ );
+ if (state.status === 'completed') {
+ if (!state.result?.syncId)
+ throw new Error('The server returned an invalid video result.');
+ const apiKey = await remoteServerManager.getApiKey(server.id);
+ const transfer = RNFS.downloadFile({
+ fromUrl: endpoint(
+ server,
+ `/v1/videos/${encodeURIComponent(job.request_id)}/content`,
+ ),
+ toFile: outputPath,
+ headers: remoteAuthorizationHeaders(server.endpoint, apiKey),
+ connectionTimeout: 15000,
+ readTimeout: 60000,
+ });
+ const stopTransfer = () => RNFS.stopDownload(transfer.jobId);
+ options.signal.addEventListener('abort', stopTransfer, {
+ once: true,
+ });
+ try {
+ if (options.signal.aborted) {
+ stopTransfer();
+ throw new Error('Video generation stopped.');
+ }
+ const result = await transfer.promise;
+ if (result.statusCode !== 200 || result.bytesWritten <= 0)
+ throw new Error('Could not download the generated video.');
+ } finally {
+ options.signal.removeEventListener('abort', stopTransfer);
+ }
+ return {
+ ...state.result,
+ provenance: isRecordProvenance(state.result.provenance)
+ ? state.result.provenance
+ : undefined,
+ path: outputPath,
+ };
+ }
+ await new Promise((resolve, reject) => {
+ const abort = () => {
+ clearTimeout(timer);
+ reject(new Error('Video generation stopped.'));
+ };
+ const timer = setTimeout(() => {
+ options.signal.removeEventListener('abort', abort);
+ resolve();
+ }, 1000);
+ options.signal.addEventListener('abort', abort, { once: true });
+ if (options.signal.aborted) abort();
+ });
+ }
+ } finally {
+ stopPreview();
+ await previewRequest;
+ options.signal.removeEventListener('abort', stopPreview);
+ options.signal.removeEventListener('abort', cancel);
+ }
+ },
+
async generateImage(
server: RemoteServer,
input: { prompt: string; size?: string },
@@ -97,32 +295,42 @@ export const remoteMediaRuntime = {
const desktop = server.modelManagement === 'offgrid-desktop-v1';
type ImagePayload = {
data?: Array<{ b64_json?: string; url?: string }>;
- choices?: Array<{ message?: { images?: Array<{ image_url?: { url?: string } }> } }>;
+ choices?: Array<{
+ message?: { images?: Array<{ image_url?: { url?: string } }> };
+ }>;
};
- const payload = await request({
- server,
- path: openRouter ? '/v1/chat/completions' : '/v1/images/generations',
- init: {
- method: 'POST',
- headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify(openRouter
- ? {
- model: requiredModel(server, 'image'),
- messages: [{ role: 'user', content: input.prompt }],
- modalities: ['image', 'text'],
- stream: false,
- }
- : {
- model: requiredModel(server, 'image'),
- prompt: input.prompt,
- size: input.size ?? '1024x1024',
- response_format: 'b64_json',
- ...(desktop ? { async: true } : {}),
- ...(options.override ? { allow_unsafe_memory_override: true } : {}),
- }),
+ const payload = await request(
+ {
+ server,
+ path: openRouter ? '/v1/chat/completions' : '/v1/images/generations',
+ init: {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(
+ openRouter
+ ? {
+ model: requiredModel(server, 'image'),
+ messages: [{ role: 'user', content: input.prompt }],
+ modalities: ['image', 'text'],
+ stream: false,
+ }
+ : {
+ model: requiredModel(server, 'image'),
+ prompt: input.prompt,
+ size: input.size ?? '1024x1024',
+ response_format: 'b64_json',
+ ...(desktop ? { async: true } : {}),
+ ...(options.override
+ ? { allow_unsafe_memory_override: true }
+ : {}),
+ },
+ ),
+ },
+ signal: options.signal,
},
- signal: options.signal,
- }, response => response.json() as Promise);
+ response =>
+ response.json() as Promise,
+ );
let result: ImagePayload = payload;
if (desktop && payload.request_id) {
while (true) {
@@ -194,31 +402,40 @@ export const remoteMediaRuntime = {
options: RemoteMediaRequestOptions = {},
): Promise {
const openRouter = new URL(server.endpoint).hostname === 'openrouter.ai';
- const voice = input.voice || (openRouter
- ? (await remoteMediaRuntime.listVoices(server, options))[0]
- : undefined);
- if (openRouter && !voice) throw new Error('This remote model has no available speakers.');
- return request({
- server,
- path: '/v1/audio/speech',
- init: {
- method: 'POST',
- headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify({
- model: requiredModel(server, 'voice'),
- input: input.text,
- ...(voice ? { voice } : {}),
- ...(openRouter ? { response_format: 'mp3' } : {}),
- }),
+ const voice =
+ input.voice ||
+ (openRouter
+ ? (await remoteMediaRuntime.listVoices(server, options))[0]
+ : undefined);
+ if (openRouter && !voice)
+ throw new Error('This remote model has no available speakers.');
+ return request(
+ {
+ server,
+ path: '/v1/audio/speech',
+ init: {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ model: requiredModel(server, 'voice'),
+ input: input.text,
+ ...(voice ? { voice } : {}),
+ ...(openRouter ? { response_format: 'mp3' } : {}),
+ }),
+ },
+ signal: options.signal,
},
- signal: options.signal,
- }, async response => ({
- audio: await response.arrayBuffer(),
- contentType: response.headers.get('content-type') ?? 'audio/mpeg',
- }));
+ async response => ({
+ audio: await response.arrayBuffer(),
+ contentType: response.headers.get('content-type') ?? 'audio/mpeg',
+ }),
+ );
},
- async listVoices(server: RemoteServer, options: RemoteMediaRequestOptions = {}): Promise {
+ async listVoices(
+ server: RemoteServer,
+ options: RemoteMediaRequestOptions = {},
+ ): Promise {
try {
const modelId = requiredModel(server, 'voice');
const catalog = await request({
@@ -244,7 +461,9 @@ export const remoteMediaRuntime = {
signal: options.signal,
}, response => response.json() as Promise<{ voices?: unknown }>);
return Array.isArray(payload.voices)
- ? payload.voices.filter((voice): voice is string => typeof voice === 'string')
+ ? payload.voices.filter(
+ (voice): voice is string => typeof voice === 'string',
+ )
: [];
},
};
diff --git a/src/services/remoteModelSelection.ts b/src/services/remoteModelSelection.ts
index 78f01aa7b..ffe5c5367 100644
--- a/src/services/remoteModelSelection.ts
+++ b/src/services/remoteModelSelection.ts
@@ -1,3 +1,4 @@
+import { videoModelDisplayName } from '../utils/modelHelpers';
import type {
RemoteModelCategory,
RemoteModelOption,
@@ -27,16 +28,24 @@ export function remoteServerModelOptions(
servers: RemoteServer[],
category: RemoteModelCategory,
): RemoteServerModelOption[] {
- return servers.flatMap(server => {
- const reported = server.modelCatalog?.[category] ?? [];
- const options =
- reported.length > 0 ? reported : configuredOption(server, category);
- return options.map(option => ({
- ...option,
- serverId: server.id,
- serverName: server.name,
- }));
- });
+ return servers
+ .filter(
+ server =>
+ category !== 'video' || server.modelManagement === 'offgrid-desktop-v1',
+ )
+ .flatMap(server => {
+ const reported = server.modelCatalog?.[category] ?? [];
+ const options =
+ reported.length > 0 ? reported : configuredOption(server, category);
+ return options.map(option => ({
+ ...option,
+ name: category === 'video'
+ ? videoModelDisplayName(option.id, option.name)
+ : option.name,
+ serverId: server.id,
+ serverName: server.name,
+ }));
+ });
}
export function selectedRemoteModelName(
@@ -45,8 +54,8 @@ export function selectedRemoteModelName(
): string | null {
const selectedId = server?.mediaModels?.[category]?.trim();
if (!server || !selectedId) return null;
- return (
+ const name =
server.modelCatalog?.[category]?.find(model => model.id === selectedId)
- ?.name ?? displayModelName(selectedId)
- );
+ ?.name ?? displayModelName(selectedId);
+ return category === 'video' ? videoModelDisplayName(selectedId, name) : name;
}
diff --git a/src/services/remoteServerManager.ts b/src/services/remoteServerManager.ts
index a69d78041..274acad89 100644
--- a/src/services/remoteServerManager.ts
+++ b/src/services/remoteServerManager.ts
@@ -250,6 +250,7 @@ class RemoteServerManager {
store.setActiveServerId(null);
store.setActiveRemoteTextModelId(null);
store.setActiveRemoteImageModelId(null);
+ store.setActiveRemoteMediaServerId('video', null);
store.setActiveRemoteMediaServerId('image', null);
store.setActiveRemoteMediaServerId('transcription', null);
store.setActiveRemoteMediaServerId('voice', null);
@@ -296,7 +297,9 @@ class RemoteServerManager {
}> {
let discovered: DiscoveredServer[];
const savedEndpoints = new Set(
- useRemoteServerStore.getState().servers.map(server => trimSlash(server.endpoint)),
+ useRemoteServerStore
+ .getState()
+ .servers.map(server => trimSlash(server.endpoint)),
);
try {
discovered = await discoverLANServers(undefined, {
@@ -314,7 +317,9 @@ class RemoteServerManager {
}
return {
moved: [],
- found: discovered.filter(server => !savedEndpoints.has(trimSlash(server.endpoint))),
+ found: discovered.filter(
+ server => !savedEndpoints.has(trimSlash(server.endpoint)),
+ ),
};
}
diff --git a/src/services/sync/messageContext.ts b/src/services/sync/messageContext.ts
index 77a01cfff..507264414 100644
--- a/src/services/sync/messageContext.ts
+++ b/src/services/sync/messageContext.ts
@@ -41,6 +41,7 @@ export function serializeMessageContext(
| 'generationMeta'
| 'isSystemInfo'
| 'turnStatus'
+ | 'turnKind'
>,
): string | null {
return serializeSyncedMessageContext({
@@ -100,6 +101,7 @@ export function serializeMessageContext(
...(message.generationTimeMs !== undefined
? { durationMs: message.generationTimeMs }
: {}),
+ turnKind: message.turnKind,
status: message.turnStatus ?? 'completed',
});
}
diff --git a/src/services/sync/mutation.ts b/src/services/sync/mutation.ts
index 2ff7cc451..9c8950e02 100644
--- a/src/services/sync/mutation.ts
+++ b/src/services/sync/mutation.ts
@@ -1,3 +1,4 @@
+import { VIDEO_SETTING_KEYS, acceptsVideoSetting } from '@offgrid/models';
import { callHook, HOOKS } from '../../bootstrap/hookRegistry';
import {
KNOWLEDGE_DOCUMENT_ENTITY,
@@ -62,6 +63,15 @@ const integerInRange =
const MODEL_SETTING_DESCRIPTORS: Readonly<
Record
> = {
+ ...Object.fromEntries(
+ VIDEO_SETTING_KEYS.map(key => [
+ key,
+ {
+ localKey: key,
+ accepts: (value: unknown) => acceptsVideoSetting(key, value),
+ },
+ ]),
+ ),
temperature: { localKey: 'temperature', accepts: finiteInRange(0, 2) },
ctxSize: {
localKey: 'contextLength',
diff --git a/src/services/toolEmbeddingRouter.ts b/src/services/toolEmbeddingRouter.ts
index e63373afe..a259b5bea 100644
--- a/src/services/toolEmbeddingRouter.ts
+++ b/src/services/toolEmbeddingRouter.ts
@@ -142,11 +142,9 @@ function discoveryBoost(tool: RoutableTool): number {
async function embedTool(tool: RoutableTool, expectedDim?: number): Promise {
const text = `${tool.function.name}: ${firstLine(tool.function.description)}`;
- const hash = hashText(text);
+ const hash = `${embeddingService.getModelId()}:${hashText(text)}`;
const cached = toolEmbeddingCache.get(tool.function.name);
- // A cache hit needs BOTH the text hash AND the current embedding dimension to match.
- // After an embedding-model swap the dimension changes while the text is identical, so
- // the dimension check stops a stale-dim vector from poisoning cosineSimilarity with NaN.
+ // Include model identity even when two models have the same vector size.
if (cached && cached.h === hash && (expectedDim == null || cached.v.length === expectedDim)) {
return cached.v;
}
@@ -169,21 +167,23 @@ export async function selectToolsByEmbedding(
if (tools.length <= topK || !query.trim()) {
return tools.map(t => t.function.name);
}
- await hydrateCache();
- await embeddingService.load();
- const queryVec = await embeddingService.embed(query);
- const tokens = queryTokens(query);
- const scored: Array<{ name: string; score: number }> = [];
- for (const tool of tools) {
- const vec = await embedTool(tool, queryVec.length);
- // Hybrid: semantic similarity + lexical (provider/verb word) + discovery boost.
- const score = cosineSimilarity(queryVec, vec) + lexicalBoost(tokens, tool) + discoveryBoost(tool);
- scored.push({ name: tool.function.name, score });
- }
- scored.sort((a, b) => b.score - a.score);
- const selected = scored.slice(0, topK).map(s => s.name);
- logger.log(`[ToolRouter] hybrid-routed ${tools.length} → ${selected.length}: [${selected.join(', ')}]`);
- return selected;
+ return embeddingService.runExclusive(async () => {
+ await hydrateCache();
+ await embeddingService.load();
+ const queryVec = await embeddingService.embed(query);
+ const tokens = queryTokens(query);
+ const scored: Array<{ name: string; score: number }> = [];
+ for (const tool of tools) {
+ const vec = await embedTool(tool, queryVec.length);
+ // Hybrid: semantic similarity + lexical (provider/verb word) + discovery boost.
+ const score = cosineSimilarity(queryVec, vec) + lexicalBoost(tokens, tool) + discoveryBoost(tool);
+ scored.push({ name: tool.function.name, score });
+ }
+ scored.sort((a, b) => b.score - a.score);
+ const selected = scored.slice(0, topK).map(s => s.name);
+ logger.log(`[ToolRouter] hybrid-routed ${tools.length} → ${selected.length}: [${selected.join(', ')}]`);
+ return selected;
+ });
}
/** Test helper: clear the in-memory cache and re-arm hydration. */
diff --git a/src/services/tools/handlers.ts b/src/services/tools/handlers.ts
index b7dd6c36d..8814dcdad 100644
--- a/src/services/tools/handlers.ts
+++ b/src/services/tools/handlers.ts
@@ -1,11 +1,22 @@
+import { videoGenerationService } from '../videoGenerationService';
import { Platform } from 'react-native';
import DeviceInfo from 'react-native-device-info';
import { ToolCall, ToolResult } from './types';
import type { RagSearchResult } from '../rag';
import logger from '../../utils/logger';
-function makeResult(call: ToolCall, start: number, opts: { content: string; error?: string }): ToolResult {
- return { toolCallId: call.id, name: call.name, content: opts.content, error: opts.error, durationMs: Date.now() - start };
+function makeResult(
+ call: ToolCall,
+ start: number,
+ opts: { content: string; error?: string },
+): ToolResult {
+ return {
+ toolCallId: call.id,
+ name: call.name,
+ content: opts.content,
+ error: opts.error,
+ durationMs: Date.now() - start,
+ };
}
function requireString(call: ToolCall, param: string): string | null {
const val = call.arguments[param];
@@ -25,6 +36,13 @@ export async function executeToolCall(call: ToolCall): Promise {
async function dispatchTool(call: ToolCall): Promise {
switch (call.name) {
+ case 'generate_video': {
+ const prompt = requireString(call, 'prompt');
+ if (!prompt || !call.context?.conversationId)
+ throw new Error('A video prompt and chat are required.');
+ videoGenerationService.defer(call.context.conversationId, { prompt });
+ return 'Video generation is queued after this reply. The video will appear in this chat when complete.';
+ }
case 'web_search': {
const q = requireString(call, 'query');
if (!q) throw new Error('Missing required parameter: query');
@@ -60,8 +78,9 @@ async function handleWebSearch(query: string): Promise {
const response = await fetch(url, {
signal: controller.signal,
headers: {
- 'User-Agent': 'Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36',
- 'Accept': 'text/html',
+ 'User-Agent':
+ 'Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36',
+ Accept: 'text/html',
},
});
const html = await response.text();
@@ -139,7 +158,12 @@ function parseBraveResults(html: string): SearchResult[] {
}
const NAMED_HTML_ENTITIES: Readonly> = {
- amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ',
+ amp: '&',
+ lt: '<',
+ gt: '>',
+ quot: '"',
+ apos: "'",
+ nbsp: ' ',
};
function decodeHTMLEntities(text: string): string {
@@ -171,7 +195,10 @@ function evaluateExpression(expr: string): number {
function parseTerm(): number {
let left = parsePower();
- while (pos < str.length && (str[pos] === '*' || str[pos] === '/' || str[pos] === '%')) {
+ while (
+ pos < str.length &&
+ (str[pos] === '*' || str[pos] === '/' || str[pos] === '%')
+ ) {
const op = str[pos++];
const right = parsePower();
if (op === '*') left *= right;
@@ -234,8 +261,14 @@ function handleCalculator(expression: string): string {
function handleGetDatetime(timezone?: string): string {
const now = new Date();
const options: Intl.DateTimeFormatOptions = {
- weekday: 'long', year: 'numeric', month: 'long', day: 'numeric',
- hour: '2-digit', minute: '2-digit', second: '2-digit', timeZoneName: 'long',
+ weekday: 'long',
+ year: 'numeric',
+ month: 'long',
+ day: 'numeric',
+ hour: '2-digit',
+ minute: '2-digit',
+ second: '2-digit',
+ timeZoneName: 'long',
...(timezone ? { timeZone: timezone } : {}),
};
try {
@@ -250,7 +283,8 @@ function handleGetDatetime(timezone?: string): string {
}
async function collectDeviceSection(
- label: string, fetcher: () => Promise,
+ label: string,
+ fetcher: () => Promise,
): Promise {
try { return await fetcher(); } catch { return `${label}: unavailable`; }
}
@@ -323,8 +357,19 @@ function htmlToMarkdown(html: string): string {
const root = parse(html);
// strip boilerplate
- ['script','style','nav','header','footer','aside','noscript','iframe','form','button'].forEach(
- tag => root.querySelectorAll(tag).forEach((el: any) => el.remove()),
+ [
+ 'script',
+ 'style',
+ 'nav',
+ 'header',
+ 'footer',
+ 'aside',
+ 'noscript',
+ 'iframe',
+ 'form',
+ 'button',
+ ].forEach(tag =>
+ root.querySelectorAll(tag).forEach((el: any) => el.remove()),
);
// prefer semantic content containers
@@ -356,8 +401,9 @@ async function handleReadUrl(rawUrl: string): Promise {
const response = await fetch(url, {
signal: controller.signal,
headers: {
- 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1',
- 'Accept': 'text/html, text/plain, */*',
+ 'User-Agent':
+ 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1',
+ Accept: 'text/html, text/plain, */*',
},
});
if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
@@ -370,16 +416,25 @@ async function handleReadUrl(rawUrl: string): Promise {
} catch (e: any) {
logger.error(`[Tools] read_url FAILED for "${url}": ${e?.message || e}`);
throw e;
- } finally { clearTimeout(timeout); }
+ } finally {
+ clearTimeout(timeout);
+ }
}
-async function handleSearchKnowledgeBase(query: string, projectId?: string): Promise {
- if (!projectId) return 'No project context. Knowledge base requires an active project.';
+async function handleSearchKnowledgeBase(
+ query: string,
+ projectId?: string,
+): Promise {
+ if (!projectId)
+ return 'No project context. Knowledge base requires an active project.';
const { ragService } = require('../rag'); // NOSONAR
const result = await ragService.searchProject(projectId, query);
if (result.chunks.length === 0) return `No results found for "${query}" in the knowledge base.`;
return result.chunks
- .map((c: RagSearchResult, i: number) => `[${i + 1}] ${c.name} (part ${c.position + 1}):\n${c.content}`)
+ .map(
+ (c: RagSearchResult, i: number) =>
+ `[${i + 1}] ${c.name} (part ${c.position + 1}):\n${c.content}`,
+ )
.join('\n\n---\n\n');
}
diff --git a/src/services/tools/registry.ts b/src/services/tools/registry.ts
index ab683cd4f..8e5e59548 100644
--- a/src/services/tools/registry.ts
+++ b/src/services/tools/registry.ts
@@ -1,11 +1,23 @@
import { ToolDefinition } from './types';
export const AVAILABLE_TOOLS: ToolDefinition[] = [
+ {
+ id: 'generate_video',
+ name: 'generate_video',
+ displayName: 'Generate Video',
+ icon: 'video',
+ description:
+ 'Request a short silent video from the selected video model. Generation starts after this text reply finishes. Describe the requested subject and motion.',
+ parameters: {
+ prompt: { type: 'string', description: 'Video prompt', required: true },
+ },
+ },
{
id: 'web_search',
name: 'web_search',
displayName: 'Web Search',
- description: 'Search the live web and return real-time result titles, snippets, and URLs. Use this for any question about current events, prices, weather, news, or anything that requires up-to-date information. When the snippet is insufficient, call read_url on the most relevant result URL to get the full page content.',
+ description:
+ 'Search the live web and return real-time result titles, snippets, and URLs. Use this for any question about current events, prices, weather, news, or anything that requires up-to-date information. When the snippet is insufficient, call read_url on the most relevant result URL to get the full page content.',
icon: 'globe',
requiresNetwork: true,
parameters: {
@@ -75,7 +87,8 @@ export const AVAILABLE_TOOLS: ToolDefinition[] = [
id: 'read_url',
name: 'read_url',
displayName: 'URL Reader',
- description: 'Fetch the full live content of any URL. Use this after web_search to read the complete text of a result page, or directly when the user shares a link.',
+ description:
+ 'Fetch the full live content of any URL. Use this after web_search to read the complete text of a result page, or directly when the user shares a link.',
icon: 'link',
requiresNetwork: true,
parameters: {
@@ -89,9 +102,8 @@ export const AVAILABLE_TOOLS: ToolDefinition[] = [
];
export function getToolsAsOpenAISchema(enabledToolIds: readonly string[]) {
- return AVAILABLE_TOOLS
- .filter(tool => enabledToolIds.includes(tool.id))
- .map(tool => ({
+ return AVAILABLE_TOOLS.filter(tool => enabledToolIds.includes(tool.id)).map(
+ tool => ({
type: 'function' as const,
function: {
name: tool.name,
@@ -113,7 +125,8 @@ export function getToolsAsOpenAISchema(enabledToolIds: readonly string[]) {
.map(([key]) => key),
},
},
- }));
+ }),
+ );
}
export function buildToolSystemPromptHint(enabledToolIds: string[]): string {
diff --git a/src/services/userModelEjection.ts b/src/services/userModelEjection.ts
index a0d6e2cbd..9540c4e06 100644
--- a/src/services/userModelEjection.ts
+++ b/src/services/userModelEjection.ts
@@ -1,3 +1,4 @@
+import { videoGenerationService } from './videoGenerationService';
import { activeModelService } from './activeModelService';
import { generationService } from './generationService';
import { imageGenerationService } from './imageGenerationService';
@@ -11,6 +12,7 @@ export async function ejectAllModelsForUser(): Promise<{ count: number }> {
await Promise.all([
generationService.stopGeneration(),
imageGenerationService.cancelGeneration(),
+ videoGenerationService.cancelGeneration(),
]);
return activeModelService.ejectAll();
}
diff --git a/src/services/videoGenerationService.ts b/src/services/videoGenerationService.ts
new file mode 100644
index 000000000..fe78bef8d
--- /dev/null
+++ b/src/services/videoGenerationService.ts
@@ -0,0 +1,622 @@
+import { Alert, DevSettings, Platform } from 'react-native';
+import { videoGenerationMeta } from '../utils/modelHelpers';
+import { resolveDocumentPath } from '../utils/resolveDocumentPath';
+import logger from '../utils/logger';
+import AsyncStorage from '@react-native-async-storage/async-storage';
+import { useRemoteServerStore } from '../stores/remoteServerStore';
+import { remoteMediaRuntime } from './remoteMediaRuntime';
+import {
+ generateStandalone,
+ getActiveEngineService,
+ isRemoteTextModelActive,
+} from './engines';
+import { activeModelService } from './activeModelService';
+import RNFS from 'react-native-fs';
+import { generateId as uuid } from '../utils/generateId';
+import {
+ resolveVideoRequest,
+ videoArchitecture,
+ type ModelEntry,
+ type ResolvedVideoRequest,
+ type VideoGenerationJobContract,
+ type VideoGenerationRequestContract,
+} from '@offgrid/models';
+import { useAppStore, useChatStore } from '../stores';
+import { videoGenerator } from './videoGenerator';
+import { resolveVideoPack } from './videoModelFiles';
+import { modelResidencyManager } from './modelResidency';
+import { OverridableMemoryError } from './modelLoadErrors';
+import { reportModelFailure, clearModelFailure } from './modelFailureHandler';
+import { reasonFromLoadError } from './modelFailureReasons';
+import { generationSession } from './generationSession';
+import type { GeneratedVideo } from '../types';
+
+const JOURNAL_KEY = 'offgrid.video.active-job.v1';
+type VideoInput = VideoGenerationRequestContract & { conversationId?: string };
+interface VideoJournal {
+ input: VideoInput;
+ id: string;
+ messageId: string;
+ remoteServerId?: string;
+ remoteJobId?: string;
+ startedAt: number;
+ result?: GeneratedVideo;
+}
+const EMPTY: VideoGenerationJobContract = {
+ id: null,
+ phase: 'idle',
+ conversationId: null,
+ projectId: null,
+ stage: null,
+ enhancedPrompt: '',
+ progress: null,
+ preview: null,
+ outputPath: null,
+ error: null,
+ startedAt: null,
+ finishedAt: null,
+};
+class VideoGenerationService {
+ private state = EMPTY;
+ private journal: VideoJournal | null = null;
+ private persistence: Promise = Promise.resolve();
+ private persist() {
+ const value = this.journal ? JSON.stringify(this.journal) : null;
+ this.persistence = this.persistence
+ .catch(() => {})
+ .then(() =>
+ value
+ ? AsyncStorage.setItem(JOURNAL_KEY, value)
+ : AsyncStorage.removeItem(JOURNAL_KEY),
+ );
+ return this.persistence;
+ }
+ async restore(): Promise {
+ if (this.completion) return;
+ const raw = await AsyncStorage.getItem(JOURNAL_KEY);
+ if (!raw) return;
+ const saved = JSON.parse(raw) as VideoJournal;
+ if (
+ !saved.input ||
+ typeof saved.id !== 'string' ||
+ !/^[a-zA-Z0-9_-]+$/.test(saved.id) ||
+ typeof saved.messageId !== 'string'
+ )
+ return;
+ resolveVideoRequest(saved.input);
+ if (
+ saved.result &&
+ saved.result.videoPath ===
+ `${RNFS.DocumentDirectoryPath}/generated-videos/${saved.id}.mp4` &&
+ (await RNFS.exists(saved.result.videoPath))
+ ) {
+ this.publishResult(saved.result, saved.messageId, saved.startedAt);
+ await AsyncStorage.removeItem(JOURNAL_KEY);
+ return;
+ }
+ if (
+ useAppStore
+ .getState()
+ .generatedVideos.some(video => video.id === saved.id)
+ ) {
+ await AsyncStorage.removeItem(JOURNAL_KEY);
+ return;
+ }
+ this.journal = saved;
+ this.messageId = saved.messageId;
+ const output = `${RNFS.DocumentDirectoryPath}/generated-videos/${saved.id}.mp4`;
+ if (!saved.remoteServerId && await videoGenerator.getStatus(output)) {
+ this.completion = this.run(saved.input, saved, { nativeRecovery: true }).finally(() => {
+ this.completion = null;
+ });
+ void this.completion.catch(error => logger.warn('[Video] Recovered job failed', error));
+ return;
+ }
+ this.update({
+ ...EMPTY,
+ id: saved.id,
+ conversationId: saved.input.conversationId ?? null,
+ phase: 'failed',
+ error: saved.remoteJobId
+ ? 'The connection was interrupted. Retry to check the same OGAD job.'
+ : 'Video generation was interrupted. Retry to start it again.',
+ startedAt: saved.startedAt,
+ finishedAt: Date.now(),
+ });
+ this.reportFailure(new Error(this.state.error!), !!saved.remoteServerId);
+ }
+ private publishResult(
+ result: GeneratedVideo,
+ messageId: string,
+ startedAt: number,
+ ) {
+ useAppStore.getState().addGeneratedVideo(result);
+ const conversation = useChatStore
+ .getState()
+ .conversations.find(c => c.id === result.conversationId);
+ if (
+ !conversation ||
+ conversation.messages.some(message => message.uuid === messageId)
+ )
+ return;
+ useChatStore
+ .getState()
+ .addMessage(conversation.id, {
+ role: 'assistant',
+ uuid: messageId,
+ content: `Generated video for: "${result.prompt}"`,
+ turnStatus: 'completed',
+ turnKind: 'video',
+ attachments: [
+ {
+ id: result.id,
+ type: 'video',
+ uri: `file://${result.videoPath}`,
+ width: result.width,
+ height: result.height,
+ },
+ ],
+ generationTimeMs: Date.now() - startedAt,
+ generationMeta: videoGenerationMeta(result),
+ });
+ }
+ async deleteVideo(id: string): Promise {
+ const video = useAppStore
+ .getState()
+ .generatedVideos.find(item => item.id === id);
+ if (!video) return;
+ const path = resolveDocumentPath(video.videoPath);
+ const root = RNFS.DocumentDirectoryPath + '/';
+ if (!path.startsWith(root) || path.split('/').includes('..'))
+ throw new Error('The video path is outside app storage.');
+ if (await RNFS.exists(path)) await RNFS.unlink(path);
+ useChatStore.getState().removeMediaAttachment(id);
+ useAppStore.getState().removeGeneratedVideo(id);
+ }
+ private reportFailure(error: unknown, remote = false) {
+ const reason = reasonFromLoadError(error);
+ const interrupted =
+ (error as { code?: string } | null)?.code === 'VIDEO_BACKGROUND_INTERRUPTED';
+ const generationFailed =
+ reason === 'load-threw' &&
+ this.state.stage !== null &&
+ this.state.stage !== 'preparing';
+ const detail = error instanceof Error ? error.message : String(error);
+ reportModelFailure('video', error, {
+ remote,
+ ...(interrupted || generationFailed
+ ? {
+ title: interrupted ? 'Video generation interrupted' : 'Video generation failed',
+ message: detail,
+ }
+ : {}),
+ onRetry: () => {
+ void (async () => {
+ if (!remote && reason === 'insufficient-memory')
+ await activeModelService.ejectAll();
+ await this.retry();
+ })().catch(() => {});
+ },
+ onLoadAnyway: () => {
+ void this.retry({ override: true }).catch(() => {});
+ },
+ });
+ }
+ async retry(options?: { override?: boolean }): Promise {
+ if (!this.journal) return;
+ if (this.completion)
+ throw new Error('Video generation is already running.');
+ if (generationSession.getConversationId())
+ throw new Error(
+ 'Wait for the current generation to finish before retrying.',
+ );
+ this.completion = this.run(this.journal.input, { ...this.journal, startedAt: Date.now() }, options).finally(() => {
+ this.completion = null;
+ });
+ return this.completion;
+ }
+ private listeners = new Set<() => void>();
+ private cancelled = false;
+ private abort = new AbortController();
+ private messageId: string | null = null;
+ getMessageId = () => this.messageId;
+ private pending = new Map();
+ private completion: Promise | null = null;
+ getState = () => this.state;
+ subscribe = (listener: () => void) => {
+ this.listeners.add(listener);
+ return () => {
+ this.listeners.delete(listener);
+ };
+ };
+ private update(patch: Partial) {
+ if (this.cancelled && patch.stage !== undefined) return;
+ this.state = { ...this.state, ...patch };
+ for (const listener of this.listeners) listener();
+ }
+ defer(conversationId: string, input: VideoGenerationRequestContract) {
+ resolveVideoRequest(input);
+ if (this.pending.has(conversationId))
+ throw new Error('A video is already requested for this turn.');
+ this.pending.set(conversationId, input);
+ }
+ discardDeferred(conversationId: string) {
+ this.pending.delete(conversationId);
+ }
+ hasPending() {
+ return this.pending.size > 0 || this.state.phase === 'running';
+ }
+ async finishDeferred(conversationId: string, stopped: boolean) {
+ const input = this.pending.get(conversationId);
+ this.pending.delete(conversationId);
+ if (input && !stopped) await this.generate({ ...input, conversationId });
+ }
+ async cancelGeneration() {
+ this.pending.clear();
+ this.cancelled = true;
+ this.abort.abort();
+ if (this.state.phase === 'running') {
+ this.update({ error: 'Video stop requested.', progress: null, preview: null });
+ await videoGenerator.cancel();
+ if (this.state.stage === 'enhancing')
+ await getActiveEngineService()?.stopGeneration();
+ await this.completion;
+ }
+ }
+ generate(
+ input: VideoGenerationRequestContract & { conversationId?: string },
+ ): Promise {
+ if (this.completion)
+ return Promise.reject(new Error('Video generation is already running.'));
+ this.completion = this.run(input).finally(() => {
+ this.completion = null;
+ });
+ return this.completion;
+ }
+ private async reserveVideo(
+ model: ModelEntry | undefined,
+ request: ResolvedVideoRequest,
+ modelId: string,
+ options?: { override?: boolean; nativeRecovery?: boolean },
+ ): Promise {
+ return modelResidencyManager.runExclusive('video-generation', async () => {
+ const sizeMB =
+ (model?.files.reduce((sum, file) => sum + (file.sizeBytes ?? 0), 0) ??
+ 0) /
+ 1048576 +
+ (request.width * request.height * request.frames * 12) / 1048576 +
+ 1024;
+ const spec = {
+ key: 'video',
+ type: 'video' as const,
+ modelId,
+ sizeMB,
+ dirtyMemory: true,
+ canEvict: () => false,
+ };
+ // A recovered native worker already owns its memory.
+ if (!options?.nativeRecovery) {
+ const fit = await modelResidencyManager.makeRoomFor(spec, options);
+ if (!fit.fits)
+ throw new OverridableMemoryError(
+ 'Not enough available memory for this video model and clip size.',
+ );
+ }
+ if (this.cancelled) throw new Error('Video generation stopped.');
+ return modelResidencyManager.register(spec, () =>
+ this.cancelGeneration(),
+ );
+ });
+ }
+ async diagnose(backend: 'auto' | 'gpu' | 'cpu'): Promise {
+ if (!__DEV__ || Platform.OS !== 'android')
+ throw new Error('Video diagnostics require an Android debug build.');
+ if (this.completion || generationSession.getConversationId())
+ throw new Error('Wait for the current generation to finish.');
+ const app = useAppStore.getState();
+ const model = app.downloadedVideoModels.find(
+ m => m.id === app.activeVideoModelId,
+ );
+ const primary = model?.files.find(file => file.role === 'primary')?.name;
+ if (!model || !primary || videoArchitecture(primary) !== 'wan21')
+ throw new Error('Select an installed Wan 2.1 video model first.');
+ // Public validation remains unchanged. Only this explicit developer action uses one frame.
+ const request = {
+ ...resolveVideoRequest({
+ model: primary,
+ prompt:
+ 'A red ball rolls slowly across a wooden table. Natural daylight, fixed camera.',
+ width: 832,
+ height: 480,
+ frames: 9,
+ fps: 8,
+ steps: 4,
+ guidance: 6,
+ seed: 597089194,
+ }),
+ frames: 1,
+ steps: 2,
+ };
+ const output = `${
+ RNFS.DocumentDirectoryPath
+ }/generated-videos/diagnostic-${backend}-${Date.now()}.mp4`;
+ this.cancelled = false;
+ this.abort = new AbortController();
+ this.update({
+ ...EMPTY,
+ phase: 'running',
+ stage: 'preparing',
+ startedAt: Date.now(),
+ });
+ this.completion = (async () => {
+ let registration: symbol | undefined;
+ try {
+ const pack = await resolveVideoPack(model);
+ registration = await this.reserveVideo(model, request, model.id, {
+ override: true,
+ });
+ await RNFS.mkdir(`${RNFS.DocumentDirectoryPath}/generated-videos`);
+ logger.log('[VideoDiagnostic] start', { ...request, backend, output });
+ await videoGenerator.generate(
+ request,
+ pack,
+ output,
+ update => this.update(update),
+ undefined,
+ backend,
+ );
+ if (this.cancelled) throw new Error('Video generation stopped.');
+ const metadata = {
+ ...request,
+ requestedBackend: backend,
+ output,
+ size: (await RNFS.stat(output)).size,
+ durationMs: Date.now() - (this.state.startedAt ?? Date.now()),
+ };
+ await RNFS.writeFile(
+ `${output}.json`,
+ JSON.stringify(metadata, null, 2),
+ 'utf8',
+ );
+ logger.log('[VideoDiagnostic] complete', metadata);
+ this.update({
+ phase: 'succeeded',
+ outputPath: output,
+ finishedAt: Date.now(),
+ });
+ } catch (error) {
+ this.update({
+ phase: this.cancelled ? 'cancelled' : 'failed',
+ error: String(error),
+ finishedAt: Date.now(),
+ });
+ throw error;
+ } finally {
+ if (registration)
+ modelResidencyManager.unregister('video', registration);
+ await RNFS.unlink(`${output}.preview.png`).catch(() => {});
+ }
+ return undefined;
+ })().finally(() => {
+ this.completion = null;
+ });
+ await this.completion;
+ return output;
+ }
+ private async run(
+ input: VideoInput,
+ resumed?: VideoJournal,
+ options?: { override?: boolean; nativeRecovery?: boolean },
+ ): Promise {
+ const app = useAppStore.getState();
+ const model = app.downloadedVideoModels.find(
+ m => m.id === (input.model ?? app.activeVideoModelId) ||
+ (options?.nativeRecovery && m.files.some(file => file.role === 'primary' && file.name === input.model)),
+ );
+ const remoteState = useRemoteServerStore.getState();
+ const server = options?.nativeRecovery ? undefined : remoteState.servers.find(
+ s =>
+ s.id ===
+ (resumed?.remoteServerId ??
+ remoteState.activeRemoteMediaServerIds.video),
+ );
+ if (resumed?.remoteServerId && !server)
+ throw new Error('The OGAD server for this job is no longer configured.');
+ if (!model && !server && !options?.nativeRecovery) throw new Error('Select a video model in Models.');
+ const modelId =
+ (options?.nativeRecovery ? model?.id : resumed?.input.model) ?? server?.mediaModels?.video ?? model!.id;
+ const primary =
+ server?.mediaModels?.video ??
+ model?.files.find(f => f.role === 'primary')?.name ?? input.model!;
+ let request = resolveVideoRequest({ ...input, model: primary }, {
+ ...(app.settings.videoParams?.[primary] ??
+ app.settings.videoParams?.default),
+ seed: app.settings.videoSeed ?? -1,
+ negativePrompt: app.settings.videoNegative ?? '',
+ });
+ const id = resumed?.id ?? uuid(),
+ messageId = resumed?.messageId ?? uuid();
+ const directory = `${RNFS.DocumentDirectoryPath}/generated-videos`,
+ output = `${directory}/${id}.mp4`;
+ clearModelFailure('video');
+ this.cancelled = false;
+ this.abort = new AbortController();
+ this.messageId = messageId;
+ this.update({
+ ...EMPTY,
+ id,
+ phase: 'running',
+ conversationId: input.conversationId ?? null,
+ stage: 'preparing',
+ startedAt: resumed?.startedAt ?? Date.now(),
+ });
+ if (input.conversationId) generationSession.begin(input.conversationId);
+ let registration: symbol | undefined;
+ this.journal = resumed ?? {
+ input: { ...input, ...request, model: modelId },
+ id,
+ messageId,
+ remoteServerId: server?.id,
+ startedAt: Date.now(),
+ };
+ try {
+ await this.persist();
+ if (
+ !resumed &&
+ (input.enhancePrompt ?? app.settings.enhanceVideoPrompts)
+ ) {
+ this.update({ stage: 'enhancing' });
+ const selectedText = activeModelService.selectedTextModelId();
+ if (
+ !isRemoteTextModelActive() &&
+ !getActiveEngineService()?.isModelLoaded() &&
+ selectedText
+ )
+ await activeModelService.loadTextModel(selectedText);
+ if (
+ isRemoteTextModelActive() ||
+ getActiveEngineService()?.isModelLoaded()
+ ) {
+ const enhanced = await generateStandalone([
+ {
+ id: uuid(),
+ role: 'system',
+ content:
+ 'Rewrite the request as one short video prompt. Preserve the subject and action. Describe useful motion and camera movement. Return only the prompt.',
+ timestamp: Date.now(),
+ },
+ {
+ id: uuid(),
+ role: 'user',
+ content: request.prompt,
+ timestamp: Date.now(),
+ },
+ ]);
+ if (enhanced.trim())
+ request = { ...request, prompt: enhanced.trim() };
+ }
+ }
+ if (this.cancelled) throw new Error('Video generation stopped.');
+ this.update({ stage: 'preparing', enhancedPrompt: request.prompt });
+ this.journal.input = { ...this.journal.input, ...request, model: modelId };
+ await this.persist();
+ await RNFS.mkdir(directory);
+ let path = output,
+ resultId = id;
+ let provenance: GeneratedVideo['provenance'];
+ if (server) {
+ const remote = await remoteMediaRuntime.generateVideo(
+ server,
+ request,
+ output,
+ {
+ signal: this.abort.signal,
+ model: modelId,
+ jobId: resumed?.remoteJobId,
+ onJobStarted: async jobId => {
+ if (this.journal) {
+ this.journal.remoteJobId = jobId;
+ await this.persist();
+ }
+ },
+ onProgress: (progress, stage) =>
+ this.update({ stage: stage ?? 'generating', progress }),
+ onPreview: preview => this.update({ preview }),
+ },
+ );
+ path = remote.path;
+ resultId = remote.provenance ? remote.syncId : id;
+ provenance = remote.provenance;
+ } else {
+ const pack = options?.nativeRecovery ? null : await resolveVideoPack(model!);
+ registration = await this.reserveVideo(model, request, modelId, options);
+ if (this.cancelled) throw new Error('Video generation stopped.');
+ path = options?.nativeRecovery
+ ? await videoGenerator.recover(request, output, update => this.update(update))
+ : await videoGenerator.generate(
+ request, pack!, output,
+ update => this.update(update),
+ reason => this.update({ error: reason, progress: null, preview: null }),
+ );
+ }
+ if (this.cancelled) throw new Error('Video generation stopped.');
+ const result: GeneratedVideo = {
+ id: resultId,
+ provenance,
+ ...request,
+ modelId,
+ videoPath: path,
+ fileName: `${id}.mp4`,
+ durationSeconds: request.frames / request.fps,
+ createdAt: new Date().toISOString(),
+ conversationId: input.conversationId,
+ };
+ this.journal.result = result;
+ await this.persist();
+ this.publishResult(result, messageId, this.state.startedAt ?? Date.now());
+ this.journal = null;
+ await this.persist().catch(error =>
+ logger.warn('[Video] Could not clear finished job', error),
+ );
+ this.update({
+ phase: 'succeeded',
+ outputPath: path,
+ finishedAt: Date.now(),
+ });
+ return result;
+ } catch (error) {
+ if (
+ (error as { code?: string })?.code === 'VIDEO_REMOTE_FAILED' &&
+ this.journal
+ ) {
+ delete this.journal.remoteJobId;
+ await this.persist().catch(error =>
+ logger.warn('[Video] Could not save failed job', error),
+ );
+ }
+ if ((error as { code?: string })?.code === 'VIDEO_CANCELLED')
+ this.cancelled = true;
+ if (this.cancelled) {
+ this.journal = null;
+ await this.persist().catch(error =>
+ logger.warn('[Video] Could not clear stopped job', error),
+ );
+ }
+ await RNFS.unlink(output).catch(() => {});
+ this.update({
+ phase: this.cancelled ? 'cancelled' : 'failed',
+ error:
+ error instanceof Error ? error.message : 'Video generation failed.',
+ finishedAt: Date.now(),
+ });
+ if (!this.cancelled) {
+ this.reportFailure(error, !!server);
+ throw error;
+ }
+ } finally {
+ this.update({ preview: null });
+ await RNFS.unlink(`${output}.preview.png`).catch(() => {});
+ if (registration) modelResidencyManager.unregister('video', registration);
+ if (generationSession.isGeneratingFor(input.conversationId))
+ generationSession.end('video-finished');
+ }
+ }
+}
+export const videoGenerationService = new VideoGenerationService();
+
+if (__DEV__ && Platform.OS === 'android') {
+ for (const backend of ['auto', 'gpu', 'cpu'] as const) {
+ DevSettings.addMenuItem(
+ `Video diagnostic: ${backend} · 1 frame, 2 steps`,
+ () => {
+ void videoGenerationService.diagnose(backend).then(
+ path => Alert.alert('Video diagnostic saved', path),
+ error => Alert.alert('Video diagnostic stopped', String(error)),
+ );
+ },
+ );
+ }
+ DevSettings.addMenuItem('Stop video diagnostic', () => {
+ void videoGenerationService.cancelGeneration();
+ });
+}
diff --git a/src/services/videoGenerator.ts b/src/services/videoGenerator.ts
new file mode 100644
index 000000000..3b206ad59
--- /dev/null
+++ b/src/services/videoGenerator.ts
@@ -0,0 +1,121 @@
+import logger from '../utils/logger';
+import { NativeEventEmitter, NativeModules } from 'react-native';
+import { videoArchitecture } from '@offgrid/models';
+import type {
+ ResolvedVideoRequest,
+ VideoModelPack,
+ VideoGenerationUpdateContract,
+} from '@offgrid/models';
+const native = NativeModules.VideoGenerationModule;
+type NativeVideoStatus = {
+ path: string;
+ phase: 'running' | 'succeeded' | 'failed';
+ stage: VideoGenerationUpdateContract['stage'];
+ step: number;
+ total: number;
+ backend?: VideoGenerationUpdateContract['backend'];
+ preview?: VideoGenerationUpdateContract['preview'];
+ code?: string;
+ error?: string;
+};
+export const videoGenerator = {
+ available: () => !!native,
+ async getStatus(outputPath: string): Promise {
+ return native?.getVideoStatus ? native.getVideoStatus(outputPath) : null;
+ },
+ async recover(
+ request: ResolvedVideoRequest,
+ outputPath: string,
+ onUpdate: (update: VideoGenerationUpdateContract) => void,
+ ): Promise {
+ for (;;) {
+ const status = await videoGenerator.getStatus(outputPath);
+ if (!status) throw new Error('The native video job is no longer available.');
+ if (status.stage !== 'generating' || status.total === request.steps) onUpdate({
+ stage: status.stage,
+ backend: status.backend ?? null,
+ progress: status.total > 0 ? { step: status.step, total: status.total } : null,
+ ...(status.preview?.path === `${outputPath}.preview.png` ? { preview: status.preview } : {}),
+ });
+ if (status.phase === 'succeeded') return status.path;
+ if (status.phase === 'failed') {
+ throw Object.assign(new Error(status.error ?? 'Video generation failed.'), { code: status.code });
+ }
+ await new Promise