feat: on the fly thumbnail generation (#3523)

## 🎯 Goal

This PR adds native video thumbnail generation for local assets and
wires it into the attachment flows. The rationale' behind the PR is the
fact that starting RN 0.84 and onwards, `ImageBackground` can no longer
reliably render `ph:///` video assets on iOS (and probably other edge
cases too).

Thumbnails are now generated natively on iOS and Android for local video
assets coming from the attachment picker / image picker / camera flows,
cached in the app cache directory, and reused if they were already
generated before. The native API is batched, so we can generate multiple
thumbnails in one go instead of doing them one by one. The module only
handles local assets. If thumbnail generation fails for a single video,
we soft fail that item only and fall back to the existing placeholder /
no thumbnail UI instead of failing the whole batch.

So in other words:

- added native video thumbnail generation on iOS and Android under
shared-native
- exposed it through a new TurboModule `StreamVideoThumbnail` API
- batched thumbnail generation for `getPhotos`, `pickImage`, and
`takePhoto`
- added deterministic cache reuse for generated thumbnails
- limited concurrent native generation to avoid decode storms
- made thumbnail generation best-effort per item:
    - native returns `{ uri, error }` per requested asset
    - failed items no longer fail the whole batch
- restricted `thumbnail` generation for local assets only
    - Android: raw path, `file://`, `content://`
    - iOS: raw path, `file://`, `ph://`
- switched attachment picker items and image upload previews away from
ImageBackground to plain Image

Notes

- this is new architecture only
- thumbnails are written to the app cache dir, so they can survive
relaunches but are still disposable cache
- existing cached thumbnails from the old cache folder name will not be
reused


## 🛠 Implementation details

<!-- Provide a description of the implementation -->

## 🎨 UI Changes

<!-- Add relevant screenshots -->

<details>
<summary>iOS</summary>


<table>
    <thead>
        <tr>
            <td>Before</td>
            <td>After</td>
        </tr>
    </thead>
    <tbody>
        <tr>
            <td>
                <!--<img src="" /> -->
            </td>
            <td>
                <!--<img src="" /> -->
            </td>
        </tr>
    </tbody>
</table>
</details>


<details>
<summary>Android</summary>

<table>
    <thead>
        <tr>
            <td>Before</td>
            <td>After</td>
        </tr>
    </thead>
    <tbody>
        <tr>
            <td>
                <!--<img src="" /> -->
            </td>
            <td>
                <!--<img src="" /> -->
            </td>
        </tr>
    </tbody>
</table>
</details>

## 🧪 Testing

<!-- Explain how this change can be tested (or why it can't be tested)
-->

## ☑️ Checklist

- [ ] I have signed the [Stream
CLA](https://docs.google.com/forms/d/e/1FAIpQLScFKsKkAJI7mhCr7K9rEIOpqIDThrWxuvxnwUq2XkHyG154vQ/viewform)
(required)
- [ ] PR targets the `develop` branch
- [ ] Documentation is updated
- [ ] New code is tested in main example apps, including all possible
scenarios
  - [ ] SampleApp iOS and Android
  - [ ] Expo iOS and Android

Author: isekovanic

.github/workflows/changelog-preview.yml

@@ -17,6 +17,8 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+      - name: Install Linux build tools
+        run: sudo apt-get update && sudo apt-get install -y build-essential
       - uses: actions/checkout@v2
         with:
           ref: develop

.github/workflows/check-pr.yml

@@ -23,6 +23,8 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+      - name: Install Linux build tools
+        run: sudo apt-get update && sudo apt-get install -y build-essential
       - name: Install && Build - SDK and Sample App
         uses: ./.github/actions/install-and-build-sdk
       - name: Lint

.github/workflows/release.yml

@@ -31,6 +31,8 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
           registry-url: 'https://registry.npmjs.org'
+      - name: Install Linux build tools
+        run: sudo apt-get update && sudo apt-get install -y build-essential
 
       - name: Prepare git
         run: |

.github/workflows/sample-distribution.yml

@@ -69,6 +69,8 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
       - uses: actions/checkout@v2
+      - name: Install Linux build tools
+        run: sudo apt-get update && sudo apt-get install -y build-essential
       - uses: actions/setup-java@v3
         with:
           distribution: 'zulu'

examples/SampleApp/android/app/build.gradle

@@ -4,6 +4,18 @@ apply plugin: 'com.google.gms.google-services'
 apply plugin: 'com.google.firebase.crashlytics'
 apply plugin: "com.facebook.react"
 
+// Some libraries look up the React Native location on the application project
+// instead of the root project. Mirror the root setting here so they never need
+// to shell out to a bare `node` binary just to resolve react-native.
+ext.REACT_NATIVE_NODE_MODULES_DIR =
+    rootProject.ext.has("REACT_NATIVE_NODE_MODULES_DIR")
+        ? rootProject.ext.get("REACT_NATIVE_NODE_MODULES_DIR")
+        : file("$rootDir/../node_modules/react-native").absolutePath
+ext.REACT_NATIVE_WORKLETS_NODE_MODULES_DIR =
+    rootProject.ext.has("REACT_NATIVE_WORKLETS_NODE_MODULES_DIR")
+        ? rootProject.ext.get("REACT_NATIVE_WORKLETS_NODE_MODULES_DIR")
+        : file("$rootDir/../node_modules/react-native-worklets").absolutePath
+
 /**
  * This is the configuration block to customize your React Native Android app.
  * By default you don't need to apply any configuration, just uncomment the lines you need.

examples/SampleApp/android/build.gradle

@@ -26,6 +26,21 @@ buildscript {
     }
 }
 
+ext.REACT_NATIVE_NODE_MODULES_DIR = file("$rootDir/../node_modules/react-native").absolutePath
+ext.REACT_NATIVE_WORKLETS_NODE_MODULES_DIR =
+    file("$rootDir/../node_modules/react-native-worklets").absolutePath
+
+subprojects { subproject ->
+    if (subproject.path != ":app") {
+        evaluationDependsOn(":app")
+        project(":app").tasks.matching { task ->
+            task.name.startsWith("configureCMake")
+        }.configureEach {
+            dependsOn(subproject.tasks.matching { it.name == "preBuild" })
+        }
+    }
+}
+
 allprojects {
     repositories {
         maven {
@@ -39,15 +54,8 @@ allprojects {
 }
 
 project(':app') {
-    afterEvaluate {
-        if (tasks.findByName("preBuild")) {
-            tasks.preBuild.doFirst {
-                exec {
-                    workingDir rootDir
-                    commandLine './gradlew', 'generateCodegenArtifactsFromSchema'
-                }
-            }
-        }
+    tasks.matching { it.name == "preBuild" || it.name.startsWith("configureCMake") }.configureEach {
+        dependsOn("generateCodegenArtifactsFromSchema")
     }
 }
 

examples/SampleApp/package.json

@@ -37,9 +37,9 @@
     "@react-native-async-storage/async-storage": "^2.2.0",
     "@react-native-camera-roll/camera-roll": "^7.10.0",
     "@react-native-clipboard/clipboard": "^1.16.3",
+    "@react-native-community/blur": "^4.4.1",
     "@react-native-community/geolocation": "^3.4.0",
     "@react-native-community/netinfo": "^11.4.1",
-    "@react-native-community/blur": "^4.4.1",
     "@react-native-documents/picker": "^10.1.3",
     "@react-native-firebase/app": "22.2.1",
     "@react-native-firebase/messaging": "22.2.1",

examples/SampleApp/yarn.lock

@@ -2190,9 +2190,9 @@
     merge-options "^3.0.4"
 
 "@react-native-camera-roll/camera-roll@^7.10.0":
-  version "7.10.0"
-  resolved "https://registry.yarnpkg.com/@react-native-camera-roll/camera-roll/-/camera-roll-7.10.0.tgz#5e9518d78a9cd87ddc8e68d03e31a608df5033ab"
-  integrity sha512-Zm1yHxxTQS2APsnnxUFoLnK+DMMTPqmIQ2z2pGtNyHRXAG40Nt4MLVB3tDJTWnuJLAG87BpTCEvpz49+u0YkUw==
+  version "7.10.2"
+  resolved "https://registry.yarnpkg.com/@react-native-camera-roll/camera-roll/-/camera-roll-7.10.2.tgz#af2234f60f0b55aff9afb60888ce7f0669d52593"
+  integrity sha512-XgJQJDFUycmqSX+MH7vTcRigQwEIQNLIu1GvOngCZRwlSV2mF61UzeruSmmHwkBcGnHZFXkKg9fil0FQVfyglw==
 
 "@react-native-clipboard/clipboard@^1.16.3":
   version "1.16.3"

package/expo-package/android/src/main/java/com/streamchatexpo/StreamChatExpoPackage.java

@@ -14,16 +14,34 @@
 import java.util.Map;
 
 public class StreamChatExpoPackage extends TurboReactPackage {
+  private static final String STREAM_VIDEO_THUMBNAIL_MODULE = "StreamVideoThumbnail";
+
   @Nullable
   @Override
   public NativeModule getModule(String name, ReactApplicationContext reactContext) {
+    if (name.equals(STREAM_VIDEO_THUMBNAIL_MODULE) && BuildConfig.IS_NEW_ARCHITECTURE_ENABLED) {
+      return createNewArchModule("com.streamchatexpo.StreamVideoThumbnailModule", reactContext);
+    }
+
     return null;
   }
 
   @Override
   public ReactModuleInfoProvider getReactModuleInfoProvider() {
     return () -> {
       final Map<String, ReactModuleInfo> moduleInfos = new HashMap<>();
+      boolean isTurboModule = BuildConfig.IS_NEW_ARCHITECTURE_ENABLED;
+      moduleInfos.put(
+              STREAM_VIDEO_THUMBNAIL_MODULE,
+              new ReactModuleInfo(
+                      STREAM_VIDEO_THUMBNAIL_MODULE,
+                      STREAM_VIDEO_THUMBNAIL_MODULE,
+                      false, // canOverrideExistingModule
+                      false, // needsEagerInit
+                      false, // hasConstants
+                      false, // isCxxModule
+                      isTurboModule // isTurboModule
+              ));
       return moduleInfos;
     };
   }
@@ -32,4 +50,19 @@ public ReactModuleInfoProvider getReactModuleInfoProvider() {
   public List<ViewManager> createViewManagers(ReactApplicationContext reactContext) {
     return Collections.<ViewManager>singletonList(new StreamShimmerViewManager());
   }
+
+  @Nullable
+  private NativeModule createNewArchModule(
+          String className,
+          ReactApplicationContext reactContext
+  ) {
+    try {
+      Class<?> moduleClass = Class.forName(className);
+      return (NativeModule) moduleClass
+              .getConstructor(ReactApplicationContext.class)
+              .newInstance(reactContext);
+    } catch (Throwable ignored) {
+      return null;
+    }
+  }
 }

package/expo-package/android/src/newarch/com/streamchatexpo/StreamVideoThumbnailModule.kt

@@ -0,0 +1,50 @@
+package com.streamchatexpo
+
+import com.facebook.react.bridge.Arguments
+import com.facebook.react.bridge.Promise
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.bridge.ReadableArray
+import com.streamchatreactnative.shared.StreamVideoThumbnailGenerator
+import java.util.concurrent.Executors
+
+class StreamVideoThumbnailModule(
+  reactContext: ReactApplicationContext,
+) : NativeStreamVideoThumbnailSpec(reactContext) {
+  override fun getName(): String = NAME
+
+  override fun createVideoThumbnails(urls: ReadableArray, promise: Promise) {
+    val urlList = mutableListOf<String>()
+    for (index in 0 until urls.size()) {
+      urlList.add(urls.getString(index) ?: "")
+    }
+
+    executor.execute {
+      try {
+        val thumbnails = StreamVideoThumbnailGenerator.generateThumbnails(reactApplicationContext, urlList)
+        val result = Arguments.createArray()
+        thumbnails.forEach { thumbnail ->
+          val thumbnailMap = Arguments.createMap()
+          if (thumbnail.uri != null) {
+            thumbnailMap.putString("uri", thumbnail.uri)
+          } else {
+            thumbnailMap.putNull("uri")
+          }
+          if (thumbnail.error != null) {
+            thumbnailMap.putString("error", thumbnail.error)
+          } else {
+            thumbnailMap.putNull("error")
+          }
+          result.pushMap(thumbnailMap)
+        }
+        promise.resolve(result)
+      } catch (error: Throwable) {
+        promise.reject("stream_video_thumbnail_error", error.message, error)
+      }
+    }
+  }
+
+  companion object {
+    const val NAME = "StreamVideoThumbnail"
+    private val executor = Executors.newCachedThreadPool()
+  }
+}