feat: add a WASM fallback backend

.github/workflows/ci.yml

@@ -0,0 +1,36 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [20, 22]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      # Pinned to the version this repo's scripts/build-wasm.sh was built
+      # and verified against -- see that file's header comment.
+      - uses: mymindstorm/setup-emsdk@v14
+        with:
+          version: 6.0.5
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Build native addon
+        run: npm run build
+
+      - name: Build WASM backend
+        run: npm run build:wasm
+
+      - name: Test
+        run: npm test

appveyor.yml

@@ -1,17 +1,15 @@
-os: Visual Studio 2015
+os: Visual Studio 2022
 version: "{build}"
 build: off
 platform: x64
 environment:
   matrix:
-  - nodejs_version: "8"
+  - nodejs_version: "20"
     nodejs_arch: "x64"
-  - nodejs_version: "10"
-    nodejs_arch: "x64"
-  - nodejs_version: "12"
+  - nodejs_version: "22"
     nodejs_arch: "x64"
 install:
   - ps: Install-Product node $env:nodejs_version $env:nodejs_arch
-  - npm install --msvs_version=2015
+  - npm install
 test_script:
   - npm test

bin/build-native.js

@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+// Runs as the package's "install" script. A failed native build here must
+// NOT fail the overall `npm install` -- lib/backend.js falls back to the
+// WASM backend at runtime when the native addon isn't present, but that
+// fallback is unreachable if npm never finishes installing in the first
+// place. So: try the native build, but always exit 0.
+
+const { spawnSync } = require('child_process');
+
+const result = spawnSync('node-gyp', ['rebuild'], {
+  stdio: 'inherit',
+  shell: true
+});
+
+if (result.status !== 0) {
+  console.warn(
+    '\n[cld] Native build failed (no working C++ toolchain for this ' +
+    'platform?) -- this is not fatal. cld will use its WASM fallback ' +
+    'backend at runtime instead of the native addon.\n'
+  );
+}
+
+process.exit(0);

bin/postinstall.js

@@ -8,7 +8,7 @@ deleteBuildFiles(function(err) {
 
 function deleteBuildFiles(cb) {
   var pattern = path.resolve(__dirname, '..', 'build', '**', '*');
-  glob(pattern, {nodir:true}, function(err, files) {
+  glob.globSync(pattern, {nodir:true}, function(err, files) {
     if (err) {
       return cb(err);
     }

binding.gyp

@@ -9,7 +9,7 @@
         "deps/cld/public",
         "<!@(node -p \"require('node-addon-api').include\")"
       ],
-      "sources": ["src/constants.cc", "src/cld.cc"],
+      "sources": ["src/constants.cc", "src/cld_core.cc", "src/cld.cc"],
       "cflags!": ["-fno-exceptions"],
       "cflags_cc!": ["-Wall", "-fno-exceptions"],
       "xcode_settings": {

index.d.ts

@@ -29,6 +29,11 @@ export const LANGUAGES: {[name: string]: string};
 export const DETECTED_LANGUAGES: string[];
 export const ENCODINGS: string[];
 
+// Browser entry point only (./wasm/browser-entry.mjs). Overrides where the
+// WASM backend fetches cld.web.wasm from -- has no effect on the native
+// backend. Must be called before the first detect() call.
+export declare function setWasmModuleOptions(options: { locateFile?: (path: string, scriptDirectory: string) => string }): void;
+
 export declare function detect(text: string, options: Options, callback: (err: string, result: DetectLanguage) => void): void;
 export declare function detect(text: string, callback: (err: string, result: DetectLanguage) => void): void;
 export declare function detect(text: string, options: Options): Promise<DetectLanguage>;

index.js

@@ -1,92 +1,10 @@
-const _    = require('underscore');
-const cld2 = require('./build/Release/cld');
+const meta             = require('./lib/metadata.json');
+const backend          = require('./lib/backend');
+const { createDetect } = require('./lib/detect-shape');
 
 module.exports = {
-  LANGUAGES          : cld2.LANGUAGES,
-  DETECTED_LANGUAGES : cld2.DETECTED_LANGUAGES,
-  ENCODINGS          : cld2.ENCODINGS,
-
-  async detect(text, options) {
-    let cb = arguments[2];
-    if (typeof cb !== 'function' && typeof options === 'function') {
-      cb = options;
-      options = {};
-    }
-
-    try {
-      if (arguments.length < 1) {
-        throw new Error('Not enough arguments provided');
-      }
-
-      if (!_.isString(text) || text.length < 1) {
-        throw new Error('Empty or invalid text');
-      }
-
-      const defaults = {
-        isHTML       : false,
-        languageHint : '',
-        encodingHint : '',
-        tldHint      : '',
-        httpHint     : '',
-        bestEffort   : false
-      };
-      options = _.defaults({}, options, defaults);
-
-      if (!_.isBoolean(options.isHTML)) {
-        throw new Error('Invalid isHTML value');
-      }
-      if (!_.isBoolean(options.bestEffort)) {
-        throw new Error('Invalid bestEffort value');
-      }
-      if (!_.isString(options.languageHint)) {
-        throw new Error('Invalid languageHint');
-      }
-      if (!_.isString(options.encodingHint)) {
-        throw new Error('Invalid encodingHint');
-      }
-      if (!_.isString(options.tldHint)) {
-        throw new Error('Invalid tldHint');
-      }
-      if (!_.isString(options.httpHint)) {
-        throw new Error('Invalid httpHint');
-      }
-      if (options.encodingHint.length > 0 &&
-        !~cld2.ENCODINGS.indexOf(options.encodingHint)) {
-
-        throw new Error('Invalid encodingHint, see ENCODINGS');
-      }
-      if (options.languageHint.length > 0 &&
-        !~_.keys(cld2.LANGUAGES).indexOf(options.languageHint) &&
-        !~_.values(cld2.LANGUAGES).indexOf(options.languageHint)) {
-
-        throw new Error('Invalid languageHint, see LANGUAGES');
-      }
-
-      const result = await cld2.detectAsync(
-        text,
-        !options.isHTML,
-        options.languageHint,
-        options.encodingHint,
-        options.tldHint,
-        options.httpHint,
-        options.bestEffort
-      );
-
-      if (result.languages.length < 1) {
-        throw new Error('Failed to identify language');
-      }
-
-      if (cb) {
-        return cb(null, result);
-      } else {
-        return result;
-      }
-    } catch (error) {
-      if (cb) {
-        cb(error);
-      } else {
-        throw error;
-      }
-    }
-  }
+  LANGUAGES          : meta.LANGUAGES,
+  DETECTED_LANGUAGES : meta.DETECTED_LANGUAGES,
+  ENCODINGS          : meta.ENCODINGS,
+  detect             : createDetect(backend.loadBackend, meta)
 };

lib/backend.js

@@ -0,0 +1,30 @@
+// Resolves which compiled backend (native N-API addon, or the WASM
+// fallback) actually runs detectAsync(). Both backends expose the same
+// {detectAsync(text, isPlainText, languageHint, encodingHint, tldHint,
+// httpHint, bestEffort)} shape, so index.js needs no branching beyond
+// awaiting loadBackend() once.
+
+const { wrapWasmModule } = require('./wasm-wrap');
+
+let backendPromise = null;
+
+function loadBackend() {
+  if (!backendPromise) {
+    backendPromise = (async () => {
+      try {
+        return require('../build/Release/cld');
+      } catch (nativeErr) {
+        return loadWasmBackend();
+      }
+    })();
+  }
+  return backendPromise;
+}
+
+async function loadWasmBackend() {
+  const createCldModule = require('../wasm/dist/cld.node.js');
+  const mod = await createCldModule();
+  return wrapWasmModule(mod);
+}
+
+module.exports = { loadBackend, loadWasmBackend };

lib/detect-shape.js

@@ -0,0 +1,95 @@
+// Shared detect() implementation used by both index.js (Node, native-first
+// with WASM fallback) and wasm/browser-entry.mjs (bundler/browser, WASM
+// only) -- the validation/defaults/error-mapping logic is identical for
+// both, only how the backend is loaded differs.
+
+const _ = require('underscore');
+
+function createDetect(loadBackend, meta) {
+  return async function detect(text, options) {
+    let cb = arguments[2];
+    if (typeof cb !== 'function' && typeof options === 'function') {
+      cb = options;
+      options = {};
+    }
+
+    try {
+      if (arguments.length < 1) {
+        throw new Error('Not enough arguments provided');
+      }
+
+      if (!_.isString(text) || text.length < 1) {
+        throw new Error('Empty or invalid text');
+      }
+
+      const defaults = {
+        isHTML       : false,
+        languageHint : '',
+        encodingHint : '',
+        tldHint      : '',
+        httpHint     : '',
+        bestEffort   : false
+      };
+      options = _.defaults({}, options, defaults);
+
+      if (!_.isBoolean(options.isHTML)) {
+        throw new Error('Invalid isHTML value');
+      }
+      if (!_.isBoolean(options.bestEffort)) {
+        throw new Error('Invalid bestEffort value');
+      }
+      if (!_.isString(options.languageHint)) {
+        throw new Error('Invalid languageHint');
+      }
+      if (!_.isString(options.encodingHint)) {
+        throw new Error('Invalid encodingHint');
+      }
+      if (!_.isString(options.tldHint)) {
+        throw new Error('Invalid tldHint');
+      }
+      if (!_.isString(options.httpHint)) {
+        throw new Error('Invalid httpHint');
+      }
+      if (options.encodingHint.length > 0 &&
+        !~meta.ENCODINGS.indexOf(options.encodingHint)) {
+
+        throw new Error('Invalid encodingHint, see ENCODINGS');
+      }
+      if (options.languageHint.length > 0 &&
+        !~_.keys(meta.LANGUAGES).indexOf(options.languageHint) &&
+        !~_.values(meta.LANGUAGES).indexOf(options.languageHint)) {
+
+        throw new Error('Invalid languageHint, see LANGUAGES');
+      }
+
+      const cld2 = await loadBackend();
+      const result = await cld2.detectAsync(
+        text,
+        !options.isHTML,
+        options.languageHint,
+        options.encodingHint,
+        options.tldHint,
+        options.httpHint,
+        options.bestEffort
+      );
+
+      if (result.languages.length < 1) {
+        throw new Error('Failed to identify language');
+      }
+
+      if (cb) {
+        return cb(null, result);
+      } else {
+        return result;
+      }
+    } catch (error) {
+      if (cb) {
+        cb(error);
+      } else {
+        throw error;
+      }
+    }
+  };
+}
+
+module.exports = { createDetect };